def wait_for_health_check(config: dict): default_port = get_default_port(config) health_check_port = config.get('health_check_port', default_port) health_check_path = config.get('health_check_path') health_check_timeout_seconds = get_or(config, 'health_check_timeout_seconds', 60) if not health_check_path: logging.info('Health check path is not configured, not waiting for health check') return if not health_check_port: logging.warning('Health check port is not configured, skipping health check') return url = 'http://localhost:{}{}'.format(health_check_port, health_check_path) start = time.time() while time.time() < start + health_check_timeout_seconds: logging.info('Waiting for health check :{}{}..'.format(health_check_port, health_check_path)) try: response = requests.get(url, timeout=5) if response.status_code == 200: logging.info('Health check returned OK') return except: pass time.sleep(2) logging.error('Timeout of {}s expired for health check :{}{}'.format( health_check_timeout_seconds, health_check_port, health_check_path)) sys.exit(2)
def get_other_options(config: dict): if not config.get('root'): # Docker only accepts UNIX user IDs (not names) entry = pwd.getpwnam('application') yield '-u' yield str(entry.pw_uid) for t in 'add', 'drop': for cap in get_or(config, 'capabilities_{}'.format(t), []): yield '--cap-{}={}'.format(t, cap) if config.get('hostname'): yield '--hostname={}'.format(config.get('hostname')) if config.get('networking'): yield '--net={}'.format(config.get('networking')) if config.get('privileged'): yield '--privileged' # Mount the container's root filesystem as read only if config.get('read_only'): yield '--read-only' # set shm_size if config.get('shm_size'): yield '--shm-size={}'.format(config.get('shm_size'))
def get_volume_options(config: dict): '''build Docker volume mount options''' for path, mount in get_or(config, 'mounts', {}).items(): yield '-v' # /opt/taupage/init.d/10-prepare-disks.py will mount the path below "/mounts" on the host system yield '{}:{}'.format('/mounts{}'.format(path), path) # meta directory, e.g. containing application credentials retrieved by berry yield '-v' # mount the meta directory as read-only filesystem yield '/meta:/meta:ro' # mount logdirectory as read-only if config.get('mount_var_log'): yield '-v' yield '/var/log:/var/log:ro' # if NewRelic Agent exisits than mount the agent to the docker container if 'newrelic_account_key' in config: # thats deprecated and has to be removed in some print('DEPRECATED WARNING: /data/newrelic will be removed please use /agents/newrelic instead ') yield '-v' yield '/opt/proprietary/newrelic:/data/newrelic:rw' yield '-v' yield '/opt/proprietary/newrelic:/agents/newrelic:rw' if 'appdynamics_application' in config: yield '-v' yield '/opt/proprietary/appdynamics-jvm:/agents/appdynamics-jvm:rw' yield '-e' yield 'CREDENTIALS_DIR={}'.format(CREDENTIALS_DIR)
def get_other_options(config: dict): if not config.get('root'): # Docker only accepts UNIX user IDs (not names) entry = pwd.getpwnam('application') yield '-u' yield str(entry.pw_uid) for t in 'add', 'drop': for cap in get_or(config, 'capabilities_{}'.format(t), []): yield '--cap-{}={}'.format(t, cap) if config.get('hostname'): yield '--hostname={}'.format(config.get('hostname')) if config.get('networking'): yield '--net={}'.format(config.get('networking')) if config.get('privileged'): yield '--privileged' # Mount the container's root filesystem as read only if config.get('read_only'): yield '--read-only' yield '--log-driver={}'.format(config.get('docker', {}).get('log_driver', 'syslog')) if config.get('docker', {}).get('container_name'): yield '--name={}'.format(config.get('docker', {}).get('container_name')) if config.get('docker', {}).get('log_opt'): for key, value in config.get('docker').get('log_opt').items(): yield '--log-opt' yield '{}={}'.format(key, value)
def get_env_options(config: dict): '''build Docker environment options''' # set OAuth2 token info URL # https://github.com/zalando-stups/taupage/issues/177 # NOTE: do this before processing "environment" # so users can overwrite TOKENINFO_URL tokeninfo_url = config.get('tokeninfo_url') if tokeninfo_url: yield '-e' yield 'TOKENINFO_URL={}'.format(tokeninfo_url) for key, val in get_or(config, 'environment', {}).items(): yield '-e' yield '{}={}'.format(key, decrypt(val)) if config.get('etcd_discovery_domain'): # TODO: use dynamic IP of docker0 yield '-e' yield 'ETCD_URL=http://172.17.0.1:2379' if config.get('appdynamics_application'): # set appdynamics analytics url yield '-e' yield 'APPDYNAMICS_ANALYTICS_URL=http://172.17.0.1:9090/v1/sinks/bt' # set APPLICATION_ID and APPLICATION_VERSION for convenience # NOTE: we should not add other environment variables here (even if it sounds tempting), # esp. EC2 metadata should not be passed as env. variables! for key in ('application_id', 'application_version'): yield '-e' yield '{}={}'.format(key.upper(), config.get(key))
def decrypt(val, config: dict): if str(val).startswith(AWS_KMS_PREFIX): ciphertext_blob = val[len(AWS_KMS_PREFIX):] ciphertext_blob = base64.b64decode(ciphertext_blob) conn = boto.kms.connect_to_region(get_region()) data = conn.decrypt(ciphertext_blob, encryption_context=get_or(config, 'encryption_context', None)) if 'Plaintext' not in data: raise Exception('KMS decrypt failed') return data['Plaintext'].decode('utf-8') else: return val
def get_volume_options(config: dict): '''build Docker volume mount options''' for path, mount in get_or(config, 'mounts', {}).items(): yield '-v' # /opt/taupage/init.d/10-prepare-disks.py will mount the path below "/mounts" on the host system yield '{}:{}'.format('/mounts{}'.format(path), path) # meta directory, e.g. containing application credentials retrieved by berry yield '-v' # mount the meta directory as read-only filesystem yield '/meta:/meta:ro' yield '-e' yield 'CREDENTIALS_DIR={}'.format(CREDENTIALS_DIR)
def get_volume_options(config: dict): '''build Docker volume mount options''' for path, mount in get_or(config, 'mounts', {}).items(): yield '-v' # /opt/taupage/init.d/10-prepare-disks.py will mount the path below "/mounts" on the host system yield '{}:{}'.format('/mounts{}'.format(path), path) # meta directory, e.g. containing application credentials retrieved by berry yield '-v' # mount the meta directory as read-only filesystem yield '/meta:/meta:ro' # mount newrelic agent into docker print('DEPRECATED WARNING: /data/newrelic will be removed please use /agents/newrelic instead ') yield '-v' yield '/opt/proprietary/newrelic:/data/newrelic:rw' yield '-v' yield '/opt/proprietary/newrelic:/agents/newrelic:rw' # mount logdirectory as read-only if config.get('mount_var_log'): yield '-v' yield '/var/log:/var/log:ro' # mount certs dir as read-only. 'private' is currently empty on Taupage if config.get('mount_certs'): yield '-v' yield '/etc/ssl/certs:/etc/ssl/certs:ro' # if NewRelic Agent exisits than mount the agent to the docker container # if 'newrelic_account_key' in config: # thats deprecated and has to be removed in some # if 'appdynamics_application' in config: # yield '-v' # yield '/opt/proprietary/appdynamics-jvm:/agents/appdynamics-jvm:rw' # typically, for continuous integration/delivery systems, you need to be able to builder # Docker images and there is no better solution currently. if config.get('docker_daemon_access'): yield '-v' yield '/var/run/docker.sock:/var/run/docker.sock' yield '-v' yield '/usr/bin/docker:/usr/bin/docker' yield '-e' yield 'CREDENTIALS_DIR={}'.format(CREDENTIALS_DIR)
def get_env_options(config: dict): '''build Docker environment options''' for key, val in get_or(config, 'environment', {}).items(): yield '-e' yield '{}={}'.format(key, decrypt(val)) if config.get('etcd_discovery_domain'): # TODO: use dynamic IP of docker0 yield '-e' yield 'ETCD_URL=http://172.17.42.1:2379' # set APPLICATION_ID and APPLICATION_VERSION for convenience # NOTE: we should not add other environment variables here (even if it sounds tempting), # esp. EC2 metadata should not be passed as env. variables! for key in ('application_id', 'application_version'): yield '-e' yield '{}={}'.format(key.upper(), config.get(key))
def get_volume_options(config: dict): '''build Docker volume mount options''' for path, mount in get_or(config, 'mounts', {}).items(): yield '-v' # /opt/taupage/init.d/10-prepare-disks.py will mount the path below "/mounts" on the host system yield '{}:{}'.format('/mounts{}'.format(path), path) # meta directory, e.g. containing application credentials retrieved by berry yield '-v' # mount the meta directory as read-only filesystem yield '/meta:/meta:ro' if config.get('newrelic_account_key'): # mount newrelic agent into docker print('DEPRECATED WARNING: /data/newrelic will be removed please use /agents/newrelic instead ') yield '-v' yield '/opt/proprietary/newrelic:/data/newrelic:rw' yield '-v' yield '/opt/proprietary/newrelic:/agents/newrelic:rw' # mount logdirectory as read-only if config.get('mount_var_log'): yield '-v' yield '/var/log:/var/log:ro' # mount certs dir as read-only. 'private' is currently empty on Taupage if config.get('mount_certs'): yield '-v' yield '/etc/ssl/certs:/etc/ssl/certs:ro' # if AppDynamics applicationname is in the config mount the agent to the container if 'appdynamics_application' in config: yield '-v' yield '/opt/proprietary/appdynamics-jvm:/agents/appdynamics-jvm:rw' # typically, for continuous integration/delivery systems, you need to be able to builder # Docker images and there is no better solution currently. if config.get('docker_daemon_access'): yield '-v' yield '/var/run/docker.sock:/var/run/docker.sock' yield '-v' yield '/usr/bin/docker:/usr/bin/docker' yield '-e' yield 'CREDENTIALS_DIR={}'.format(CREDENTIALS_DIR)
def get_port_options(config: dict): ''' >>> list(get_port_options({})) [] >>> list(get_port_options({'ports': {80: 8080}})) ['-p', '80:8080'] >>> list(get_port_options({'ports': {'80/udp': 8080}})) ['-p', '80:8080/udp'] ''' for host_port, container_port in get_or(config, 'ports', {}).items(): protocol = None if '/' in str(host_port): host_port, protocol = str(host_port).split('/') if protocol and '/' not in str(container_port): container_port = '{}/{}'.format(container_port, protocol) yield '-p' yield '{}:{}'.format(host_port, container_port)
def get_other_options(config: dict): if not config.get('root'): # Docker only accepts UNIX user IDs (not names) entry = pwd.getpwnam('application') yield '-u' yield str(entry.pw_uid) for t in 'add', 'drop': for cap in get_or(config, 'capabilities_{}'.format(t), []): yield '--cap-{}={}'.format(t, cap) if config.get('hostname'): yield '--hostname={}'.format(config.get('hostname')) if config.get('networking'): yield '--net={}'.format(config.get('networking')) if config.get('privileged'): yield '--privileged'
def get_env_options(config: dict): '''build Docker environment options''' for key, val in get_or(config, 'environment', {}).items(): yield '-e' yield '{}={}'.format(key, decrypt(val)) if config.get('etcd_discovery_domain'): # TODO: use dynamic IP of docker0 yield '-e' yield 'ETCD_URL=http://172.17.42.1:2379' # if config.get('appdynamics_application'): # set appdynamics analytics url # yield '-e' # yield 'APPDYNAMICS_ANALYTICS_URL=http://172.17.42.1:9090/v1/sinks/bt' # set APPLICATION_ID and APPLICATION_VERSION for convenience # NOTE: we should not add other environment variables here (even if it sounds tempting), # esp. EC2 metadata should not be passed as env. variables! for key in ('application_id', 'application_version'): yield '-e' yield '{}={}'.format(key.upper(), config.get(key))
def get_volume_options(config: dict): '''build Docker volume mount options''' for path, mount in get_or(config, 'mounts', {}).items(): yield '-v' # /opt/taupage/init.d/10-prepare-disks.py will mount the path below "/mounts" on the host system yield '{}:{}'.format('/mounts{}'.format(path), path) # meta directory, e.g. containing application credentials retrieved by berry yield '-v' # mount the meta directory as read-only filesystem yield '/meta:/meta:ro' # mount logdirectory as read-only if config.get('mount_var_log'): yield '-v' yield '/var/log:/var/log:ro' if 'appdynamics_application' in config: yield '-v' yield '/opt/proprietary/appdynamics-jvm:/agents/appdynamics-jvm:rw' yield '-e' yield 'CREDENTIALS_DIR={}'.format(CREDENTIALS_DIR)
def get_other_options(config: dict): if not config.get('root'): # Docker only accepts UNIX user IDs (not names) entry = pwd.getpwnam('application') yield '-u' yield str(entry.pw_uid) for t in 'add', 'drop': for cap in get_or(config, 'capabilities_{}'.format(t), []): yield '--cap-{}={}'.format(t, cap) if config.get('hostname'): yield '--hostname={}'.format(config.get('hostname')) if config.get('networking'): yield '--net={}'.format(config.get('networking')) if config.get('privileged'): yield '--privileged' # Mount the container's root filesystem as read only if config.get('read_only'): yield '--read-only' yield '--log-driver={}'.format( config.get('docker', {}).get('log_driver', 'syslog')) if config.get('docker', {}).get('container_name'): yield '--name={}'.format( config.get('docker', {}).get('container_name')) if config.get('docker', {}).get('log_opt'): for key, value in config.get('docker').get('log_opt').items(): yield '--log-opt' yield '{}={}'.format(key, value) if config.get('docker', {}).get('stop_timeout'): yield '--stop-timeout' yield str(config.get('docker', {}).get('stop_timeout'))
def get_volume_options(config: dict): '''build Docker volume mount options''' for path, mount in get_or(config, 'mounts', {}).items(): yield '-v' # /opt/taupage/init.d/10-prepare-disks.py will mount the path below "/mounts" on the host system yield '{}:{}'.format('/mounts{}'.format(path), path) # meta directory, e.g. containing application credentials retrieved by berry yield '-v' # mount the meta directory as read-only filesystem yield '/meta:/meta:ro' # if NewRelic Agent exisits than mount the agent to the docker container # TODO move newrelic to /opt/proprietary and mount to /agents/newrelic-jvm if 'newrelic_account_key' in config: yield '-v' yield '/opt/proprietary/newrelic:/data/newrelic:rw' if 'appdynamics_application' in config: yield '-v' yield '/opt/proprietary/appdynamics-jvm:/agents/appdynamics-jvm:rw' yield '-e' yield 'CREDENTIALS_DIR={}'.format(CREDENTIALS_DIR)
def get_volume_options(config: dict): '''build Docker volume mount options''' for path, mount in get_or(config, 'mounts', {}).items(): yield '-v' # /opt/taupage/init.d/10-prepare-disks.py will mount the path below "/mounts" on the host system yield '{}:{}'.format('/mounts{}'.format(path), path) # meta directory, e.g. containing application credentials retrieved by berry yield '-v' # mount the meta directory as read-only filesystem yield '/meta:/meta:ro' if config.get('newrelic_account_key'): # mount newrelic agent into docker print('DEPRECATED WARNING: /data/newrelic will be removed please use /agents/newrelic instead ') yield '-v' yield '/opt/proprietary/newrelic:/data/newrelic:rw' yield '-v' yield '/opt/proprietary/newrelic:/agents/newrelic:rw' # mount logdirectory as read-only if config.get('mount_var_log'): yield '-v' yield '/var/log:/var/log-host:ro' # mount custom log dir as read-write if config.get('mount_custom_log'): yield '-v' yield '/var/log-custom:/var/log:rw' # mount certs dir as read-only. 'private' is currently empty on Taupage if config.get('mount_certs'): yield '-v' yield '/etc/ssl/certs:/etc/ssl/certs:ro' # if AppDynamics applicationname is in the config and directory exists mount the agent & jobfiles to the container if 'appdynamics_application' in config: if os.path.isdir('/opt/proprietary/appdynamics-jvm'): yield '-v' yield '/opt/proprietary/appdynamics-jvm:/agents/appdynamics-jvm:rw' if os.path.isdir('/opt/proprietary/appdynamics-machine/monitors/analytics-agent/conf/job'): yield '-v' yield '/opt/proprietary/appdynamics-machine/monitors/analytics-agent/conf/job:/agents/jobfiles:rw' # typically, for continuous integration/delivery systems, you need to be able to build # Docker images and there is no better solution currently. if config.get('docker_daemon_access'): yield '-v' yield '/var/run/docker.sock:/var/run/docker.sock' yield '-v' yield '/usr/bin/docker:/usr/bin/docker' yield '-v' yield '/lib/x86_64-linux-gnu/libsystemd-journal.so.0:/lib/x86_64-linux-gnu/libsystemd-journal.so.0' yield '-v' yield '/lib/x86_64-linux-gnu/libcgmanager.so.0:/lib/x86_64-linux-gnu/libcgmanager.so.0' yield '-v' yield '/lib/x86_64-linux-gnu/libnih.so.1:/lib/x86_64-linux-gnu/libnih.so.1' yield '-v' yield '/lib/x86_64-linux-gnu/libnih-dbus.so.1:/lib/x86_64-linux-gnu/libnih-dbus.so.1' yield '-v' yield '/lib/x86_64-linux-gnu/libdbus-1.so.3:/lib/x86_64-linux-gnu/libdbus-1.so.3' yield '-v' yield '/lib/x86_64-linux-gnu/libgcrypt.so.11:/lib/x86_64-linux-gnu/libgcrypt.so.11' yield '-v' yield '/usr/lib/x86_64-linux-gnu/libapparmor.so.1:/usr/lib/x86_64-linux-gnu/libapparmor.so.1' yield '-v' yield '/usr/lib/x86_64-linux-gnu/libltdl.so.7:/usr/lib/x86_64-linux-gnu/libltdl.so.7' yield '-e' yield 'CREDENTIALS_DIR={}'.format(CREDENTIALS_DIR)