class Sloth(cls): def __init__(self, config): super().__init__(config) self._docker_config = extension['config'] self._docker_client = Client(self._docker_config.get('base_url'), timeout=10) self._docker_client._version = str( self._docker_config.get('version') or self._docker_client._version) self._docker_image = self._docker_config.get('image') or slugify( self.listen_point) def execute(self, action): '''Execute an action inside a container, then commit the changes to the image and remove the container. :param action: action to be executed :returns: True if the execution was successful; raises exception otherwise ''' self.processing_logger.info('Executing action: %s', action) try: container_id = self._docker_client.create_container( self._docker_image, command=action, working_dir=self.config.get('work_dir') or '.', mem_limit=self._docker_config.get('memory_limit') or 0, cpu_shares=self._docker_config.get('cpu_share') or None)['Id'] self._docker_client.start(container_id) for log in self._docker_client.attach(container_id, logs=True, stream=True): self.processing_logger.debug('%s', log) self._docker_client.commit(container_id, self._docker_image, message=action) self._docker_client.remove_container(container_id) self.processing_logger.info('Action executed: %s', action) return True except Exception: raise
class Docker_interface: def __init__(self, net_name='tosker_net', tmp_dir='/tmp', socket='unix://var/run/docker.sock'): self._log = Logger.get(__name__) self._net_name = net_name self._cli = Client(base_url=os.environ.get('DOCKER_HOST') or socket) self._tmp_dir = tmp_dir # TODO: aggiungere un parametro per eliminare i container se esistono gia'! def create(self, con, cmd=None, entrypoint=None, saved_image=False): def create_container(): tmp_dir = path.join(self._tmp_dir, con.name) try: os.makedirs(tmp_dir) except: pass saved_img_name = '{}/{}'.format(self._net_name, con.name) img_name = con.image if saved_image and self.inspect(saved_img_name): img_name = saved_img_name self._log.debug('container: {}'.format(con.get_str_obj())) con.id = self._cli.create_container( name=con.name, image=img_name, entrypoint=entrypoint if entrypoint else con.entrypoint, command=cmd if cmd else con.cmd, environment=con.env, detach=True, # stdin_open=True, ports=[key for key in con.ports.keys()] if con.ports else None, volumes=['/tmp/dt'] + ([k for k, v in con.volume.items()] if con.volume else []), networking_config=self._cli.create_networking_config({ self._net_name: self._cli.create_endpoint_config(links=con.link # ,aliases=['db'] ) }), host_config=self._cli.create_host_config( port_bindings=con.ports, # links=con.link, binds=[tmp_dir + ':/tmp/dt'] + ([v + ':' + k for k, v in con.volume.items()] if con.volume else []), )).get('Id') assert isinstance(con, Container) if con.to_build: self._log.debug('start building..') # utility.print_json( self._cli.build(path='/'.join(con.dockerfile.split('/')[0:-1]), dockerfile='./' + con.dockerfile.split('/')[-1], tag=con.image, pull=True, quiet=True) # ) self._log.debug('stop building..') elif not saved_image: # TODO: da evitare se si deve utilizzare un'immagine custom self._log.debug('start pulling.. {}'.format(con.image)) utility.print_json(self._cli.pull(con.image, stream=True), self._log.debug) self._log.debug('end pulling..') try: create_container() except errors.APIError as e: self._log.debug(e) # self.stop(con) self.delete(con) create_container() # raise e def stop(self, container): name = self._get_name(container) try: return self._cli.stop(name) except errors.NotFound as e: self._log.error(e) def start(self, container, wait=False): name = self._get_name(container) self._cli.start(name) if wait: self._log.debug('wait container..') self._cli.wait(name) utility.print_byte(self._cli.logs(name, stream=True), self._log.debug) def delete(self, container): name = self._get_name(container) try: self._cli.remove_container(name, v=True) except (errors.NotFound, errors.APIError) as e: self._log.error(e) raise e def exec_cmd(self, container, cmd): name = self._get_name(container) if not self.is_running(name): return False try: exec_id = self._cli.exec_create(name, cmd) status = self._cli.exec_start(exec_id) # TODO: verificare attendibilita' di questo check! check = 'rpc error:' != status[:10].decode("utf-8") self._log.debug('check: {}'.format(check)) return check except errors.APIError as e: self._log.error(e) return False except requests.exceptions.ConnectionError as e: # TODO: questo errore arriva dopo un timeout di 10 secodi self._log.error(e) return False def create_volume(self, volume): assert isinstance(volume, Volume) self._log.debug('volume opt: {}'.format(volume.get_all_opt())) return self._cli.create_volume(volume.name, volume.driver, volume.get_all_opt()) def delete_volume(self, volume): name = self._get_name(volume) return self._cli.remove_volume(name) def get_containers(self, all=False): return self._cli.containers(all=all) def get_volumes(self): volumes = self._cli.volumes() return volumes['Volumes'] or [] def inspect(self, item): name = self._get_name(item) try: return self._cli.inspect_container(name) except errors.NotFound: pass try: return self._cli.inspect_image(name) except errors.NotFound: pass try: return self._cli.inspect_volume(name) except errors.NotFound: return None def remove_all_containers(self): for c in self.get_containers(all=True): self.stop(c['Id']) self.delete(c['Id']) def remove_all_volumes(self): for v in self.get_volumes(): self.delete_volume(v['Name']) def create_network(self, name, subnet='172.25.0.0/16'): # docker network create -d bridge --subnet 172.25.0.0/16 isolated_nw # self.delete_network(name) try: self._cli.create_network(name=name, driver='bridge', ipam={'subnet': subnet}, check_duplicate=True) except errors.APIError: self._log.debug('network already exists!') def delete_network(self, name): assert isinstance(name, str) try: self._cli.remove_network(name) except errors.APIError: self._log.debug('network not exists!') def delete_image(self, name): assert isinstance(name, str) try: self._cli.remove_image(name) except errors.NotFound: pass # TODO: splittare questo metodo in due, semantica non chiara! def update_container(self, node, cmd, saved_image=True): assert isinstance(node, Container) # self._log.debug('container_conf: {}'.format(node.host_container)) stat = self.inspect(node.image) old_cmd = stat['Config']['Cmd'] or None old_entry = stat['Config']['Entrypoint'] or None if self.inspect(node): self.stop(node) self.delete(node) self.create(node, cmd=cmd, entrypoint='', saved_image=saved_image) self.start(node.id, wait=True) self.stop(node.id) name = '{}/{}'.format(self._net_name, node.name) self._cli.commit(node.id, name) self.stop(node) self.delete(node) self.create(node, cmd=node.cmd or old_cmd, entrypoint=node.entrypoint or old_entry, saved_image=True) self._cli.commit(node.id, name) def is_running(self, container): name = self._get_name(container) stat = self.inspect(name) stat = stat is not None and stat['State']['Running'] is True self._log.debug('State: {}'.format(stat)) return stat def _get_name(self, name): if isinstance(name, six.string_types): return name else: assert isinstance(name, (Container, Volume)) return name.name
class DockerCluster(object): IMAGE_NAME_BASE = os.path.join('teradatalabs', 'pa_test') BARE_CLUSTER_TYPE = 'bare' """Start/stop/control/query arbitrary clusters of docker containers. This class is aimed at product test writers to create docker containers for testing purposes. """ def __init__(self, master_host, slave_hosts, local_mount_dir, docker_mount_dir): # see PyDoc for all_internal_hosts() for an explanation on the # difference between an internal and regular host self.internal_master = master_host self.internal_slaves = slave_hosts self.master = master_host + '-' + str(uuid.uuid4()) self.slaves = [ slave + '-' + str(uuid.uuid4()) for slave in slave_hosts ] # the root path for all local mount points; to get a particular # container mount point call get_local_mount_dir() self.local_mount_dir = local_mount_dir self.mount_dir = docker_mount_dir kwargs = kwargs_from_env() if 'tls' in kwargs: kwargs['tls'].assert_hostname = False kwargs['timeout'] = 240 self.client = Client(**kwargs) self._DOCKER_START_TIMEOUT = 30 DockerCluster.__check_if_docker_exists() def all_hosts(self): return self.slaves + [self.master] def get_master(self): return self.master def all_internal_hosts(self): """The difference between this method and all_hosts() is that all_hosts() returns the unique, "outside facing" hostnames that docker uses. On the other hand all_internal_hosts() returns the more human readable host aliases for the containers used internally between containers. For example the unique master host will look something like 'master-07d1774e-72d7-45da-bf84-081cfaa5da9a', whereas the internal master host will be 'master'. Returns: List of all internal hosts with the random suffix stripped out. """ return [host.split('-')[0] for host in self.all_hosts()] def get_local_mount_dir(self, host): return os.path.join(self.local_mount_dir, self.__get_unique_host(host)) def get_dist_dir(self, unique): if unique: return os.path.join(DIST_DIR, self.master) else: return DIST_DIR def __get_unique_host(self, host): matches = [ unique_host for unique_host in self.all_hosts() if unique_host.startswith(host) ] if matches: return matches[0] elif host in self.all_hosts(): return host else: raise DockerClusterException( 'Specified host: {0} does not exist.'.format(host)) @staticmethod def __check_if_docker_exists(): try: subprocess.call(['docker', '--version']) except OSError: sys.exit('Docker is not installed. Try installing it with ' 'presto-admin/bin/install-docker.sh.') def create_image(self, path_to_dockerfile_dir, image_tag, base_image, base_image_tag=None): self.fetch_image_if_not_present(base_image, base_image_tag) output = self._execute_and_wait(self.client.build, path=path_to_dockerfile_dir, tag=image_tag, rm=True) if not self._is_image_present_locally(image_tag, 'latest'): raise OSError('Unable to build image %s: %s' % (image_tag, output)) def fetch_image_if_not_present(self, image, tag=None): if not tag and not self.client.images(image): self._execute_and_wait(self.client.pull, image) elif tag and not self._is_image_present_locally(image, tag): self._execute_and_wait(self.client.pull, image, tag) def _is_image_present_locally(self, image_name, tag): image_name_and_tag = image_name + ':' + tag images = self.client.images(image_name) if images: for image in images: if image_name_and_tag in image['RepoTags']: return True return False def start_containers(self, master_image, slave_image=None, cmd=None, **kwargs): self.tear_down() self._create_host_mount_dirs() self._create_and_start_containers(master_image, slave_image, cmd, **kwargs) self._ensure_docker_containers_started(master_image) def tear_down(self): for container_name in self.all_hosts(): self._tear_down_container(container_name) self._remove_host_mount_dirs() def _tear_down_container(self, container_name): try: shutil.rmtree(self.get_dist_dir(unique=True)) except OSError as e: # no such file or directory if e.errno != errno.ENOENT: raise try: self.stop_host(container_name) self.client.remove_container(container_name, v=True, force=True) except APIError as e: # container does not exist if e.response.status_code != 404: raise def stop_host(self, container_name): self.client.stop(container_name) self.client.wait(container_name) def start_host(self, container_name): self.client.start(container_name) def get_down_hostname(self, host_name): return host_name def _remove_host_mount_dirs(self): for container_name in self.all_hosts(): try: shutil.rmtree(self.get_local_mount_dir(container_name)) except OSError as e: # no such file or directory if e.errno != errno.ENOENT: raise def _create_host_mount_dirs(self): for container_name in self.all_hosts(): try: os.makedirs(self.get_local_mount_dir(container_name)) except OSError as e: # file exists if e.errno != errno.EEXIST: raise @staticmethod def _execute_and_wait(func, *args, **kwargs): ret = func(*args, **kwargs) # go through all lines in returned stream to ensure func finishes output = '' for line in ret: output += line return output def _create_and_start_containers(self, master_image, slave_image=None, cmd=None, **kwargs): if slave_image: for container_name in self.slaves: container_mount_dir = \ self.get_local_mount_dir(container_name) self._create_container(slave_image, container_name, container_name.split('-')[0], cmd) self.client.start(container_name, binds={ container_mount_dir: { 'bind': self.mount_dir, 'ro': False } }, **kwargs) master_mount_dir = self.get_local_mount_dir(self.master) self._create_container(master_image, self.master, hostname=self.internal_master, cmd=cmd) self.client.start( self.master, binds={master_mount_dir: { 'bind': self.mount_dir, 'ro': False }}, links=zip(self.slaves, self.slaves), **kwargs) self._add_hostnames_to_slaves() def _create_container(self, image, container_name, hostname=None, cmd=None): self._execute_and_wait(self.client.create_container, image, detach=True, name=container_name, hostname=hostname, volumes=self.local_mount_dir, command=cmd, mem_limit='2g') def _add_hostnames_to_slaves(self): ips = self.get_ip_address_dict() additions_to_etc_hosts = '' for host in self.all_internal_hosts(): additions_to_etc_hosts += '%s\t%s\n' % (ips[host], host) for host in self.slaves: self.exec_cmd_on_host( host, 'bin/bash -c \'echo "%s" >> /etc/hosts\'' % additions_to_etc_hosts) def _ensure_docker_containers_started(self, image): centos_based_images = [BASE_TD_IMAGE_NAME] timeout = 0 is_host_started = {} for host in self.all_hosts(): is_host_started[host] = False while timeout < self._DOCKER_START_TIMEOUT: for host in self.all_hosts(): atomic_is_started = True atomic_is_started &= \ self.client.inspect_container(host)['State']['Running'] if image in centos_based_images or \ image.startswith(self.IMAGE_NAME_BASE): atomic_is_started &= \ self._are_centos_container_services_up(host) is_host_started[host] = atomic_is_started if not DockerCluster._are_all_hosts_started(is_host_started): timeout += 1 sleep(1) else: break if timeout is self._DOCKER_START_TIMEOUT: raise DockerClusterException( 'Docker container timed out on start.' + str(is_host_started)) @staticmethod def _are_all_hosts_started(host_started_map): all_started = True for host in host_started_map.keys(): all_started &= host_started_map[host] return all_started def _are_centos_container_services_up(self, host): """Some essential services in our CentOS containers take some time to start after the container itself is up. This function checks whether those services are up and returns a boolean accordingly. Specifically, we check that the app-admin user has been created and that the ssh daemon is up. Args: host: the host to check. Returns: True if the specified services have started, False otherwise. """ ps_output = self.exec_cmd_on_host(host, 'ps') # also ensure that the app-admin user exists try: user_output = self.exec_cmd_on_host(host, 'grep app-admin /etc/passwd') user_output += self.exec_cmd_on_host(host, 'stat /home/app-admin') except OSError: user_output = '' if 'sshd_bootstrap' in ps_output or 'sshd\n' not in ps_output\ or not user_output: return False return True def exec_cmd_on_host(self, host, cmd, raise_error=True, tty=False): ex = self.client.exec_create(self.__get_unique_host(host), cmd, tty=tty) output = self.client.exec_start(ex['Id'], tty=tty) exit_code = self.client.exec_inspect(ex['Id'])['ExitCode'] if raise_error and exit_code: raise OSError(exit_code, output) return output @staticmethod def _get_master_image_name(cluster_type): return os.path.join(DockerCluster.IMAGE_NAME_BASE, '%s_master' % (cluster_type)) @staticmethod def _get_slave_image_name(cluster_type): return os.path.join(DockerCluster.IMAGE_NAME_BASE, '%s_slave' % (cluster_type)) @staticmethod def start_bare_cluster(): dc = DockerCluster master_name = dc._get_master_image_name(dc.BARE_CLUSTER_TYPE) slave_name = dc._get_slave_image_name(dc.BARE_CLUSTER_TYPE) centos_cluster = DockerCluster('master', ['slave1', 'slave2', 'slave3'], DEFAULT_LOCAL_MOUNT_POINT, DEFAULT_DOCKER_MOUNT_POINT) if not dc._check_for_images(master_name, slave_name): centos_cluster.create_image(BASE_TD_DOCKERFILE_DIR, master_name, BASE_IMAGE_NAME, BASE_IMAGE_TAG) centos_cluster.create_image(BASE_TD_DOCKERFILE_DIR, slave_name, BASE_IMAGE_NAME, BASE_IMAGE_TAG) centos_cluster.start_containers(master_name, slave_name) return centos_cluster @staticmethod def start_existing_images(cluster_type): dc = DockerCluster master_name = dc._get_master_image_name(cluster_type) slave_name = dc._get_slave_image_name(cluster_type) if not dc._check_for_images(master_name, slave_name): return None centos_cluster = DockerCluster('master', ['slave1', 'slave2', 'slave3'], DEFAULT_LOCAL_MOUNT_POINT, DEFAULT_DOCKER_MOUNT_POINT) centos_cluster.start_containers(master_name, slave_name) return centos_cluster @staticmethod def _check_for_images(master_image_name, slave_image_name): client = Client(timeout=180) images = client.images() has_master_image = False has_slave_image = False for image in images: if master_image_name in image['RepoTags'][0]: has_master_image = True if slave_image_name in image['RepoTags'][0]: has_slave_image = True return has_master_image and has_slave_image def commit_images(self, cluster_type): self.client.commit(self.master, self._get_master_image_name(cluster_type)) self.client.commit(self.slaves[0], self._get_slave_image_name(cluster_type)) def run_script_on_host(self, script_contents, host): temp_script = '/tmp/tmp.sh' self.write_content_to_host('#!/bin/bash\n%s' % script_contents, temp_script, host) self.exec_cmd_on_host(host, 'chmod +x %s' % temp_script) return self.exec_cmd_on_host(host, temp_script, tty=True) def write_content_to_host(self, content, path, host): filename = os.path.basename(path) dest_dir = os.path.dirname(path) host_local_mount_point = self.get_local_mount_dir(host) local_path = os.path.join(host_local_mount_point, filename) with open(local_path, 'w') as config_file: config_file.write(content) self.exec_cmd_on_host(host, 'mkdir -p ' + dest_dir) self.exec_cmd_on_host( host, 'cp %s %s' % (os.path.join(self.mount_dir, filename), dest_dir)) def copy_to_host(self, source_path, dest_host): shutil.copy(source_path, self.get_local_mount_dir(dest_host)) def get_ip_address_dict(self): ip_addresses = {} for host, internal_host in zip(self.all_hosts(), self.all_internal_hosts()): inspect = self.client.inspect_container(host) ip_addresses[host] = inspect['NetworkSettings']['IPAddress'] ip_addresses[internal_host] = \ inspect['NetworkSettings']['IPAddress'] return ip_addresses def _post_presto_install(self): for worker in self.slaves: self.run_script_on_host( 'sed -i /node.id/d /etc/presto/node.properties; ' 'uuid=$(uuidgen); ' 'echo node.id=$uuid >> /etc/presto/node.properties', worker) def postinstall(self, installer): from tests.product.standalone.presto_installer \ import StandalonePrestoInstaller _post_install_hooks = { StandalonePrestoInstaller: DockerCluster._post_presto_install } hook = _post_install_hooks.get(installer, None) if hook: hook(self)
class Docker(SnapshotableContainerBackend, SuspendableContainerBackend): """ Docker container backend powered by docker-py bindings. """ """ The prefix that is prepended to the name of created containers. """ CONTAINER_NAME_PREFIX = 'coco-' """ The prefix that is prepended to the name of created container snapshots. """ CONTAINER_SNAPSHOT_NAME_PREFIX = 'snapshot-' def __init__(self, base_url='unix://var/run/docker.sock', version=None, registry=None): """ Initialize a new Docker container backend. :param base_url: The URL or unix path to the Docker API endpoint. :param version: The Docker API version number (see docker version). :param registry: If set, created images will be pushed to this registery. """ try: self._client = Client(base_url=base_url, timeout=600, version=version) self._registry = registry except Exception as ex: raise ConnectionError(ex) def container_exists(self, container, **kwargs): """ :inherit. """ try: self._client.inspect_container(container) return True except DockerError as ex: if ex.response.status_code == requests.codes.not_found: return False raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def container_image_exists(self, image, **kwargs): """ :inherit. """ try: image = self._client.inspect_image(image) return True except DockerError as ex: if ex.response.status_code == requests.codes.not_found: return False raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def container_is_running(self, container, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError try: return self._client.inspect_container(container).get( 'State', {}).get('Running', {}) is True except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def container_is_suspended(self, container, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError try: return self._client.inspect_container(container).get( 'State', {}).get('Paused', {}) is True except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def container_snapshot_exists(self, snapshot, **kwargs): """ :inherit. """ return self.container_image_exists(snapshot, **kwargs) def create_container(self, username, uid, name, ports, volumes, cmd=None, base_url=None, image=None, clone_of=None, **kwargs): """ :inherit. """ name = "%su%i-%s" % (self.CONTAINER_NAME_PREFIX, uid, name) if self.container_exists(name): raise ContainerBackendError( "A container with that name already exists") if clone_of is not None and not self.container_exists(clone_of): raise ContainerNotFoundError( "Base container for the clone does not exist") # cloning if clone_of: # TODO: some way to ensure no regular image is created with that name image = self.create_container_image(clone_of, 'for-clone-' + name + '-at-' + str(int(time.time())), push=False) image_pk = image.get(ContainerBackend.KEY_PK) else: image_pk = image # bind mounts mount_points = [ vol.get(ContainerBackend.VOLUME_KEY_TARGET) for vol in volumes ] binds = map( lambda bind: "%s:%s" % (bind.get(ContainerBackend.VOLUME_KEY_SOURCE), bind.get(ContainerBackend.VOLUME_KEY_TARGET)), volumes) # port mappings port_mappings = {} for port in ports: port_mappings[port.get( ContainerBackend.PORT_MAPPING_KEY_INTERNAL)] = ( port.get(ContainerBackend.PORT_MAPPING_KEY_ADDRESS), port.get(ContainerBackend.PORT_MAPPING_KEY_EXTERNAL)) container = None try: if self._registry and not clone_of: parts = image_pk.split('/') if len(parts) > 2: # includes registry repository = parts[0] + '/' + parts[1] + '/' + parts[ 2].split(':')[0] tag = parts[2].split(':')[1] else: repository = image_pk.split(':')[0] tag = image_pk.split(':')[1] # FIXME: should be done automatically self._client.pull(repository=repository, tag=tag) container = self._client.create_container( image=image_pk, command=cmd, name=name, ports=[ port.get(ContainerBackend.PORT_MAPPING_KEY_INTERNAL) for port in ports ], volumes=mount_points, host_config=docker_utils.create_host_config( binds=binds, port_bindings=port_mappings), environment={ 'OWNER': username, 'BASE_URL': base_url }, detach=True) container = self.get_container(container.get('Id')) self.start_container(container.get(ContainerBackend.KEY_PK)) except Exception as ex: raise ContainerBackendError(ex) if clone_of is None: ret = container else: ret = { ContainerBackend.CONTAINER_KEY_CLONE_CONTAINER: container, ContainerBackend.CONTAINER_KEY_CLONE_IMAGE: image } return ret def create_container_image(self, container, name, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError full_image_name = self.get_internal_container_image_name( container, name) if self.container_image_exists(full_image_name): raise ContainerBackendError( "An image with that name already exists for the given container" ) if self._registry: parts = full_image_name.split('/') registry = parts[0] repository = parts[1] + '/' + parts[2].split(':')[0] tag = parts[2].split(':')[1] commit_name = registry + '/' + repository else: repository = full_image_name.split(':')[0] tag = full_image_name.split(':')[1] commit_name = repository try: self._client.commit(container=container, repository=commit_name, tag=tag) if self._registry and kwargs.get('push', True): self._client.push( repository=full_image_name, stream=False, insecure_registry=True # TODO: constructor? ) return {ContainerBackend.KEY_PK: full_image_name} except Exception as ex: print ex raise ContainerBackendError(ex) def create_container_snapshot(self, container, name, **kwargs): """ :inherit. """ return self.create_container_image( container, self.CONTAINER_SNAPSHOT_NAME_PREFIX + name, push=False) def delete_container(self, container, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError try: if self.container_is_suspended(container): self.resume_container(container) if self.container_is_running(container): self.stop_container(container) except: pass try: return self._client.remove_container(container=container, force=True) except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def delete_container_image(self, image, **kwargs): """ :inherit. """ if not self.container_image_exists(image): raise ContainerImageNotFoundError try: self._client.remove_image(image=image, force=True) except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerImageNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def delete_container_snapshot(self, snapshot, **kwargs): """ :inherit. """ try: self.delete_container_image(snapshot, **kwargs) except ContainerImageNotFoundError as ex: raise ContainerSnapshotNotFoundError except ContainerBackendError as ex: raise ex except Exception as ex: raise ContainerBackendError(ex) def exec_in_container(self, container, cmd, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError if not self.container_is_running( container) or self.container_is_suspended(container): raise IllegalContainerStateError try: exec_id = self._client.exec_create(container=container, cmd=cmd) return self._client.exec_start(exec_id=exec_id, stream=False) except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def get_container(self, container, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError try: container = self._client.inspect_container(container) return self.make_container_contract_conform(container) except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def get_container_image(self, image, **kwargs): """ :inherit. """ if not self.container_image_exists(image): raise ContainerImageNotFoundError try: self._client.inspect_image(image) return {ContainerBackend.KEY_PK: image} except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerImageNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def get_container_images(self, **kwargs): """ :inherit. """ try: images = [] for image in self._client.images(): if not self.is_container_snapshot(image): images.append( {ContainerBackend.KEY_PK: image.get('RepoTags')[0]}) return images except Exception as ex: raise ContainerBackendError(ex) def get_container_logs(self, container, **kwargs): """ :inherit. :param timestamps: If true, the log messages' timestamps are included. """ if not self.container_exists(container): raise ContainerNotFoundError timestamps = kwargs.get('timestamps') try: logs = self._client.logs(container=container, stream=False, timestamps=(timestamps is True)) return filter(lambda x: len(x) > 0, logs.split('\n')) # remove empty lines except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def get_container_snapshot(self, snapshot, **kwargs): """ :inherit. """ if not self.container_snapshot_exists(snapshot): raise ContainerSnapshotNotFoundError return next(sh for sh in self.get_container_snapshots() if sh.get(ContainerBackend.KEY_PK).startswith(snapshot)) def get_internal_container_image_name(self, container, name): """ Return the name how the image with name `name` for the given container is named internally. :param container: The container the snapshot belongs to. :param name: The image's name. """ if not self.container_exists(container): raise ContainerNotFoundError try: container = self._client.inspect_container(container) container_name = container.get('Name') container_name = re.sub( # i.e. coco-u2500-ipython r'^/' + self.CONTAINER_NAME_PREFIX + r'u(\d+)-(.+)$', # i.e. coco-u2500/ipython:shared-name self.CONTAINER_NAME_PREFIX + r'u\g<1>' + '/' + r'\g<2>' + ':' + name, container_name) if self._registry: container_name = self._registry + '/' + container_name return container_name except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def get_container_snapshots(self, **kwargs): """ :inherit. """ try: snapshots = [] for image in self._client.images(): if self.is_container_snapshot(image): snapshots.append( self.make_snapshot_contract_conform(image)) return snapshots except Exception as ex: raise ContainerBackendError(ex) def get_containers(self, only_running=False, **kwargs): """ :inherit. """ try: containers = [] for container in self._client.containers(all=(not only_running)): containers.append( self.make_container_contract_conform(container)) return containers except Exception as ex: raise ContainerBackendError(ex) def get_containers_snapshots(self, container, **kwargs): """ TODO: implement. """ raise NotImplementedError def get_status(self): """ :inherit. """ try: self._client.info() return ContainerBackend.BACKEND_STATUS_OK except Exception: return ContainerBackend.BACKEND_STATUS_ERROR def is_container_snapshot(self, image): """ Return true if `image` is internally used as a container snapshot. :param image: The image to check. """ parts = image.get('RepoTags', [' : '])[0].split(':') if len(parts) > 1: return parts[1].startswith(self.CONTAINER_SNAPSHOT_NAME_PREFIX) return False def make_container_contract_conform(self, container): """ Ensure the container dict returned from Docker is confirm with that the contract requires. :param container: The container to make conform. """ if not self.container_is_running(container.get('Id')): status = ContainerBackend.CONTAINER_STATUS_STOPPED elif self.container_is_suspended(container.get('Id')): status = SuspendableContainerBackend.CONTAINER_STATUS_SUSPENDED else: status = ContainerBackend.CONTAINER_STATUS_RUNNING return { ContainerBackend.KEY_PK: container.get('Id'), ContainerBackend.CONTAINER_KEY_STATUS: status } def make_snapshot_contract_conform(self, snapshot): """ Ensure the snapshot dict returned from Docker is confirm with that the contract requires. :param snapshot: The snapshot to make conform. """ return self.make_image_contract_conform(snapshot) def restart_container(self, container, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError try: return self._client.restart(container=container, timeout=0) except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerImageNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def restore_container_snapshot(self, container, snapshot, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError if not self.container_snapshot_exists(snapshot): raise ContainerSnapshotNotFoundError raise NotImplementedError def resume_container(self, container, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError if not self.container_is_running( container) or not self.container_is_suspended(container): raise IllegalContainerStateError try: return self._client.unpause(container=container) except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerImageNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def start_container(self, container, **kwargs): """ :inherit. :param kwargs: All optional arguments the docker-py library accepts as well. """ if not self.container_exists(container): raise ContainerNotFoundError # if self.container_is_running(container): # raise IllegalContainerStateError try: return self._client.start(container=container, **kwargs) except Exception as ex: raise ContainerBackendError(ex) def stop_container(self, container, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError # if not self.container_is_running(container): # raise IllegalContainerStateError try: self.resume_container(container) except: pass try: return self._client.stop(container=container, timeout=0) except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerImageNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def suspend_container(self, container, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError if not self.container_is_running( container): # or self.container_is_suspended(container): raise IllegalContainerStateError try: return self._client.pause(container=container) except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerImageNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex)
class Docker(SnapshotableContainerBackend, SuspendableContainerBackend): """ Docker container backend powered by docker-py bindings. """ """ The prefix that is prepended to the name of created containers. """ CONTAINER_NAME_PREFIX = 'coco-' """ The prefix that is prepended to the name of created container snapshots. """ CONTAINER_SNAPSHOT_NAME_PREFIX = 'snapshot-' def __init__(self, base_url='unix://var/run/docker.sock', version=None, registry=None ): """ Initialize a new Docker container backend. :param base_url: The URL or unix path to the Docker API endpoint. :param version: The Docker API version number (see docker version). :param registry: If set, created images will be pushed to this registery. """ try: self._client = Client( base_url=base_url, timeout=600, version=version ) self._registry = registry except Exception as ex: raise ConnectionError(ex) def container_exists(self, container, **kwargs): """ :inherit. """ try: self._client.inspect_container(container) return True except DockerError as ex: if ex.response.status_code == requests.codes.not_found: return False raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def container_image_exists(self, image, **kwargs): """ :inherit. """ try: image = self._client.inspect_image(image) return True except DockerError as ex: if ex.response.status_code == requests.codes.not_found: return False raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def container_is_running(self, container, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError try: return self._client.inspect_container(container).get('State', {}).get('Running', {}) is True except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def container_is_suspended(self, container, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError try: return self._client.inspect_container(container).get('State', {}).get('Paused', {}) is True except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def container_snapshot_exists(self, snapshot, **kwargs): """ :inherit. """ return self.container_image_exists(snapshot, **kwargs) def create_container(self, username, uid, name, ports, volumes, cmd=None, base_url=None, image=None, clone_of=None, **kwargs): """ :inherit. """ name = "%su%i-%s" % (self.CONTAINER_NAME_PREFIX, uid, name) if self.container_exists(name): raise ContainerBackendError("A container with that name already exists") if clone_of is not None and not self.container_exists(clone_of): raise ContainerNotFoundError("Base container for the clone does not exist") # cloning if clone_of: # TODO: some way to ensure no regular image is created with that name image = self.create_container_image(clone_of, 'for-clone-' + name + '-at-' + str(int(time.time())), push=False) image_pk = image.get(ContainerBackend.KEY_PK) else: image_pk = image # bind mounts mount_points = [vol.get(ContainerBackend.VOLUME_KEY_TARGET) for vol in volumes] binds = map( lambda bind: "%s:%s" % ( bind.get(ContainerBackend.VOLUME_KEY_SOURCE), bind.get(ContainerBackend.VOLUME_KEY_TARGET) ), volumes ) # port mappings port_mappings = {} for port in ports: port_mappings[port.get(ContainerBackend.PORT_MAPPING_KEY_INTERNAL)] = ( port.get(ContainerBackend.PORT_MAPPING_KEY_ADDRESS), port.get(ContainerBackend.PORT_MAPPING_KEY_EXTERNAL) ) container = None try: if self._registry and not clone_of: parts = image_pk.split('/') if len(parts) > 2: # includes registry repository = parts[0] + '/' + parts[1] + '/' + parts[2].split(':')[0] tag = parts[2].split(':')[1] else: repository = image_pk.split(':')[0] tag = image_pk.split(':')[1] # FIXME: should be done automatically self._client.pull( repository=repository, tag=tag ) container = self._client.create_container( image=image_pk, command=cmd, name=name, ports=[port.get(ContainerBackend.PORT_MAPPING_KEY_INTERNAL) for port in ports], volumes=mount_points, host_config=docker_utils.create_host_config( binds=binds, port_bindings=port_mappings ), environment={ 'OWNER': username, 'BASE_URL': base_url }, detach=True ) container = self.get_container(container.get('Id')) self.start_container(container.get(ContainerBackend.KEY_PK)) except Exception as ex: raise ContainerBackendError(ex) if clone_of is None: ret = container else: ret = { ContainerBackend.CONTAINER_KEY_CLONE_CONTAINER: container, ContainerBackend.CONTAINER_KEY_CLONE_IMAGE: image } return ret def create_container_image(self, container, name, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError full_image_name = self.get_internal_container_image_name(container, name) if self.container_image_exists(full_image_name): raise ContainerBackendError("An image with that name already exists for the given container") if self._registry: parts = full_image_name.split('/') registry = parts[0] repository = parts[1] + '/' + parts[2].split(':')[0] tag = parts[2].split(':')[1] commit_name = registry + '/' + repository else: repository = full_image_name.split(':')[0] tag = full_image_name.split(':')[1] commit_name = repository try: self._client.commit( container=container, repository=commit_name, tag=tag ) if self._registry and kwargs.get('push', True): self._client.push( repository=full_image_name, stream=False, insecure_registry=True # TODO: constructor? ) return { ContainerBackend.KEY_PK: full_image_name } except Exception as ex: print ex raise ContainerBackendError(ex) def create_container_snapshot(self, container, name, **kwargs): """ :inherit. """ return self.create_container_image(container, self.CONTAINER_SNAPSHOT_NAME_PREFIX + name, push=False) def delete_container(self, container, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError try: if self.container_is_suspended(container): self.resume_container(container) if self.container_is_running(container): self.stop_container(container) except: pass try: return self._client.remove_container(container=container, force=True) except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def delete_container_image(self, image, **kwargs): """ :inherit. """ if not self.container_image_exists(image): raise ContainerImageNotFoundError try: self._client.remove_image(image=image, force=True) except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerImageNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def delete_container_snapshot(self, snapshot, **kwargs): """ :inherit. """ try: self.delete_container_image(snapshot, **kwargs) except ContainerImageNotFoundError as ex: raise ContainerSnapshotNotFoundError except ContainerBackendError as ex: raise ex except Exception as ex: raise ContainerBackendError(ex) def exec_in_container(self, container, cmd, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError if not self.container_is_running(container) or self.container_is_suspended(container): raise IllegalContainerStateError try: exec_id = self._client.exec_create(container=container, cmd=cmd) return self._client.exec_start(exec_id=exec_id, stream=False) except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def get_container(self, container, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError try: container = self._client.inspect_container(container) return self.make_container_contract_conform(container) except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def get_container_image(self, image, **kwargs): """ :inherit. """ if not self.container_image_exists(image): raise ContainerImageNotFoundError try: self._client.inspect_image(image) return { ContainerBackend.KEY_PK: image } except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerImageNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def get_container_images(self, **kwargs): """ :inherit. """ try: images = [] for image in self._client.images(): if not self.is_container_snapshot(image): images.append({ ContainerBackend.KEY_PK: image.get('RepoTags')[0] }) return images except Exception as ex: raise ContainerBackendError(ex) def get_container_logs(self, container, **kwargs): """ :inherit. :param timestamps: If true, the log messages' timestamps are included. """ if not self.container_exists(container): raise ContainerNotFoundError timestamps = kwargs.get('timestamps') try: logs = self._client.logs( container=container, stream=False, timestamps=(timestamps is True) ) return filter(lambda x: len(x) > 0, logs.split('\n')) # remove empty lines except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def get_container_snapshot(self, snapshot, **kwargs): """ :inherit. """ if not self.container_snapshot_exists(snapshot): raise ContainerSnapshotNotFoundError return next(sh for sh in self.get_container_snapshots() if sh.get(ContainerBackend.KEY_PK).startswith(snapshot)) def get_internal_container_image_name(self, container, name): """ Return the name how the image with name `name` for the given container is named internally. :param container: The container the snapshot belongs to. :param name: The image's name. """ if not self.container_exists(container): raise ContainerNotFoundError try: container = self._client.inspect_container(container) container_name = container.get('Name') container_name = re.sub( # i.e. coco-u2500-ipython r'^/' + self.CONTAINER_NAME_PREFIX + r'u(\d+)-(.+)$', # i.e. coco-u2500/ipython:shared-name self.CONTAINER_NAME_PREFIX + r'u\g<1>' + '/' + r'\g<2>' + ':' + name, container_name ) if self._registry: container_name = self._registry + '/' + container_name return container_name except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def get_container_snapshots(self, **kwargs): """ :inherit. """ try: snapshots = [] for image in self._client.images(): if self.is_container_snapshot(image): snapshots.append(self.make_snapshot_contract_conform(image)) return snapshots except Exception as ex: raise ContainerBackendError(ex) def get_containers(self, only_running=False, **kwargs): """ :inherit. """ try: containers = [] for container in self._client.containers(all=(not only_running)): containers.append(self.make_container_contract_conform(container)) return containers except Exception as ex: raise ContainerBackendError(ex) def get_containers_snapshots(self, container, **kwargs): """ TODO: implement. """ raise NotImplementedError def get_status(self): """ :inherit. """ try: self._client.info() return ContainerBackend.BACKEND_STATUS_OK except Exception: return ContainerBackend.BACKEND_STATUS_ERROR def is_container_snapshot(self, image): """ Return true if `image` is internally used as a container snapshot. :param image: The image to check. """ parts = image.get('RepoTags', [' : '])[0].split(':') if len(parts) > 1: return parts[1].startswith(self.CONTAINER_SNAPSHOT_NAME_PREFIX) return False def make_container_contract_conform(self, container): """ Ensure the container dict returned from Docker is confirm with that the contract requires. :param container: The container to make conform. """ if not self.container_is_running(container.get('Id')): status = ContainerBackend.CONTAINER_STATUS_STOPPED elif self.container_is_suspended(container.get('Id')): status = SuspendableContainerBackend.CONTAINER_STATUS_SUSPENDED else: status = ContainerBackend.CONTAINER_STATUS_RUNNING return { ContainerBackend.KEY_PK: container.get('Id'), ContainerBackend.CONTAINER_KEY_STATUS: status } def make_snapshot_contract_conform(self, snapshot): """ Ensure the snapshot dict returned from Docker is confirm with that the contract requires. :param snapshot: The snapshot to make conform. """ return self.make_image_contract_conform(snapshot) def restart_container(self, container, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError try: return self._client.restart(container=container, timeout=0) except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerImageNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def restore_container_snapshot(self, container, snapshot, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError if not self.container_snapshot_exists(snapshot): raise ContainerSnapshotNotFoundError raise NotImplementedError def resume_container(self, container, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError if not self.container_is_running(container) or not self.container_is_suspended(container): raise IllegalContainerStateError try: return self._client.unpause(container=container) except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerImageNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def start_container(self, container, **kwargs): """ :inherit. :param kwargs: All optional arguments the docker-py library accepts as well. """ if not self.container_exists(container): raise ContainerNotFoundError # if self.container_is_running(container): # raise IllegalContainerStateError try: return self._client.start(container=container, **kwargs) except Exception as ex: raise ContainerBackendError(ex) def stop_container(self, container, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError # if not self.container_is_running(container): # raise IllegalContainerStateError try: self.resume_container(container) except: pass try: return self._client.stop(container=container, timeout=0) except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerImageNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex) def suspend_container(self, container, **kwargs): """ :inherit. """ if not self.container_exists(container): raise ContainerNotFoundError if not self.container_is_running(container): # or self.container_is_suspended(container): raise IllegalContainerStateError try: return self._client.pause(container=container) except DockerError as ex: if ex.response.status_code == requests.codes.not_found: raise ContainerImageNotFoundError raise ContainerBackendError(ex) except Exception as ex: raise ContainerBackendError(ex)
class DockerApi(object): """ """ def __init__(self, url): self.url = url self.cli = Client(base_url='%s:2375' % self.url, version='1.20', timeout=120) ################################################ #列出容器 # quiet (bool): Only display numeric Ids # all (bool): Show all containers. Only running containers are shown by default # trunc (bool): Truncate output # latest (bool): Show only the latest created container, include non-running ones. # since (str): Show only containers created since Id or Name, include non-running ones # before (str): Show only container created before Id or Name, include non-running ones # limit (int): Show limit last created containers, include non-running ones # size (bool): Display sizes # filters (dict): Filters to be processed on the image list. Available filters: # exited (int): Only containers with specified exit code # status (str): One of restarting, running, paused, exited # label (str): format either "key" or "key=value" # Returns (dict): The system's containers # >>> from docker import Client # >>> cli = Client(base_url='tcp://127.0.0.1:2375') # >>> cli.containers() # [{'Command': '/bin/sleep 30', # 'Created': 1412574844, # 'Id': '6e276c9e6e5759e12a6a9214efec6439f80b4f37618e1a6547f28a3da34db07a', # 'Image': 'busybox:buildroot-2014.02', # 'Names': ['/grave_mayer'], # 'Ports': [], # 'Status': 'Up 1 seconds'}] ################################################# def containers(self): container_all = self.cli.containers(all=True) return container_all ############################################### # exec_create # Sets up an exec instance in a running container. # Params: # container (str): Target container where exec instance will be created # cmd (str or list): Command to be executed # stdout (bool): Attach to stdout of the exec command if true. Default: True # stderr (bool): Attach to stderr of the exec command if true. Default: True # tty (bool): Allocate a pseudo-TTY. Default: False # user (str): User to execute command as. Default: root # Returns (dict): A dictionary with an exec 'Id' key. ############################################### def exec_create(self, container_name, cmd): return self.cli.exec_create(container=container_name, cmd=cmd) ############################################### # exec_start # Start a previously set up exec instance. # Params: # exec_id (str): ID of the exec instance # detach (bool): If true, detach from the exec command. Default: False # tty (bool): Allocate a pseudo-TTY. Default: False # stream (bool): Stream response data. Default: False ############################################### def exec_start(self, exec_id): return self.cli.exec_start(exec_id=exec_id) ################################################# #创建容器 # image (str): The image to run # command (str or list): The command to be run in the container # hostname (str): Optional hostname for the container # user (str or int): Username or UID # detach (bool): Detached mode: run container in the background and print new container Id # stdin_open (bool): Keep STDIN open even if not attached # tty (bool): Allocate a pseudo-TTY # mem_limit (float or str): Memory limit (format: [number][optional unit], where unit = b, k, m, or g) # ports (list of ints): A list of port numbers # environment (dict or list): A dictionary or a list of strings in the following format ["PASSWORD=xxx"] or {"PASSWORD": "******"}. # dns (list): DNS name servers # volumes (str or list): # volumes_from (str or list): List of container names or Ids to get volumes from. Optionally a single string joining container id's with commas # network_disabled (bool): Disable networking # name (str): A name for the container # entrypoint (str or list): An entrypoint # cpu_shares (int): CPU shares (relative weight) # working_dir (str): Path to the working directory # domainname (str or list): Set custom DNS search domains # memswap_limit (int): # host_config (dict): A HostConfig dictionary # mac_address (str): The Mac Address to assign the container # labels (dict or list): A dictionary of name-value labels (e.g. {"label1": "value1", "label2": "value2"}) or a list of names of labels to set with empty values (e.g. ["label1", "label2"]) # volume_driver (str): The name of a volume driver/plugin. # Returns (dict): A dictionary with an image 'Id' key and a 'Warnings' key. # >>> from docker import Client # >>> cli = Client(base_url='tcp://127.0.0.1:2375') # >>> container = cli.create_container(image='busybox:latest', command='/bin/sleep 30') # >>> print(container) # {'Id': '8a61192da2b3bb2d922875585e29b74ec0dc4e0117fcbf84c962204e97564cd7', # 'Warnings': None} ######################################################## def create_container(self, image, command, password): try: container = self.cli.create_container( image=image, command=command, environment={"PASSWORD": password}) return container except: return None ########################################################## # remove_container # Remove a container. Similar to the docker rm command. # Params: # container (str): The container to remove # v (bool): Remove the volumes associated with the container # link (bool): Remove the specified link and not the underlying container # force (bool): Force the removal of a running container (uses SIGKILL) ########################################################## def remove_container(self, container): return self.cli.remove_container(container=container, force=True) ########################################################## # container (str): The container to start # response = cli.start(container=container.get('Id')) # >>> print(response) ########################################################## def start(self, container_name): try: req = self.cli.start(container=container_name) return rep except: return None ########################################################## # container (str): The container to stop # timeout (int): Timeout in seconds to wait for the container to stop before sending a SIGKILL ########################################################## def stop(self, container_name): try: req = self.cli.stop(container=container_name) return rep except: return None ########################################################## #重启 #Restart a container. Similar to the docker restart command. # If container a dict, the Id key is used. # Params: # container (str or dict): The container to restart # timeout (int): Number of seconds to try to stop for before killing the container. Once killed it will then be restarted. Default is 10 seconds. ########################################################### def restart(self, container_name): try: req = self.cli.restart(container=container_name) return rep except: return None ################################################ # images # List images. Identical to the docker images command. # Params: # name (str): Only show images belonging to the repository name # quiet (bool): Only show numeric Ids. Returns a list # all (bool): Show all images (by default filter out the intermediate image layers) # filters (dict): Filters to be processed on the image list. Available filters: # dangling (bool) # label (str): format either "key" or "key=value" # Returns (dict or list): A list if quiet=True, otherwise a di ################################################ def images(self): # print type(self.cli.images()) return self.cli.images() ########################################################## # remove_image # Remove an image. Similar to the docker rmi command. # Params: # image (str): The image to remove # force (bool): Force removal of the image # noprune (bool): Do not delete untagged parents ########################################################## def remove_image(self, image): # print type(self.cli.images()) return self.cli.remove_image(image=image, force=True) ######################################################### # path (str): Path to the directory containing the Dockerfile # tag (str): A tag to add to the final image # quiet (bool): Whether to return the status # fileobj: A file object to use as the Dockerfile. (Or a file-like object) # nocache (bool): Don't use the cache when set to True # rm (bool): Remove intermediate containers. The docker build command now defaults to --rm=true, but we have kept the old default of False to preserve backward compatibility # stream (bool): Deprecated for API version > 1.8 (always True). Return a blocking generator you can iterate over to retrieve build output as it happens # timeout (int): HTTP timeout # custom_context (bool): Optional if using fileobj # encoding (str): The encoding for a stream. Set to gzip for compressing # pull (bool): Downloads any updates to the FROM image in Dockerfiles # forcerm (bool): Always remove intermediate containers, even after unsuccessful builds # dockerfile (str): path within the build context to the Dockerfile # container_limits (dict): A dictionary of limits applied to each container created by the build process. Valid keys: # memory (int): set memory limit for build # memswap (int): Total memory (memory + swap), -1 to disable swap # cpushares (int): CPU shares (relative weight) # cpusetcpus (str): CPUs in which to allow execution, e.g., "0-3", "0,1" # decode (bool): If set to True, the returned stream will be decoded into dicts on the fly. Default False. # >>> from io import BytesIO # >>> from docker import Client # >>> dockerfile = ''' # ... # Shared Volume # ... FROM busybox:buildroot-2014.02 # ... MAINTAINER first last, [email protected] # ... VOLUME /data # ... CMD ["/bin/sh"] # ... ''' # >>> f = BytesIO(dockerfile.encode('utf-8')) # >>> cli = Client(base_url='tcp://127.0.0.1:2375') # >>> response = [line for line in cli.build( # ... fileobj=f, rm=True, tag='yourname/volume' # ... )] # >>> response # ['{"stream":" ---\\u003e a9eb17255234\\n"}', ######################################################################## def build(self, dockerfile, tag): try: f = BytesIO(dockerfile.encode('utf-8')) res = [ line for line in self.cli.build(fileobj=f, rm=True, tag=tag) ] return res except: print traceback.format_exc() return None ########################################################################### #commit # container (str): The image hash of the container # repository (str): The repository to push the image to # tag (str): The tag to push # message (str): A commit message # author (str): The name of the author # conf (dict): The configuration for the container. See the Docker remote api for full details. ########################################################################### def commit(self, container, repository): try: res = self.cli.commit(container=container, repository=repository) return res except: return None ############################################################################ #pull # repository (str): The repository to pull # tag (str): The tag to pull # stream (bool): Stream the output as a generator # insecure_registry (bool): Use an insecure registry # auth_config (dict): Override the credentials that Client.login has set for this request auth_config should contain the username and password keys to be valid. # Returns (generator or str): The output # >>> from docker import Client # >>> cli = Client(base_url='tcp://127.0.0.1:2375') # >>> for line in cli.pull('busybox', stream=True): # ... print(json.dumps(json.loads(line), indent=4)) # { # "status": "Pulling image (latest) from busybox", # "progressDetail": {}, # "id": "e72ac664f4f0" # } # { # "status": "Pulling image (latest) from busybox, endpoint: ...", # "progressDetail": {}, # "id": "e72ac664f4f0" # } ############################################################################ def pull(self, repository): try: # res = cli.pull(repository,stream=True) res = [line for line in self.cli.pull(repository, stream=True)] return res except: print traceback.format_exc() return None ############################################################################ #push # repository (str): The repository to push to # tag (str): An optional tag to push # stream (bool): Stream the output as a blocking generator # insecure_registry (bool): Use http:// to connect to the registry # Returns (generator or str): The output of the upload # >>> from docker import Client # >>> cli = Client(base_url='tcp://127.0.0.1:2375') # >>> response = [line for line in cli.push('yourname/app', stream=True)] # >>> response # ['{"status":"Pushing repository yourname/app (1 tags)"}\\n', # '{"status":"Pushing","progressDetail":{},"id":"511136ea3c5a"}\\n', # '{"status":"Image already pushed, skipping","progressDetail":{}, # "id":"511136ea3c5a"}\\n', # ... # '{"status":"Pushing tag for rev [918af568e6e5] on { # https://cdn-registry-1.docker.io/v1/repositories/ # yourname/app/tags/latest}"}\\n'] ############################################################################ def push(self, repository): try: # res = cli.push(repository,stream=True) res = [ line for line in self.cli.push(repository=repository, stream=True) ] return res except: print traceback.format_exc() return None
class DockerCluster(BaseCluster): IMAGE_NAME_BASE = os.path.join('teradatalabs', 'pa_test') BARE_CLUSTER_TYPE = 'bare' """Start/stop/control/query arbitrary clusters of docker containers. This class is aimed at product test writers to create docker containers for testing purposes. """ def __init__(self, master_host, slave_hosts, local_mount_dir, docker_mount_dir): # see PyDoc for all_internal_hosts() for an explanation on the # difference between an internal and regular host self.internal_master = master_host self.internal_slaves = slave_hosts self._master = master_host + '-' + str(uuid.uuid4()) self.slaves = [slave + '-' + str(uuid.uuid4()) for slave in slave_hosts] # the root path for all local mount points; to get a particular # container mount point call get_local_mount_dir() self.local_mount_dir = local_mount_dir self._mount_dir = docker_mount_dir kwargs = kwargs_from_env() if 'tls' in kwargs: kwargs['tls'].assert_hostname = False kwargs['timeout'] = 300 self.client = Client(**kwargs) self._user = '******' DockerCluster.__check_if_docker_exists() def all_hosts(self): return self.slaves + [self.master] def all_internal_hosts(self): return [host.split('-')[0] for host in self.all_hosts()] def get_local_mount_dir(self, host): return os.path.join(self.local_mount_dir, self.__get_unique_host(host)) def get_dist_dir(self, unique): if unique: return os.path.join(DIST_DIR, self.master) else: return DIST_DIR def __get_unique_host(self, host): matches = [unique_host for unique_host in self.all_hosts() if unique_host.startswith(host)] if matches: return matches[0] elif host in self.all_hosts(): return host else: raise DockerClusterException( 'Specified host: {0} does not exist.'.format(host)) @staticmethod def __check_if_docker_exists(): try: subprocess.call(['docker', '--version']) except OSError: sys.exit('Docker is not installed. Try installing it with ' 'presto-admin/bin/install-docker.sh.') def _is_image_present_locally(self, image_name, tag): image_name_and_tag = image_name + ':' + tag images = self.client.images(image_name) if images: for image in images: if image_name_and_tag in image['RepoTags']: return True return False def start_containers(self, master_image, slave_image=None, cmd=None, **kwargs): self._create_host_mount_dirs() self._create_and_start_containers(master_image, slave_image, cmd, **kwargs) self._ensure_docker_containers_started(master_image) def tear_down(self): for container_name in self.all_hosts(): self._tear_down_container(container_name) self._remove_host_mount_dirs() if self.client: self.client.close() self.client = None def _tear_down_container(self, container_name): try: shutil.rmtree(self.get_dist_dir(unique=True)) except OSError as e: # no such file or directory if e.errno != errno.ENOENT: raise try: self.stop_host(container_name) self.client.remove_container(container_name, v=True, force=True) except APIError as e: # container does not exist if e.response.status_code != 404: raise def stop_host(self, container_name): self.client.stop(container_name) self.client.wait(container_name) def start_host(self, container_name): self.client.start(container_name) def get_down_hostname(self, host_name): return host_name def _remove_host_mount_dirs(self): for container_name in self.all_hosts(): try: shutil.rmtree( self.get_local_mount_dir(container_name)) except OSError as e: # no such file or directory if e.errno != errno.ENOENT: raise def _create_host_mount_dirs(self): for container_name in self.all_hosts(): try: os.makedirs( self.get_local_mount_dir(container_name)) except OSError as e: # file exists if e.errno != errno.EEXIST: raise @staticmethod def _execute_and_wait(func, *args, **kwargs): ret = func(*args, **kwargs) # go through all lines in returned stream to ensure func finishes output = '' for line in ret: output += line return output def _create_and_start_containers(self, master_image, slave_image=None, cmd=None, **kwargs): if slave_image: for container_name in self.slaves: container_mount_dir = \ self.get_local_mount_dir(container_name) self._create_container( slave_image, container_name, container_name.split('-')[0], cmd ) self.client.start(container_name, binds={container_mount_dir: {'bind': self.mount_dir, 'ro': False}}, **kwargs) master_mount_dir = self.get_local_mount_dir(self.master) self._create_container( master_image, self.master, hostname=self.internal_master, cmd=cmd ) self.client.start(self.master, binds={master_mount_dir: {'bind': self.mount_dir, 'ro': False}}, links=zip(self.slaves, self.slaves), **kwargs) self._add_hostnames_to_slaves() def _create_container(self, image, container_name, hostname=None, cmd=None): self._execute_and_wait(self.client.create_container, image, detach=True, name=container_name, hostname=hostname, volumes=self.local_mount_dir, command=cmd, host_config={'mem_limit': '2g'}) def _add_hostnames_to_slaves(self): ips = self.get_ip_address_dict() additions_to_etc_hosts = '' for host in self.all_internal_hosts(): additions_to_etc_hosts += '%s\t%s\n' % (ips[host], host) for host in self.slaves: self.exec_cmd_on_host( host, 'bin/bash -c \'echo "%s" >> /etc/hosts\'' % additions_to_etc_hosts ) @retry(stop_max_delay=_DOCKER_START_TIMEOUT, wait_fixed=_DOCKER_START_WAIT) def _ensure_docker_containers_started(self, image): # Strip off the tag, if there is one. We don't want to have to update # the NO_WAIT_SSH_IMAGES list every time we update the docker images. image_no_tag = image.split(':')[0] host_started = {} for host in self.all_hosts(): host_started[host] = False for host in host_started.keys(): if host_started[host]: continue is_started = True is_started &= \ self.client.inspect_container(host)['State']['Running'] if is_started and image_no_tag not in NO_WAIT_SSH_IMAGES: is_started &= self._are_centos_container_services_up(host) host_started[host] = is_started not_started = [host for (host, started) in host_started.items() if not started] if len(not_started): raise NotStartedException(not_started) @staticmethod def _are_all_hosts_started(host_started_map): all_started = True for host in host_started_map.keys(): all_started &= host_started_map[host] return all_started def _are_centos_container_services_up(self, host): """Some essential services in our CentOS containers take some time to start after the container itself is up. This function checks whether those services are up and returns a boolean accordingly. Specifically, we check that the app-admin user has been created and that the ssh daemon is up, as well as that the SSH keys are in the right place. Args: host: the host to check. Returns: True if the specified services have started, False otherwise. """ ps_output = self.exec_cmd_on_host(host, 'ps') # also ensure that the app-admin user exists try: user_output = self.exec_cmd_on_host( host, 'grep app-admin /etc/passwd' ) user_output += self.exec_cmd_on_host(host, 'stat /home/app-admin') except OSError: user_output = '' if 'sshd_bootstrap' in ps_output or 'sshd\n' not in ps_output\ or not user_output: return False # check for .ssh being in the right place try: ssh_output = self.exec_cmd_on_host(host, 'ls /home/app-admin/.ssh') if 'id_rsa' not in ssh_output: return False except OSError: return False return True def exec_cmd_on_host(self, host, cmd, user=None, raise_error=True, tty=False, invoke_sudo=False): ex = self.client.exec_create(self.__get_unique_host(host), ['sh', '-c', cmd], tty=tty, user=user) output = self.client.exec_start(ex['Id'], tty=tty) exit_code = self.client.exec_inspect(ex['Id'])['ExitCode'] if raise_error and exit_code: raise OSError(exit_code, output) return output @staticmethod def _get_tag_basename(bare_image_provider, cluster_type, ms): return '_'.join( [bare_image_provider.get_tag_decoration(), cluster_type, ms]) @staticmethod def _get_master_image_name(bare_image_provider, cluster_type): return os.path.join(DockerCluster.IMAGE_NAME_BASE, DockerCluster._get_tag_basename( bare_image_provider, cluster_type, 'master')) @staticmethod def _get_slave_image_name(bare_image_provider, cluster_type): return os.path.join(DockerCluster.IMAGE_NAME_BASE, DockerCluster._get_tag_basename( bare_image_provider, cluster_type, 'slave')) @staticmethod def _get_image_names(bare_image_provider, cluster_type): dc = DockerCluster return (dc._get_master_image_name(bare_image_provider, cluster_type), dc._get_slave_image_name(bare_image_provider, cluster_type)) @staticmethod def start_cluster(bare_image_provider, cluster_type, master_host='master', slave_hosts=None, **kwargs): if slave_hosts is None: slave_hosts = ['slave1', 'slave2', 'slave3'] created_bare = False dc = DockerCluster centos_cluster = DockerCluster(master_host, slave_hosts, DEFAULT_LOCAL_MOUNT_POINT, DEFAULT_DOCKER_MOUNT_POINT) master_name, slave_name = dc._get_image_names( bare_image_provider, cluster_type) if not dc._check_for_images(master_name, slave_name): master_name, slave_name = dc._get_image_names( bare_image_provider, dc.BARE_CLUSTER_TYPE) if not dc._check_for_images(master_name, slave_name): bare_image_provider.create_bare_images( centos_cluster, master_name, slave_name) created_bare = True centos_cluster.start_containers(master_name, slave_name, **kwargs) return centos_cluster, created_bare @staticmethod def _check_for_images(master_image_name, slave_image_name, tag='latest'): master_repotag = '%s:%s' % (master_image_name, tag) slave_repotag = '%s:%s' % (slave_image_name, tag) with Client(timeout=180) as client: images = client.images() has_master_image = False has_slave_image = False for image in images: if image['RepoTags'] is not None and master_repotag in image['RepoTags']: has_master_image = True if image['RepoTags'] is not None and slave_repotag in image['RepoTags']: has_slave_image = True return has_master_image and has_slave_image def commit_images(self, bare_image_provider, cluster_type): self.client.commit(self.master, self._get_master_image_name(bare_image_provider, cluster_type)) if self.slaves: self.client.commit(self.slaves[0], self._get_slave_image_name(bare_image_provider, cluster_type)) def run_script_on_host(self, script_contents, host, tty=True): temp_script = '/tmp/tmp.sh' self.write_content_to_host('#!/bin/bash\n%s' % script_contents, temp_script, host) self.exec_cmd_on_host(host, 'chmod +x %s' % temp_script) return self.exec_cmd_on_host(host, temp_script, tty=tty) def write_content_to_host(self, content, path, host): filename = os.path.basename(path) dest_dir = os.path.dirname(path) host_local_mount_point = self.get_local_mount_dir(host) local_path = os.path.join(host_local_mount_point, filename) with open(local_path, 'w') as config_file: config_file.write(content) self.exec_cmd_on_host(host, 'mkdir -p ' + dest_dir) self.exec_cmd_on_host( host, 'cp %s %s' % (os.path.join(self.mount_dir, filename), dest_dir)) def copy_to_host(self, source_path, dest_host, **kwargs): shutil.copy(source_path, self.get_local_mount_dir(dest_host)) def get_ip_address_dict(self): ip_addresses = {} for host, internal_host in zip(self.all_hosts(), self.all_internal_hosts()): inspect = self.client.inspect_container(host) ip_addresses[host] = inspect['NetworkSettings']['IPAddress'] ip_addresses[internal_host] = \ inspect['NetworkSettings']['IPAddress'] return ip_addresses def _post_presto_install(self): for worker in self.slaves: self.run_script_on_host( 'sed -i /node.id/d /etc/presto/node.properties; ' 'uuid=$(uuidgen); ' 'echo node.id=$uuid >> /etc/presto/node.properties', worker ) def postinstall(self, installer): from tests.product.standalone.presto_installer \ import StandalonePrestoInstaller _post_install_hooks = { StandalonePrestoInstaller: DockerCluster._post_presto_install } hook = _post_install_hooks.get(installer, None) if hook: hook(self) @property def rpm_cache_dir(self): return self._mount_dir @property def mount_dir(self): return self._mount_dir @property def user(self): return self._user @property def master(self): return self._master
class DockerClient: '''A singleton class to ensure there is only one client''' _instance = None def __new__(cls, *args, **kwargs): if not cls._instance: cls._instance = super(DockerClient, cls).__new__(cls) return cls._instance def __init__(self): self.client = Client(**kwargs_from_env(assert_hostname=False)) try: self.client.info() except Exception as e: self.client = None def _is_image_avail(self, image): images = sum([x['RepoTags'] for x in self.client.images()], []) return (':' in image and image in images) or \ (':' not in image and '{}:latest'.format(image) in images) def stream(self, line): # properly output streamed output try: sys.stdout.write(json.loads(line).get('stream', '')) except ValueError: # sometimes all the data is sent on a single line ???? # # ValueError: Extra data: line 1 column 87 - line 1 column # 33268 (char 86 - 33267) # This ONLY works because every line is formatted as # {"stream": STRING} for obj in re.findall('{\s*"stream"\s*:\s*"[^"]*"\s*}', line): sys.stdout.write(json.loads(obj).get('stream', '')) def build(self, script, **kwargs): if not self.client: raise RuntimeError('Cannot connect to the Docker daemon. Is the docker daemon running on this host?') if script is not None: f = BytesIO(script.encode('utf-8')) for line in self.client.build(fileobj=f, **kwargs): self.stream(line.decode()) else: for line in self.client.build(**kwargs): self.stream(line.decode()) # if a tag is given, check if the image is built if 'tag' in kwargs and not self._is_image_avail(kwargs['tag']): raise RuntimeError('Image with tag {} is not created.'.format(kwargs['tag'])) def import_image(self, image, **kwargs): if not self.client: raise RuntimeError('Cannot connect to the Docker daemon. Is the docker daemon running on this host?') env.logger.info('docker import {}'.format(image)) self.client.import_image(image, **kwargs) def pull(self, image): if not self.client: raise RuntimeError('Cannot connect to the Docker daemon. Is the docker daemon running on this host?') # if image is specified, check if it is available locally. If not, pull it ret = 0 if not self._is_image_avail(image): env.logger.info('docker pull {}'.format(image)) # using subprocess instead of docker-py's pull function because this would have # much better progress bar display ret = subprocess.call('docker pull {}'.format(image), shell=True) #for line in self.client.pull(image, stream=True): # self.stream(line) if not self._is_image_avail(image): raise RuntimeError('Failed to pull image {}'.format(image)) return ret def commit(self, **kwargs): if not self.client: raise RuntimeError('Cannot connect to the Docker daemon. Is the docker daemon running on this host?') for line in self.client.commit(**kwargs): self.stream(line.decode()) return 0 def run(self, image, script='', interpreter='', suffix='.sh', **kwargs): if self.client is None: raise RuntimeError('Cannot connect to the Docker daemon. Is the docker daemon running on this host?') env.logger.debug('docker_run with keyword args {}'.format(kwargs)) # now, write a temporary file to a tempoary directory under the current directory, this is because # we need to share the directory to ... with tempfile.TemporaryDirectory(dir=os.getcwd()) as tempdir: # keep the temporary script for debugging purposes # tempdir = tempfile.mkdtemp(dir=os.getcwd()) if script: tempscript = 'docker_run_{}{}'.format(os.getpid(), suffix) with open(os.path.join(tempdir, tempscript), 'w') as script_file: script_file.write(script) # binds = [] if 'volumes' in kwargs: volumes = [kwargs['volumes']] if isinstance(kwargs['volumes'], str) else kwargs['volumes'] for vol in volumes: if not vol: continue if vol.count(':') != 1: raise RuntimeError('Please specify columes in the format of host_dir:mnt_dir') host_dir, mnt_dir = vol.split(':') if platform.system() == 'Darwin': # under Darwin, host_dir must be under /Users if not os.path.abspath(host_dir).startswith('/Users'): raise RuntimeError('hostdir ({}) under MacOSX must be under /Users to be usable in docker container'.format(host_dir)) binds.append('{}:{}'.format(os.path.abspath(host_dir), mnt_dir)) # we also need to mount the script if script and interpreter: binds.append('{}:{}'.format(os.path.join(tempdir, tempscript), '/var/lib/sos/{}'.format(tempscript))) cmd = interpreter.replace('{}', '/var/lib/sos/{}'.format(tempscript)) else: cmd = '' command = 'docker run -t --rm {} {} {}'.format(' '.join('-v ' +x for x in binds), image, cmd) env.logger.info(command) ret = subprocess.call(command, shell=True) if ret != 0: raise RuntimeError('Executing script in docker returns an error') return 0
class dockerizer(default_logger): def __init__(self, args): default_logger.__init__(self, "dockerizer") self.args = args self.config = self.get_config() self.args["eula"] = self.config["installer"]["eula"] self.directory = (os.path.dirname(os.path.realpath(__file__))) self.base_image_name = args["base_image"] self.docker = Client(base_url='unix://var/run/docker.sock') self.installer = jedox_installer(args) self.installer.start() sleep(15) self.installer.stop() sleep(15) self.patch() self.add() self.build_base_image(self.base_image_name) self.base_container = self.docker.create_container( self.base_image_name) self.docker.start(self.base_container) self.docker_exec(self.base_container, self.config["docker"]["exec"]) self.commit(self.args["docker_repository"], self.args["docker_tag"]) #remove intermediate container self.logger.info("removing base container") self.docker.remove_container(container=self.base_container, force=True) def get_config(self): try: config_file = self.args["config"] version = self.args["jedox_version"] j = json.load(open(config_file)) return j[version] except KeyError as e: self.logger.exception(e) self.logger.error( "Could not find the right config for version=%s in file=%s \n Aborting..." % (version, config_file)) sys.exit(1) def patch(self): self.logger.info("patching files from installer") self.change_working_directory("patch") for p in self.config["patch"]: target = os.path.join(self.args["jedox_home"], p["target"]) description = p.get("description", p["target"]) self.logger.info("patching : %s" % description) subprocess.check_call("patch %s < %s" % (target, p["source"]), shell=True) def add(self): self.logger.info("adding additional content to installation") self.change_working_directory("add") for a in self.config["add"]: target = os.path.join(self.args["jedox_home"], a["target"]) self.logger.info("copy %s to %s" % (a["source"], target)) shutil.copy(a["source"], target) def change_working_directory(self, area): working_directory = os.path.join(self.directory, area, self.args["jedox_version"]) self.logger.info("working dir is now %s" % working_directory) os.chdir(working_directory) def build_base_image(self, image_name="jedox/base"): os.chdir(self.args["jedox_home"]) self.logger.info( "Import Jedox Suite into intermediate docker image '%s'" % image_name) subprocess.check_call( """tar --to-stdout --numeric-owner --exclude=/proc --exclude=/sys --exclude='*.tar.gz' --exclude='*.log' -c ./ | docker import --change "CMD while true; do ping 8.8.8.8; done" --change "ENV TERM=xterm" - %s""" % image_name, shell=True) self.logger.info("successfully create basecontainer %s" % image_name) def docker_exec(self, myContainer, exec_list): self.docker.timeout = 300 for e in exec_list: if "description" in e: #print description in logs if available self.logger.info(e["description"]) exec_c = self.docker.exec_create(myContainer, e["cmd"], stdout=True, stderr=True) output = self.docker.exec_start(exec_c) self.logger.debug(self.docker.exec_inspect(exec_c)) self.logger.info(output) self.logger.debug("all exec done") def commit(self, repository, tag): tag = Template(self.args["docker_tag"]).safe_substitute( jedox_version=self.args["jedox_version"]) self.logger.info("commiting finale image %s to %s : %s" % (self.base_container, repository, tag)) config = { "CMD": "/entrypoint", "EXPOSE": "[80,7777]", } self.docker.commit(self.base_container, repository, tag, conf=config)
class DockerApi(object): """ """ def __init__(self,url): self.url = url self.cli = Client(base_url='%s:2375'%self.url,version='1.20', timeout=120) ################################################ #列出容器 # quiet (bool): Only display numeric Ids # all (bool): Show all containers. Only running containers are shown by default # trunc (bool): Truncate output # latest (bool): Show only the latest created container, include non-running ones. # since (str): Show only containers created since Id or Name, include non-running ones # before (str): Show only container created before Id or Name, include non-running ones # limit (int): Show limit last created containers, include non-running ones # size (bool): Display sizes # filters (dict): Filters to be processed on the image list. Available filters: # exited (int): Only containers with specified exit code # status (str): One of restarting, running, paused, exited # label (str): format either "key" or "key=value" # Returns (dict): The system's containers # >>> from docker import Client # >>> cli = Client(base_url='tcp://127.0.0.1:2375') # >>> cli.containers() # [{'Command': '/bin/sleep 30', # 'Created': 1412574844, # 'Id': '6e276c9e6e5759e12a6a9214efec6439f80b4f37618e1a6547f28a3da34db07a', # 'Image': 'busybox:buildroot-2014.02', # 'Names': ['/grave_mayer'], # 'Ports': [], # 'Status': 'Up 1 seconds'}] ################################################# def containers(self): container_all = self.cli.containers(all=True) return container_all ############################################### # exec_create # Sets up an exec instance in a running container. # Params: # container (str): Target container where exec instance will be created # cmd (str or list): Command to be executed # stdout (bool): Attach to stdout of the exec command if true. Default: True # stderr (bool): Attach to stderr of the exec command if true. Default: True # tty (bool): Allocate a pseudo-TTY. Default: False # user (str): User to execute command as. Default: root # Returns (dict): A dictionary with an exec 'Id' key. ############################################### def exec_create(self,container_name,cmd): return self.cli.exec_create(container=container_name,cmd=cmd) ############################################### # exec_start # Start a previously set up exec instance. # Params: # exec_id (str): ID of the exec instance # detach (bool): If true, detach from the exec command. Default: False # tty (bool): Allocate a pseudo-TTY. Default: False # stream (bool): Stream response data. Default: False ############################################### def exec_start(self,exec_id): return self.cli.exec_start(exec_id=exec_id) ################################################# #创建容器 # image (str): The image to run # command (str or list): The command to be run in the container # hostname (str): Optional hostname for the container # user (str or int): Username or UID # detach (bool): Detached mode: run container in the background and print new container Id # stdin_open (bool): Keep STDIN open even if not attached # tty (bool): Allocate a pseudo-TTY # mem_limit (float or str): Memory limit (format: [number][optional unit], where unit = b, k, m, or g) # ports (list of ints): A list of port numbers # environment (dict or list): A dictionary or a list of strings in the following format ["PASSWORD=xxx"] or {"PASSWORD": "******"}. # dns (list): DNS name servers # volumes (str or list): # volumes_from (str or list): List of container names or Ids to get volumes from. Optionally a single string joining container id's with commas # network_disabled (bool): Disable networking # name (str): A name for the container # entrypoint (str or list): An entrypoint # cpu_shares (int): CPU shares (relative weight) # working_dir (str): Path to the working directory # domainname (str or list): Set custom DNS search domains # memswap_limit (int): # host_config (dict): A HostConfig dictionary # mac_address (str): The Mac Address to assign the container # labels (dict or list): A dictionary of name-value labels (e.g. {"label1": "value1", "label2": "value2"}) or a list of names of labels to set with empty values (e.g. ["label1", "label2"]) # volume_driver (str): The name of a volume driver/plugin. # Returns (dict): A dictionary with an image 'Id' key and a 'Warnings' key. # >>> from docker import Client # >>> cli = Client(base_url='tcp://127.0.0.1:2375') # >>> container = cli.create_container(image='busybox:latest', command='/bin/sleep 30') # >>> print(container) # {'Id': '8a61192da2b3bb2d922875585e29b74ec0dc4e0117fcbf84c962204e97564cd7', # 'Warnings': None} ######################################################## def create_container(self,image,command,password): try: container = self.cli.create_container(image=image,command=command,environment={"PASSWORD": password}) return container except: return None ########################################################## # remove_container # Remove a container. Similar to the docker rm command. # Params: # container (str): The container to remove # v (bool): Remove the volumes associated with the container # link (bool): Remove the specified link and not the underlying container # force (bool): Force the removal of a running container (uses SIGKILL) ########################################################## def remove_container(self,container): return self.cli.remove_container(container=container,force=True) ########################################################## # container (str): The container to start # response = cli.start(container=container.get('Id')) # >>> print(response) ########################################################## def start(self,container_name): try: req = self.cli.start(container=container_name) return rep except: return None ########################################################## # container (str): The container to stop # timeout (int): Timeout in seconds to wait for the container to stop before sending a SIGKILL ########################################################## def stop(self,container_name): try: req = self.cli.stop(container=container_name) return rep except: return None ########################################################## #重启 #Restart a container. Similar to the docker restart command. # If container a dict, the Id key is used. # Params: # container (str or dict): The container to restart # timeout (int): Number of seconds to try to stop for before killing the container. Once killed it will then be restarted. Default is 10 seconds. ########################################################### def restart(self,container_name): try: req = self.cli.restart(container=container_name) return rep except: return None ################################################ # images # List images. Identical to the docker images command. # Params: # name (str): Only show images belonging to the repository name # quiet (bool): Only show numeric Ids. Returns a list # all (bool): Show all images (by default filter out the intermediate image layers) # filters (dict): Filters to be processed on the image list. Available filters: # dangling (bool) # label (str): format either "key" or "key=value" # Returns (dict or list): A list if quiet=True, otherwise a di ################################################ def images(self): # print type(self.cli.images()) return self.cli.images() ########################################################## # remove_image # Remove an image. Similar to the docker rmi command. # Params: # image (str): The image to remove # force (bool): Force removal of the image # noprune (bool): Do not delete untagged parents ########################################################## def remove_image(self,image): # print type(self.cli.images()) return self.cli.remove_image(image=image,force=True) ######################################################### # path (str): Path to the directory containing the Dockerfile # tag (str): A tag to add to the final image # quiet (bool): Whether to return the status # fileobj: A file object to use as the Dockerfile. (Or a file-like object) # nocache (bool): Don't use the cache when set to True # rm (bool): Remove intermediate containers. The docker build command now defaults to --rm=true, but we have kept the old default of False to preserve backward compatibility # stream (bool): Deprecated for API version > 1.8 (always True). Return a blocking generator you can iterate over to retrieve build output as it happens # timeout (int): HTTP timeout # custom_context (bool): Optional if using fileobj # encoding (str): The encoding for a stream. Set to gzip for compressing # pull (bool): Downloads any updates to the FROM image in Dockerfiles # forcerm (bool): Always remove intermediate containers, even after unsuccessful builds # dockerfile (str): path within the build context to the Dockerfile # container_limits (dict): A dictionary of limits applied to each container created by the build process. Valid keys: # memory (int): set memory limit for build # memswap (int): Total memory (memory + swap), -1 to disable swap # cpushares (int): CPU shares (relative weight) # cpusetcpus (str): CPUs in which to allow execution, e.g., "0-3", "0,1" # decode (bool): If set to True, the returned stream will be decoded into dicts on the fly. Default False. # >>> from io import BytesIO # >>> from docker import Client # >>> dockerfile = ''' # ... # Shared Volume # ... FROM busybox:buildroot-2014.02 # ... MAINTAINER first last, [email protected] # ... VOLUME /data # ... CMD ["/bin/sh"] # ... ''' # >>> f = BytesIO(dockerfile.encode('utf-8')) # >>> cli = Client(base_url='tcp://127.0.0.1:2375') # >>> response = [line for line in cli.build( # ... fileobj=f, rm=True, tag='yourname/volume' # ... )] # >>> response # ['{"stream":" ---\\u003e a9eb17255234\\n"}', ######################################################################## def build(self,dockerfile,tag): try: f = BytesIO(dockerfile.encode('utf-8')) res = [line for line in self.cli.build(fileobj=f, rm=True, tag=tag)] return res except: print traceback.format_exc() return None ########################################################################### #commit # container (str): The image hash of the container # repository (str): The repository to push the image to # tag (str): The tag to push # message (str): A commit message # author (str): The name of the author # conf (dict): The configuration for the container. See the Docker remote api for full details. ########################################################################### def commit(self,container,repository): try: res = self.cli.commit(container=container,repository=repository) return res except: return None ############################################################################ #pull # repository (str): The repository to pull # tag (str): The tag to pull # stream (bool): Stream the output as a generator # insecure_registry (bool): Use an insecure registry # auth_config (dict): Override the credentials that Client.login has set for this request auth_config should contain the username and password keys to be valid. # Returns (generator or str): The output # >>> from docker import Client # >>> cli = Client(base_url='tcp://127.0.0.1:2375') # >>> for line in cli.pull('busybox', stream=True): # ... print(json.dumps(json.loads(line), indent=4)) # { # "status": "Pulling image (latest) from busybox", # "progressDetail": {}, # "id": "e72ac664f4f0" # } # { # "status": "Pulling image (latest) from busybox, endpoint: ...", # "progressDetail": {}, # "id": "e72ac664f4f0" # } ############################################################################ def pull(self,repository): try: # res = cli.pull(repository,stream=True) res = [line for line in self.cli.pull(repository,stream=True)] return res except: print traceback.format_exc() return None ############################################################################ #push # repository (str): The repository to push to # tag (str): An optional tag to push # stream (bool): Stream the output as a blocking generator # insecure_registry (bool): Use http:// to connect to the registry # Returns (generator or str): The output of the upload # >>> from docker import Client # >>> cli = Client(base_url='tcp://127.0.0.1:2375') # >>> response = [line for line in cli.push('yourname/app', stream=True)] # >>> response # ['{"status":"Pushing repository yourname/app (1 tags)"}\\n', # '{"status":"Pushing","progressDetail":{},"id":"511136ea3c5a"}\\n', # '{"status":"Image already pushed, skipping","progressDetail":{}, # "id":"511136ea3c5a"}\\n', # ... # '{"status":"Pushing tag for rev [918af568e6e5] on { # https://cdn-registry-1.docker.io/v1/repositories/ # yourname/app/tags/latest}"}\\n'] ############################################################################ def push(self,repository): try: # res = cli.push(repository,stream=True) res = [line for line in self.cli.push(repository =repository, stream=True)] return res except: print traceback.format_exc() return None
def main(self, args): # !! TODO needs to implement login if using that directory = args.metadata_path directory = os.path.expanduser(directory) if not os.path.exists(directory): os.makedirs(directory) list_args = Object() list_args.metadata_path = args.metadata_path list_args.z = True list_a = list.list.main(list_args) host_args = Object() host_args.metadata_path = args.metadata_path host_args.z = True host_a = hosts.hosts.main(host_args) commit = str(uuid.uuid4()) snapshot = {} found = 0 for container in list_a: if args.CONTAINER in container: host = container.split(",")[1] cont = container.split(",")[0] commit_name = "bowl-snapshot-"+commit try: for h in host_a: if host in h: c = Client(**kwargs_from_env()) #c = docker.Client(base_url='tcp://'+h, # version='1.12', # timeout=2) c.commit(cont, repository=commit_name) snapshot[cont] = host+":"+commit_name except: if not args.z: print "unable to connect to "+host return False found = 1 if found: try: with open(os.path.join(directory, "snapshots"), 'a') as f: for container in snapshot: f.write("{" + "'container_id': '"+container+"'," + " 'snapshot_id': '"+snapshot[container].split(":")[1]+"'," + " 'host': '"+snapshot[container].split(":")[0]+"'" + "}\n") if not args.z: print "snapshotted", print container + " on", print snapshot[container].split(":")[0], print "to repository;", print snapshot[container].split(":")[1] except: if not args.z: print "unable to snapshot container" return False else: if not args.z: print args.CONTAINER, "is not a running container" return False return True
class dockerizer(default_logger): def __init__(self,args): default_logger.__init__(self,"dockerizer") self.args=args self.config=self.get_config() self.args["eula"]=self.config["installer"]["eula"] self.directory=(os.path.dirname(os.path.realpath(__file__))) self.base_image_name=args["base_image"] self.docker=Client(base_url='unix://var/run/docker.sock') self.installer=jedox_installer(args) self.installer.start() sleep(15) self.installer.stop() sleep(15) self.patch() self.add() self.build_base_image(self.base_image_name) self.base_container=self.docker.create_container(self.base_image_name) self.docker.start(self.base_container) self.docker_exec(self.base_container,self.config["docker"]["exec"]) self.commit(self.args["docker_repository"],self.args["docker_tag"]) #remove intermediate container self.logger.info("removing base container") self.docker.remove_container(container=self.base_container,force=True) def get_config(self): try : config_file=self.args["config"] version=self.args["jedox_version"] j=json.load(open(config_file)) return j[version] except KeyError as e: self.logger.exception(e) self.logger.error("Could not find the right config for version=%s in file=%s \n Aborting..." % (version,config_file)) sys.exit(1) def patch(self): self.logger.info("patching files from installer") self.change_working_directory("patch") for p in self.config["patch"]: target=os.path.join(self.args["jedox_home"],p["target"]) description=p.get("description",p["target"]) self.logger.info("patching : %s" % description) subprocess.check_call("patch %s < %s" % (target,p["source"]),shell=True) def add(self): self.logger.info("adding additional content to installation") self.change_working_directory("add") for a in self.config["add"]: target=os.path.join(self.args["jedox_home"],a["target"]) self.logger.info("copy %s to %s" % (a["source"],target)) shutil.copy(a["source"],target) def change_working_directory(self,area): working_directory=os.path.join(self.directory,area,self.args["jedox_version"]) self.logger.info("working dir is now %s" % working_directory) os.chdir(working_directory) def build_base_image(self,image_name="jedox/base"): os.chdir(self.args["jedox_home"]) self.logger.info("Import Jedox Suite into intermediate docker image '%s'" % image_name) subprocess.check_call("""tar --to-stdout --numeric-owner --exclude=/proc --exclude=/sys --exclude='*.tar.gz' --exclude='*.log' -c ./ | docker import --change "CMD while true; do ping 8.8.8.8; done" --change "ENV TERM=xterm" - %s""" % image_name, shell=True) self.logger.info("successfully create basecontainer %s" % image_name) def docker_exec(self,myContainer,exec_list): self.docker.timeout=300 for e in exec_list: if "description" in e : #print description in logs if available self.logger.info(e["description"]) exec_c=self.docker.exec_create(myContainer,e["cmd"],stdout=True,stderr=True) output=self.docker.exec_start(exec_c) self.logger.debug(self.docker.exec_inspect(exec_c)) self.logger.info(output) self.logger.debug("all exec done") def commit(self,repository,tag): tag=Template(self.args["docker_tag"]).safe_substitute(jedox_version=self.args["jedox_version"]) self.logger.info("commiting finale image %s to %s : %s" % (self.base_container,repository,tag)) config={"CMD":"/entrypoint", "EXPOSE": "[80,7777]", } self.docker.commit(self.base_container,repository,tag,conf=config)
class DockerClient: '''A singleton class to ensure there is only one client''' _instance = None def __new__(cls, *args, **kwargs): if not cls._instance: cls._instance = super(DockerClient, cls).__new__(cls) return cls._instance def __init__(self): kwargs = kwargs_from_env(assert_hostname=False) kwargs.update({'version': 'auto'}) self.client = Client(**kwargs) try: self.client.info() # mount the /Volumes folder under mac, please refer to # https://github.com/bpeng2000/SOS/wiki/SoS-Docker-guide # for details. self.has_volumes = False if platform.system() == 'Darwin': try: # this command log in to the docker machine, check if /Volumes has been mounted, # and try to mount it if possible. This requires users to configure subprocess.call("""docker-machine ssh "{}" 'mount | grep /Volumes || {{ echo "mounting /Volumes"; sudo mount -t vboxsf Volumes /Volumes; }}' """.format(os.environ['DOCKER_MACHINE_NAME']), shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) env.logger.trace('Sucessfully mount /Volumes to virtual machine') self.has_volumes = True except Exception as e: env.logger.trace('Failed to mount /Volumes to virtual machine: {}'.format(e)) except Exception as e: env.logger.debug('Docker client init fail: {}'.format(e)) self.client = None def total_memory(self, image='ubuntu'): '''Get the available ram fo the docker machine in Kb''' try: ret = subprocess.check_output( '''docker run -t {} cat /proc/meminfo | grep MemTotal'''.format(image), shell=True, stdin=subprocess.DEVNULL) # ret: MemTotal: 30208916 kB self.tot_mem = int(ret.split()[1]) except: # some system does not have cat or grep self.tot_mem = None return self.tot_mem def _is_image_avail(self, image): images = sum([x['RepoTags'] for x in self.client.images() if x['RepoTags']], []) # some earlier version of docker-py returns docker.io/ for global repositories images = [x[10:] if x.startswith('docker.io/') else x for x in images] return (':' in image and image in images) or \ (':' not in image and '{}:latest'.format(image) in images) def stream(self, line): # properly output streamed output try: sys.stdout.write(json.loads(line).get('stream', '')) except ValueError: # sometimes all the data is sent on a single line ???? # # ValueError: Extra data: line 1 column 87 - line 1 column # 33268 (char 86 - 33267) # This ONLY works because every line is formatted as # {"stream": STRING} for obj in re.findall('{\s*"stream"\s*:\s*"[^"]*"\s*}', line): sys.stdout.write(json.loads(obj).get('stream', '')) def build(self, script, **kwargs): if not self.client: raise RuntimeError('Cannot connect to the Docker daemon. Is the docker daemon running on this host?') if script is not None: f = BytesIO(script.encode('utf-8')) for line in self.client.build(fileobj=f, **kwargs): self.stream(line.decode()) else: for line in self.client.build(**kwargs): self.stream(line.decode()) # if a tag is given, check if the image is built if 'tag' in kwargs and not self._is_image_avail(kwargs['tag']): raise RuntimeError('Image with tag {} is not created.'.format(kwargs['tag'])) def import_image(self, image, **kwargs): if not self.client: raise RuntimeError('Cannot connect to the Docker daemon. Is the docker daemon running on this host?') env.logger.info('docker import {}'.format(image)) self.client.import_image(image, **kwargs) def pull(self, image): if not self.client: raise RuntimeError('Cannot connect to the Docker daemon. Is the docker daemon running on this host?') # if image is specified, check if it is available locally. If not, pull it ret = 0 if not self._is_image_avail(image): env.logger.info('docker pull {}'.format(image)) # using subprocess instead of docker-py's pull function because this would have # much better progress bar display ret = subprocess.call('docker pull {}'.format(image), shell=True) #for line in self.client.pull(image, stream=True): # self.stream(line) if not self._is_image_avail(image): raise RuntimeError('Failed to pull image {}'.format(image)) return ret def commit(self, **kwargs): if not self.client: raise RuntimeError('Cannot connect to the Docker daemon. Is the docker daemon running on this host?') for line in self.client.commit(**kwargs): self.stream(line.decode()) return 0 def run(self, image, script='', interpreter='', args='', suffix='.sh', **kwargs): if self.client is None: raise RuntimeError('Cannot connect to the Docker daemon. Is the docker daemon running on this host?') # env.logger.debug('docker_run with keyword args {}'.format(kwargs)) # # now, write a temporary file to a tempoary directory under the current directory, this is because # we need to share the directory to ... with tempfile.TemporaryDirectory(dir=os.getcwd()) as tempdir: # keep the temporary script for debugging purposes # tempdir = tempfile.mkdtemp(dir=os.getcwd()) if script: tempscript = 'docker_run_{}{}'.format(os.getpid(), suffix) with open(os.path.join(tempdir, tempscript), 'w') as script_file: script_file.write(script) # # if there is an interpreter and with args if not args: args = '${filename!q}' if not interpreter: interpreter = '/bin/bash' # if there is a shebang line, we ... if script.startswith('#!'): # make the script executable env.logger.warning('Shebang line in a docker-run script is ignored') # binds = [] if 'volumes' in kwargs: volumes = [kwargs['volumes']] if isinstance(kwargs['volumes'], str) else kwargs['volumes'] for vol in volumes: if not vol: continue if vol.count(':') != 1: raise RuntimeError('Please specify columes in the format of host_dir:mnt_dir') host_dir, mnt_dir = vol.split(':') if platform.system() == 'Darwin': # under Darwin, host_dir must be under /Users if not os.path.abspath(host_dir).startswith('/Users') and not (self.has_volumes and os.path.abspath(host_dir).startswith('/Volumes')): raise RuntimeError('hostdir ({}) under MacOSX must be under /Users or /Volumes (if properly configured, see https://github.com/bpeng2000/SOS/wiki/SoS-Docker-guide for details) to be usable in docker container'.format(host_dir)) binds.append('{}:{}'.format(os.path.abspath(host_dir), mnt_dir)) # volumes_opt = ' '.join('-v {}'.format(x) for x in binds) # under mac, we by default share /Users within docker if platform.system() == 'Darwin': if not any(x.startswith('/Users:') for x in binds): volumes_opt += ' -v /Users:/Users' if self.has_volumes: volumes_opt += ' -v /Volumes:/Volumes' if not any(x.startswith('/tmp:') for x in binds): volumes_opt += ' -v /tmp:/tmp' # mem_limit_opt = '' if 'mem_limit' in kwargs: mem_limit_opt = '--memory={}'.format(kwargs['mem_limit']) # volumes_from_opt = '' if 'volumes_from' in kwargs: if isinstance(kwargs['volumes_from'], str): volumes_from_opt = '--volumes_from={}'.format(kwargs['volumes_from']) elif isinstance(kwargs['volumes_from'], list): volumes_from_opt = ' '.join('--volumes_from={}'.format(x) for x in kwargs['volumes_from']) else: raise RuntimeError('Option volumes_from only accept a string or list of string'.format(kwargs['volumes_from'])) # we also need to mount the script cmd_opt = '' if script and interpreter: volumes_opt += ' -v {}:{}'.format(os.path.join(tempdir, tempscript), '/var/lib/sos/{}'.format(tempscript)) cmd_opt = interpolate('{} {}'.format(interpreter, args), '${ }', {'filename': '/var/lib/sos/{}'.format(tempscript)}) # working_dir_opt = '-w={}'.format(os.path.abspath(os.getcwd())) if 'working_dir' in kwargs: if not os.path.isabs(kwargs['working_dir']): env.logger.warning('An absolute path is needed for -w option of docker run command. "{}" provided, "{}" used.' .format(kwargs['working_dir'], os.path.abspath(os.path.expanduser(kwargs['working_dir'])))) working_dir_opt = '-w={}'.format(os.path.abspath(os.path.expanduser(kwargs['working_dir']))) else: working_dir_opt = '-w={}'.format(kwargs['working_dir']) # env_opt = '' if 'environment' in kwargs: if isinstance(kwargs['environment'], dict): env_opt = ' '.join('-e {}={}'.format(x,y) for x,y in kwargs['environment'].items()) elif isinstance(kwargs['environment'], list): env_opt = ' '.join('-e {}'.format(x) for x in kwargs['environment']) elif isinstance(kwargs['environment'], str): env_opt = '-e {}'.format(kwargs['environment']) else: raise RuntimeError('Invalid value for option environment (str, list, or dict is allowd, {} provided)'.format(kwargs['environment'])) # port_opt = '-P' if 'port' in kwargs: if isinstance(kwargs['port'], (str, int)): port_opt = '-p {}'.format(kwargs['port']) elif isinstance(kwargs['port'], list): port_opt = ' '.join('-p {}'.format(x) for x in kwargs['port']) else: raise RuntimeError('Invalid value for option port (a list of intergers), {} provided'.format(kwargs['port'])) # name_opt = '' if 'name' in kwargs: name_opt = '--name={}'.format(kwargs['name']) # stdin_opt = '' if 'stdin_open' in kwargs and kwargs['stdin_optn']: stdin_opt = '-i' # tty_opt = '-t' if 'tty' in kwargs and not kwargs['tty']: tty_opt = '' # user_opt = '' if 'user' in kwargs: user_opt = '-u {}'.format(kwargs['user']) # extra_opt = '' if 'extra_args' in kwargs: extra_opt = kwargs['extra_args'] # security_opt = '' if platform.system() == 'Linux': # this is for a selinux problem when /var/sos/script cannot be executed security_opt = '--security-opt label:disable' command = 'docker run --rm {} {} {} {} {} {} {} {} {} {} {} {} {} {}'.format( security_opt, # security option volumes_opt, # volumes volumes_from_opt, # volumes_from name_opt, # name stdin_opt, # stdin_optn tty_opt, # tty port_opt, # port working_dir_opt, # working dir user_opt, # user env_opt, # environment mem_limit_opt, # memory limit extra_opt, # any extra parameters image, # image cmd_opt ) env.logger.info(command) ret = subprocess.call(command, shell=True) if ret != 0: msg = 'The script has been saved to .sos/{} so that you can execute it using the following command:\n{}'.format( tempscript, command.replace(tempdir, os.path.abspath('./.sos'))) shutil.copy(os.path.join(tempdir, tempscript), '.sos') if ret == 125: raise RuntimeError('Docker daemon failed (exitcode=125). ' + msg) elif ret == 126: raise RuntimeError('Failed to invoke specified command (exitcode=126). ' + msg) elif ret == 127: raise RuntimeError('Failed to locate specified command (exitcode=127). ' + msg) elif ret == 137: if not hasattr(self, 'tot_mem'): self.tot_mem = self.total_memory(image) if self.tot_mem is None: raise RuntimeError('Script killed by docker. ' + msg) else: raise RuntimeError('Script killed by docker, probably because of lack of RAM (available RAM={:.1f}GB, exitcode=137). '.format(self.tot_mem/1024/1024) + msg) else: raise RuntimeError('Executing script in docker returns an error (exitcode={}). '.format(ret) + msg) return 0
class BaseProductTestCase(BaseTestCase): default_workers_config_ = """coordinator=false discovery.uri=http://master:8080 http-server.http.port=8080 task.max-memory=1GB\n""" default_node_properties_ = """node.data-dir=/var/lib/presto/data node.environment=presto plugin.config-dir=/etc/presto/catalog plugin.dir=/usr/lib/presto/lib/plugin\n""" default_jvm_config_ = """-server -Xmx1G -XX:-UseBiasedLocking -XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+HeapDumpOnOutOfMemoryError -XX:+UseGCOverheadLimit -XX:OnOutOfMemoryError=kill -9 %p\n""" default_coordinator_config_ = """coordinator=true discovery-server.enabled=true discovery.uri=http://master:8080 http-server.http.port=8080 task.max-memory=1GB\n""" down_node_connection_error = ( r"(\nWarning: (\[%(host)s\] )?Low level socket " r"error connecting to host %(host)s on " r"port 22: No route to host " r"\(tried 1 time\)\n\nUnderlying " r"exception:\n No route to host\n" r"|\nWarning: (\[%(host)s] )?Timed out trying " r"to connect to %(host)s \(tried 1 " r"time\)\n\nUnderlying exception:" r"\n timed out\n)" ) len_down_node_error = 6 def setUp(self): super(BaseProductTestCase, self).setUp() self.maxDiff = None self.docker_client = Client(timeout=180) self.presto_rpm_filename = self.detect_presto_rpm() def detect_presto_rpm(self): """ Detects the Presto RPM in the main directory of presto-admin. Returns the name of the RPM, if it exists, else returns None. """ rpm_names = fnmatch.filter(os.listdir(prestoadmin.main_dir), PRESTO_RPM_GLOB) if rpm_names: # Choose the last RPM name if you sort the list, since if there # are multiple RPMs, the last one is probably the latest return sorted(rpm_names)[-1] else: # TODO: once the RPM is on Maven Central, pull the RPM from there rpm_filename = "presto-0.101-1.0.x86_64.rpm" rpm_path = os.path.join(prestoadmin.main_dir, rpm_filename) urllib.urlretrieve( "http://teradata-download.s3.amazonaws.com/" "aster/presto/lib/presto-0.101-1.0.x86_64.rpm", rpm_path ) return rpm_filename def setup_docker_cluster(self, cluster_type="centos"): cluster_types = ["presto", "centos"] if cluster_type not in cluster_types: self.fail( "{0} is not a supported cluster type. Must choose one" " from {1}".format(cluster_type, cluster_types) ) try: are_presto_images_present = DockerCluster.check_for_presto_images() if cluster_type == "presto" and are_presto_images_present: self.docker_cluster = DockerCluster.start_presto_cluster() return self.docker_cluster = DockerCluster.start_centos_cluster() if cluster_type == "presto" and not are_presto_images_present: self.install_presto_admin(self.docker_cluster) self.upload_topology() self.server_install() self.docker_client.commit(self.docker_cluster.master, INSTALLED_PRESTO_TEST_MASTER_IMAGE) self.docker_client.commit(self.docker_cluster.slaves[0], INSTALLED_PRESTO_TEST_SLAVE_IMAGE) except DockerClusterException as e: self.fail(e.msg) def tearDown(self): self.restore_stdout_stderr_keep_open() if hasattr(locals()["self"], "docker_cluster"): self.docker_cluster.tear_down_containers() super(BaseProductTestCase, self).tearDown() def build_dist_if_necessary(self, cluster=None, unique=False): if not cluster: cluster = self.docker_cluster if not os.path.isdir(cluster.get_dist_dir(unique)) or not fnmatch.filter( os.listdir(cluster.get_dist_dir(unique)), "prestoadmin-*.tar.bz2" ): cluster.clean_up_presto_test_images() self.build_installer_in_docker(cluster=cluster, unique=unique) return cluster.get_dist_dir(unique) def build_installer_in_docker(self, online_installer=False, cluster=None, unique=False): if not cluster: cluster = self.docker_cluster container_name = "installer" installer_container = DockerCluster(container_name, [], DEFAULT_LOCAL_MOUNT_POINT, DEFAULT_DOCKER_MOUNT_POINT) try: installer_container.create_image( os.path.join(LOCAL_RESOURCES_DIR, "centos6-ssh-test"), "teradatalabs/centos6-ssh-test", "jdeathe/centos-ssh", ) installer_container.start_containers("teradatalabs/centos6-ssh-test") except DockerClusterException as e: installer_container.tear_down_containers() self.fail(e.msg) try: shutil.copytree( prestoadmin.main_dir, os.path.join(installer_container.get_local_mount_dir(container_name), "presto-admin"), ignore=shutil.ignore_patterns("tmp", ".git", "presto*.rpm"), ) installer_container.run_script( "-e\n" "pip install --upgrade pip\n" "pip install --upgrade wheel\n" "pip install --upgrade setuptools\n" "mv %s/presto-admin ~/\n" "cd ~/presto-admin\n" "make %s\n" "cp dist/prestoadmin-*.tar.bz2 %s" % ( installer_container.docker_mount_dir, "dist" if not online_installer else "dist-online", installer_container.docker_mount_dir, ), container_name, ) try: os.makedirs(cluster.get_dist_dir(unique)) except OSError, e: if e.errno != errno.EEXIST: raise local_container_dist_dir = os.path.join( prestoadmin.main_dir, installer_container.get_local_mount_dir(container_name) ) installer_file = fnmatch.filter(os.listdir(local_container_dist_dir), "prestoadmin-*.tar.bz2")[0] shutil.copy(os.path.join(local_container_dist_dir, installer_file), cluster.get_dist_dir(unique)) finally:
class DockerCluster(BaseCluster): IMAGE_NAME_BASE = os.path.join('teradatalabs', 'pa_test') BARE_CLUSTER_TYPE = 'bare' """Start/stop/control/query arbitrary clusters of docker containers. This class is aimed at product test writers to create docker containers for testing purposes. """ def __init__(self, master_host, slave_hosts, local_mount_dir, docker_mount_dir): # see PyDoc for all_internal_hosts() for an explanation on the # difference between an internal and regular host self.internal_master = master_host self.internal_slaves = slave_hosts self.master = master_host + '-' + str(uuid.uuid4()) self.slaves = [slave + '-' + str(uuid.uuid4()) for slave in slave_hosts] # the root path for all local mount points; to get a particular # container mount point call get_local_mount_dir() self.local_mount_dir = local_mount_dir self.mount_dir = docker_mount_dir kwargs = kwargs_from_env() if 'tls' in kwargs: kwargs['tls'].assert_hostname = False kwargs['timeout'] = 300 self.client = Client(**kwargs) DockerCluster.__check_if_docker_exists() def all_hosts(self): return self.slaves + [self.master] def get_master(self): return self.master def all_internal_hosts(self): return [host.split('-')[0] for host in self.all_hosts()] def get_local_mount_dir(self, host): return os.path.join(self.local_mount_dir, self.__get_unique_host(host)) def get_dist_dir(self, unique): if unique: return os.path.join(DIST_DIR, self.master) else: return DIST_DIR def __get_unique_host(self, host): matches = [unique_host for unique_host in self.all_hosts() if unique_host.startswith(host)] if matches: return matches[0] elif host in self.all_hosts(): return host else: raise DockerClusterException( 'Specified host: {0} does not exist.'.format(host)) @staticmethod def __check_if_docker_exists(): try: subprocess.call(['docker', '--version']) except OSError: sys.exit('Docker is not installed. Try installing it with ' 'presto-admin/bin/install-docker.sh.') def fetch_image_if_not_present(self, image, tag=None): if not tag and not self.client.images(image): self._execute_and_wait(self.client.pull, image) elif tag and not self._is_image_present_locally(image, tag): self._execute_and_wait(self.client.pull, image, tag) def _is_image_present_locally(self, image_name, tag): image_name_and_tag = image_name + ':' + tag images = self.client.images(image_name) if images: for image in images: if image_name_and_tag in image['RepoTags']: return True return False def start_containers(self, master_image, slave_image=None, cmd=None, **kwargs): self._create_host_mount_dirs() self._create_and_start_containers(master_image, slave_image, cmd, **kwargs) self._ensure_docker_containers_started(master_image) def tear_down(self): for container_name in self.all_hosts(): self._tear_down_container(container_name) self._remove_host_mount_dirs() if self.client: self.client.close() self.client = None def _tear_down_container(self, container_name): try: shutil.rmtree(self.get_dist_dir(unique=True)) except OSError as e: # no such file or directory if e.errno != errno.ENOENT: raise try: self.stop_host(container_name) self.client.remove_container(container_name, v=True, force=True) except APIError as e: # container does not exist if e.response.status_code != 404: raise def stop_host(self, container_name): self.client.stop(container_name) self.client.wait(container_name) def start_host(self, container_name): self.client.start(container_name) def get_down_hostname(self, host_name): return host_name def _remove_host_mount_dirs(self): for container_name in self.all_hosts(): try: shutil.rmtree( self.get_local_mount_dir(container_name)) except OSError as e: # no such file or directory if e.errno != errno.ENOENT: raise def _create_host_mount_dirs(self): for container_name in self.all_hosts(): try: os.makedirs( self.get_local_mount_dir(container_name)) except OSError as e: # file exists if e.errno != errno.EEXIST: raise @staticmethod def _execute_and_wait(func, *args, **kwargs): ret = func(*args, **kwargs) # go through all lines in returned stream to ensure func finishes output = '' for line in ret: output += line return output def _create_and_start_containers(self, master_image, slave_image=None, cmd=None, **kwargs): if slave_image: for container_name in self.slaves: container_mount_dir = \ self.get_local_mount_dir(container_name) self._create_container( slave_image, container_name, container_name.split('-')[0], cmd ) self.client.start(container_name, binds={container_mount_dir: {'bind': self.mount_dir, 'ro': False}}, **kwargs) master_mount_dir = self.get_local_mount_dir(self.master) self._create_container( master_image, self.master, hostname=self.internal_master, cmd=cmd ) self.client.start(self.master, binds={master_mount_dir: {'bind': self.mount_dir, 'ro': False}}, links=zip(self.slaves, self.slaves), **kwargs) self._add_hostnames_to_slaves() def _create_container(self, image, container_name, hostname=None, cmd=None): self._execute_and_wait(self.client.create_container, image, detach=True, name=container_name, hostname=hostname, volumes=self.local_mount_dir, command=cmd, host_config={'mem_limit': '2g'}) def _add_hostnames_to_slaves(self): ips = self.get_ip_address_dict() additions_to_etc_hosts = '' for host in self.all_internal_hosts(): additions_to_etc_hosts += '%s\t%s\n' % (ips[host], host) for host in self.slaves: self.exec_cmd_on_host( host, 'bin/bash -c \'echo "%s" >> /etc/hosts\'' % additions_to_etc_hosts ) @retry(stop_max_delay=_DOCKER_START_TIMEOUT, wait_fixed=_DOCKER_START_WAIT) def _ensure_docker_containers_started(self, image): host_started = {} for host in self.all_hosts(): host_started[host] = False for host in host_started.keys(): if host_started[host]: continue is_started = True is_started &= \ self.client.inspect_container(host)['State']['Running'] if is_started and image not in NO_WAIT_SSH_IMAGES: is_started &= self._are_centos_container_services_up(host) host_started[host] = is_started not_started = [host for (host, started) in host_started.items() if not started] if len(not_started): raise NotStartedException(not_started) @staticmethod def _are_all_hosts_started(host_started_map): all_started = True for host in host_started_map.keys(): all_started &= host_started_map[host] return all_started def _are_centos_container_services_up(self, host): """Some essential services in our CentOS containers take some time to start after the container itself is up. This function checks whether those services are up and returns a boolean accordingly. Specifically, we check that the app-admin user has been created and that the ssh daemon is up, as well as that the SSH keys are in the right place. Args: host: the host to check. Returns: True if the specified services have started, False otherwise. """ ps_output = self.exec_cmd_on_host(host, 'ps') # also ensure that the app-admin user exists try: user_output = self.exec_cmd_on_host( host, 'grep app-admin /etc/passwd' ) user_output += self.exec_cmd_on_host(host, 'stat /home/app-admin') except OSError: user_output = '' if 'sshd_bootstrap' in ps_output or 'sshd\n' not in ps_output\ or not user_output: return False # check for .ssh being in the right place try: ssh_output = self.exec_cmd_on_host(host, 'ls /home/app-admin/.ssh') if 'id_rsa' not in ssh_output: return False except OSError: return False return True def exec_cmd_on_host(self, host, cmd, user=None, raise_error=True, tty=False): ex = self.client.exec_create(self.__get_unique_host(host), cmd, tty=tty, user=user) output = self.client.exec_start(ex['Id'], tty=tty) exit_code = self.client.exec_inspect(ex['Id'])['ExitCode'] if raise_error and exit_code: raise OSError(exit_code, output) return output @staticmethod def _get_tag_basename(bare_image_provider, cluster_type, ms): return '_'.join( [bare_image_provider.get_tag_decoration(), cluster_type, ms]) @staticmethod def _get_master_image_name(bare_image_provider, cluster_type): return os.path.join(DockerCluster.IMAGE_NAME_BASE, DockerCluster._get_tag_basename( bare_image_provider, cluster_type, 'master')) @staticmethod def _get_slave_image_name(bare_image_provider, cluster_type): return os.path.join(DockerCluster.IMAGE_NAME_BASE, DockerCluster._get_tag_basename( bare_image_provider, cluster_type, 'slave')) @staticmethod def _get_image_names(bare_image_provider, cluster_type): dc = DockerCluster return (dc._get_master_image_name(bare_image_provider, cluster_type), dc._get_slave_image_name(bare_image_provider, cluster_type)) @staticmethod def start_cluster(bare_image_provider, cluster_type, master_host='master', slave_hosts=None, **kwargs): if slave_hosts is None: slave_hosts = ['slave1', 'slave2', 'slave3'] created_bare = False dc = DockerCluster centos_cluster = DockerCluster(master_host, slave_hosts, DEFAULT_LOCAL_MOUNT_POINT, DEFAULT_DOCKER_MOUNT_POINT) master_name, slave_name = dc._get_image_names( bare_image_provider, cluster_type) if not dc._check_for_images(master_name, slave_name): master_name, slave_name = dc._get_image_names( bare_image_provider, dc.BARE_CLUSTER_TYPE) if not dc._check_for_images(master_name, slave_name): bare_image_provider.create_bare_images( centos_cluster, master_name, slave_name) created_bare = True centos_cluster.start_containers(master_name, slave_name, **kwargs) return centos_cluster, created_bare @staticmethod def _check_for_images(master_image_name, slave_image_name, tag='latest'): master_repotag = '%s:%s' % (master_image_name, tag) slave_repotag = '%s:%s' % (slave_image_name, tag) with Client(timeout=180) as client: images = client.images() has_master_image = False has_slave_image = False for image in images: if master_repotag in image['RepoTags']: has_master_image = True if slave_repotag in image['RepoTags']: has_slave_image = True return has_master_image and has_slave_image def commit_images(self, bare_image_provider, cluster_type): self.client.commit(self.master, self._get_master_image_name(bare_image_provider, cluster_type)) if self.slaves: self.client.commit(self.slaves[0], self._get_slave_image_name(bare_image_provider, cluster_type)) def run_script_on_host(self, script_contents, host): temp_script = '/tmp/tmp.sh' self.write_content_to_host('#!/bin/bash\n%s' % script_contents, temp_script, host) self.exec_cmd_on_host(host, 'chmod +x %s' % temp_script) return self.exec_cmd_on_host(host, temp_script, tty=True) def write_content_to_host(self, content, path, host): filename = os.path.basename(path) dest_dir = os.path.dirname(path) host_local_mount_point = self.get_local_mount_dir(host) local_path = os.path.join(host_local_mount_point, filename) with open(local_path, 'w') as config_file: config_file.write(content) self.exec_cmd_on_host(host, 'mkdir -p ' + dest_dir) self.exec_cmd_on_host( host, 'cp %s %s' % (os.path.join(self.mount_dir, filename), dest_dir)) def copy_to_host(self, source_path, dest_host, **kwargs): shutil.copy(source_path, self.get_local_mount_dir(dest_host)) def get_ip_address_dict(self): ip_addresses = {} for host, internal_host in zip(self.all_hosts(), self.all_internal_hosts()): inspect = self.client.inspect_container(host) ip_addresses[host] = inspect['NetworkSettings']['IPAddress'] ip_addresses[internal_host] = \ inspect['NetworkSettings']['IPAddress'] return ip_addresses def _post_presto_install(self): for worker in self.slaves: self.run_script_on_host( 'sed -i /node.id/d /etc/presto/node.properties; ' 'uuid=$(uuidgen); ' 'echo node.id=$uuid >> /etc/presto/node.properties', worker ) def postinstall(self, installer): from tests.product.standalone.presto_installer \ import StandalonePrestoInstaller _post_install_hooks = { StandalonePrestoInstaller: DockerCluster._post_presto_install } hook = _post_install_hooks.get(installer, None) if hook: hook(self)
class DockerCluster(object): IMAGE_NAME_BASE = os.path.join('teradatalabs', 'pa_test') BARE_CLUSTER_TYPE = 'bare' """Start/stop/control/query arbitrary clusters of docker containers. This class is aimed at product test writers to create docker containers for testing purposes. """ def __init__(self, master_host, slave_hosts, local_mount_dir, docker_mount_dir): # see PyDoc for all_internal_hosts() for an explanation on the # difference between an internal and regular host self.internal_master = master_host self.internal_slaves = slave_hosts self.master = master_host + '-' + str(uuid.uuid4()) self.slaves = [slave + '-' + str(uuid.uuid4()) for slave in slave_hosts] # the root path for all local mount points; to get a particular # container mount point call get_local_mount_dir() self.local_mount_dir = local_mount_dir self.mount_dir = docker_mount_dir kwargs = kwargs_from_env() if 'tls' in kwargs: kwargs['tls'].assert_hostname = False kwargs['timeout'] = 240 self.client = Client(**kwargs) self._DOCKER_START_TIMEOUT = 30 DockerCluster.__check_if_docker_exists() def all_hosts(self): return self.slaves + [self.master] def get_master(self): return self.master def all_internal_hosts(self): """The difference between this method and all_hosts() is that all_hosts() returns the unique, "outside facing" hostnames that docker uses. On the other hand all_internal_hosts() returns the more human readable host aliases for the containers used internally between containers. For example the unique master host will look something like 'master-07d1774e-72d7-45da-bf84-081cfaa5da9a', whereas the internal master host will be 'master'. Returns: List of all internal hosts with the random suffix stripped out. """ return [host.split('-')[0] for host in self.all_hosts()] def get_local_mount_dir(self, host): return os.path.join(self.local_mount_dir, self.__get_unique_host(host)) def get_dist_dir(self, unique): if unique: return os.path.join(DIST_DIR, self.master) else: return DIST_DIR def __get_unique_host(self, host): matches = [unique_host for unique_host in self.all_hosts() if unique_host.startswith(host)] if matches: return matches[0] elif host in self.all_hosts(): return host else: raise DockerClusterException( 'Specified host: {0} does not exist.'.format(host)) @staticmethod def __check_if_docker_exists(): try: subprocess.call(['docker', '--version']) except OSError: sys.exit('Docker is not installed. Try installing it with ' 'presto-admin/bin/install-docker.sh.') def create_image(self, path_to_dockerfile_dir, image_tag, base_image, base_image_tag=None): self.fetch_image_if_not_present(base_image, base_image_tag) output = self._execute_and_wait(self.client.build, path=path_to_dockerfile_dir, tag=image_tag, rm=True) if not self._is_image_present_locally(image_tag, 'latest'): raise OSError('Unable to build image %s: %s' % (image_tag, output)) def fetch_image_if_not_present(self, image, tag=None): if not tag and not self.client.images(image): self._execute_and_wait(self.client.pull, image) elif tag and not self._is_image_present_locally(image, tag): self._execute_and_wait(self.client.pull, image, tag) def _is_image_present_locally(self, image_name, tag): image_name_and_tag = image_name + ':' + tag images = self.client.images(image_name) if images: for image in images: if image_name_and_tag in image['RepoTags']: return True return False def start_containers(self, master_image, slave_image=None, cmd=None, **kwargs): self.tear_down() self._create_host_mount_dirs() self._create_and_start_containers(master_image, slave_image, cmd, **kwargs) self._ensure_docker_containers_started(master_image) def tear_down(self): for container_name in self.all_hosts(): self._tear_down_container(container_name) self._remove_host_mount_dirs() def _tear_down_container(self, container_name): try: shutil.rmtree(self.get_dist_dir(unique=True)) except OSError as e: # no such file or directory if e.errno != errno.ENOENT: raise try: self.stop_host(container_name) self.client.remove_container(container_name, v=True, force=True) except APIError as e: # container does not exist if e.response.status_code != 404: raise def stop_host(self, container_name): self.client.stop(container_name) self.client.wait(container_name) def start_host(self, container_name): self.client.start(container_name) def get_down_hostname(self, host_name): return host_name def _remove_host_mount_dirs(self): for container_name in self.all_hosts(): try: shutil.rmtree( self.get_local_mount_dir(container_name)) except OSError as e: # no such file or directory if e.errno != errno.ENOENT: raise def _create_host_mount_dirs(self): for container_name in self.all_hosts(): try: os.makedirs( self.get_local_mount_dir(container_name)) except OSError as e: # file exists if e.errno != errno.EEXIST: raise @staticmethod def _execute_and_wait(func, *args, **kwargs): ret = func(*args, **kwargs) # go through all lines in returned stream to ensure func finishes output = '' for line in ret: output += line return output def _create_and_start_containers(self, master_image, slave_image=None, cmd=None, **kwargs): if slave_image: for container_name in self.slaves: container_mount_dir = \ self.get_local_mount_dir(container_name) self._create_container( slave_image, container_name, container_name.split('-')[0], cmd ) self.client.start(container_name, binds={container_mount_dir: {'bind': self.mount_dir, 'ro': False}}, **kwargs) master_mount_dir = self.get_local_mount_dir(self.master) self._create_container( master_image, self.master, hostname=self.internal_master, cmd=cmd ) self.client.start(self.master, binds={master_mount_dir: {'bind': self.mount_dir, 'ro': False}}, links=zip(self.slaves, self.slaves), **kwargs) self._add_hostnames_to_slaves() def _create_container(self, image, container_name, hostname=None, cmd=None): self._execute_and_wait(self.client.create_container, image, detach=True, name=container_name, hostname=hostname, volumes=self.local_mount_dir, command=cmd, mem_limit='2g') def _add_hostnames_to_slaves(self): ips = self.get_ip_address_dict() additions_to_etc_hosts = '' for host in self.all_internal_hosts(): additions_to_etc_hosts += '%s\t%s\n' % (ips[host], host) for host in self.slaves: self.exec_cmd_on_host( host, 'bin/bash -c \'echo "%s" >> /etc/hosts\'' % additions_to_etc_hosts ) def _ensure_docker_containers_started(self, image): centos_based_images = [BASE_TD_IMAGE_NAME] timeout = 0 is_host_started = {} for host in self.all_hosts(): is_host_started[host] = False while timeout < self._DOCKER_START_TIMEOUT: for host in self.all_hosts(): atomic_is_started = True atomic_is_started &= \ self.client.inspect_container(host)['State']['Running'] if image in centos_based_images or \ image.startswith(self.IMAGE_NAME_BASE): atomic_is_started &= \ self._are_centos_container_services_up(host) is_host_started[host] = atomic_is_started if not DockerCluster._are_all_hosts_started(is_host_started): timeout += 1 sleep(1) else: break if timeout is self._DOCKER_START_TIMEOUT: raise DockerClusterException( 'Docker container timed out on start.' + str(is_host_started)) @staticmethod def _are_all_hosts_started(host_started_map): all_started = True for host in host_started_map.keys(): all_started &= host_started_map[host] return all_started def _are_centos_container_services_up(self, host): """Some essential services in our CentOS containers take some time to start after the container itself is up. This function checks whether those services are up and returns a boolean accordingly. Specifically, we check that the app-admin user has been created and that the ssh daemon is up. Args: host: the host to check. Returns: True if the specified services have started, False otherwise. """ ps_output = self.exec_cmd_on_host(host, 'ps') # also ensure that the app-admin user exists try: user_output = self.exec_cmd_on_host( host, 'grep app-admin /etc/passwd' ) user_output += self.exec_cmd_on_host(host, 'stat /home/app-admin') except OSError: user_output = '' if 'sshd_bootstrap' in ps_output or 'sshd\n' not in ps_output\ or not user_output: return False return True def exec_cmd_on_host(self, host, cmd, raise_error=True, tty=False): ex = self.client.exec_create(self.__get_unique_host(host), cmd, tty=tty) output = self.client.exec_start(ex['Id'], tty=tty) exit_code = self.client.exec_inspect(ex['Id'])['ExitCode'] if raise_error and exit_code: raise OSError(exit_code, output) return output @staticmethod def _get_master_image_name(cluster_type): return os.path.join(DockerCluster.IMAGE_NAME_BASE, '%s_master' % (cluster_type)) @staticmethod def _get_slave_image_name(cluster_type): return os.path.join(DockerCluster.IMAGE_NAME_BASE, '%s_slave' % (cluster_type)) @staticmethod def start_bare_cluster(): dc = DockerCluster master_name = dc._get_master_image_name(dc.BARE_CLUSTER_TYPE) slave_name = dc._get_slave_image_name(dc.BARE_CLUSTER_TYPE) centos_cluster = DockerCluster('master', ['slave1', 'slave2', 'slave3'], DEFAULT_LOCAL_MOUNT_POINT, DEFAULT_DOCKER_MOUNT_POINT) if not dc._check_for_images(master_name, slave_name): centos_cluster.create_image( BASE_TD_DOCKERFILE_DIR, master_name, BASE_IMAGE_NAME, BASE_IMAGE_TAG ) centos_cluster.create_image( BASE_TD_DOCKERFILE_DIR, slave_name, BASE_IMAGE_NAME, BASE_IMAGE_TAG ) centos_cluster.start_containers(master_name, slave_name) return centos_cluster @staticmethod def start_existing_images(cluster_type): dc = DockerCluster master_name = dc._get_master_image_name(cluster_type) slave_name = dc._get_slave_image_name(cluster_type) if not dc._check_for_images(master_name, slave_name): return None centos_cluster = DockerCluster('master', ['slave1', 'slave2', 'slave3'], DEFAULT_LOCAL_MOUNT_POINT, DEFAULT_DOCKER_MOUNT_POINT) centos_cluster.start_containers(master_name, slave_name) return centos_cluster @staticmethod def _check_for_images(master_image_name, slave_image_name): client = Client(timeout=180) images = client.images() has_master_image = False has_slave_image = False for image in images: if master_image_name in image['RepoTags'][0]: has_master_image = True if slave_image_name in image['RepoTags'][0]: has_slave_image = True return has_master_image and has_slave_image def commit_images(self, cluster_type): self.client.commit(self.master, self._get_master_image_name(cluster_type)) self.client.commit(self.slaves[0], self._get_slave_image_name(cluster_type)) def run_script_on_host(self, script_contents, host): temp_script = '/tmp/tmp.sh' self.write_content_to_host('#!/bin/bash\n%s' % script_contents, temp_script, host) self.exec_cmd_on_host(host, 'chmod +x %s' % temp_script) return self.exec_cmd_on_host(host, temp_script, tty=True) def write_content_to_host(self, content, path, host): filename = os.path.basename(path) dest_dir = os.path.dirname(path) host_local_mount_point = self.get_local_mount_dir(host) local_path = os.path.join(host_local_mount_point, filename) with open(local_path, 'w') as config_file: config_file.write(content) self.exec_cmd_on_host(host, 'mkdir -p ' + dest_dir) self.exec_cmd_on_host( host, 'cp %s %s' % (os.path.join(self.mount_dir, filename), dest_dir)) def copy_to_host(self, source_path, dest_host): shutil.copy(source_path, self.get_local_mount_dir(dest_host)) def get_ip_address_dict(self): ip_addresses = {} for host, internal_host in zip(self.all_hosts(), self.all_internal_hosts()): inspect = self.client.inspect_container(host) ip_addresses[host] = inspect['NetworkSettings']['IPAddress'] ip_addresses[internal_host] = \ inspect['NetworkSettings']['IPAddress'] return ip_addresses def _post_presto_install(self): for worker in self.slaves: self.run_script_on_host( 'sed -i /node.id/d /etc/presto/node.properties; ' 'uuid=$(uuidgen); ' 'echo node.id=$uuid >> /etc/presto/node.properties', worker ) def postinstall(self, installer): from tests.product.standalone.presto_installer \ import StandalonePrestoInstaller _post_install_hooks = { StandalonePrestoInstaller: DockerCluster._post_presto_install } hook = _post_install_hooks.get(installer, None) if hook: hook(self)
class DockerClient: '''A singleton class to ensure there is only one client''' _instance = None def __new__(cls, *args, **kwargs): if not cls._instance: cls._instance = super(DockerClient, cls).__new__(cls) return cls._instance def __init__(self): kwargs = kwargs_from_env(assert_hostname=False) kwargs.update({'version': 'auto'}) self.client = Client(**kwargs) try: self.client.info() # mount the /Volumes folder under mac, please refer to # https://github.com/bpeng2000/SOS/wiki/SoS-Docker-guide # for details. self.has_volumes = False if platform.system() == 'Darwin': try: # this command log in to the docker machine, check if /Volumes has been mounted, # and try to mount it if possible. This requires users to configure subprocess.call( """docker-machine ssh "{}" 'mount | grep /Volumes || {{ echo "mounting /Volumes"; sudo mount -t vboxsf Volumes /Volumes; }}' """ .format(os.environ['DOCKER_MACHINE_NAME']), shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) env.logger.trace( 'Sucessfully mount /Volumes to virtual machine') self.has_volumes = True except Exception as e: env.logger.trace( 'Failed to mount /Volumes to virtual machine: {}'. format(e)) except Exception as e: env.logger.debug('Docker client init fail: {}'.format(e)) self.client = None def total_memory(self, image='ubuntu'): '''Get the available ram fo the docker machine in Kb''' try: ret = subprocess.check_output( '''docker run -t {} cat /proc/meminfo | grep MemTotal'''. format(image), shell=True, stdin=subprocess.DEVNULL) # ret: MemTotal: 30208916 kB self.tot_mem = int(ret.split()[1]) except: # some system does not have cat or grep self.tot_mem = None return self.tot_mem def _is_image_avail(self, image): images = sum( [x['RepoTags'] for x in self.client.images() if x['RepoTags']], []) # some earlier version of docker-py returns docker.io/ for global repositories images = [x[10:] if x.startswith('docker.io/') else x for x in images] return (':' in image and image in images) or \ (':' not in image and '{}:latest'.format(image) in images) def stream(self, line): # properly output streamed output try: sys.stdout.write(json.loads(line).get('stream', '')) except ValueError: # sometimes all the data is sent on a single line ???? # # ValueError: Extra data: line 1 column 87 - line 1 column # 33268 (char 86 - 33267) # This ONLY works because every line is formatted as # {"stream": STRING} for obj in re.findall('{\s*"stream"\s*:\s*"[^"]*"\s*}', line): sys.stdout.write(json.loads(obj).get('stream', '')) def build(self, script, **kwargs): if not self.client: raise RuntimeError( 'Cannot connect to the Docker daemon. Is the docker daemon running on this host?' ) if script is not None: f = BytesIO(script.encode('utf-8')) for line in self.client.build(fileobj=f, **kwargs): self.stream(line.decode()) else: for line in self.client.build(**kwargs): self.stream(line.decode()) # if a tag is given, check if the image is built if 'tag' in kwargs and not self._is_image_avail(kwargs['tag']): raise RuntimeError('Image with tag {} is not created.'.format( kwargs['tag'])) def import_image(self, image, **kwargs): if not self.client: raise RuntimeError( 'Cannot connect to the Docker daemon. Is the docker daemon running on this host?' ) env.logger.info('docker import {}'.format(image)) self.client.import_image(image, **kwargs) def pull(self, image): if not self.client: raise RuntimeError( 'Cannot connect to the Docker daemon. Is the docker daemon running on this host?' ) # if image is specified, check if it is available locally. If not, pull it ret = 0 if not self._is_image_avail(image): env.logger.info('docker pull {}'.format(image)) # using subprocess instead of docker-py's pull function because this would have # much better progress bar display ret = subprocess.call('docker pull {}'.format(image), shell=True) #for line in self.client.pull(image, stream=True): # self.stream(line) if not self._is_image_avail(image): raise RuntimeError('Failed to pull image {}'.format(image)) return ret def commit(self, **kwargs): if not self.client: raise RuntimeError( 'Cannot connect to the Docker daemon. Is the docker daemon running on this host?' ) for line in self.client.commit(**kwargs): self.stream(line.decode()) return 0 def run(self, image, script='', interpreter='', suffix='.sh', **kwargs): if self.client is None: raise RuntimeError( 'Cannot connect to the Docker daemon. Is the docker daemon running on this host?' ) # env.logger.debug('docker_run with keyword args {}'.format(kwargs)) # # now, write a temporary file to a tempoary directory under the current directory, this is because # we need to share the directory to ... with tempfile.TemporaryDirectory(dir=os.getcwd()) as tempdir: # keep the temporary script for debugging purposes # tempdir = tempfile.mkdtemp(dir=os.getcwd()) if script: tempscript = 'docker_run_{}{}'.format(os.getpid(), suffix) with open(os.path.join(tempdir, tempscript), 'w') as script_file: script_file.write(script) # binds = [] if 'volumes' in kwargs: volumes = [kwargs['volumes']] if isinstance( kwargs['volumes'], str) else kwargs['volumes'] for vol in volumes: if not vol: continue if vol.count(':') != 1: raise RuntimeError( 'Please specify columes in the format of host_dir:mnt_dir' ) host_dir, mnt_dir = vol.split(':') if platform.system() == 'Darwin': # under Darwin, host_dir must be under /Users if not os.path.abspath(host_dir).startswith( '/Users') and not ( self.has_volumes and os.path.abspath( host_dir).startswith('/Volumes')): raise RuntimeError( 'hostdir ({}) under MacOSX must be under /Users or /Volumes (if properly configured, see https://github.com/bpeng2000/SOS/wiki/SoS-Docker-guide for details) to be usable in docker container' .format(host_dir)) binds.append('{}:{}'.format(os.path.abspath(host_dir), mnt_dir)) # volumes_opt = ' '.join('-v {}'.format(x) for x in binds) # under mac, we by default share /Users within docker if platform.system() == 'Darwin': if not any(x.startswith('/Users:') for x in binds): volumes_opt += ' -v /Users:/Users' if self.has_volumes: volumes_opt += ' -v /Volumes:/Volumes' if not any(x.startswith('/tmp:') for x in binds): volumes_opt += ' -v /tmp:/tmp' # mem_limit_opt = '' if 'mem_limit' in kwargs: mem_limit_opt = '--memory={}'.format(kwargs['mem_limit']) # volumes_from_opt = '' if 'volumes_from' in kwargs: if isinstance(kwargs['volumes_from'], str): volumes_from_opt = '--volumes_from={}'.format( kwargs['volumes_from']) elif isinstance(kwargs['volumes_from'], list): volumes_from_opt = ' '.join( '--volumes_from={}'.format(x) for x in kwargs['volumes_from']) else: raise RuntimeError( 'Option volumes_from only accept a string or list of string' .format(kwargs['volumes_from'])) # we also need to mount the script cmd_opt = '' if script and interpreter: volumes_opt += ' -v {}:{}'.format( os.path.join(tempdir, tempscript), '/var/lib/sos/{}'.format(tempscript)) cmd_opt = interpreter.replace( '{}', '/var/lib/sos/{}'.format(tempscript)) # working_dir_opt = '-w={}'.format(os.path.abspath(os.getcwd())) if 'working_dir' in kwargs: if not os.path.isabs(kwargs['working_dir']): env.logger.warning( 'An absolute path is needed for -w option of docker run command. "{}" provided, "{}" used.' .format( kwargs['working_dir'], os.path.abspath( os.path.expanduser(kwargs['working_dir'])))) working_dir_opt = '-w={}'.format( os.path.abspath( os.path.expanduser(kwargs['working_dir']))) else: working_dir_opt = '-w={}'.format(kwargs['working_dir']) # env_opt = '' if 'environment' in kwargs: if isinstance(kwargs['environment'], dict): env_opt = ' '.join( '-e {}={}'.format(x, y) for x, y in kwargs['environment'].items()) elif isinstance(kwargs['environment'], list): env_opt = ' '.join('-e {}'.format(x) for x in kwargs['environment']) elif isinstance(kwargs['environment'], str): env_opt = '-e {}'.format(kwargs['environment']) else: raise RuntimeError( 'Invalid value for option environment (str, list, or dict is allowd, {} provided)' .format(kwargs['environment'])) # port_opt = '-P' if 'port' in kwargs: if isinstance(kwargs['port'], (str, int)): port_opt = '-p {}'.format(kwargs['port']) elif isinstance(kwargs['port'], list): port_opt = ' '.join('-p {}'.format(x) for x in kwargs['port']) else: raise RuntimeError( 'Invalid value for option port (a list of intergers), {} provided' .format(kwargs['port'])) # name_opt = '' if 'name' in kwargs: name_opt = '--name={}'.format(kwargs['name']) # stdin_opt = '' if 'stdin_open' in kwargs and kwargs['stdin_optn']: stdin_opt = '-i' # tty_opt = '-t' if 'tty' in kwargs and not kwargs['tty']: tty_opt = '' # user_opt = '' if 'user' in kwargs: user_opt = '-u {}'.format(kwargs['user']) # extra_opt = '' if 'extra_args' in kwargs: extra_opt = kwargs['extra_args'] # security_opt = '' if platform.system() == 'Linux': # this is for a selinux problem when /var/sos/script cannot be executed security_opt = '--security-opt label:disable' command = 'docker run --rm {} {} {} {} {} {} {} {} {} {} {} {} {} {}'.format( security_opt, # security option volumes_opt, # volumes volumes_from_opt, # volumes_from name_opt, # name stdin_opt, # stdin_optn tty_opt, # tty port_opt, # port working_dir_opt, # working dir user_opt, # user env_opt, # environment mem_limit_opt, # memory limit extra_opt, # any extra parameters image, # image cmd_opt) env.logger.info(command) ret = subprocess.call(command, shell=True) if ret != 0: msg = 'The script has been saved to .sos/{} so that you can execute it using the following command:\n{}'.format( tempscript, command.replace(tempdir, os.path.abspath('./.sos'))) shutil.copy(os.path.join(tempdir, tempscript), '.sos') if ret == 125: raise RuntimeError( 'Docker daemon failed (exitcode=125). ' + msg) elif ret == 126: raise RuntimeError( 'Failed to invoke specified command (exitcode=126). ' + msg) elif ret == 127: raise RuntimeError( 'Failed to locate specified command (exitcode=127). ' + msg) elif ret == 137: if not hasattr(self, 'tot_mem'): self.tot_mem = self.total_memory(image) if self.tot_mem is None: raise RuntimeError('Script killed by docker. ' + msg) else: raise RuntimeError( 'Script killed by docker, probably because of lack of RAM (available RAM={:.1f}GB, exitcode=137). ' .format(self.tot_mem / 1024 / 1024) + msg) else: raise RuntimeError( 'Executing script in docker returns an error (exitcode={}). ' .format(ret) + msg) return 0