class Sloth(cls): def __init__(self, config): super().__init__(config) self._docker_config = extension['config'] self._docker_client = Client(self._docker_config.get('base_url'), timeout=10) self._docker_client._version = str( self._docker_config.get('version') or self._docker_client._version) self._docker_image = self._docker_config.get('image') or slugify( self.listen_point) def execute(self, action): '''Execute an action inside a container, then commit the changes to the image and remove the container. :param action: action to be executed :returns: True if the execution was successful; raises exception otherwise ''' self.processing_logger.info('Executing action: %s', action) try: container_id = self._docker_client.create_container( self._docker_image, command=action, working_dir=self.config.get('work_dir') or '.', mem_limit=self._docker_config.get('memory_limit') or 0, cpu_shares=self._docker_config.get('cpu_share') or None)['Id'] self._docker_client.start(container_id) for log in self._docker_client.attach(container_id, logs=True, stream=True): self.processing_logger.debug('%s', log) self._docker_client.commit(container_id, self._docker_image, message=action) self._docker_client.remove_container(container_id) self.processing_logger.info('Action executed: %s', action) return True except Exception: raise
class DockerObject(object): def __init__(self, repo, tag=None): self.client = Client(base_url='unix://var/run/docker.sock', version='auto') self.logger = logging.getLogger(LOGGER) self.repo = repo self.tag = tag self.command = None self.image = self.repo if self.tag: self.image = self.image + ':' + self.tag self.__container = None self.environment = {} self.port_bindings = None self.privileged = None self.binds = None self.hostname = None self.links = [] # internal containers are created, started, and destryed along with this container. self.internal_containers = [] self.exit_code = None # login is global in docker and should not be implemented here. # self.login = False self.volumes_from = None self.insecure_registry = False def enable_debug(self): ch = logging.StreamHandler() self.logger.addHandler(ch) self.logger.setLevel(logging.DEBUG) def wait(self, timeout): if self.exit_code is None: self.logger.debug('waiting for container to end.') self.exit_code = self.client.wait(self.get_container(), timeout) return self.exit_code def get_exit_code(self): return self.exit_code def add_volumes_from(self, container): if self.volumes_from is None: self.volumes_from = [] if isinstance(container, DockerObject): self.volumes_from.append(container.get_container()) else: self.volumes_from.append(container) def set_volumes(self, binds): self.binds = binds def set_hostname(self, hostname): self.hostname = hostname def set_command(self, command): self.command = command def set_privileged(self, privileged): self.privileged = privileged def add_link(self, name, container, internal=False): if isinstance(container, DockerObject): self.links.append((container, name)) if internal: self.internal_containers.append(container) else: raise RuntimeError("illegal argument") def add_environment(self, key, value): self.environment[key] = value def set_port_bindings(self, port_bindings): self.port_bindings = port_bindings def add_port_binding(self, port): if self.port_bindings is None: self.port_bindings = {} self.port_bindings[port] = ('0.0.0.0', ) def expose_all_ports(self): self.pull_if_needed(repository=self.repo, tag=self.tag, insecure_registry=self.insecure_registry) image = self.client.inspect_image(image=self.image) ports = image["Config"]["ExposedPorts"] for port in ports: self.add_port_binding(port) def add_volume(self, local, container, ro=False): if self.binds is None: self.binds = {} local = os.path.abspath(local) self.binds[local] = {"ro": ro, "bind": container} def set_container(self, container): self.__container = container def get_container(self): return self.__container def get_tag(self): return self.tag def get_repository(self): return self.repo def create(self): # create only if needed if not self.should_create(): return if self.internal_containers: self.logger.debug('creating linked containers') for c in self.internal_containers: c.create() self.pull_if_needed(repository=self.repo, tag=self.tag, insecure_registry=True) ports = None if self.port_bindings: ports = [k for k in self.port_bindings] volume_to_mount = None if self.binds: volume_to_mount = [self.binds[k]['bind'] for k in self.binds] container = self.client.create_container( image=self.image, hostname=self.hostname, ports=ports, environment=self.environment, volumes=volume_to_mount, command=self.command).get('Id') self.set_container(container) self.logger.debug('Container %s created %s', self.repo, container) def start_container(self): self.logger.debug('Starting container %s (%s)', self.repo, self.get_container()) links = {} for container, name in self.links: links[container.get_container()] = name self.client.start(container=self.get_container(), links=links, port_bindings=self.port_bindings, privileged=self.privileged, binds=self.binds, volumes_from=self.volumes_from) def pull_if_needed(self, repository, tag=None, insecure_registry=False): images = self.client.images(name=repository) pull = False if len(images) == 0: self.logger.debug('Pulling: no images for %s', repository) pull = True elif tag != None: l = [x['RepoTags'] for x in images] tags = [item for sublist in l for item in sublist] repo_and_tag = repository + ':' + tag # TODO: check if tags are case sensitive # tags = [x.lower() for x in tags] # repo_and_tag = repo_and_tag.lower() if repo_and_tag not in tags: self.logger.debug('Pullin: %s not in %s', repo_and_tag, tags) pull = True if pull: self.logger.debug('Pulling %s', repository) self.client.pull(repository=repository, tag=tag, insecure_registry=insecure_registry) def should_create(self): return self.__container == None def destroy(self): if self.get_container() == None: return self.logger.debug('destroying container %s', self.repo) self.client.remove_container(container=self.get_container(), force=True) self.set_container(None) if self.internal_containers: self.logger.debug('destroying linked containers') for c in reversed(self.internal_containers): c.destroy() def start(self, wait=True): if self.should_create(): self.create() if self.internal_containers: self.logger.debug('starting linked containers') for c in self.internal_containers: c.start(wait=True) self.start_container() self.exit_code = None if wait: self.wait_for_container() def stop(self): # do not stop linked containers. as it is not a must self.logger.debug('Stopping container %s', self.repo) self.client.stop(container=self.get_container(), timeout=2) def should_start(self): if self.should_create(): return True return not self.client.inspect_container( container=self.__container)['State']['Running'] def get_port(self, port): return self.client.port(container=self.get_container(), private_port=port) def attach(self, stdout=True, stderr=True, stream=False, logs=True): return self.client.attach(container=self.get_container(), stdout=stdout, stderr=stderr, stream=stream, logs=logs) def get_hostname(self): return self.client.inspect_container( container=self.__container)['Config']['Hostname'] def get_ip(self): return self.client.inspect_container( container=self.__container)["NetworkSettings"]["IPAddress"] def inspect(self): return self.client.inspect_container(container=self.__container) def __enter__(self): self.create() return self def __exit__(self, type_, value_, tb): self.destroy() def __del__(self): self.destroy() def get_host_port(self, port): ports = self.get_port(port) host, port = 'localhost', ports[0]['HostPort'] port = int(port) return host, port def check_port_open(self, port): host, port = self.get_host_port(port) import socket sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) result = sock.connect_ex((host, port)) sock.close() if result == 0: return True return False def random_password(self, size=8, chars=string.ascii_uppercase + string.ascii_lowercase + string.digits): # http://stackoverflow.com/questions/2257441/random-string-generation-with-upper-case-letters-and-digits-in-python return ''.join(random.choice(chars) for _ in range(size)) def wait_for_container(self): raise NotImplementedError()
class DockerObject(object): def __init__(self, repo, tag = None): self.client = Client(base_url='unix://var/run/docker.sock', version='auto') self.logger = logging.getLogger(LOGGER) self.repo = repo self.tag = tag self.command = None self.image = self.repo if self.tag: self.image = self.image + ':' + self.tag self.__container = None self.environment = {} self.port_bindings = None self.privileged = None self.binds = None self.hostname = None self.links = [] # internal containers are created, started, and destryed along with this container. self.internal_containers = [] self.exit_code = None # login is global in docker and should not be implemented here. # self.login = False self.volumes_from = None self.insecure_registry = False def enable_debug(self): ch = logging.StreamHandler() self.logger.addHandler(ch) self.logger.setLevel(logging.DEBUG) def wait(self, timeout): if self.exit_code is None: self.logger.debug('waiting for container to end.') self.exit_code = self.client.wait(self.get_container(), timeout) return self.exit_code def get_exit_code(self): return self.exit_code def add_volumes_from(self, container): if self.volumes_from is None: self.volumes_from = [] if isinstance(container, DockerObject): self.volumes_from.append(container.get_container()) else: self.volumes_from.append(container) def set_volumes(self, binds): self.binds = binds def set_hostname(self, hostname): self.hostname = hostname def set_command(self, command): self.command = command def set_privileged(self, privileged): self.privileged = privileged def add_link(self, name, container, internal = False): if isinstance(container, DockerObject): self.links.append((container, name)) if internal: self.internal_containers.append(container) else: raise RuntimeError("illegal argument") def add_environment(self, key, value): self.environment[key] = value def set_port_bindings(self, port_bindings): self.port_bindings = port_bindings def add_port_binding(self, port): if self.port_bindings is None: self.port_bindings = {} self.port_bindings[port] = ('0.0.0.0',) def expose_all_ports(self): self.pull_if_needed(repository=self.repo, tag=self.tag, insecure_registry = self.insecure_registry) image = self.client.inspect_image(image=self.image) ports = image["Config"]["ExposedPorts"] for port in ports: self.add_port_binding(port) def add_volume(self, local, container, ro=False): if self.binds is None: self.binds = {} local = os.path.abspath(local) self.binds[local] = {"ro":ro, "bind": container} def set_container(self, container): self.__container = container def get_container(self): return self.__container def get_tag(self): return self.tag def get_repository(self): return self.repo def create(self): # create only if needed if not self.should_create(): return if self.internal_containers: self.logger.debug('creating linked containers') for c in self.internal_containers: c.create() self.pull_if_needed(repository=self.repo, tag=self.tag, insecure_registry = True) ports = None if self.port_bindings: ports = [k for k in self.port_bindings] volume_to_mount = None if self.binds: volume_to_mount = [self.binds[k]['bind'] for k in self.binds] container = self.client.create_container(image=self.image, hostname=self.hostname, ports=ports, environment=self.environment, volumes=volume_to_mount, command=self.command).get('Id') self.set_container(container) self.logger.debug('Container %s created %s', self.repo, container) def start_container(self): self.logger.debug('Starting container %s (%s)', self.repo, self.get_container()) links = {} for container, name in self.links: links[container.get_container()] = name self.client.start(container=self.get_container(), links=links, port_bindings=self.port_bindings, privileged=self.privileged, binds=self.binds, volumes_from=self.volumes_from) def pull_if_needed(self, repository, tag = None, insecure_registry = False): images = self.client.images(name=repository) pull = False if len(images) == 0: self.logger.debug('Pulling: no images for %s', repository) pull = True elif tag != None: l = [ x['RepoTags'] for x in images ] tags = [ item for sublist in l for item in sublist ] repo_and_tag = repository + ':' + tag # TODO: check if tags are case sensitive # tags = [x.lower() for x in tags] # repo_and_tag = repo_and_tag.lower() if repo_and_tag not in tags: self.logger.debug('Pullin: %s not in %s', repo_and_tag, tags) pull = True if pull: self.logger.debug('Pulling %s', repository) self.client.pull(repository=repository, tag=tag, insecure_registry=insecure_registry) def should_create(self): return self.__container == None def destroy(self): if self.get_container() == None: return self.logger.debug('destroying container %s', self.repo) self.client.remove_container(container=self.get_container(), force=True) self.set_container(None) if self.internal_containers: self.logger.debug('destroying linked containers') for c in reversed(self.internal_containers): c.destroy() def start(self, wait = True): if self.should_create(): self.create() if self.internal_containers: self.logger.debug('starting linked containers') for c in self.internal_containers: c.start(wait=True) self.start_container() self.exit_code = None if wait: self.wait_for_container() def stop(self): # do not stop linked containers. as it is not a must self.logger.debug('Stopping container %s', self.repo) self.client.stop(container=self.get_container(), timeout=2) def should_start(self): if self.should_create(): return True return not self.client.inspect_container(container=self.__container)['State']['Running'] def get_port(self, port): return self.client.port(container=self.get_container(), private_port=port) def attach(self, stdout=True, stderr=True, stream=False, logs=True): return self.client.attach(container=self.get_container(), stdout=stdout, stderr=stderr, stream=stream, logs=logs) def get_hostname(self): return self.client.inspect_container(container=self.__container)['Config']['Hostname'] def get_ip(self): return self.client.inspect_container(container=self.__container)["NetworkSettings"]["IPAddress"] def inspect(self): return self.client.inspect_container(container=self.__container) def __enter__(self): self.create() return self def __exit__(self, type_, value_, tb): self.destroy() def __del__(self): self.destroy() def get_host_port(self, port): ports = self.get_port(port) host, port = 'localhost', ports[0]['HostPort'] port = int(port) return host, port def check_port_open(self, port): host, port = self.get_host_port(port) import socket sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) result = sock.connect_ex((host, port)) sock.close() if result == 0: return True return False def random_password(self,size=8, chars=string.ascii_uppercase + string.ascii_lowercase + string.digits): # http://stackoverflow.com/questions/2257441/random-string-generation-with-upper-case-letters-and-digits-in-python return ''.join(random.choice(chars) for _ in range(size)) def wait_for_container(self): raise NotImplementedError()
class Builder: def __init__(self, workdir, yaml_file=".drone.yml"): self.__base_build_docker_dir = "/home/wlsuser/gavin/tmp/build_docker" self.workspace = workdir self.yaml_file = path.join(workdir, yaml_file) self.ydata, self.is_err = load_yaml(self.yaml_file) self.cli = Client(base_url=BASE_URL) self.errmsg='' #can not have number self.image_name = 'mytest' def create_docker_dir(self): if self.is_err: return folder_name = create_folder_name(self.ydata.get('project')) self.build_docker_dir = path.join(self.__base_build_docker_dir, folder_name) remove_folder(self.build_docker_dir) try: mkdir(self.build_docker_dir) except FileNotFoundError: print ("mkdir error\n") self.is_err = True self.errmsg = "mkdir %s error"%(self.build_docker_dir) #self.image_name = folder_name def create_docker_file(self): if self.is_err: return docker_creat = new_Dockerfile(self.build_docker_dir) image_name = self.ydata.get("image") docker_creat.write_From(image_name) envs = self.ydata.get("env", None) if envs: for env in envs: docker_creat.write_Env(env) pre_scripts = self.ydata.get("scripts", None) if pre_scripts: for pre_script in pre_scripts: docker_creat.write_Run(pre_script) #docker_creat.write_Volume("/data") docker_creat.write_Done() def create_run_file(self): if self.is_err: return runfile_creat = new_Runfile(self.workspace) try: build_scripts = self.ydata.get("build").get("scripts") except AttributeError: build_scripts = None if build_scripts: for build_script in build_scripts: runfile_creat.write_Command(build_script) runfile_creat.write_Done() self.runfile = runfile_creat.get_Runfile() #+x cmd = "chmod +x %s"%(self.runfile) run_Command(cmd) def build_docker_image(self): if self.is_err: return self.__remove_self_image() print("PATH:", self.build_docker_dir) print("tag name:", self.image_name) docker_build = self.cli.build(path=self.build_docker_dir, rm=True, forcerm=True, tag=self.image_name, stream=True) for line in docker_build: line=eval(line) print(line) if line.get('errorDetail', None): print("build docker image error, errmsg:", line.get('errorDetail')) self.is_err=True self.errmsg = line.get('errorDetail').get('message') def build_sources(self): if self.is_err: return cid = self.cli.create_container(self.image_name, command='/bin/bash ./drun.sh', volumes=['/tmp/drone'], working_dir='/tmp/drone', host_config=self.cli.create_host_config(binds={self.workspace:{'bind':'/tmp/drone', 'mode': 'rw',}})) print (cid.get('Id')) response = self.cli.start(container=cid.get('Id')) attachs = self.cli.attach(cid.get('Id'), stream=True) for attach in attachs: print(attach) #logs=self.cli.logs(container=cid.get('Id'), stdout=True, stderr=False) #print("Logs:", logs) self.__remove_self_image() self.errmsg=self.cli.logs(container=cid.get('Id'), stdout=False, stderr=True) if self.errmsg: self.is_err=True print("Build source error, error message:%s"%(self.errmsg)) return print("Compile source code sucessfully!") def __remove_self_image(self): if is_Image_Exist(self.image_name): self.cli.remove_image(image=self.image_name,force=True)