class TestSystem(unittest.TestCase): podman = None # initialized podman configuration for tests service = None # podman service instance topContainerId = "" def setUp(self): super().setUp() self.client = DockerClient(base_url="tcp://127.0.0.1:8080", timeout=15) TestSystem.podman.restore_image_from_cache(self.client) TestSystem.topContainerId = common.run_top_container(self.client) def tearDown(self): common.remove_all_containers(self.client) common.remove_all_images(self.client) self.client.close() return super().tearDown() @classmethod def setUpClass(cls): super().setUpClass() TestSystem.podman = Podman() TestSystem.service = TestSystem.podman.open("system", "service", "tcp:127.0.0.1:8080", "--time=0") # give the service some time to be ready... time.sleep(2) returncode = TestSystem.service.poll() if returncode is not None: raise subprocess.CalledProcessError(returncode, "podman system service") @classmethod def tearDownClass(cls): TestSystem.service.terminate() stdout, stderr = TestSystem.service.communicate(timeout=0.5) if stdout: sys.stdout.write("\nImages Service Stdout:\n" + stdout.decode("utf-8")) if stderr: sys.stderr.write("\nImAges Service Stderr:\n" + stderr.decode("utf-8")) TestSystem.podman.tear_down() return super().tearDownClass() def test_Info(self): self.assertIsNotNone(self.client.info()) def test_info_container_details(self): info = self.client.info() self.assertEqual(info["Containers"], 1) self.client.containers.create(image=constant.ALPINE) info = self.client.info() self.assertEqual(info["Containers"], 2) def test_version(self): version = self.client.version() self.assertIsNotNone(version["Platform"]["Name"])
def _is_docker_running(client: docker.DockerClient) -> bool: """ Check if docker binary and docker daemon are available :param client: DockerClient instance :return: true or false """ try: client.info() return True except (ImportError, IOError, DockerException): return False
def docker(): client = DockerClient("unix:///var/run/docker.sock") already_swarm = client.info()["Swarm"]["LocalNodeState"] == "active" if not already_swarm: client.swarm.init() network = client.networks.create("dockerspawner-test-network", driver="overlay", attachable=True) network.connect("dockerspawner-test") try: yield client finally: for service in client.services.list(): if service.name.startswith("jupyterhub-client"): service.scale(0) for _ in range(10): if not service.tasks(): break sleep(1) service.remove() network.disconnect("dockerspawner-test") network.remove() if not already_swarm: client.swarm.leave(True)
def is_swarm_manager(client: DockerClient) -> bool: """ Check if the given client is connected to a Docker Engine running in Swarm Mode and has the manager role. :param client: DockerClient :rtype: bool :return: true if Docker Engine is in Swarm mode, else false """ info = client.info() swarm = info['Swarm'] return swarm['LocalNodeState'] == 'active' and swarm['ControlAvailable']
class DockerController: def __init__(self): self.client = DockerClient(base_url='tcp://10.200.10.1:2375') def _info(self): print(self.client.info()) def show_running_containers(self): return self.client.containers.list() def build_image(self, path, tag): # path = os.path.join(os.getcwd(), 'files', dir_name) self.client.build(path=path, tag=tag) def show_images(self): return self.client.images() def run_container(self, image: str, internal_web_port: int, build_name=None, command=None): ssh_port = get_no_port_being_used() web_port = get_no_port_being_used() container = self.client.containers.run(image=image, ports={ '22/tcp': ssh_port, f'{internal_web_port}/tcp': web_port }, name=build_name, command=command, detach=True, pids_limit=MAX_PID) logging.log(logging.INFO, f"ssh port is {ssh_port}, web port is {web_port}") container_info = {} container_info['id'] = container.id container_info['ssh_port'] = ssh_port container_info['web_port'] = web_port return container_info def rm_container(self, container_id: str): container = self.client.containers.get(container_id) # container.stop() # 直接删除 container.remove(force=True) logging.log(logging.INFO, f'{container.id} has been removed') def exec_container(self, container_id: str, command): self.client.containers.get(container_id).exec_run(command)
class DockerPreliminaryInformation: def __init__(self, unixsock): self.clinobjc = DockerClient(base_url=unixsock) def get_docker_info(self): """ Returns container station information """ return self.clinobjc.info() def get_docker_version(self): """ Returns container station versioning """ return self.clinobjc.version()
class DockerEnvironmentDriver(EnvironmentDriver): """ This EnvironmentDriver handles environment management in the project using docker Parameters ---------- filepath : str, optional home filepath for project (default is empty) docker_execpath : str, optional execution path for docker (default is "docker" which defers to system) docker_socket : str, optional socket path to docker daemon to connect (default is None, this takes the default path for the system) Attributes ---------- filepath : str home filepath for project docker_execpath : str docker execution path for the system docker_socket : str specific socket for docker (default is None, which means system default is used by docker) client : DockerClient docker python api client cpu_prefix : list list of strings for the prefix command for all docker commands info : dict information about the docker daemon connection is_connected : bool True if connected to daemon else False type : str type of EnvironmentDriver """ def __init__(self, filepath="", docker_execpath="docker", docker_socket=None): if not docker_socket: if platform.system() != "Windows": docker_socket = "unix:///var/run/docker.sock" super(DockerEnvironmentDriver, self).__init__() self.filepath = filepath # Check if filepath exists if not os.path.exists(self.filepath): raise PathDoesNotExist( __("error", "controller.environment.driver.docker.__init__.dne", filepath)) # TODO: separate methods for instantiation into init function below # Initiate Docker execution self.docker_execpath = docker_execpath try: self.docker_socket = docker_socket if self.docker_socket: self.client = DockerClient(base_url=self.docker_socket) self.cpu_prefix = [ self.docker_execpath, "-H", self.docker_socket ] else: self.client = DockerClient() self.cpu_prefix = [self.docker_execpath] self.info = self.client.info() except Exception: raise EnvironmentInitFailed( __("error", "controller.environment.driver.docker.__init__", platform.system())) self.is_connected = True if self.info["Images"] != None else False self._is_initialized = self.is_initialized self.type = "docker" # execute a docker info command # make sure it passed # if gpu_requested == True: # pass # if gpu is requested then # make sure docker info confirms that it"s available @property def is_initialized(self): # TODO: Check if Docker is up and running if self.is_connected: self._is_initialized = True return self._is_initialized self._is_initialized = False return self._is_initialized def create(self, path=None, output_path=None, language=None): if not path: path = os.path.join(self.filepath, "Dockerfile") if not output_path: directory, filename = os.path.split(path) output_path = os.path.join(directory, "datmo" + filename) if not language: language = "python3" requirements_filepath = None if not os.path.isfile(path): if language == "python3": # Create requirements txt file for python requirements_filepath = self.create_requirements_file() # Create Dockerfile for ubuntu path = self.create_default_dockerfile( requirements_filepath=requirements_filepath, language=language) else: raise EnvironmentDoesNotExist( __("error", "controller.environment.driver.docker.create.dne", path)) if os.path.isfile(output_path): raise FileAlreadyExistsException( __("error", "controller.environment.driver.docker.create.exists", output_path)) success = self.form_datmo_definition_file( input_definition_path=path, output_definition_path=output_path) return success, path, output_path, requirements_filepath def build(self, name, path): return self.build_image(name, path) def run(self, name, options, log_filepath): run_return_code, run_id = \ self.run_container(image_name=name, **options) log_return_code, logs = self.log_container(run_id, filepath=log_filepath) final_return_code = run_return_code and log_return_code return final_return_code, run_id, logs def stop(self, run_id, force=False): stop_result = self.stop_container(run_id) remove_run_result = self.remove_container(run_id, force=force) return stop_result and remove_run_result def remove(self, name, force=False): stop_and_remove_containers_result = \ self.stop_remove_containers_by_term(name, force=force) remove_image_result = self.remove_image(name, force=force) return stop_and_remove_containers_result and \ remove_image_result def init(self): # TODO: Fill in to start up Docker try: # Startup Docker pass except Exception as e: raise EnvironmentExecutionException( __("error", "controller.environment.driver.docker.init", str(e))) return True def get_tags_for_docker_repository(self, repo_name): # TODO: Use more common CLI command (e.g. curl instead of wget) """Method to get tags for docker repositories Parameters ---------- repo_name: str Docker repository name Returns ------- list List of tags available for that docker repo """ docker_repository_tag_cmd = "wget -q https://registry.hub.docker.com/v1/repositories/" + repo_name + "/tags -O -" string_repository_tags = subprocess.check_output( docker_repository_tag_cmd, shell=True) string_repository_tags = string_repository_tags.decode().strip() repository_tags = ast.literal_eval(string_repository_tags) list_tag_names = [] for repository_tag in repository_tags: list_tag_names.append(repository_tag["name"]) return list_tag_names def build_image(self, tag, definition_path="Dockerfile"): """Builds docker image Parameters ---------- tag : str name to tag image with definition_path : str absolute file path to the definition Returns ------- bool True if success Raises ------ EnvironmentExecutionException """ try: docker_shell_cmd_list = list(self.cpu_prefix) docker_shell_cmd_list.append("build") # Passing tag name for the image docker_shell_cmd_list.append("-t") docker_shell_cmd_list.append(tag) # Passing path of Dockerfile docker_shell_cmd_list.append("-f") docker_shell_cmd_list.append(definition_path) dockerfile_dirpath = os.path.split(definition_path)[0] docker_shell_cmd_list.append(str(dockerfile_dirpath)) # Remove intermediate containers after a successful build docker_shell_cmd_list.append("--rm") process_returncode = subprocess.Popen(docker_shell_cmd_list).wait() if process_returncode == 0: return True elif process_returncode == 1: raise EnvironmentExecutionException( __("error", "controller.environment.driver.docker.build_image", "Docker subprocess failed")) except Exception as e: raise EnvironmentExecutionException( __("error", "controller.environment.driver.docker.build_image", str(e))) def get_image(self, image_name): return self.client.images.get(image_name) def list_images(self, name=None, all_images=False, filters=None): return self.client.images.list(name=name, all=all_images, filters=filters) def search_images(self, term): return self.client.images.search(term=term) def remove_image(self, image_id_or_name, force=False): try: if force: docker_image_remove_cmd = list(self.cpu_prefix) docker_image_remove_cmd.extend(["rmi", "-f", image_id_or_name]) else: docker_image_remove_cmd = list(self.cpu_prefix) docker_image_remove_cmd.extend(["rmi", image_id_or_name]) subprocess.check_output(docker_image_remove_cmd).decode().strip() except Exception as e: raise EnvironmentExecutionException( __("error", "controller.environment.driver.docker.remove_image", str(e))) return True def remove_images(self, name=None, all=False, filters=None, force=False): """Remove multiple images """ try: images = self.list_images(name=name, all_images=all, filters=filters) for image in images: self.remove_image(image.id, force=force) except Exception as e: raise EnvironmentExecutionException( __("error", "controller.environment.driver.docker.remove_images", str(e))) return True def run_container(self, image_name, command=None, ports=None, name=None, volumes=None, detach=False, stdin_open=False, tty=False, gpu=False, api=False): """Run Docker container with parameters given as defined below Parameters ---------- image_name : str Docker image name command : list, optional List with complete user-given command (e.g. ["python3", "cool.py"]) ports : list, optional Here are some example ports used for common applications. * "jupyter notebook" - 8888 * flask API - 5000 * tensorboard - 6006 An example input for the above would be ["8888:8888", "5000:5000", "6006:6006"] which maps the running host port (right) to that of the environment (left) name : str, optional User given name for container volumes : dict, optional Includes storage volumes for docker (e.g. { outsidepath1 : {"bind", containerpath2, "mode", MODE} }) detach : bool, optional True if container is to be detached else False stdin_open : bool, optional True if stdin is open else False tty : bool, optional True to connect pseudo-terminal with stdin / stdout else False gpu : bool, optional True if GPU should be enabled else False api : bool, optional True if Docker python client should be used else use subprocess Returns ------- if api=False: return_code: int integer success code of command container_id: str output container id if api=True & if detach=True: container_obj: Container object from Docker python api with details about container if api=True & if detach=False: logs: str output logs for the run function Raises ------ EnvironmentExecutionException error in running the environment command """ try: container_id = None if api: # calling the docker client via the API # TODO: Test this out for the API (need to verify ports work) if detach: command = " ".join(command) if command else command container = \ self.client.containers.run(image_name, command, ports=ports, name=name, volumes=volumes, detach=detach, stdin_open=stdin_open) return container else: command = " ".join(command) if command else command logs = self.client.containers.run(image_name, command, ports=ports, name=name, volumes=volumes, detach=detach, stdin_open=stdin_open) return logs.decode() else: # if calling run function with the shell commands if gpu: docker_shell_cmd_list = list(self.cpu_prefix) else: docker_shell_cmd_list = list(self.cpu_prefix) docker_shell_cmd_list.append("run") if name: docker_shell_cmd_list.append("--name") docker_shell_cmd_list.append(name) if stdin_open: docker_shell_cmd_list.append("-i") if tty: docker_shell_cmd_list.append("-t") if detach: docker_shell_cmd_list.append("-d") # Volume if volumes: # Mounting volumes for key in list(volumes): docker_shell_cmd_list.append("-v") volume_mount = key + ":" + volumes[key]["bind"] + ":" + \ volumes[key]["mode"] docker_shell_cmd_list.append(volume_mount) if ports: # Mapping ports for mapping in ports: docker_shell_cmd_list.append("-p") docker_shell_cmd_list.append(mapping) docker_shell_cmd_list.append(image_name) if command: docker_shell_cmd_list.extend(command) return_code = subprocess.call(docker_shell_cmd_list) if return_code != 0: raise EnvironmentExecutionException( __( "error", "controller.environment.driver.docker.run_container", docker_shell_cmd_list)) list_process_cmd = list(self.cpu_prefix) list_process_cmd.extend(["ps", "-q", "-l"]) container_id = subprocess.check_output(list_process_cmd) container_id = container_id.decode().strip() except Exception as e: raise EnvironmentExecutionException( __("error", "controller.environment.driver.docker.run_container", str(e))) return return_code, container_id def get_container(self, container_id): return self.client.containers.get(container_id) def list_containers(self, all=False, before=None, filters=None, limit=-1, since=None): return self.client.containers.list(all=all, before=before, filters=filters, limit=limit, since=since) def stop_container(self, container_id): try: docker_container_stop_cmd = list(self.cpu_prefix) docker_container_stop_cmd.extend(["stop", container_id]) subprocess.check_output(docker_container_stop_cmd).decode().strip() except Exception as e: raise EnvironmentExecutionException( __("error", "controller.environment.driver.docker.stop_container", str(e))) return True def remove_container(self, container_id, force=False): try: docker_container_remove_cmd_list = list(self.cpu_prefix) if force: docker_container_remove_cmd_list.extend( ["rm", "-f", container_id]) else: docker_container_remove_cmd_list.extend(["rm", container_id]) subprocess.check_output( docker_container_remove_cmd_list).decode().strip() except Exception as e: raise EnvironmentExecutionException( __("error", "controller.environment.driver.docker.remove_container", str(e))) return True def log_container(self, container_id, filepath, api=False, follow=True): """Log capture at a particular point `docker logs`. Can also use `--follow` for real time logs Parameters ---------- container_id : str Docker container id filepath : str Filepath to store log file api : bool True to use the docker python api follow : bool Tail the output Returns ------- return_code : str Process return code for the container logs : str Output logs read into a string format """ # TODO: Fix function to better accomodate all logs in the same way if api: # calling the docker client via the API with open(filepath, "w") as log_file: for line in self.client.containers.get(container_id).logs( stream=True): log_file.write(to_unicode(line.strip() + "\n")) else: command = list(self.cpu_prefix) if follow: command.extend(["logs", "--follow", str(container_id)]) else: command.extend(["logs", str(container_id)]) process = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True) with open(filepath, "w") as log_file: while True: output = process.stdout.readline() if output == "" and process.poll() is not None: break if output: printable_output = output.strip().replace("\x08", " ") log_file.write(to_unicode(printable_output + "\n")) return_code = process.poll() with open(filepath, "r") as log_file: logs = log_file.read() return return_code, logs def stop_remove_containers_by_term(self, term, force=False): """Stops and removes containers by term """ try: running_docker_container_cmd_list = list(self.cpu_prefix) running_docker_container_cmd_list.extend([ "ps", "-a", "|", "grep", "'", term, "'", "|", "awk '{print $1}'" ]) running_docker_container_cmd_str = str( " ".join(running_docker_container_cmd_list)) output = subprocess.Popen(running_docker_container_cmd_str, shell=True, stdout=subprocess.PIPE) out_list_cmd, err_list_cmd = output.communicate() # checking for running container id before stopping any if out_list_cmd: docker_container_stop_cmd_list = list(self.cpu_prefix) docker_container_stop_cmd_list = docker_container_stop_cmd_list + \ ["stop", "$("] + running_docker_container_cmd_list + \ [")"] docker_container_stop_cmd_str = str( " ".join(docker_container_stop_cmd_list)) output = subprocess.Popen(docker_container_stop_cmd_str, shell=True, stdout=subprocess.PIPE) _, _ = output.communicate() # rechecking for container id after stopping them to ensure no errors output = subprocess.Popen(running_docker_container_cmd_str, shell=True, stdout=subprocess.PIPE) out_list_cmd, _ = output.communicate() if out_list_cmd: docker_container_remove_cmd_list = list(self.cpu_prefix) if force: docker_container_remove_cmd_list = docker_container_remove_cmd_list + \ ["rm", "-f", "$("] + running_docker_container_cmd_list + \ [")"] else: docker_container_remove_cmd_list = docker_container_remove_cmd_list + \ ["rm", "$("] + running_docker_container_cmd_list + \ [")"] docker_container_remove_cmd_str = str( " ".join(docker_container_remove_cmd_list)) output = subprocess.Popen(docker_container_remove_cmd_str, shell=True, stdout=subprocess.PIPE) _, _ = output.communicate() except Exception as e: raise EnvironmentExecutionException( __( "error", "controller.environment.driver.docker.stop_remove_containers_by_term", str(e))) return True def create_requirements_file(self, execpath="pipreqs"): """Create python requirements txt file for the project Parameters ---------- execpath : str, optional execpath for the pipreqs command to form requirements.txt file (default is "pipreqs") Returns ------- str absolute filepath for requirements file Raises ------ EnvironmentDoesNotExist no python requirements found for environment EnvironmentRequirementsCreateException error in running pipreqs command to extract python requirements """ try: subprocess.check_output([execpath, self.filepath, "--force"], cwd=self.filepath).strip() requirements_filepath = os.path.join(self.filepath, "requirements.txt") except Exception as e: raise EnvironmentRequirementsCreateException( __("error", "controller.environment.requirements.create", str(e))) if open(requirements_filepath, "r").read() == "\n": raise EnvironmentDoesNotExist() return requirements_filepath def create_default_dockerfile(self, requirements_filepath, language): """Create a default Dockerfile for a given language Parameters ---------- requirements_filepath : str path for the requirements txt file language : str programming language used ("python2" and "python3" currently supported) Returns ------- str absolute path for the new Dockerfile using requirements txt file """ language_dockerfile = "%sDockerfile" % language base_dockerfile_filepath = os.path.join( os.path.dirname(os.path.abspath(__file__)), "templates", language_dockerfile) # Combine dockerfile destination_dockerfile = os.path.join(self.filepath, "Dockerfile") destination = open(destination_dockerfile, "w") shutil.copyfileobj(open(base_dockerfile_filepath, "r"), destination) destination.write( to_unicode( str("COPY %s /tmp/requirements.txt\n") % os.path.split(requirements_filepath)[-1])) destination.write( to_unicode( str("RUN pip install --no-cache-dir -r /tmp/requirements.txt\n" ))) destination.close() return destination_dockerfile def form_datmo_definition_file(self, input_definition_path="Dockerfile", output_definition_path="datmoDockerfile"): """ In order to create intermediate dockerfile to run """ base_dockerfile_filepath = os.path.join( os.path.dirname(os.path.abspath(__file__)), "templates", "baseDockerfile") # Combine dockerfiles destination = open(os.path.join(self.filepath, output_definition_path), "w") shutil.copyfileobj(open(input_definition_path, "r"), destination) shutil.copyfileobj(open(base_dockerfile_filepath, "r"), destination) destination.close() return True
def nvidia_gpu(client: DockerClient) -> bool: info = client.info() return 'Runtimes' in info and 'nvidia' in info['Runtimes']
class DockerManager(): def __init__(self, docker_url=None, verbose=False): self._client = DockerClient(base_url=docker_url) self._docker_url = docker_url self._verbose = verbose try: self._client.ping() except APIError as err: logger.exception(err) raise DockerOperationError( 'Failed to connect to the Docker Engine.') def get_docker_info(self): """ Returns info about the local docker engine """ try: return { 'info': self._client.info(), 'version': self._client.version(), } except APIError as err: logger.exception(err) raise DockerOperationError( 'Failed to retrieve the Docker Engine info') def search_runnable_commands(self, image, tag, pull=True): """ Searching for ENTRYPOINT or CMD in a given image """ def _bind_shell(cmd_list): """ e.g. ["/bin/sh", "-c", "echo", "hello!"] """ if cmd_list[0] in SUPPORTED_SHELLS: return cmd_list else: # add a default shell on head return (DEFAULT_SHELL + cmd_list) full_name = '{0}:{1}'.format(image, tag) logger.info('Analyzing the [{}] image'.format(full_name)) image = self._pull_image_with_auth(image, tag) if image is not None: output = self._client.api.inspect_image(full_name) if not output or 'Config' not in output: raise FatalError( 'Failed to inspect {} image'.format(full_name)) else: entrypoint = output['Config']['Entrypoint'] cmd = output['Config']['Cmd'] commands = list() if entrypoint and not cmd: logger.debug( 'Detected only an ENTRYPOINT: {}'.format(entrypoint)) commands = _bind_shell(entrypoint) elif not entrypoint and cmd: logger.debug('Detected only a CMD: {}'.format(cmd)) commands = _bind_shell(cmd) elif entrypoint and cmd: logger.debug('Detected both ENTRYPOINT: \ {0} and CMD: {1}'.format(entrypoint, cmd)) # combining entrypoint (first) with cmd commands = _bind_shell(entrypoint) commands += cmd else: logger.info('the {} image is not runnable. \ Adding an infinite sleep'.format(full_name)) commands = DEFAULT_SHELL + ['sleep', 'infinity'] # add quotes # [0]: /bin/bash [1]: -c commands[2] = "\'" + commands[2] commands[-1] = commands[-1] + "\'" return ' '.join(commands) def _pull_image_with_auth(self, image, tag): try: return self._client.images.pull(image, tag) except ImageNotFound: # it should be an authentication error return self._image_authentication(image, tag) def _image_authentication(self, src_image, src_tag=None, auth=None): """ Handling Docker authentication if the image is hosted on a private repository Args: src_image (str): The original image that needs authentication for being fetched. src_tag (str): The tag of the image above. """ if src_tag is None: src_tag = 'latest' logger.warning('[{0}:{1}] image may not exist or authentication \ is required'.format(src_image, src_tag)) res = '' while res != 'YES': res = (input('[{0}:{1}] is correct? [Yes] Continue \ [No] Abort\n'.format(src_image, src_tag))).upper() if res == 'NO': logger.error( 'Docker image [{0}:{1}] cannot be found, the operation \ is aborted by the user.\n(Hint: Check the TOSCA manifest.)' .format(src_image, src_tag)) raise OperationAbortedByUser( CommonErrorMessages._DEFAULT_OPERATION_ABORTING_ERROR_MSG) attempts = 3 while attempts > 0: logger.info('Authenticate with the Docker Repository..') try: if auth is None: auth = create_auth_interactive( user_text='Enter the username: '******'Enter the password: '******'UNAUTHORIZED' or 'NOT FOUND' in msg: logger.info('Invalid username/password.') else: logger.exception(err) raise FatalError( CommonErrorMessages._DEFAULT_FATAL_ERROR_MSG) attempts -= 1 logger.info('Authentication failed. \ You have [{}] more attempts.'.format(attempts)) # Check authentication failure if attempts == 0: logger.error( 'You have used all the authentication attempts. Abort.') raise DockerAuthenticationFailedError( 'Authentication failed. Abort.') def _push_image(self, image, tag=None, auth=None, attempts=0): """ Push an image to a remote Docker Registry. Args: image (str): The name of the Docker Image to be pushed. tag (str): An optional tag for the Docker Image (default: 'latest') auth: An optional Dict for the authentication. attempts (int): The number of unsuccessful authentication attempts. """ if tag is None: tag = 'latest' if attempts == MAX_PUSH_ATTEMPTS: err = 'Reached max attempts for pushing \ [{0}] with tag [{1}]'.format(image, tag) logger.error(err) raise FatalError(err) logger.info('Pushing [{0}] with tag [{1}]'.format(image, tag)) result = self._client.images.push(image, tag=tag, auth_config=auth, stream=True, decode=True) # Access Denied detection for line in result: # an error is occurred if 'error' in line: # access denied if 'denied' or 'unauthorized' in line['error']: logger.info('Access to the repository denied. \ Authentication failed.') auth = create_auth_interactive() self._push_image(image, tag=tag, auth=auth, attempts=(attempts + 1)) else: logger.error( 'Unknown error during push of [{0}]: {1}'.format( image, line['error'])) raise FatalError( CommonErrorMessages._DEFAULT_FATAL_ERROR_MSG) logger.info('Image [{0}] with tag [{1}] successfully pushed.'.format( image, tag)) def _validate_build(self, build_logs): """ Validate the output of the building process. Args: build_logs (str): The output logs of a docker image building process. """ for log in build_logs: if 'stream' in log and self._verbose: print(log['stream']) if 'error' in log: logger.error('Error building the Docker Image:\ \n{}'.format(log['error'])) raise DockerOperationError('Failed to "toskosing" \ the Docker Image. Abort.') def _toskose_image_availability(self, toskose_image, toskose_tag='latest'): """ Check the availability of the official Docker Toskose image used during the "toskosing" process. Args: toskose_image (str): The official Toskose Docker image. toskose_tag (str): The tag of the image (default: 'latest'). """ try: _ = self._client.images.pull(toskose_image, tag=toskose_tag) except ImageNotFound: logger.error( 'Failed to retrieve the official Toskose image [{0}:{1}]. \ Abort'.format(toskose_image, toskose_tag)) raise DockerOperationError( 'The official Toskose image {0} not found. Try later.\ \nIf the problem persist, open an issue at {1}'.format( toskose_image, toskose_tag)) def _remove_previous_toskosed(self, image, tag=None): """ Remove previous toskosed images. Note: docker rmi doesn't remove an image if there are multiple tags referencing it. Args: image (str): The name of the Docker image. tag (str): The tag of the Docker Image (default: 'latest'). """ if tag is None: tag = 'latest' def print_well(tags): out = '' for tag in tags: out += '- {}\n'.format(tag) return out try: logger.info( 'Searching for previous toskosed images [{0}:{1}]'.format( image, tag)) image_found = self._client.images.get(image) logger.debug( 'Image [{0}] found. It\'s referenced by the following \ tags:\n\n{1}'.format(image, print_well(image_found.tags))) full_name = '{0}:{1}'.format(image, tag) if full_name in image_found.tags: self._client.images.remove(image=full_name, force=True) logger.info('Removed [{0}] reference from [{1}] image'.format( full_name, image)) try: image_found = self._client.images.get(image) logger.debug('Image [{0}][ID: {1}] still exist'.format( image, image_found.id)) except ImageNotFound: logger.info('Image [{0}][ID: {1}] doesn\'t have any \ references yet. Removed.'.format( image, image_found.id)) except ImageNotFound: logger.info('No previous image found.') def toskose_image(self, src_image, src_tag, dst_image, dst_tag, context, process_type, app_name, toskose_dockerfile=None, toskose_image=None, toskose_tag=None, enable_push=True): """ The process of "toskosing" the component(s) of a multi-component TOSCA-defined application. The "toskosing" process consists in merging contexts (e.g. artifacts, lifecycle scripts) of TOSCA software component(s) and a TOSCA container node in which they are hosted on in a new docker image. The TOSCA container node comes with a suggested docker image, which is "enriched" by the Toskose logic fetched remotely from the official Toskose Docker image, using a template Dockerfile by means of docker multi-stage features. This logic permits to handle the lifecycle of multiple components within the same container. (More details on how this process works can be found in the template Dockerfile in dockerfiles/ directory or in the official Toskose GitHub repository) In other words, this function consists in receiving a name of an existing docker image and a path to a build context, then by means of the docker client, the toskose template dockerfile is built and a new fresh docker image is generated. The new docker image includes the content of the original image, the contexts of the TOSCA software components and the logic of toskose for managing them. Args: src_image (str): The name of the image to be "toskosed". src_tag (str): The tag of the image to be "toskosed". dst_image (str): The name of the "toskosed" image. dst_tag (str): The tag of the "toskosed" image. context (str): The path of the application context. process_type (enum): The type of "toskosing" process. [unit/manager/free] app_name (str): The name of the TOSCA application. toskose_dockerfile (str): The path of the template dockerfile used in the process. toskose_image (str): The Docker Toskose base-image used in the "toskosing" process. toskose_tag (str): The tag of the Docker Toskose base-image. enable_push (bool): enable/disable pushing of the "toskosed" image. (default: True) """ if toskose_dockerfile is not None: if not os.path.exists(toskose_dockerfile): raise ValueError('The given toskose template dockerfile \ {} doesn\'t exist'.format(toskose_dockerfile)) if not app_name: raise ValueError('A name associated to the TOSCA application \ must be given') template_dir = os.path.join(os.path.dirname(__file__), DOCKERFILE_TEMPLATES_PATH) # TODO can be enanched if process_type == ToskosingProcessType.TOSKOSE_UNIT: if toskose_image is None: toskose_image = constants.DEFAULT_TOSKOSE_UNIT_BASE_IMAGE if toskose_tag is None: toskose_tag = constants.DEFAULT_TOSKOSE_UNIT_BASE_TAG if toskose_dockerfile is None: toskose_dockerfile = os.path.join( template_dir, DOCKERFILE_TOSKOSE_UNIT_TEMPLATE) elif process_type == ToskosingProcessType.TOSKOSE_MANAGER: if toskose_image is None: toskose_image = constants.DEFAULT_MANAGER_BASE_IMAGE if toskose_tag is None: toskose_tag = constants.DEFAULT_MANAGER_BASE_TAG if toskose_dockerfile is None: toskose_dockerfile = os.path.join( template_dir, DOCKERFILE_TOSKOSE_MANAGER_TEMPLATE) src_image = toskose_image src_tag = toskose_tag elif process_type == ToskosingProcessType.TOSKOSE_FREE: pass else: raise ValueError('Cannot recognize the "toskosing" process \ {}'.format(process_type)) if toskose_image is not None: self._toskose_image_availability(toskose_image, toskose_tag) # Check if the original image exists and needs authentication # to be fetched. # note: docker client does not distinguish between authentication # error or invalid image name (?) logger.info('Pulling [{0}:{1}]'.format(src_image, src_tag)) self._pull_image_with_auth(src_image, src_tag) # TODO can be removed? is it really necessary? self._remove_previous_toskosed(dst_image, dst_tag) logger.info('Toskosing [{0}:{1}] image'.format(src_image, src_tag)) try: build_args = { 'TOSCA_SRC_IMAGE': '{0}:{1}'.format(src_image, src_tag), 'TOSKOSE_BASE_IMG': '{0}:{1}'.format(toskose_image, toskose_tag) } image, build_logs = self._client.images.build( path=context, tag='{0}:{1}'.format(dst_image, dst_tag), buildargs=build_args, dockerfile=toskose_dockerfile, rm=True, # remove intermediate containers ) self._validate_build(build_logs) # push the "toskosed" image if enable_push: self._push_image(dst_image, tag=dst_tag) logger.info( '[{0}:{1}] image successfully toskosed in [{2}:{3}].'.format( src_image, src_tag, dst_image, dst_tag)) except (BuildError, APIError) as err: logger.exception(err) raise DockerOperationError('Failed to build image') def close(self): self._client.close()
class DockerEnvironmentDriver(EnvironmentDriver): """ This EnvironmentDriver handles environment management in the project using docker Parameters ---------- root : str, optional home filepath for project (default is empty) docker_execpath : str, optional execution path for docker (default is "docker" which defers to system) docker_socket : str, optional socket path to docker daemon to connect (default is None, this takes the default path for the system) Attributes ---------- root : str home filepath for project datmo_directory_name : str datmo directory name for project docker_execpath : str, optional docker execution path for the system docker_socket : str, optional specific socket for docker (default is None, which means system default is used by docker) client : DockerClient docker python api client cpu_prefix : list list of strings for the prefix command for all docker commands info : dict information about the docker daemon connection is_connected : bool True if connected to daemon else False type : str type of EnvironmentDriver """ def __init__(self, root, datmo_directory_name, docker_execpath="docker", docker_socket=None): super(DockerEnvironmentDriver, self).__init__() if not docker_socket: if platform.system() != "Windows": docker_socket = "unix:///var/run/docker.sock" self.root = root # Check if filepath exists if not os.path.exists(self.root): raise PathDoesNotExist( __("error", "controller.environment.driver.docker.__init__.dne", root)) self._datmo_directory_name = datmo_directory_name self._datmo_directory_path = os.path.join(self.root, self._datmo_directory_name) self.environment_directory_name = "environment" self.environment_directory_path = os.path.join( self._datmo_directory_path, self.environment_directory_name) self.docker_execpath = docker_execpath self.docker_socket = docker_socket if self.docker_socket: self.client = DockerClient(base_url=self.docker_socket) self.prefix = [self.docker_execpath, "-H", self.docker_socket] else: self.client = DockerClient() self.prefix = [self.docker_execpath] self._is_connected = False self._is_initialized = self.is_initialized self.type = "docker" with open(docker_config_filepath) as f: self.docker_config = json.load(f) @property def is_initialized(self): if self.exists_environment_dir(): self._is_initialized = True return self._is_initialized self._is_initialized = False return self._is_initialized @property def is_connected(self): # TODO: Check if Docker is up and running return self._is_connected def init(self): try: # Ensure the environment files structure exists self.ensure_environment_dir() except Exception as e: raise EnvironmentInitFailed( __("error", "controller.environment.driver.docker.init", str(e))) return True def connect(self): # TODO: Fill in to start up Docker # Startup Docker try: pass except Exception as e: raise EnvironmentExecutionError( __("error", "controller.environment.driver.docker.init", str(e))) # Initiate Docker execution try: self.info = self.client.info() self._is_connected = True if self.info["Images"] != None else False except Exception: raise EnvironmentConnectFailed( __("error", "controller.environment.driver.docker.__init__", platform.system())) return True # Environment directory def create_environment_dir(self): if not os.path.isdir(self._datmo_directory_path): raise FileStructureError( __("error", "controller.file.driver.local.create_collections_dir")) if not os.path.isdir(self.environment_directory_path): os.makedirs(self.environment_directory_path) return True def exists_environment_dir(self): return os.path.isdir(self.environment_directory_path) def ensure_environment_dir(self): if not self.exists_environment_dir(): self.create_environment_dir() return True def delete_environment_dir(self): shutil.rmtree(self.environment_directory_path) return True def list_environment_files(self): if not self.is_initialized: raise FileStructureError( __("error", "controller.file.driver.local.list_file_collections")) environment_files_list = os.listdir(self.environment_directory_path) return environment_files_list # Other functions def get_environment_types(self): # To get the current environment type return list(self.docker_config["environment_types"]) def get_supported_frameworks(self, environment_type): # To get the current environments environment_names = [] for environment_name in self.docker_config["environment_types"][ environment_type]["environment_frameworks"]: environment_names.append([ environment_name, self.docker_config["environment_types"][environment_type] ["environment_frameworks"][environment_name]["info"] ]) return environment_names def get_supported_languages(self, environment_type, environment_framework): # To get the current environments return self.docker_config["environment_types"][environment_type][ "environment_frameworks"][environment_framework][ "environment_languages"] def setup(self, options, definition_path): environment_type = options.get("environment_type", None) environment_framework = options.get("environment_framework", None) environment_language = options.get("environment_language", None) environment_types = self.get_environment_types() if environment_type not in environment_types: raise EnvironmentDoesNotExist( __("error", "controller.environment.driver.docker.setup.dne", environment_type)) available_environment_info = self.get_supported_frameworks( environment_type) available_environments = [ item[0] for item in available_environment_info ] if environment_framework not in available_environments: raise EnvironmentDoesNotExist( __("error", "controller.environment.driver.docker.setup.dne", environment_framework)) available_environment_languages = self.get_supported_languages( environment_type, environment_framework) if available_environment_languages and environment_language not in available_environment_languages: raise EnvironmentDoesNotExist( __("error", "controller.environment.driver.docker.setup.dne", environment_language)) # Validate the given definition path exists if not os.path.isdir(definition_path): raise PathDoesNotExist() # To setup the environment definition file definition_filepath = os.path.join(definition_path, "Dockerfile") with open(definition_filepath, "wb") as f: if environment_language: f.write( to_bytes("FROM datmo/%s:%s-%s%s%s" % (environment_framework, environment_type, environment_language, os.linesep, os.linesep))) else: f.write( to_bytes("FROM datmo/%s:%s%s%s" % (environment_framework, environment_type, os.linesep, os.linesep))) return True def create(self, path=None, output_path=None, workspace=None): if not path: path = os.path.join(self.root, "Dockerfile") if not output_path: directory, filename = os.path.split(path) output_path = os.path.join(directory, "datmo" + filename) if not os.path.isfile(path): raise EnvironmentDoesNotExist( __("error", "controller.environment.driver.docker.create.dne", path)) if os.path.isfile(output_path): raise FileAlreadyExistsError( __("error", "controller.environment.driver.docker.create.exists", output_path)) success = self.create_datmo_definition( input_definition_path=path, output_definition_path=output_path, workspace=workspace) return success, path, output_path # running daemon needed def build(self, name, path, workspace=None): return self.build_image(name, path, workspace) # running daemon needed def run(self, name, options, log_filepath): if "gpu" in options: gpu_ready = self.gpu_enabled() if options["gpu"] is True: if not gpu_ready: raise GPUSupportNotEnabled('nvidia') else: options['runtime'] = 'nvidia' options.pop("gpu", None) run_return_code, run_id = self.run_container(image_name=name, **options) log_return_code, logs = self.log_container(run_id, filepath=log_filepath) final_return_code = run_return_code and log_return_code return final_return_code, run_id, logs # running daemon needed def stop(self, run_id, force=False): stop_result = self.stop_container(run_id) remove_run_result = self.remove_container(run_id, force=force) return stop_result and remove_run_result # running daemon needed def remove(self, name, force=False): stop_and_remove_containers_result = \ self.stop_remove_containers_by_term(name, force=force) try: self.get_image(name) remove_image_result = self.remove_image(name, force=force) except EnvironmentImageNotFound: remove_image_result = True return stop_and_remove_containers_result and \ remove_image_result # running daemon needed def gpu_enabled(self): # test if this images works # docker run --runtime=nvidia --rm nvidia/cuda nvidia-smi process = subprocess.Popen([ "docker", "run", "--runtime=nvidia", "--rm", "nvidia/cuda", "nvidia-smi", ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() stderr = stderr.decode("utf-8") if "Unknown runtime specified nvidia" in stderr: return False if "OCI runtime create failed" in stderr: return False if len(stderr) > 2: raise GPUSupportNotEnabled(stderr) # this may mean we're good to go. Untested though. return True # running daemon needed def get_tags_for_docker_repository(self, repo_name): # TODO: Use more common CLI command (e.g. curl instead of wget) """Method to get tags for docker repositories Parameters ---------- repo_name: str Docker repository name Returns ------- list List of tags available for that docker repo """ docker_repository_tag_cmd = "wget -q https://registry.hub.docker.com/v1/repositories/" + repo_name + "/tags -O -" try: process = subprocess.Popen(docker_repository_tag_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if process.returncode > 0: raise EnvironmentExecutionError( __("error", "controller.environment.driver.docker.get_tags", str(stderr))) string_repository_tags = stdout.decode().strip() except subprocess.CalledProcessError as e: raise EnvironmentExecutionError( __("error", "controller.environment.driver.docker.get_tags", str(e))) repository_tags = ast.literal_eval(string_repository_tags) list_tag_names = [] for repository_tag in repository_tags: list_tag_names.append(repository_tag["name"]) return list_tag_names # running daemon needed def build_image(self, tag, definition_path="Dockerfile", workspace=None): """Builds docker image Parameters ---------- tag : str name to tag image with definition_path : str absolute file path to the definition workspace : str workspace to be used for the run Returns ------- bool True if success Raises ------ EnvironmentExecutionError """ try: docker_shell_cmd_list = list(self.prefix) docker_shell_cmd_list.append("build") # Passing tag name for the image docker_shell_cmd_list.append("-t") docker_shell_cmd_list.append(tag) # Passing path of Dockerfile # Creating datmoDockerfile for new build dockerfile_dirpath = os.path.split(definition_path)[0] input_dockerfile = os.path.split(definition_path)[1] output_dockerfile_path = os.path.join(dockerfile_dirpath, "datmo%s" % input_dockerfile) self.create(definition_path, output_dockerfile_path, workspace) docker_shell_cmd_list.append("-f") docker_shell_cmd_list.append(output_dockerfile_path) docker_shell_cmd_list.append(str(dockerfile_dirpath)) # Remove intermediate containers after a successful build docker_shell_cmd_list.append("--rm") process_returncode = subprocess.Popen(docker_shell_cmd_list).wait() if process_returncode == 0: return True elif process_returncode == 1: raise EnvironmentExecutionError( __("error", "controller.environment.driver.docker.build_image", "Docker subprocess failed")) except Exception as e: raise EnvironmentExecutionError( __("error", "controller.environment.driver.docker.build_image", str(e))) # running daemon needed def get_image(self, image_name): try: return self.client.images.get(image_name) except errors.ImageNotFound: raise EnvironmentImageNotFound() # running daemon needed def list_images(self, name=None, all_images=False, filters=None): return self.client.images.list(name=name, all=all_images, filters=filters) # running daemon needed def search_images(self, term): return self.client.images.search(term=term) # running daemon needed def remove_image(self, image_id_or_name, force=False): try: docker_image_remove_cmd = list(self.prefix) if force: docker_image_remove_cmd.extend(["rmi", "-f", image_id_or_name]) else: docker_image_remove_cmd.extend(["rmi", image_id_or_name]) process = subprocess.Popen(docker_image_remove_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if process.returncode > 0 or stderr: raise EnvironmentExecutionError( __("error", "controller.environment.driver.docker.remove_image", str(stderr))) except subprocess.CalledProcessError as e: raise EnvironmentExecutionError( __("error", "controller.environment.driver.docker.remove_image", str(e))) return True # running daemon needed def remove_images(self, name=None, all=False, filters=None, force=False): """Remove multiple images """ try: images = self.list_images(name=name, all_images=all, filters=filters) for image in images: self.remove_image(image.id, force=force) except Exception as e: raise EnvironmentExecutionError( __("error", "controller.environment.driver.docker.remove_images", str(e))) return True def extract_workspace_url(self, container_name, workspace=None): """Extract workspace url from the container Parameters ---------- container_name : str name of the container being run workspace : str workspace being used for the run Returns ------- str web url for the workspace being run, None if it doesn't exist """ if workspace in ['notebook', 'jupyterlab']: docker_shell_cmd_list = list(self.prefix) docker_shell_cmd_list.append("exec") docker_shell_cmd_list.append(container_name) docker_shell_cmd_list.append("jupyter") docker_shell_cmd_list.append("notebook") docker_shell_cmd_list.append("list") workspace_url = None timeout_count = 0 while workspace_url is None and timeout_count < 10: process = subprocess.Popen(docker_shell_cmd_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if process.returncode > 0: time.sleep(1) timeout_count += 1 result = re.search("(?P<url>https?://[^\s]+)", stdout.decode('utf-8')) if stdout else None workspace_url = result.group("url") if result else None return workspace_url elif workspace == 'rstudio': # Having a pause for two seconds before returning url time.sleep(2) workspace_url = 'http://localhost:8787' else: workspace_url = None return workspace_url # running daemon needed def run_container(self, image_name, command=None, ports=None, name=None, volumes=None, mem_limit=None, runtime=None, detach=False, stdin_open=False, tty=False, api=False): """Run Docker container with parameters given as defined below Parameters ---------- image_name : str Docker image name command : list, optional List with complete user-given command (e.g. ["python3", "cool.py"]) ports : list, optional Here are some example ports used for common applications. * "jupyter notebook" - 8888 * flask API - 5000 * tensorboard - 6006 An example input for the above would be ["8888:8888", "5000:5000", "6006:6006"] which maps the running host port (right) to that of the environment (left) name : str, optional User given name for container volumes : dict, optional Includes storage volumes for docker (e.g. { outsidepath1 : {"bind", containerpath2, "mode", MODE} }) mem_limit : str, optional maximum amount of memory the container can use (these options take a positive integer, followed by a suffix of b, k, m, g, to indicate bytes, kilobytes, megabytes, or gigabytes. memory limit is contrained by total memory of the VM in which docker runs) detach : bool, optional True if container is to be detached else False stdin_open : bool, optional True if stdin is open else False tty : bool, optional True to connect pseudo-terminal with stdin / stdout else False api : bool, optional True if Docker python client should be used else use subprocess Returns ------- if api=False: return_code: int integer success code of command container_id: str output container id if api=True & if detach=True: container_obj: Container object from Docker python api with details about container if api=True & if detach=False: logs: str output logs for the run function Raises ------ EnvironmentExecutionError error in running the environment command """ try: if api: # calling the docker client via the API # TODO: Test this out for the API (need to verify ports work) if detach: command = " ".join(command) if command else command container = \ self.client.containers.run(image_name, command, ports=ports, name=name, volumes=volumes, mem_limit=mem_limit, detach=detach, stdin_open=stdin_open) return container else: command = " ".join(command) if command else command logs = self.client.containers.run(image_name, command, ports=ports, name=name, volumes=volumes, mem_limit=mem_limit, detach=detach, stdin_open=stdin_open) return logs.decode() else: # if calling run function with the shell commands docker_shell_cmd_list = list(self.prefix) docker_shell_cmd_list.append("run") if name: docker_shell_cmd_list.append("--name") docker_shell_cmd_list.append(name) if runtime: docker_shell_cmd_list.append("--runtime") docker_shell_cmd_list.append(runtime) if mem_limit: docker_shell_cmd_list.append("-m") docker_shell_cmd_list.append(mem_limit) docker_shell_cmd_list.append("--memory-swap") docker_shell_cmd_list.append("-1") if stdin_open: docker_shell_cmd_list.append("-i") if tty: docker_shell_cmd_list.append("-t") if detach: docker_shell_cmd_list.append("-d") # Volume if volumes: # Mounting volumes for key in list(volumes): docker_shell_cmd_list.append("-v") volume_mount = key + ":" + volumes[key]["bind"] + ":" + \ volumes[key]["mode"] docker_shell_cmd_list.append(volume_mount) if ports: # Mapping ports for mapping in ports: docker_shell_cmd_list.append("-p") docker_shell_cmd_list.append(mapping) docker_shell_cmd_list.append(image_name) if command: docker_shell_cmd_list.extend(command) return_code = subprocess.call(docker_shell_cmd_list) if return_code != 0: raise EnvironmentExecutionError( __( "error", "controller.environment.driver.docker.run_container", str(docker_shell_cmd_list))) list_process_cmd = list(self.prefix) list_process_cmd.extend(["ps", "-q", "-l"]) process = subprocess.Popen(list_process_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if process.returncode > 0: raise EnvironmentExecutionError( __( "error", "controller.environment.driver.docker.run_container", str(stderr))) container_id = stdout.decode().strip() except subprocess.CalledProcessError as e: raise EnvironmentExecutionError( __("error", "controller.environment.driver.docker.run_container", str(e))) return return_code, container_id # running daemon needed def get_container(self, container_id): try: return self.client.containers.get(container_id) except errors.NotFound: raise EnvironmentContainerNotFound() # running daemon needed def list_containers(self, all=False, before=None, filters=None, limit=-1, since=None): return self.client.containers.list(all=all, before=before, filters=filters, limit=limit, since=since) # running daemon needed def stop_container(self, container_id): try: docker_container_stop_cmd = list(self.prefix) docker_container_stop_cmd.extend(["stop", container_id]) process = subprocess.Popen(docker_container_stop_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if process.returncode > 0: raise EnvironmentExecutionError( __("error", "controller.environment.driver.docker.stop_container", str(stderr))) except subprocess.CalledProcessError as e: raise EnvironmentExecutionError( __("error", "controller.environment.driver.docker.stop_container", str(e))) return True # running daemon needed def remove_container(self, container_id, force=False): try: docker_container_remove_cmd_list = list(self.prefix) if force: docker_container_remove_cmd_list.extend( ["rm", "-f", container_id]) else: docker_container_remove_cmd_list.extend(["rm", container_id]) process = subprocess.Popen(docker_container_remove_cmd_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if process.returncode > 0: raise EnvironmentExecutionError( __( "error", "controller.environment.driver.docker.remove_container", str(stderr))) except subprocess.CalledProcessError as e: raise EnvironmentExecutionError( __("error", "controller.environment.driver.docker.remove_container", str(e))) return True # running daemon needed def log_container(self, container_id, filepath, api=False, follow=True): """Log capture at a particular point `docker logs`. Can also use `--follow` for real time logs Parameters ---------- container_id : str Docker container id filepath : str Filepath to store log file api : bool True to use the docker python api follow : bool Tail the output Returns ------- return_code : str Process return code for the container logs : str Output logs read into a string format """ # TODO: Fix function to better accomodate all logs in the same way if api: # calling the docker client via the API with open(filepath, "wb") as log_file: for line in self.client.containers.get(container_id).logs( stream=True): log_file.write(to_bytes(line.strip() + os.linesep)) else: command = list(self.prefix) if follow: command.extend(["logs", "--follow", str(container_id)]) else: command.extend(["logs", str(container_id)]) process = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True) with open(filepath, "wb") as log_file: while True: output = process.stdout.readline() if output == "" and process.poll() is not None: break if output: printable_output = output.strip().replace("\x08", " ") log_file.write(to_bytes(printable_output + os.linesep)) return_code = process.poll() with open(filepath, "rb") as log_file: logs = log_file.read() if type(logs) != str: # handle for python 3x logs = logs.decode("utf-8") return return_code, logs # running daemon needed def stop_remove_containers_by_term(self, term, force=False): """Stops and removes containers by term """ # TODO: split out the find containers function from stop / remove try: running_docker_container_cmd_list = list(self.prefix) running_docker_container_cmd_list.extend([ "ps", "-a", "|", "grep", "'%s'" % term, "|", "awk '{print $1}'" ]) running_docker_container_cmd_str = str( " ".join(running_docker_container_cmd_list)) process = subprocess.Popen(running_docker_container_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out_list_cmd, err_list_cmd = process.communicate() if process.returncode > 0: raise EnvironmentExecutionError( __( "error", "controller.environment.driver.docker.stop_remove_containers_by_term", str(err_list_cmd))) # checking for running container id before stopping any if out_list_cmd: docker_container_stop_cmd_list = list(self.prefix) docker_container_stop_cmd_list = docker_container_stop_cmd_list + \ ["stop", "$("] + running_docker_container_cmd_list + \ [")"] docker_container_stop_cmd_str = str( " ".join(docker_container_stop_cmd_list)) process = subprocess.Popen(docker_container_stop_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) _, _ = process.communicate() # rechecking for container id after stopping them to ensure no errors process = subprocess.Popen(running_docker_container_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out_list_cmd, err_list_cmd = process.communicate() if process.returncode > 0: raise EnvironmentExecutionError( __( "error", "controller.environment.driver.docker.stop_remove_containers_by_term", str(err_list_cmd))) if out_list_cmd: docker_container_remove_cmd_list = list(self.prefix) if force: docker_container_remove_cmd_list = docker_container_remove_cmd_list + \ ["rm", "-f", "$("] + running_docker_container_cmd_list + \ [")"] else: docker_container_remove_cmd_list = docker_container_remove_cmd_list + \ ["rm", "$("] + running_docker_container_cmd_list + \ [")"] docker_container_remove_cmd_str = str( " ".join(docker_container_remove_cmd_list)) process = subprocess.Popen(docker_container_remove_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) _, err_list_cmd = process.communicate() if process.returncode > 0: raise EnvironmentExecutionError( __( "error", "controller.environment.driver.docker.stop_remove_containers_by_term", str(err_list_cmd))) except subprocess.CalledProcessError as e: raise EnvironmentExecutionError( __( "error", "controller.environment.driver.docker.stop_remove_containers_by_term", str(e))) return True def create_requirements_file(self, package_manager="pip"): """Create python requirements txt file for the project Parameters ---------- package_manager : str, optional the package manager being used during the snapshot creation Returns ------- str absolute filepath for requirements file Raises ------ EnvironmentRequirementsCreateError error in running package manager command to extract environment requirements """ if package_manager == "pip": try: requirements_filepath = os.path.join(self.root, "datmorequirements.txt") outfile_requirements = open(requirements_filepath, "wb") process = subprocess.Popen(["pip", "freeze"], cwd=self.root, stdout=outfile_requirements, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if process.returncode > 0: raise EnvironmentRequirementsCreateError( __("error", "controller.environment.requirements.create", str(stderr))) except Exception as e: raise EnvironmentRequirementsCreateError( __("error", "controller.environment.requirements.create", str(e))) if not os.path.isfile(requirements_filepath): return None return requirements_filepath else: raise EnvironmentRequirementsCreateError( __("error", "controller.environment.requirements.create", "no such package manager")) @staticmethod def create_default_definition(directory, language="python3"): language_dockerfile = "%sDockerfile" % language default_dockerfile_filepath = os.path.join( os.path.dirname(os.path.abspath(__file__)), "templates", language_dockerfile) destination_dockerfile = os.path.join(directory, "Dockerfile") with open(default_dockerfile_filepath, "rb") as input_file: with open(destination_dockerfile, "wb") as output_file: for line in input_file: if to_bytes(os.linesep) in line: output_file.write(line.strip() + to_bytes(os.linesep)) else: output_file.write(line.strip()) return destination_dockerfile def get_default_definition_filename(self): return "Dockerfile" def get_datmo_definition_filenames(self): return ["hardware_info"] def get_hardware_info(self): # Extract hardware info of the container (currently taking from system platform) # TODO: extract hardware information directly from the container (system, node, release, version, machine, processor) = platform.uname() return { 'system': system, 'node': node, 'release': release, 'version': version, 'machine': machine, 'processor': processor } @staticmethod def create_datmo_definition(input_definition_path, output_definition_path, workspace=None): """ Creates a datmo dockerfiles to run at the output path specified """ datmo_base_dockerfile_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "templates", "baseDockerfile") # Combine dockerfiles with open(input_definition_path, "rb") as input_file: with open(datmo_base_dockerfile_path, "rb") as datmo_base_file: with open(output_definition_path, "wb") as output_file: for line in input_file: bool_workspace_update = (to_bytes('FROM datmo/') in line.strip() and workspace) if to_bytes("\n") in line and bool_workspace_update: updated_line = line.strip() + to_bytes( "-%s%s" % (workspace, os.linesep)) elif to_bytes(os.linesep) in line: updated_line = line.strip() + to_bytes(os.linesep) else: updated_line = line.strip() output_file.write(updated_line) for line in datmo_base_file: if to_bytes("\n") in line: output_file.write(line.strip() + to_bytes(os.linesep)) else: output_file.write(line.strip()) return True
class BugZoo(object): """ Used to interact with and manage a local BugZoo installation. """ def __init__(self, path=None, base_url_docker='unix:///var/run/docker.sock') -> None: """ Creates a new BugZoo installation manager. Args: path: the absolute path of a BugZoo installation on this machine. If unspecified, the value of the environmental variable :code:`BUGZOO_PATH` will be used, unless unspecified, in which case :code:`./${HOME}/.bugzoo` will be used instead. """ # TODO support windows if path is None: default_path = os.path.join(os.environ['HOME'], '.bugzoo') path = os.environ.get('BUGZOO_PATH', default_path) self.__path = path logger.debug("using BugZoo directory: %s", path) logger.debug("preparing BugZoo directory") if not os.path.exists(self.path): os.makedirs(self.path) if not os.path.exists(self.coverage_path): os.makedirs(self.coverage_path) logger.debug("prepared BugZoo directory") logger.debug("connecting to Docker at %s", base_url_docker) self.__base_url_docker = base_url_docker try: self.__docker = DockerClient(base_url=base_url_docker, timeout=120) assert self.__docker.ping() except (docker.errors.APIError, AssertionError): logger.exception("failed to connect to Docker") raise # FIXME add DockerConnectionFailure logger.debug("connected to Docker") logger.debug("Docker version: %s", self.__docker.version()) logger.debug("Docker server info: %s", self.__docker.info()) self.__mgr_build = BuildManager(self.__docker) self.__bugs = BugManager(self) self.__tools = ToolManager(self) self.__sources = SourceManager(self) self.__containers = ContainerManager(self) self.__files = FileManager(self.__bugs, self.__containers) self.__coverage = CoverageManager(self) def shutdown(self) -> None: logger.info("Shutting down daemon...") self.__containers.clear() logger.info("Shut down daemon") @property def docker(self) -> DockerClient: """ The Docker client used by this server. """ return self.__docker @property def base_url_docker(self) -> str: """ The base URL of the Docker server to which BugZoo is connected. """ return self.__base_url_docker @property def path(self) -> str: """ The absolute path to the local installation of BugZoo. """ return self.__path @property def coverage_path(self) -> str: """ The absolute path to the directory used to store cached coverage information for bugs provided by this BugZoo installation. """ return os.path.join(self.path, "coverage") def rescan(self): self.__sources.scan() @property def build(self) -> BuildManager: return self.__mgr_build @property def coverage(self) -> CoverageManager: """ The coverage manager is used to compute line coverage information. """ return self.__coverage @property def files(self) -> FileManager: """ Provides access to the file system within running containers. """ return self.__files @property def sources(self) -> SourceManager: """ The sources registered with this BugZoo installation. """ return self.__sources @property def tools(self) -> ToolManager: """ The tools registered with this BugZoo installation. """ return self.__tools @property def bugs(self) -> BugManager: """ The bugs registered with this BugZoo installation. """ return self.__bugs @property def containers(self) -> ContainerManager: """ The containers that are running on this server. """ return self.__containers
def __init__(self, client: docker.DockerClient, domains): super().__init__(client, domains) self.addresses = [] self.domains = domains for manager in client.info()['Swarm']['RemoteManagers']: self.addresses.append(manager['Addr'].split(':')[0])