コード例 #1
0
ファイル: dockerenv.py プロジェクト: yyht/datmo
    def build_image(self, tag, definition_path="Dockerfile", workspace=None):
        """Builds docker image

        Parameters
        ----------
        tag : str
            name to tag image with
        definition_path : str
            absolute file path to the definition
        workspace : str
            workspace to be used for the run

        Returns
        -------
        bool
            True if success

        Raises
        ------
        EnvironmentExecutionError

        """
        try:
            docker_shell_cmd_list = list(self.prefix)
            docker_shell_cmd_list.append("build")

            # Passing tag name for the image
            docker_shell_cmd_list.append("-t")
            docker_shell_cmd_list.append(tag)

            # Passing path of Dockerfile
            # Creating datmoDockerfile for new build
            dockerfile_dirpath = os.path.split(definition_path)[0]
            input_dockerfile = os.path.split(definition_path)[1]
            output_dockerfile_path = os.path.join(dockerfile_dirpath,
                                                  "datmo%s" % input_dockerfile)
            self.create(definition_path, output_dockerfile_path, workspace)
            docker_shell_cmd_list.append("-f")
            docker_shell_cmd_list.append(output_dockerfile_path)
            docker_shell_cmd_list.append(str(dockerfile_dirpath))

            # Remove intermediate containers after a successful build
            docker_shell_cmd_list.append("--rm")
            process_returncode = subprocess.Popen(docker_shell_cmd_list).wait()
            if process_returncode == 0:
                return True
            elif process_returncode == 1:
                raise EnvironmentExecutionError(
                    __("error",
                       "controller.environment.driver.docker.build_image",
                       "Docker subprocess failed"))
        except Exception as e:
            raise EnvironmentExecutionError(
                __("error", "controller.environment.driver.docker.build_image",
                   str(e)))
コード例 #2
0
ファイル: dockerenv.py プロジェクト: yyht/datmo
 def stop_container(self, container_id):
     try:
         docker_container_stop_cmd = list(self.prefix)
         docker_container_stop_cmd.extend(["stop", container_id])
         process = subprocess.Popen(docker_container_stop_cmd,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
         stdout, stderr = process.communicate()
         if process.returncode > 0:
             raise EnvironmentExecutionError(
                 __("error",
                    "controller.environment.driver.docker.stop_container",
                    str(stderr)))
     except subprocess.CalledProcessError as e:
         raise EnvironmentExecutionError(
             __("error",
                "controller.environment.driver.docker.stop_container",
                str(e)))
     return True
コード例 #3
0
ファイル: dockerenv.py プロジェクト: yyht/datmo
 def remove_image(self, image_id_or_name, force=False):
     try:
         docker_image_remove_cmd = list(self.prefix)
         if force:
             docker_image_remove_cmd.extend(["rmi", "-f", image_id_or_name])
         else:
             docker_image_remove_cmd.extend(["rmi", image_id_or_name])
         process = subprocess.Popen(docker_image_remove_cmd,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
         stdout, stderr = process.communicate()
         if process.returncode > 0 or stderr:
             raise EnvironmentExecutionError(
                 __("error",
                    "controller.environment.driver.docker.remove_image",
                    str(stderr)))
     except subprocess.CalledProcessError as e:
         raise EnvironmentExecutionError(
             __("error",
                "controller.environment.driver.docker.remove_image",
                str(e)))
     return True
コード例 #4
0
ファイル: dockerenv.py プロジェクト: yyht/datmo
 def remove_container(self, container_id, force=False):
     try:
         docker_container_remove_cmd_list = list(self.prefix)
         if force:
             docker_container_remove_cmd_list.extend(
                 ["rm", "-f", container_id])
         else:
             docker_container_remove_cmd_list.extend(["rm", container_id])
         process = subprocess.Popen(docker_container_remove_cmd_list,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
         stdout, stderr = process.communicate()
         if process.returncode > 0:
             raise EnvironmentExecutionError(
                 __(
                     "error",
                     "controller.environment.driver.docker.remove_container",
                     str(stderr)))
     except subprocess.CalledProcessError as e:
         raise EnvironmentExecutionError(
             __("error",
                "controller.environment.driver.docker.remove_container",
                str(e)))
     return True
コード例 #5
0
ファイル: dockerenv.py プロジェクト: yyht/datmo
 def remove_images(self, name=None, all=False, filters=None, force=False):
     """Remove multiple images
     """
     try:
         images = self.list_images(name=name,
                                   all_images=all,
                                   filters=filters)
         for image in images:
             self.remove_image(image.id, force=force)
     except Exception as e:
         raise EnvironmentExecutionError(
             __("error",
                "controller.environment.driver.docker.remove_images",
                str(e)))
     return True
コード例 #6
0
ファイル: dockerenv.py プロジェクト: yyht/datmo
    def get_tags_for_docker_repository(self, repo_name):
        # TODO: Use more common CLI command (e.g. curl instead of wget)
        """Method to get tags for docker repositories

        Parameters
        ----------
        repo_name: str
            Docker repository name

        Returns
        -------
        list
            List of tags available for that docker repo
        """
        docker_repository_tag_cmd = "wget -q https://registry.hub.docker.com/v1/repositories/" + repo_name + "/tags -O -"
        try:
            process = subprocess.Popen(docker_repository_tag_cmd,
                                       shell=True,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE)
            stdout, stderr = process.communicate()
            if process.returncode > 0:
                raise EnvironmentExecutionError(
                    __("error",
                       "controller.environment.driver.docker.get_tags",
                       str(stderr)))
            string_repository_tags = stdout.decode().strip()
        except subprocess.CalledProcessError as e:
            raise EnvironmentExecutionError(
                __("error", "controller.environment.driver.docker.get_tags",
                   str(e)))
        repository_tags = ast.literal_eval(string_repository_tags)
        list_tag_names = []
        for repository_tag in repository_tags:
            list_tag_names.append(repository_tag["name"])
        return list_tag_names
コード例 #7
0
ファイル: dockerenv.py プロジェクト: yyht/datmo
 def connect(self):
     # TODO: Fill in to start up Docker
     # Startup Docker
     try:
         pass
     except Exception as e:
         raise EnvironmentExecutionError(
             __("error", "controller.environment.driver.docker.init",
                str(e)))
     # Initiate Docker execution
     try:
         self.info = self.client.info()
         self._is_connected = True if self.info["Images"] != None else False
     except Exception:
         raise EnvironmentConnectFailed(
             __("error", "controller.environment.driver.docker.__init__",
                platform.system()))
     return True
コード例 #8
0
ファイル: dockerenv.py プロジェクト: yyht/datmo
    def stop_remove_containers_by_term(self, term, force=False):
        """Stops and removes containers by term
        """
        # TODO: split out the find containers function from stop / remove
        try:
            running_docker_container_cmd_list = list(self.prefix)
            running_docker_container_cmd_list.extend([
                "ps", "-a", "|", "grep",
                "'%s'" % term, "|", "awk '{print $1}'"
            ])

            running_docker_container_cmd_str = str(
                " ".join(running_docker_container_cmd_list))
            process = subprocess.Popen(running_docker_container_cmd_str,
                                       shell=True,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE)
            out_list_cmd, err_list_cmd = process.communicate()
            if process.returncode > 0:
                raise EnvironmentExecutionError(
                    __(
                        "error",
                        "controller.environment.driver.docker.stop_remove_containers_by_term",
                        str(err_list_cmd)))
            # checking for running container id before stopping any
            if out_list_cmd:
                docker_container_stop_cmd_list = list(self.prefix)
                docker_container_stop_cmd_list = docker_container_stop_cmd_list + \
                                                 ["stop", "$("] + running_docker_container_cmd_list + \
                                                 [")"]
                docker_container_stop_cmd_str = str(
                    " ".join(docker_container_stop_cmd_list))
                process = subprocess.Popen(docker_container_stop_cmd_str,
                                           shell=True,
                                           stdout=subprocess.PIPE,
                                           stderr=subprocess.PIPE)
                _, _ = process.communicate()
                # rechecking for container id after stopping them to ensure no errors
                process = subprocess.Popen(running_docker_container_cmd_str,
                                           shell=True,
                                           stdout=subprocess.PIPE,
                                           stderr=subprocess.PIPE)
                out_list_cmd, err_list_cmd = process.communicate()
                if process.returncode > 0:
                    raise EnvironmentExecutionError(
                        __(
                            "error",
                            "controller.environment.driver.docker.stop_remove_containers_by_term",
                            str(err_list_cmd)))
                if out_list_cmd:
                    docker_container_remove_cmd_list = list(self.prefix)
                    if force:
                        docker_container_remove_cmd_list = docker_container_remove_cmd_list + \
                                                           ["rm", "-f", "$("] + running_docker_container_cmd_list + \
                                                           [")"]
                    else:
                        docker_container_remove_cmd_list = docker_container_remove_cmd_list + \
                                                           ["rm", "$("] + running_docker_container_cmd_list + \
                                                           [")"]
                    docker_container_remove_cmd_str = str(
                        " ".join(docker_container_remove_cmd_list))
                    process = subprocess.Popen(docker_container_remove_cmd_str,
                                               shell=True,
                                               stdout=subprocess.PIPE,
                                               stderr=subprocess.PIPE)
                    _, err_list_cmd = process.communicate()
                    if process.returncode > 0:
                        raise EnvironmentExecutionError(
                            __(
                                "error",
                                "controller.environment.driver.docker.stop_remove_containers_by_term",
                                str(err_list_cmd)))
        except subprocess.CalledProcessError as e:
            raise EnvironmentExecutionError(
                __(
                    "error",
                    "controller.environment.driver.docker.stop_remove_containers_by_term",
                    str(e)))
        return True
コード例 #9
0
ファイル: dockerenv.py プロジェクト: yyht/datmo
    def run_container(self,
                      image_name,
                      command=None,
                      ports=None,
                      name=None,
                      volumes=None,
                      mem_limit=None,
                      runtime=None,
                      detach=False,
                      stdin_open=False,
                      tty=False,
                      api=False):
        """Run Docker container with parameters given as defined below
        Parameters
        ----------
        image_name : str
            Docker image name
        command : list, optional
            List with complete user-given command (e.g. ["python3", "cool.py"])
        ports : list, optional
            Here are some example ports used for common applications.
               *  "jupyter notebook" - 8888
               *  flask API - 5000
               *  tensorboard - 6006
            An example input for the above would be ["8888:8888", "5000:5000", "6006:6006"]
            which maps the running host port (right) to that of the environment (left)
        name : str, optional
            User given name for container
        volumes : dict, optional
            Includes storage volumes for docker
            (e.g. { outsidepath1 : {"bind", containerpath2, "mode", MODE} })
        mem_limit : str, optional
            maximum amount of memory the container can use
            (these options take a positive integer, followed by a suffix of b, k, m, g, to indicate bytes, kilobytes,
            megabytes, or gigabytes. memory limit is contrained by total memory of the VM in which docker runs)
        detach : bool, optional
            True if container is to be detached else False
        stdin_open : bool, optional
            True if stdin is open else False
        tty : bool, optional
            True to connect pseudo-terminal with stdin / stdout else False
        api : bool, optional
            True if Docker python client should be used else use subprocess
        Returns
        -------
        if api=False:
        return_code: int
            integer success code of command
        container_id: str
            output container id
        if api=True & if detach=True:
        container_obj: Container
            object from Docker python api with details about container
        if api=True & if detach=False:
        logs: str
            output logs for the run function
        Raises
        ------
        EnvironmentExecutionError
             error in running the environment command
        """
        try:
            if api:  # calling the docker client via the API
                # TODO: Test this out for the API (need to verify ports work)
                if detach:
                    command = " ".join(command) if command else command
                    container = \
                        self.client.containers.run(image_name, command, ports=ports,
                                                   name=name, volumes=volumes,
                                                   mem_limit=mem_limit,
                                                   detach=detach, stdin_open=stdin_open)
                    return container
                else:
                    command = " ".join(command) if command else command
                    logs = self.client.containers.run(image_name,
                                                      command,
                                                      ports=ports,
                                                      name=name,
                                                      volumes=volumes,
                                                      mem_limit=mem_limit,
                                                      detach=detach,
                                                      stdin_open=stdin_open)
                    return logs.decode()
            else:  # if calling run function with the shell commands
                docker_shell_cmd_list = list(self.prefix)
                docker_shell_cmd_list.append("run")

                if name:
                    docker_shell_cmd_list.append("--name")
                    docker_shell_cmd_list.append(name)

                if runtime:
                    docker_shell_cmd_list.append("--runtime")
                    docker_shell_cmd_list.append(runtime)

                if mem_limit:
                    docker_shell_cmd_list.append("-m")
                    docker_shell_cmd_list.append(mem_limit)
                    docker_shell_cmd_list.append("--memory-swap")
                    docker_shell_cmd_list.append("-1")

                if stdin_open:
                    docker_shell_cmd_list.append("-i")

                if tty:
                    docker_shell_cmd_list.append("-t")

                if detach:
                    docker_shell_cmd_list.append("-d")

                # Volume
                if volumes:
                    # Mounting volumes
                    for key in list(volumes):
                        docker_shell_cmd_list.append("-v")
                        volume_mount = key + ":" + volumes[key]["bind"] + ":" + \
                                       volumes[key]["mode"]
                        docker_shell_cmd_list.append(volume_mount)

                if ports:
                    # Mapping ports
                    for mapping in ports:
                        docker_shell_cmd_list.append("-p")
                        docker_shell_cmd_list.append(mapping)

                docker_shell_cmd_list.append(image_name)
                if command:
                    docker_shell_cmd_list.extend(command)
                return_code = subprocess.call(docker_shell_cmd_list)
                if return_code != 0:
                    raise EnvironmentExecutionError(
                        __(
                            "error",
                            "controller.environment.driver.docker.run_container",
                            str(docker_shell_cmd_list)))
                list_process_cmd = list(self.prefix)
                list_process_cmd.extend(["ps", "-q", "-l"])
                process = subprocess.Popen(list_process_cmd,
                                           stdout=subprocess.PIPE,
                                           stderr=subprocess.PIPE)
                stdout, stderr = process.communicate()
                if process.returncode > 0:
                    raise EnvironmentExecutionError(
                        __(
                            "error",
                            "controller.environment.driver.docker.run_container",
                            str(stderr)))
                container_id = stdout.decode().strip()
        except subprocess.CalledProcessError as e:
            raise EnvironmentExecutionError(
                __("error",
                   "controller.environment.driver.docker.run_container",
                   str(e)))
        return return_code, container_id