def build_image(self, tag, definition_path="Dockerfile"):
"""Builds docker image
Parameters
----------
tag : str
name to tag image with
definition_path : str
absolute file path to the definition
Returns
-------
bool
True if success
Raises
------
EnvironmentExecutionException
"""
try:
docker_shell_cmd_list = list(self.cpu_prefix)
docker_shell_cmd_list.append("build")
# Passing tag name for the image
docker_shell_cmd_list.append("-t")
docker_shell_cmd_list.append(tag)
# Passing path of Dockerfile
docker_shell_cmd_list.append("-f")
docker_shell_cmd_list.append(definition_path)
dockerfile_dirpath = os.path.split(definition_path)[0]
docker_shell_cmd_list.append(str(dockerfile_dirpath))
# Remove intermediate containers after a successful build
docker_shell_cmd_list.append("--rm")
process_returncode = subprocess.Popen(docker_shell_cmd_list).wait()
if process_returncode == 0:
return True
elif process_returncode == 1:
raise EnvironmentExecutionException(
__("error",
"controller.environment.driver.docker.build_image",
"Docker subprocess failed"))
except Exception as e:
raise EnvironmentExecutionException(
__("error", "controller.environment.driver.docker.build_image",
str(e)))
def stop_remove_containers_by_term(self, term, force=False):
"""Stops and removes containers by term
"""
try:
running_docker_container_cmd_list = list(self.cpu_prefix)
running_docker_container_cmd_list.extend([
"ps", "-a", "|", "grep", "'", term, "'", "|",
"awk '{print $1}'"
])
running_docker_container_cmd_str = str(
" ".join(running_docker_container_cmd_list))
output = subprocess.Popen(running_docker_container_cmd_str,
shell=True,
stdout=subprocess.PIPE)
out_list_cmd, err_list_cmd = output.communicate()
# checking for running container id before stopping any
if out_list_cmd:
docker_container_stop_cmd_list = list(self.cpu_prefix)
docker_container_stop_cmd_list = docker_container_stop_cmd_list + \
["stop", "$("] + running_docker_container_cmd_list + \
[")"]
docker_container_stop_cmd_str = str(
" ".join(docker_container_stop_cmd_list))
output = subprocess.Popen(docker_container_stop_cmd_str,
shell=True,
stdout=subprocess.PIPE)
_, _ = output.communicate()
# rechecking for container id after stopping them to ensure no errors
output = subprocess.Popen(running_docker_container_cmd_str,
shell=True,
stdout=subprocess.PIPE)
out_list_cmd, _ = output.communicate()
if out_list_cmd:
docker_container_remove_cmd_list = list(self.cpu_prefix)
if force:
docker_container_remove_cmd_list = docker_container_remove_cmd_list + \
["rm", "-f", "$("] + running_docker_container_cmd_list + \
[")"]
else:
docker_container_remove_cmd_list = docker_container_remove_cmd_list + \
["rm", "$("] + running_docker_container_cmd_list + \
[")"]
docker_container_remove_cmd_str = str(
" ".join(docker_container_remove_cmd_list))
output = subprocess.Popen(docker_container_remove_cmd_str,
shell=True,
stdout=subprocess.PIPE)
_, _ = output.communicate()
except Exception as e:
raise EnvironmentExecutionException(
__(
"error",
"controller.environment.driver.docker.stop_remove_containers_by_term",
str(e)))
return True
def init(self):
# TODO: Fill in to start up Docker
try:
# Startup Docker
pass
except Exception as e:
raise EnvironmentExecutionException(
__("error", "controller.environment.driver.docker.init",
str(e)))
return True
def stop_container(self, container_id):
try:
docker_container_stop_cmd = list(self.prefix)
docker_container_stop_cmd.extend(["stop", container_id])
process = subprocess.Popen(docker_container_stop_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
if process.returncode > 0:
raise EnvironmentExecutionException(
__("error",
"controller.environment.driver.docker.stop_container",
str(stderr)))
except subprocess.CalledProcessError as e:
raise EnvironmentExecutionException(
__("error",
"controller.environment.driver.docker.stop_container",
str(e)))
return True
def stop_container(self, container_id):
try:
docker_container_stop_cmd = list(self.cpu_prefix)
docker_container_stop_cmd.extend(["stop", container_id])
subprocess.check_output(docker_container_stop_cmd).decode().strip()
except Exception as e:
raise EnvironmentExecutionException(
__("error",
"controller.environment.driver.docker.stop_container",
str(e)))
return True
def remove_image(self, image_id_or_name, force=False):
try:
if force:
docker_image_remove_cmd = list(self.prefix)
docker_image_remove_cmd.extend(["rmi", "-f", image_id_or_name])
else:
docker_image_remove_cmd = list(self.prefix)
docker_image_remove_cmd.extend(["rmi", image_id_or_name])
process = subprocess.Popen(docker_image_remove_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
if process.returncode > 0:
raise EnvironmentExecutionException(
__("error",
"controller.environment.driver.docker.remove_image",
str(stderr)))
except subprocess.CalledProcessError as e:
raise EnvironmentExecutionException(
__("error",
"controller.environment.driver.docker.remove_image",
str(e)))
return True
def remove_images(self, name=None, all=False, filters=None, force=False):
"""Remove multiple images
"""
try:
images = self.list_images(name=name,
all_images=all,
filters=filters)
for image in images:
self.remove_image(image.id, force=force)
except Exception as e:
raise EnvironmentExecutionException(
__("error",
"controller.environment.driver.docker.remove_images",
str(e)))
return True
def remove_image(self, image_id_or_name, force=False):
try:
if force:
docker_image_remove_cmd = list(self.cpu_prefix)
docker_image_remove_cmd.extend(["rmi", "-f", image_id_or_name])
else:
docker_image_remove_cmd = list(self.cpu_prefix)
docker_image_remove_cmd.extend(["rmi", image_id_or_name])
subprocess.check_output(docker_image_remove_cmd).decode().strip()
except Exception as e:
raise EnvironmentExecutionException(
__("error",
"controller.environment.driver.docker.remove_image",
str(e)))
return True
def remove_container(self, container_id, force=False):
try:
docker_container_remove_cmd_list = list(self.prefix)
if force:
docker_container_remove_cmd_list.extend(
["rm", "-f", container_id])
else:
docker_container_remove_cmd_list.extend(["rm", container_id])
process = subprocess.Popen(docker_container_remove_cmd_list,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
if process.returncode > 0:
raise EnvironmentExecutionException(
__(
"error",
"controller.environment.driver.docker.remove_container",
str(stderr)))
except subprocess.CalledProcessError as e:
raise EnvironmentExecutionException(
__("error",
"controller.environment.driver.docker.remove_container",
str(e)))
return True
def get_tags_for_docker_repository(self, repo_name):
# TODO: Use more common CLI command (e.g. curl instead of wget)
"""Method to get tags for docker repositories
Parameters
----------
repo_name: str
Docker repository name
Returns
-------
list
List of tags available for that docker repo
"""
docker_repository_tag_cmd = "wget -q https://registry.hub.docker.com/v1/repositories/" + repo_name + "/tags -O -"
try:
process = subprocess.Popen(docker_repository_tag_cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
if process.returncode > 0:
raise EnvironmentExecutionException(
__("error",
"controller.environment.driver.docker.get_tags",
str(stderr)))
string_repository_tags = stdout.decode().strip()
except subprocess.CalledProcessError as e:
raise EnvironmentExecutionException(
__("error", "controller.environment.driver.docker.get_tags",
str(e)))
repository_tags = ast.literal_eval(string_repository_tags)
list_tag_names = []
for repository_tag in repository_tags:
list_tag_names.append(repository_tag["name"])
return list_tag_names
def remove_container(self, container_id, force=False):
try:
docker_container_remove_cmd_list = list(self.cpu_prefix)
if force:
docker_container_remove_cmd_list.extend(
["rm", "-f", container_id])
else:
docker_container_remove_cmd_list.extend(["rm", container_id])
subprocess.check_output(
docker_container_remove_cmd_list).decode().strip()
except Exception as e:
raise EnvironmentExecutionException(
__("error",
"controller.environment.driver.docker.remove_container",
str(e)))
return True
def run_container(self,
image_name,
command=None,
ports=None,
name=None,
volumes=None,
detach=False,
stdin_open=False,
tty=False,
gpu=False,
api=False):
"""Run Docker container with parameters given as defined below
Parameters
----------
image_name : str
Docker image name
command : list, optional
List with complete user-given command (e.g. ["python3", "cool.py"])
ports : list, optional
Here are some example ports used for common applications.
* "jupyter notebook" - 8888
* flask API - 5000
* tensorboard - 6006
An example input for the above would be ["8888:8888", "5000:5000", "6006:6006"]
which maps the running host port (right) to that of the environment (left)
name : str, optional
User given name for container
volumes : dict, optional
Includes storage volumes for docker
(e.g. { outsidepath1 : {"bind", containerpath2, "mode", MODE} })
detach : bool, optional
True if container is to be detached else False
stdin_open : bool, optional
True if stdin is open else False
tty : bool, optional
True to connect pseudo-terminal with stdin / stdout else False
gpu : bool, optional
True if GPU should be enabled else False
api : bool, optional
True if Docker python client should be used else use subprocess
Returns
-------
if api=False:
return_code: int
integer success code of command
container_id: str
output container id
if api=True & if detach=True:
container_obj: Container
object from Docker python api with details about container
if api=True & if detach=False:
logs: str
output logs for the run function
Raises
------
EnvironmentExecutionException
error in running the environment command
"""
try:
container_id = None
if api: # calling the docker client via the API
# TODO: Test this out for the API (need to verify ports work)
if detach:
command = " ".join(command) if command else command
container = \
self.client.containers.run(image_name, command, ports=ports,
name=name, volumes=volumes,
detach=detach, stdin_open=stdin_open)
return container
else:
command = " ".join(command) if command else command
logs = self.client.containers.run(image_name,
command,
ports=ports,
name=name,
volumes=volumes,
detach=detach,
stdin_open=stdin_open)
return logs.decode()
else: # if calling run function with the shell commands
if gpu:
docker_shell_cmd_list = list(self.cpu_prefix)
else:
docker_shell_cmd_list = list(self.cpu_prefix)
docker_shell_cmd_list.append("run")
if name:
docker_shell_cmd_list.append("--name")
docker_shell_cmd_list.append(name)
if stdin_open:
docker_shell_cmd_list.append("-i")
if tty:
docker_shell_cmd_list.append("-t")
if detach:
docker_shell_cmd_list.append("-d")
# Volume
if volumes:
# Mounting volumes
for key in list(volumes):
docker_shell_cmd_list.append("-v")
volume_mount = key + ":" + volumes[key]["bind"] + ":" + \
volumes[key]["mode"]
docker_shell_cmd_list.append(volume_mount)
if ports:
# Mapping ports
for mapping in ports:
docker_shell_cmd_list.append("-p")
docker_shell_cmd_list.append(mapping)
docker_shell_cmd_list.append(image_name)
if command:
docker_shell_cmd_list.extend(command)
return_code = subprocess.call(docker_shell_cmd_list)
if return_code != 0:
raise EnvironmentExecutionException(
__(
"error",
"controller.environment.driver.docker.run_container",
docker_shell_cmd_list))
list_process_cmd = list(self.cpu_prefix)
list_process_cmd.extend(["ps", "-q", "-l"])
container_id = subprocess.check_output(list_process_cmd)
container_id = container_id.decode().strip()
except Exception as e:
raise EnvironmentExecutionException(
__("error",
"controller.environment.driver.docker.run_container",
str(e)))
return return_code, container_id
def stop_remove_containers_by_term(self, term, force=False):
"""Stops and removes containers by term
"""
# TODO: split out the find containers function from stop / remove
try:
running_docker_container_cmd_list = list(self.prefix)
running_docker_container_cmd_list.extend([
"ps", "-a", "|", "grep",
"'%s'" % term, "|", "awk '{print $1}'"
])
running_docker_container_cmd_str = str(
" ".join(running_docker_container_cmd_list))
process = subprocess.Popen(running_docker_container_cmd_str,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out_list_cmd, err_list_cmd = process.communicate()
if process.returncode > 0:
raise EnvironmentExecutionException(
__(
"error",
"controller.environment.driver.docker.stop_remove_containers_by_term",
str(err_list_cmd)))
# checking for running container id before stopping any
if out_list_cmd:
docker_container_stop_cmd_list = list(self.prefix)
docker_container_stop_cmd_list = docker_container_stop_cmd_list + \
["stop", "$("] + running_docker_container_cmd_list + \
[")"]
docker_container_stop_cmd_str = str(
" ".join(docker_container_stop_cmd_list))
process = subprocess.Popen(docker_container_stop_cmd_str,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
_, _ = process.communicate()
# rechecking for container id after stopping them to ensure no errors
process = subprocess.Popen(running_docker_container_cmd_str,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out_list_cmd, err_list_cmd = process.communicate()
if process.returncode > 0:
raise EnvironmentExecutionException(
__(
"error",
"controller.environment.driver.docker.stop_remove_containers_by_term",
str(err_list_cmd)))
if out_list_cmd:
docker_container_remove_cmd_list = list(self.prefix)
if force:
docker_container_remove_cmd_list = docker_container_remove_cmd_list + \
["rm", "-f", "$("] + running_docker_container_cmd_list + \
[")"]
else:
docker_container_remove_cmd_list = docker_container_remove_cmd_list + \
["rm", "$("] + running_docker_container_cmd_list + \
[")"]
docker_container_remove_cmd_str = str(
" ".join(docker_container_remove_cmd_list))
process = subprocess.Popen(docker_container_remove_cmd_str,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
_, err_list_cmd = process.communicate()
if process.returncode > 0:
raise EnvironmentExecutionException(
__(
"error",
"controller.environment.driver.docker.stop_remove_containers_by_term",
str(err_list_cmd)))
except subprocess.CalledProcessError as e:
raise EnvironmentExecutionException(
__(
"error",
"controller.environment.driver.docker.stop_remove_containers_by_term",
str(e)))
return True
