class TestSystem(unittest.TestCase):
podman = None # initialized podman configuration for tests
service = None # podman service instance
topContainerId = ""
def setUp(self):
super().setUp()
self.client = DockerClient(base_url="tcp://127.0.0.1:8080", timeout=15)
TestSystem.podman.restore_image_from_cache(self.client)
TestSystem.topContainerId = common.run_top_container(self.client)
def tearDown(self):
common.remove_all_containers(self.client)
common.remove_all_images(self.client)
self.client.close()
return super().tearDown()
@classmethod
def setUpClass(cls):
super().setUpClass()
TestSystem.podman = Podman()
TestSystem.service = TestSystem.podman.open("system", "service",
"tcp:127.0.0.1:8080",
"--time=0")
# give the service some time to be ready...
time.sleep(2)
returncode = TestSystem.service.poll()
if returncode is not None:
raise subprocess.CalledProcessError(returncode,
"podman system service")
@classmethod
def tearDownClass(cls):
TestSystem.service.terminate()
stdout, stderr = TestSystem.service.communicate(timeout=0.5)
if stdout:
sys.stdout.write("\nImages Service Stdout:\n" +
stdout.decode("utf-8"))
if stderr:
sys.stderr.write("\nImAges Service Stderr:\n" +
stderr.decode("utf-8"))
TestSystem.podman.tear_down()
return super().tearDownClass()
def test_Info(self):
self.assertIsNotNone(self.client.info())
def test_info_container_details(self):
info = self.client.info()
self.assertEqual(info["Containers"], 1)
self.client.containers.create(image=constant.ALPINE)
info = self.client.info()
self.assertEqual(info["Containers"], 2)
def test_version(self):
version = self.client.version()
self.assertIsNotNone(version["Platform"]["Name"])
def _is_docker_running(client: docker.DockerClient) -> bool:
"""
Check if docker binary and docker daemon are available
:param client: DockerClient instance
:return: true or false
"""
try:
client.info()
return True
except (ImportError, IOError, DockerException):
return False
def docker():
client = DockerClient("unix:///var/run/docker.sock")
already_swarm = client.info()["Swarm"]["LocalNodeState"] == "active"
if not already_swarm:
client.swarm.init()
network = client.networks.create("dockerspawner-test-network", driver="overlay", attachable=True)
network.connect("dockerspawner-test")
try:
yield client
finally:
for service in client.services.list():
if service.name.startswith("jupyterhub-client"):
service.scale(0)
for _ in range(10):
if not service.tasks():
break
sleep(1)
service.remove()
network.disconnect("dockerspawner-test")
network.remove()
if not already_swarm:
client.swarm.leave(True)
def is_swarm_manager(client: DockerClient) -> bool:
"""
Check if the given client is connected to a Docker Engine
running in Swarm Mode and has the manager role.
:param client: DockerClient
:rtype: bool
:return: true if Docker Engine is in Swarm mode, else false
"""
info = client.info()
swarm = info['Swarm']
return swarm['LocalNodeState'] == 'active' and swarm['ControlAvailable']
class DockerController:
def __init__(self):
self.client = DockerClient(base_url='tcp://10.200.10.1:2375')
def _info(self):
print(self.client.info())
def show_running_containers(self):
return self.client.containers.list()
def build_image(self, path, tag):
# path = os.path.join(os.getcwd(), 'files', dir_name)
self.client.build(path=path, tag=tag)
def show_images(self):
return self.client.images()
def run_container(self,
image: str,
internal_web_port: int,
build_name=None,
command=None):
ssh_port = get_no_port_being_used()
web_port = get_no_port_being_used()
container = self.client.containers.run(image=image,
ports={
'22/tcp':
ssh_port,
f'{internal_web_port}/tcp':
web_port
},
name=build_name,
command=command,
detach=True,
pids_limit=MAX_PID)
logging.log(logging.INFO,
f"ssh port is {ssh_port}, web port is {web_port}")
container_info = {}
container_info['id'] = container.id
container_info['ssh_port'] = ssh_port
container_info['web_port'] = web_port
return container_info
def rm_container(self, container_id: str):
container = self.client.containers.get(container_id)
# container.stop()
# 直接删除
container.remove(force=True)
logging.log(logging.INFO, f'{container.id} has been removed')
def exec_container(self, container_id: str, command):
self.client.containers.get(container_id).exec_run(command)
class DockerPreliminaryInformation:
def __init__(self, unixsock):
self.clinobjc = DockerClient(base_url=unixsock)
def get_docker_info(self):
"""
Returns container station information
"""
return self.clinobjc.info()
def get_docker_version(self):
"""
Returns container station versioning
"""
return self.clinobjc.version()
class DockerEnvironmentDriver(EnvironmentDriver):
"""
This EnvironmentDriver handles environment management in the project using docker
Parameters
----------
filepath : str, optional
home filepath for project
(default is empty)
docker_execpath : str, optional
execution path for docker
(default is "docker" which defers to system)
docker_socket : str, optional
socket path to docker daemon to connect
(default is None, this takes the default path for the system)
Attributes
----------
filepath : str
home filepath for project
docker_execpath : str
docker execution path for the system
docker_socket : str
specific socket for docker
(default is None, which means system default is used by docker)
client : DockerClient
docker python api client
cpu_prefix : list
list of strings for the prefix command for all docker commands
info : dict
information about the docker daemon connection
is_connected : bool
True if connected to daemon else False
type : str
type of EnvironmentDriver
"""
def __init__(self,
filepath="",
docker_execpath="docker",
docker_socket=None):
if not docker_socket:
if platform.system() != "Windows":
docker_socket = "unix:///var/run/docker.sock"
super(DockerEnvironmentDriver, self).__init__()
self.filepath = filepath
# Check if filepath exists
if not os.path.exists(self.filepath):
raise PathDoesNotExist(
__("error",
"controller.environment.driver.docker.__init__.dne",
filepath))
# TODO: separate methods for instantiation into init function below
# Initiate Docker execution
self.docker_execpath = docker_execpath
try:
self.docker_socket = docker_socket
if self.docker_socket:
self.client = DockerClient(base_url=self.docker_socket)
self.cpu_prefix = [
self.docker_execpath, "-H", self.docker_socket
]
else:
self.client = DockerClient()
self.cpu_prefix = [self.docker_execpath]
self.info = self.client.info()
except Exception:
raise EnvironmentInitFailed(
__("error", "controller.environment.driver.docker.__init__",
platform.system()))
self.is_connected = True if self.info["Images"] != None else False
self._is_initialized = self.is_initialized
self.type = "docker"
# execute a docker info command
# make sure it passed
# if gpu_requested == True:
# pass
# if gpu is requested then
# make sure docker info confirms that it"s available
@property
def is_initialized(self):
# TODO: Check if Docker is up and running
if self.is_connected:
self._is_initialized = True
return self._is_initialized
self._is_initialized = False
return self._is_initialized
def create(self, path=None, output_path=None, language=None):
if not path:
path = os.path.join(self.filepath, "Dockerfile")
if not output_path:
directory, filename = os.path.split(path)
output_path = os.path.join(directory, "datmo" + filename)
if not language:
language = "python3"
requirements_filepath = None
if not os.path.isfile(path):
if language == "python3":
# Create requirements txt file for python
requirements_filepath = self.create_requirements_file()
# Create Dockerfile for ubuntu
path = self.create_default_dockerfile(
requirements_filepath=requirements_filepath,
language=language)
else:
raise EnvironmentDoesNotExist(
__("error",
"controller.environment.driver.docker.create.dne",
path))
if os.path.isfile(output_path):
raise FileAlreadyExistsException(
__("error",
"controller.environment.driver.docker.create.exists",
output_path))
success = self.form_datmo_definition_file(
input_definition_path=path, output_definition_path=output_path)
return success, path, output_path, requirements_filepath
def build(self, name, path):
return self.build_image(name, path)
def run(self, name, options, log_filepath):
run_return_code, run_id = \
self.run_container(image_name=name, **options)
log_return_code, logs = self.log_container(run_id,
filepath=log_filepath)
final_return_code = run_return_code and log_return_code
return final_return_code, run_id, logs
def stop(self, run_id, force=False):
stop_result = self.stop_container(run_id)
remove_run_result = self.remove_container(run_id, force=force)
return stop_result and remove_run_result
def remove(self, name, force=False):
stop_and_remove_containers_result = \
self.stop_remove_containers_by_term(name, force=force)
remove_image_result = self.remove_image(name, force=force)
return stop_and_remove_containers_result and \
remove_image_result
def init(self):
# TODO: Fill in to start up Docker
try:
# Startup Docker
pass
except Exception as e:
raise EnvironmentExecutionException(
__("error", "controller.environment.driver.docker.init",
str(e)))
return True
def get_tags_for_docker_repository(self, repo_name):
# TODO: Use more common CLI command (e.g. curl instead of wget)
"""Method to get tags for docker repositories
Parameters
----------
repo_name: str
Docker repository name
Returns
-------
list
List of tags available for that docker repo
"""
docker_repository_tag_cmd = "wget -q https://registry.hub.docker.com/v1/repositories/" + repo_name + "/tags -O -"
string_repository_tags = subprocess.check_output(
docker_repository_tag_cmd, shell=True)
string_repository_tags = string_repository_tags.decode().strip()
repository_tags = ast.literal_eval(string_repository_tags)
list_tag_names = []
for repository_tag in repository_tags:
list_tag_names.append(repository_tag["name"])
return list_tag_names
def build_image(self, tag, definition_path="Dockerfile"):
"""Builds docker image
Parameters
----------
tag : str
name to tag image with
definition_path : str
absolute file path to the definition
Returns
-------
bool
True if success
Raises
------
EnvironmentExecutionException
"""
try:
docker_shell_cmd_list = list(self.cpu_prefix)
docker_shell_cmd_list.append("build")
# Passing tag name for the image
docker_shell_cmd_list.append("-t")
docker_shell_cmd_list.append(tag)
# Passing path of Dockerfile
docker_shell_cmd_list.append("-f")
docker_shell_cmd_list.append(definition_path)
dockerfile_dirpath = os.path.split(definition_path)[0]
docker_shell_cmd_list.append(str(dockerfile_dirpath))
# Remove intermediate containers after a successful build
docker_shell_cmd_list.append("--rm")
process_returncode = subprocess.Popen(docker_shell_cmd_list).wait()
if process_returncode == 0:
return True
elif process_returncode == 1:
raise EnvironmentExecutionException(
__("error",
"controller.environment.driver.docker.build_image",
"Docker subprocess failed"))
except Exception as e:
raise EnvironmentExecutionException(
__("error", "controller.environment.driver.docker.build_image",
str(e)))
def get_image(self, image_name):
return self.client.images.get(image_name)
def list_images(self, name=None, all_images=False, filters=None):
return self.client.images.list(name=name,
all=all_images,
filters=filters)
def search_images(self, term):
return self.client.images.search(term=term)
def remove_image(self, image_id_or_name, force=False):
try:
if force:
docker_image_remove_cmd = list(self.cpu_prefix)
docker_image_remove_cmd.extend(["rmi", "-f", image_id_or_name])
else:
docker_image_remove_cmd = list(self.cpu_prefix)
docker_image_remove_cmd.extend(["rmi", image_id_or_name])
subprocess.check_output(docker_image_remove_cmd).decode().strip()
except Exception as e:
raise EnvironmentExecutionException(
__("error",
"controller.environment.driver.docker.remove_image",
str(e)))
return True
def remove_images(self, name=None, all=False, filters=None, force=False):
"""Remove multiple images
"""
try:
images = self.list_images(name=name,
all_images=all,
filters=filters)
for image in images:
self.remove_image(image.id, force=force)
except Exception as e:
raise EnvironmentExecutionException(
__("error",
"controller.environment.driver.docker.remove_images",
str(e)))
return True
def run_container(self,
image_name,
command=None,
ports=None,
name=None,
volumes=None,
detach=False,
stdin_open=False,
tty=False,
gpu=False,
api=False):
"""Run Docker container with parameters given as defined below
Parameters
----------
image_name : str
Docker image name
command : list, optional
List with complete user-given command (e.g. ["python3", "cool.py"])
ports : list, optional
Here are some example ports used for common applications.
* "jupyter notebook" - 8888
* flask API - 5000
* tensorboard - 6006
An example input for the above would be ["8888:8888", "5000:5000", "6006:6006"]
which maps the running host port (right) to that of the environment (left)
name : str, optional
User given name for container
volumes : dict, optional
Includes storage volumes for docker
(e.g. { outsidepath1 : {"bind", containerpath2, "mode", MODE} })
detach : bool, optional
True if container is to be detached else False
stdin_open : bool, optional
True if stdin is open else False
tty : bool, optional
True to connect pseudo-terminal with stdin / stdout else False
gpu : bool, optional
True if GPU should be enabled else False
api : bool, optional
True if Docker python client should be used else use subprocess
Returns
-------
if api=False:
return_code: int
integer success code of command
container_id: str
output container id
if api=True & if detach=True:
container_obj: Container
object from Docker python api with details about container
if api=True & if detach=False:
logs: str
output logs for the run function
Raises
------
EnvironmentExecutionException
error in running the environment command
"""
try:
container_id = None
if api: # calling the docker client via the API
# TODO: Test this out for the API (need to verify ports work)
if detach:
command = " ".join(command) if command else command
container = \
self.client.containers.run(image_name, command, ports=ports,
name=name, volumes=volumes,
detach=detach, stdin_open=stdin_open)
return container
else:
command = " ".join(command) if command else command
logs = self.client.containers.run(image_name,
command,
ports=ports,
name=name,
volumes=volumes,
detach=detach,
stdin_open=stdin_open)
return logs.decode()
else: # if calling run function with the shell commands
if gpu:
docker_shell_cmd_list = list(self.cpu_prefix)
else:
docker_shell_cmd_list = list(self.cpu_prefix)
docker_shell_cmd_list.append("run")
if name:
docker_shell_cmd_list.append("--name")
docker_shell_cmd_list.append(name)
if stdin_open:
docker_shell_cmd_list.append("-i")
if tty:
docker_shell_cmd_list.append("-t")
if detach:
docker_shell_cmd_list.append("-d")
# Volume
if volumes:
# Mounting volumes
for key in list(volumes):
docker_shell_cmd_list.append("-v")
volume_mount = key + ":" + volumes[key]["bind"] + ":" + \
volumes[key]["mode"]
docker_shell_cmd_list.append(volume_mount)
if ports:
# Mapping ports
for mapping in ports:
docker_shell_cmd_list.append("-p")
docker_shell_cmd_list.append(mapping)
docker_shell_cmd_list.append(image_name)
if command:
docker_shell_cmd_list.extend(command)
return_code = subprocess.call(docker_shell_cmd_list)
if return_code != 0:
raise EnvironmentExecutionException(
__(
"error",
"controller.environment.driver.docker.run_container",
docker_shell_cmd_list))
list_process_cmd = list(self.cpu_prefix)
list_process_cmd.extend(["ps", "-q", "-l"])
container_id = subprocess.check_output(list_process_cmd)
container_id = container_id.decode().strip()
except Exception as e:
raise EnvironmentExecutionException(
__("error",
"controller.environment.driver.docker.run_container",
str(e)))
return return_code, container_id
def get_container(self, container_id):
return self.client.containers.get(container_id)
def list_containers(self,
all=False,
before=None,
filters=None,
limit=-1,
since=None):
return self.client.containers.list(all=all,
before=before,
filters=filters,
limit=limit,
since=since)
def stop_container(self, container_id):
try:
docker_container_stop_cmd = list(self.cpu_prefix)
docker_container_stop_cmd.extend(["stop", container_id])
subprocess.check_output(docker_container_stop_cmd).decode().strip()
except Exception as e:
raise EnvironmentExecutionException(
__("error",
"controller.environment.driver.docker.stop_container",
str(e)))
return True
def remove_container(self, container_id, force=False):
try:
docker_container_remove_cmd_list = list(self.cpu_prefix)
if force:
docker_container_remove_cmd_list.extend(
["rm", "-f", container_id])
else:
docker_container_remove_cmd_list.extend(["rm", container_id])
subprocess.check_output(
docker_container_remove_cmd_list).decode().strip()
except Exception as e:
raise EnvironmentExecutionException(
__("error",
"controller.environment.driver.docker.remove_container",
str(e)))
return True
def log_container(self, container_id, filepath, api=False, follow=True):
"""Log capture at a particular point `docker logs`. Can also use `--follow` for real time logs
Parameters
----------
container_id : str
Docker container id
filepath : str
Filepath to store log file
api : bool
True to use the docker python api
follow : bool
Tail the output
Returns
-------
return_code : str
Process return code for the container
logs : str
Output logs read into a string format
"""
# TODO: Fix function to better accomodate all logs in the same way
if api: # calling the docker client via the API
with open(filepath, "w") as log_file:
for line in self.client.containers.get(container_id).logs(
stream=True):
log_file.write(to_unicode(line.strip() + "\n"))
else:
command = list(self.cpu_prefix)
if follow:
command.extend(["logs", "--follow", str(container_id)])
else:
command.extend(["logs", str(container_id)])
process = subprocess.Popen(command,
stdout=subprocess.PIPE,
universal_newlines=True)
with open(filepath, "w") as log_file:
while True:
output = process.stdout.readline()
if output == "" and process.poll() is not None:
break
if output:
printable_output = output.strip().replace("\x08", " ")
log_file.write(to_unicode(printable_output + "\n"))
return_code = process.poll()
with open(filepath, "r") as log_file:
logs = log_file.read()
return return_code, logs
def stop_remove_containers_by_term(self, term, force=False):
"""Stops and removes containers by term
"""
try:
running_docker_container_cmd_list = list(self.cpu_prefix)
running_docker_container_cmd_list.extend([
"ps", "-a", "|", "grep", "'", term, "'", "|",
"awk '{print $1}'"
])
running_docker_container_cmd_str = str(
" ".join(running_docker_container_cmd_list))
output = subprocess.Popen(running_docker_container_cmd_str,
shell=True,
stdout=subprocess.PIPE)
out_list_cmd, err_list_cmd = output.communicate()
# checking for running container id before stopping any
if out_list_cmd:
docker_container_stop_cmd_list = list(self.cpu_prefix)
docker_container_stop_cmd_list = docker_container_stop_cmd_list + \
["stop", "$("] + running_docker_container_cmd_list + \
[")"]
docker_container_stop_cmd_str = str(
" ".join(docker_container_stop_cmd_list))
output = subprocess.Popen(docker_container_stop_cmd_str,
shell=True,
stdout=subprocess.PIPE)
_, _ = output.communicate()
# rechecking for container id after stopping them to ensure no errors
output = subprocess.Popen(running_docker_container_cmd_str,
shell=True,
stdout=subprocess.PIPE)
out_list_cmd, _ = output.communicate()
if out_list_cmd:
docker_container_remove_cmd_list = list(self.cpu_prefix)
if force:
docker_container_remove_cmd_list = docker_container_remove_cmd_list + \
["rm", "-f", "$("] + running_docker_container_cmd_list + \
[")"]
else:
docker_container_remove_cmd_list = docker_container_remove_cmd_list + \
["rm", "$("] + running_docker_container_cmd_list + \
[")"]
docker_container_remove_cmd_str = str(
" ".join(docker_container_remove_cmd_list))
output = subprocess.Popen(docker_container_remove_cmd_str,
shell=True,
stdout=subprocess.PIPE)
_, _ = output.communicate()
except Exception as e:
raise EnvironmentExecutionException(
__(
"error",
"controller.environment.driver.docker.stop_remove_containers_by_term",
str(e)))
return True
def create_requirements_file(self, execpath="pipreqs"):
"""Create python requirements txt file for the project
Parameters
----------
execpath : str, optional
execpath for the pipreqs command to form requirements.txt file
(default is "pipreqs")
Returns
-------
str
absolute filepath for requirements file
Raises
------
EnvironmentDoesNotExist
no python requirements found for environment
EnvironmentRequirementsCreateException
error in running pipreqs command to extract python requirements
"""
try:
subprocess.check_output([execpath, self.filepath, "--force"],
cwd=self.filepath).strip()
requirements_filepath = os.path.join(self.filepath,
"requirements.txt")
except Exception as e:
raise EnvironmentRequirementsCreateException(
__("error", "controller.environment.requirements.create",
str(e)))
if open(requirements_filepath, "r").read() == "\n":
raise EnvironmentDoesNotExist()
return requirements_filepath
def create_default_dockerfile(self, requirements_filepath, language):
"""Create a default Dockerfile for a given language
Parameters
----------
requirements_filepath : str
path for the requirements txt file
language : str
programming language used ("python2" and "python3" currently supported)
Returns
-------
str
absolute path for the new Dockerfile using requirements txt file
"""
language_dockerfile = "%sDockerfile" % language
base_dockerfile_filepath = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "templates",
language_dockerfile)
# Combine dockerfile
destination_dockerfile = os.path.join(self.filepath, "Dockerfile")
destination = open(destination_dockerfile, "w")
shutil.copyfileobj(open(base_dockerfile_filepath, "r"), destination)
destination.write(
to_unicode(
str("COPY %s /tmp/requirements.txt\n") %
os.path.split(requirements_filepath)[-1]))
destination.write(
to_unicode(
str("RUN pip install --no-cache-dir -r /tmp/requirements.txt\n"
)))
destination.close()
return destination_dockerfile
def form_datmo_definition_file(self,
input_definition_path="Dockerfile",
output_definition_path="datmoDockerfile"):
"""
In order to create intermediate dockerfile to run
"""
base_dockerfile_filepath = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "templates",
"baseDockerfile")
# Combine dockerfiles
destination = open(os.path.join(self.filepath, output_definition_path),
"w")
shutil.copyfileobj(open(input_definition_path, "r"), destination)
shutil.copyfileobj(open(base_dockerfile_filepath, "r"), destination)
destination.close()
return True
def nvidia_gpu(client: DockerClient) -> bool:
info = client.info()
return 'Runtimes' in info and 'nvidia' in info['Runtimes']
class DockerManager():
def __init__(self, docker_url=None, verbose=False):
self._client = DockerClient(base_url=docker_url)
self._docker_url = docker_url
self._verbose = verbose
try:
self._client.ping()
except APIError as err:
logger.exception(err)
raise DockerOperationError(
'Failed to connect to the Docker Engine.')
def get_docker_info(self):
""" Returns info about the local docker engine """
try:
return {
'info': self._client.info(),
'version': self._client.version(),
}
except APIError as err:
logger.exception(err)
raise DockerOperationError(
'Failed to retrieve the Docker Engine info')
def search_runnable_commands(self, image, tag, pull=True):
""" Searching for ENTRYPOINT or CMD in a given image """
def _bind_shell(cmd_list):
""" e.g. ["/bin/sh", "-c", "echo", "hello!"] """
if cmd_list[0] in SUPPORTED_SHELLS:
return cmd_list
else:
# add a default shell on head
return (DEFAULT_SHELL + cmd_list)
full_name = '{0}:{1}'.format(image, tag)
logger.info('Analyzing the [{}] image'.format(full_name))
image = self._pull_image_with_auth(image, tag)
if image is not None:
output = self._client.api.inspect_image(full_name)
if not output or 'Config' not in output:
raise FatalError(
'Failed to inspect {} image'.format(full_name))
else:
entrypoint = output['Config']['Entrypoint']
cmd = output['Config']['Cmd']
commands = list()
if entrypoint and not cmd:
logger.debug(
'Detected only an ENTRYPOINT: {}'.format(entrypoint))
commands = _bind_shell(entrypoint)
elif not entrypoint and cmd:
logger.debug('Detected only a CMD: {}'.format(cmd))
commands = _bind_shell(cmd)
elif entrypoint and cmd:
logger.debug('Detected both ENTRYPOINT: \
{0} and CMD: {1}'.format(entrypoint, cmd))
# combining entrypoint (first) with cmd
commands = _bind_shell(entrypoint)
commands += cmd
else:
logger.info('the {} image is not runnable. \
Adding an infinite sleep'.format(full_name))
commands = DEFAULT_SHELL + ['sleep', 'infinity']
# add quotes
# [0]: /bin/bash [1]: -c
commands[2] = "\'" + commands[2]
commands[-1] = commands[-1] + "\'"
return ' '.join(commands)
def _pull_image_with_auth(self, image, tag):
try:
return self._client.images.pull(image, tag)
except ImageNotFound:
# it should be an authentication error
return self._image_authentication(image, tag)
def _image_authentication(self, src_image, src_tag=None, auth=None):
""" Handling Docker authentication if the image
is hosted on a private repository
Args:
src_image (str): The original image that needs
authentication for being fetched.
src_tag (str): The tag of the image above.
"""
if src_tag is None:
src_tag = 'latest'
logger.warning('[{0}:{1}] image may not exist or authentication \
is required'.format(src_image, src_tag))
res = ''
while res != 'YES':
res = (input('[{0}:{1}] is correct? [Yes] Continue \
[No] Abort\n'.format(src_image, src_tag))).upper()
if res == 'NO':
logger.error(
'Docker image [{0}:{1}] cannot be found, the operation \
is aborted by the user.\n(Hint: Check the TOSCA manifest.)'
.format(src_image, src_tag))
raise OperationAbortedByUser(
CommonErrorMessages._DEFAULT_OPERATION_ABORTING_ERROR_MSG)
attempts = 3
while attempts > 0:
logger.info('Authenticate with the Docker Repository..')
try:
if auth is None:
auth = create_auth_interactive(
user_text='Enter the username: '******'Enter the password: '******'UNAUTHORIZED' or 'NOT FOUND' in msg:
logger.info('Invalid username/password.')
else:
logger.exception(err)
raise FatalError(
CommonErrorMessages._DEFAULT_FATAL_ERROR_MSG)
attempts -= 1
logger.info('Authentication failed. \
You have [{}] more attempts.'.format(attempts))
# Check authentication failure
if attempts == 0:
logger.error(
'You have used all the authentication attempts. Abort.')
raise DockerAuthenticationFailedError(
'Authentication failed. Abort.')
def _push_image(self, image, tag=None, auth=None, attempts=0):
""" Push an image to a remote Docker Registry.
Args:
image (str): The name of the Docker Image to be pushed.
tag (str): An optional tag for the Docker Image (default: 'latest')
auth: An optional Dict for the authentication.
attempts (int): The number of unsuccessful authentication attempts.
"""
if tag is None:
tag = 'latest'
if attempts == MAX_PUSH_ATTEMPTS:
err = 'Reached max attempts for pushing \
[{0}] with tag [{1}]'.format(image, tag)
logger.error(err)
raise FatalError(err)
logger.info('Pushing [{0}] with tag [{1}]'.format(image, tag))
result = self._client.images.push(image,
tag=tag,
auth_config=auth,
stream=True,
decode=True)
# Access Denied detection
for line in result:
# an error is occurred
if 'error' in line:
# access denied
if 'denied' or 'unauthorized' in line['error']:
logger.info('Access to the repository denied. \
Authentication failed.')
auth = create_auth_interactive()
self._push_image(image,
tag=tag,
auth=auth,
attempts=(attempts + 1))
else:
logger.error(
'Unknown error during push of [{0}]: {1}'.format(
image, line['error']))
raise FatalError(
CommonErrorMessages._DEFAULT_FATAL_ERROR_MSG)
logger.info('Image [{0}] with tag [{1}] successfully pushed.'.format(
image, tag))
def _validate_build(self, build_logs):
""" Validate the output of the building process.
Args:
build_logs (str): The output logs of a docker
image building process.
"""
for log in build_logs:
if 'stream' in log and self._verbose:
print(log['stream'])
if 'error' in log:
logger.error('Error building the Docker Image:\
\n{}'.format(log['error']))
raise DockerOperationError('Failed to "toskosing" \
the Docker Image. Abort.')
def _toskose_image_availability(self, toskose_image, toskose_tag='latest'):
""" Check the availability of the official Docker Toskose image used
during the "toskosing" process.
Args:
toskose_image (str): The official Toskose Docker image.
toskose_tag (str): The tag of the image (default: 'latest').
"""
try:
_ = self._client.images.pull(toskose_image, tag=toskose_tag)
except ImageNotFound:
logger.error(
'Failed to retrieve the official Toskose image [{0}:{1}]. \
Abort'.format(toskose_image, toskose_tag))
raise DockerOperationError(
'The official Toskose image {0} not found. Try later.\
\nIf the problem persist, open an issue at {1}'.format(
toskose_image, toskose_tag))
def _remove_previous_toskosed(self, image, tag=None):
""" Remove previous toskosed images.
Note: docker rmi doesn't remove an image if there are multiple tags
referencing it.
Args:
image (str): The name of the Docker image.
tag (str): The tag of the Docker Image (default: 'latest').
"""
if tag is None:
tag = 'latest'
def print_well(tags):
out = ''
for tag in tags:
out += '- {}\n'.format(tag)
return out
try:
logger.info(
'Searching for previous toskosed images [{0}:{1}]'.format(
image, tag))
image_found = self._client.images.get(image)
logger.debug(
'Image [{0}] found. It\'s referenced by the following \
tags:\n\n{1}'.format(image, print_well(image_found.tags)))
full_name = '{0}:{1}'.format(image, tag)
if full_name in image_found.tags:
self._client.images.remove(image=full_name, force=True)
logger.info('Removed [{0}] reference from [{1}] image'.format(
full_name, image))
try:
image_found = self._client.images.get(image)
logger.debug('Image [{0}][ID: {1}] still exist'.format(
image, image_found.id))
except ImageNotFound:
logger.info('Image [{0}][ID: {1}] doesn\'t have any \
references yet. Removed.'.format(
image, image_found.id))
except ImageNotFound:
logger.info('No previous image found.')
def toskose_image(self,
src_image,
src_tag,
dst_image,
dst_tag,
context,
process_type,
app_name,
toskose_dockerfile=None,
toskose_image=None,
toskose_tag=None,
enable_push=True):
""" The process of "toskosing" the component(s) of a multi-component
TOSCA-defined application.
The "toskosing" process consists in merging contexts
(e.g. artifacts, lifecycle scripts) of TOSCA software component(s)
and a TOSCA container node in which they are hosted on in a new
docker image.
The TOSCA container node comes with a suggested docker image,
which is "enriched" by the Toskose logic fetched remotely from
the official Toskose Docker image, using a template Dockerfile
by means of docker multi-stage features.
This logic permits to handle the lifecycle of multiple components
within the same container. (More details on how this process works
can be found in the template Dockerfile in dockerfiles/ directory or
in the official Toskose GitHub repository)
In other words, this function consists in receiving a name of an
existing docker image and a path to a build context, then by means
of the docker client, the toskose template dockerfile is built and
a new fresh docker image is generated. The new docker image includes
the content of the original image, the contexts of the TOSCA software
components and the logic of toskose for managing them.
Args:
src_image (str): The name of the image to be "toskosed".
src_tag (str): The tag of the image to be "toskosed".
dst_image (str): The name of the "toskosed" image.
dst_tag (str): The tag of the "toskosed" image.
context (str): The path of the application context.
process_type (enum): The type of "toskosing" process.
[unit/manager/free]
app_name (str): The name of the TOSCA application.
toskose_dockerfile (str): The path of the template dockerfile
used in the process.
toskose_image (str): The Docker Toskose base-image used in the
"toskosing" process.
toskose_tag (str): The tag of the Docker Toskose base-image.
enable_push (bool): enable/disable pushing of the "toskosed" image.
(default: True)
"""
if toskose_dockerfile is not None:
if not os.path.exists(toskose_dockerfile):
raise ValueError('The given toskose template dockerfile \
{} doesn\'t exist'.format(toskose_dockerfile))
if not app_name:
raise ValueError('A name associated to the TOSCA application \
must be given')
template_dir = os.path.join(os.path.dirname(__file__),
DOCKERFILE_TEMPLATES_PATH)
# TODO can be enanched
if process_type == ToskosingProcessType.TOSKOSE_UNIT:
if toskose_image is None:
toskose_image = constants.DEFAULT_TOSKOSE_UNIT_BASE_IMAGE
if toskose_tag is None:
toskose_tag = constants.DEFAULT_TOSKOSE_UNIT_BASE_TAG
if toskose_dockerfile is None:
toskose_dockerfile = os.path.join(
template_dir, DOCKERFILE_TOSKOSE_UNIT_TEMPLATE)
elif process_type == ToskosingProcessType.TOSKOSE_MANAGER:
if toskose_image is None:
toskose_image = constants.DEFAULT_MANAGER_BASE_IMAGE
if toskose_tag is None:
toskose_tag = constants.DEFAULT_MANAGER_BASE_TAG
if toskose_dockerfile is None:
toskose_dockerfile = os.path.join(
template_dir, DOCKERFILE_TOSKOSE_MANAGER_TEMPLATE)
src_image = toskose_image
src_tag = toskose_tag
elif process_type == ToskosingProcessType.TOSKOSE_FREE:
pass
else:
raise ValueError('Cannot recognize the "toskosing" process \
{}'.format(process_type))
if toskose_image is not None:
self._toskose_image_availability(toskose_image, toskose_tag)
# Check if the original image exists and needs authentication
# to be fetched.
# note: docker client does not distinguish between authentication
# error or invalid image name (?)
logger.info('Pulling [{0}:{1}]'.format(src_image, src_tag))
self._pull_image_with_auth(src_image, src_tag)
# TODO can be removed? is it really necessary?
self._remove_previous_toskosed(dst_image, dst_tag)
logger.info('Toskosing [{0}:{1}] image'.format(src_image, src_tag))
try:
build_args = {
'TOSCA_SRC_IMAGE': '{0}:{1}'.format(src_image, src_tag),
'TOSKOSE_BASE_IMG': '{0}:{1}'.format(toskose_image,
toskose_tag)
}
image, build_logs = self._client.images.build(
path=context,
tag='{0}:{1}'.format(dst_image, dst_tag),
buildargs=build_args,
dockerfile=toskose_dockerfile,
rm=True, # remove intermediate containers
)
self._validate_build(build_logs)
# push the "toskosed" image
if enable_push:
self._push_image(dst_image, tag=dst_tag)
logger.info(
'[{0}:{1}] image successfully toskosed in [{2}:{3}].'.format(
src_image, src_tag, dst_image, dst_tag))
except (BuildError, APIError) as err:
logger.exception(err)
raise DockerOperationError('Failed to build image')
def close(self):
self._client.close()
class DockerEnvironmentDriver(EnvironmentDriver):
"""
This EnvironmentDriver handles environment management in the project using docker
Parameters
----------
root : str, optional
home filepath for project
(default is empty)
docker_execpath : str, optional
execution path for docker
(default is "docker" which defers to system)
docker_socket : str, optional
socket path to docker daemon to connect
(default is None, this takes the default path for the system)
Attributes
----------
root : str
home filepath for project
datmo_directory_name : str
datmo directory name for project
docker_execpath : str, optional
docker execution path for the system
docker_socket : str, optional
specific socket for docker
(default is None, which means system default is used by docker)
client : DockerClient
docker python api client
cpu_prefix : list
list of strings for the prefix command for all docker commands
info : dict
information about the docker daemon connection
is_connected : bool
True if connected to daemon else False
type : str
type of EnvironmentDriver
"""
def __init__(self,
root,
datmo_directory_name,
docker_execpath="docker",
docker_socket=None):
super(DockerEnvironmentDriver, self).__init__()
if not docker_socket:
if platform.system() != "Windows":
docker_socket = "unix:///var/run/docker.sock"
self.root = root
# Check if filepath exists
if not os.path.exists(self.root):
raise PathDoesNotExist(
__("error",
"controller.environment.driver.docker.__init__.dne", root))
self._datmo_directory_name = datmo_directory_name
self._datmo_directory_path = os.path.join(self.root,
self._datmo_directory_name)
self.environment_directory_name = "environment"
self.environment_directory_path = os.path.join(
self._datmo_directory_path, self.environment_directory_name)
self.docker_execpath = docker_execpath
self.docker_socket = docker_socket
if self.docker_socket:
self.client = DockerClient(base_url=self.docker_socket)
self.prefix = [self.docker_execpath, "-H", self.docker_socket]
else:
self.client = DockerClient()
self.prefix = [self.docker_execpath]
self._is_connected = False
self._is_initialized = self.is_initialized
self.type = "docker"
with open(docker_config_filepath) as f:
self.docker_config = json.load(f)
@property
def is_initialized(self):
if self.exists_environment_dir():
self._is_initialized = True
return self._is_initialized
self._is_initialized = False
return self._is_initialized
@property
def is_connected(self):
# TODO: Check if Docker is up and running
return self._is_connected
def init(self):
try:
# Ensure the environment files structure exists
self.ensure_environment_dir()
except Exception as e:
raise EnvironmentInitFailed(
__("error", "controller.environment.driver.docker.init",
str(e)))
return True
def connect(self):
# TODO: Fill in to start up Docker
# Startup Docker
try:
pass
except Exception as e:
raise EnvironmentExecutionError(
__("error", "controller.environment.driver.docker.init",
str(e)))
# Initiate Docker execution
try:
self.info = self.client.info()
self._is_connected = True if self.info["Images"] != None else False
except Exception:
raise EnvironmentConnectFailed(
__("error", "controller.environment.driver.docker.__init__",
platform.system()))
return True
# Environment directory
def create_environment_dir(self):
if not os.path.isdir(self._datmo_directory_path):
raise FileStructureError(
__("error",
"controller.file.driver.local.create_collections_dir"))
if not os.path.isdir(self.environment_directory_path):
os.makedirs(self.environment_directory_path)
return True
def exists_environment_dir(self):
return os.path.isdir(self.environment_directory_path)
def ensure_environment_dir(self):
if not self.exists_environment_dir():
self.create_environment_dir()
return True
def delete_environment_dir(self):
shutil.rmtree(self.environment_directory_path)
return True
def list_environment_files(self):
if not self.is_initialized:
raise FileStructureError(
__("error",
"controller.file.driver.local.list_file_collections"))
environment_files_list = os.listdir(self.environment_directory_path)
return environment_files_list
# Other functions
def get_environment_types(self):
# To get the current environment type
return list(self.docker_config["environment_types"])
def get_supported_frameworks(self, environment_type):
# To get the current environments
environment_names = []
for environment_name in self.docker_config["environment_types"][
environment_type]["environment_frameworks"]:
environment_names.append([
environment_name,
self.docker_config["environment_types"][environment_type]
["environment_frameworks"][environment_name]["info"]
])
return environment_names
def get_supported_languages(self, environment_type, environment_framework):
# To get the current environments
return self.docker_config["environment_types"][environment_type][
"environment_frameworks"][environment_framework][
"environment_languages"]
def setup(self, options, definition_path):
environment_type = options.get("environment_type", None)
environment_framework = options.get("environment_framework", None)
environment_language = options.get("environment_language", None)
environment_types = self.get_environment_types()
if environment_type not in environment_types:
raise EnvironmentDoesNotExist(
__("error", "controller.environment.driver.docker.setup.dne",
environment_type))
available_environment_info = self.get_supported_frameworks(
environment_type)
available_environments = [
item[0] for item in available_environment_info
]
if environment_framework not in available_environments:
raise EnvironmentDoesNotExist(
__("error", "controller.environment.driver.docker.setup.dne",
environment_framework))
available_environment_languages = self.get_supported_languages(
environment_type, environment_framework)
if available_environment_languages and environment_language not in available_environment_languages:
raise EnvironmentDoesNotExist(
__("error", "controller.environment.driver.docker.setup.dne",
environment_language))
# Validate the given definition path exists
if not os.path.isdir(definition_path):
raise PathDoesNotExist()
# To setup the environment definition file
definition_filepath = os.path.join(definition_path, "Dockerfile")
with open(definition_filepath, "wb") as f:
if environment_language:
f.write(
to_bytes("FROM datmo/%s:%s-%s%s%s" %
(environment_framework, environment_type,
environment_language, os.linesep, os.linesep)))
else:
f.write(
to_bytes("FROM datmo/%s:%s%s%s" %
(environment_framework, environment_type,
os.linesep, os.linesep)))
return True
def create(self, path=None, output_path=None, workspace=None):
if not path:
path = os.path.join(self.root, "Dockerfile")
if not output_path:
directory, filename = os.path.split(path)
output_path = os.path.join(directory, "datmo" + filename)
if not os.path.isfile(path):
raise EnvironmentDoesNotExist(
__("error", "controller.environment.driver.docker.create.dne",
path))
if os.path.isfile(output_path):
raise FileAlreadyExistsError(
__("error",
"controller.environment.driver.docker.create.exists",
output_path))
success = self.create_datmo_definition(
input_definition_path=path,
output_definition_path=output_path,
workspace=workspace)
return success, path, output_path
# running daemon needed
def build(self, name, path, workspace=None):
return self.build_image(name, path, workspace)
# running daemon needed
def run(self, name, options, log_filepath):
if "gpu" in options:
gpu_ready = self.gpu_enabled()
if options["gpu"] is True:
if not gpu_ready:
raise GPUSupportNotEnabled('nvidia')
else:
options['runtime'] = 'nvidia'
options.pop("gpu", None)
run_return_code, run_id = self.run_container(image_name=name,
**options)
log_return_code, logs = self.log_container(run_id,
filepath=log_filepath)
final_return_code = run_return_code and log_return_code
return final_return_code, run_id, logs
# running daemon needed
def stop(self, run_id, force=False):
stop_result = self.stop_container(run_id)
remove_run_result = self.remove_container(run_id, force=force)
return stop_result and remove_run_result
# running daemon needed
def remove(self, name, force=False):
stop_and_remove_containers_result = \
self.stop_remove_containers_by_term(name, force=force)
try:
self.get_image(name)
remove_image_result = self.remove_image(name, force=force)
except EnvironmentImageNotFound:
remove_image_result = True
return stop_and_remove_containers_result and \
remove_image_result
# running daemon needed
def gpu_enabled(self):
# test if this images works
# docker run --runtime=nvidia --rm nvidia/cuda nvidia-smi
process = subprocess.Popen([
"docker",
"run",
"--runtime=nvidia",
"--rm",
"nvidia/cuda",
"nvidia-smi",
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
stderr = stderr.decode("utf-8")
if "Unknown runtime specified nvidia" in stderr:
return False
if "OCI runtime create failed" in stderr:
return False
if len(stderr) > 2:
raise GPUSupportNotEnabled(stderr)
# this may mean we're good to go. Untested though.
return True
# running daemon needed
def get_tags_for_docker_repository(self, repo_name):
# TODO: Use more common CLI command (e.g. curl instead of wget)
"""Method to get tags for docker repositories
Parameters
----------
repo_name: str
Docker repository name
Returns
-------
list
List of tags available for that docker repo
"""
docker_repository_tag_cmd = "wget -q https://registry.hub.docker.com/v1/repositories/" + repo_name + "/tags -O -"
try:
process = subprocess.Popen(docker_repository_tag_cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
if process.returncode > 0:
raise EnvironmentExecutionError(
__("error",
"controller.environment.driver.docker.get_tags",
str(stderr)))
string_repository_tags = stdout.decode().strip()
except subprocess.CalledProcessError as e:
raise EnvironmentExecutionError(
__("error", "controller.environment.driver.docker.get_tags",
str(e)))
repository_tags = ast.literal_eval(string_repository_tags)
list_tag_names = []
for repository_tag in repository_tags:
list_tag_names.append(repository_tag["name"])
return list_tag_names
# running daemon needed
def build_image(self, tag, definition_path="Dockerfile", workspace=None):
"""Builds docker image
Parameters
----------
tag : str
name to tag image with
definition_path : str
absolute file path to the definition
workspace : str
workspace to be used for the run
Returns
-------
bool
True if success
Raises
------
EnvironmentExecutionError
"""
try:
docker_shell_cmd_list = list(self.prefix)
docker_shell_cmd_list.append("build")
# Passing tag name for the image
docker_shell_cmd_list.append("-t")
docker_shell_cmd_list.append(tag)
# Passing path of Dockerfile
# Creating datmoDockerfile for new build
dockerfile_dirpath = os.path.split(definition_path)[0]
input_dockerfile = os.path.split(definition_path)[1]
output_dockerfile_path = os.path.join(dockerfile_dirpath,
"datmo%s" % input_dockerfile)
self.create(definition_path, output_dockerfile_path, workspace)
docker_shell_cmd_list.append("-f")
docker_shell_cmd_list.append(output_dockerfile_path)
docker_shell_cmd_list.append(str(dockerfile_dirpath))
# Remove intermediate containers after a successful build
docker_shell_cmd_list.append("--rm")
process_returncode = subprocess.Popen(docker_shell_cmd_list).wait()
if process_returncode == 0:
return True
elif process_returncode == 1:
raise EnvironmentExecutionError(
__("error",
"controller.environment.driver.docker.build_image",
"Docker subprocess failed"))
except Exception as e:
raise EnvironmentExecutionError(
__("error", "controller.environment.driver.docker.build_image",
str(e)))
# running daemon needed
def get_image(self, image_name):
try:
return self.client.images.get(image_name)
except errors.ImageNotFound:
raise EnvironmentImageNotFound()
# running daemon needed
def list_images(self, name=None, all_images=False, filters=None):
return self.client.images.list(name=name,
all=all_images,
filters=filters)
# running daemon needed
def search_images(self, term):
return self.client.images.search(term=term)
# running daemon needed
def remove_image(self, image_id_or_name, force=False):
try:
docker_image_remove_cmd = list(self.prefix)
if force:
docker_image_remove_cmd.extend(["rmi", "-f", image_id_or_name])
else:
docker_image_remove_cmd.extend(["rmi", image_id_or_name])
process = subprocess.Popen(docker_image_remove_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
if process.returncode > 0 or stderr:
raise EnvironmentExecutionError(
__("error",
"controller.environment.driver.docker.remove_image",
str(stderr)))
except subprocess.CalledProcessError as e:
raise EnvironmentExecutionError(
__("error",
"controller.environment.driver.docker.remove_image",
str(e)))
return True
# running daemon needed
def remove_images(self, name=None, all=False, filters=None, force=False):
"""Remove multiple images
"""
try:
images = self.list_images(name=name,
all_images=all,
filters=filters)
for image in images:
self.remove_image(image.id, force=force)
except Exception as e:
raise EnvironmentExecutionError(
__("error",
"controller.environment.driver.docker.remove_images",
str(e)))
return True
def extract_workspace_url(self, container_name, workspace=None):
"""Extract workspace url from the container
Parameters
----------
container_name : str
name of the container being run
workspace : str
workspace being used for the run
Returns
-------
str
web url for the workspace being run, None if it doesn't exist
"""
if workspace in ['notebook', 'jupyterlab']:
docker_shell_cmd_list = list(self.prefix)
docker_shell_cmd_list.append("exec")
docker_shell_cmd_list.append(container_name)
docker_shell_cmd_list.append("jupyter")
docker_shell_cmd_list.append("notebook")
docker_shell_cmd_list.append("list")
workspace_url = None
timeout_count = 0
while workspace_url is None and timeout_count < 10:
process = subprocess.Popen(docker_shell_cmd_list,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
if process.returncode > 0:
time.sleep(1)
timeout_count += 1
result = re.search("(?P<url>https?://[^\s]+)",
stdout.decode('utf-8')) if stdout else None
workspace_url = result.group("url") if result else None
return workspace_url
elif workspace == 'rstudio':
# Having a pause for two seconds before returning url
time.sleep(2)
workspace_url = 'http://localhost:8787'
else:
workspace_url = None
return workspace_url
# running daemon needed
def run_container(self,
image_name,
command=None,
ports=None,
name=None,
volumes=None,
mem_limit=None,
runtime=None,
detach=False,
stdin_open=False,
tty=False,
api=False):
"""Run Docker container with parameters given as defined below
Parameters
----------
image_name : str
Docker image name
command : list, optional
List with complete user-given command (e.g. ["python3", "cool.py"])
ports : list, optional
Here are some example ports used for common applications.
* "jupyter notebook" - 8888
* flask API - 5000
* tensorboard - 6006
An example input for the above would be ["8888:8888", "5000:5000", "6006:6006"]
which maps the running host port (right) to that of the environment (left)
name : str, optional
User given name for container
volumes : dict, optional
Includes storage volumes for docker
(e.g. { outsidepath1 : {"bind", containerpath2, "mode", MODE} })
mem_limit : str, optional
maximum amount of memory the container can use
(these options take a positive integer, followed by a suffix of b, k, m, g, to indicate bytes, kilobytes,
megabytes, or gigabytes. memory limit is contrained by total memory of the VM in which docker runs)
detach : bool, optional
True if container is to be detached else False
stdin_open : bool, optional
True if stdin is open else False
tty : bool, optional
True to connect pseudo-terminal with stdin / stdout else False
api : bool, optional
True if Docker python client should be used else use subprocess
Returns
-------
if api=False:
return_code: int
integer success code of command
container_id: str
output container id
if api=True & if detach=True:
container_obj: Container
object from Docker python api with details about container
if api=True & if detach=False:
logs: str
output logs for the run function
Raises
------
EnvironmentExecutionError
error in running the environment command
"""
try:
if api: # calling the docker client via the API
# TODO: Test this out for the API (need to verify ports work)
if detach:
command = " ".join(command) if command else command
container = \
self.client.containers.run(image_name, command, ports=ports,
name=name, volumes=volumes,
mem_limit=mem_limit,
detach=detach, stdin_open=stdin_open)
return container
else:
command = " ".join(command) if command else command
logs = self.client.containers.run(image_name,
command,
ports=ports,
name=name,
volumes=volumes,
mem_limit=mem_limit,
detach=detach,
stdin_open=stdin_open)
return logs.decode()
else: # if calling run function with the shell commands
docker_shell_cmd_list = list(self.prefix)
docker_shell_cmd_list.append("run")
if name:
docker_shell_cmd_list.append("--name")
docker_shell_cmd_list.append(name)
if runtime:
docker_shell_cmd_list.append("--runtime")
docker_shell_cmd_list.append(runtime)
if mem_limit:
docker_shell_cmd_list.append("-m")
docker_shell_cmd_list.append(mem_limit)
docker_shell_cmd_list.append("--memory-swap")
docker_shell_cmd_list.append("-1")
if stdin_open:
docker_shell_cmd_list.append("-i")
if tty:
docker_shell_cmd_list.append("-t")
if detach:
docker_shell_cmd_list.append("-d")
# Volume
if volumes:
# Mounting volumes
for key in list(volumes):
docker_shell_cmd_list.append("-v")
volume_mount = key + ":" + volumes[key]["bind"] + ":" + \
volumes[key]["mode"]
docker_shell_cmd_list.append(volume_mount)
if ports:
# Mapping ports
for mapping in ports:
docker_shell_cmd_list.append("-p")
docker_shell_cmd_list.append(mapping)
docker_shell_cmd_list.append(image_name)
if command:
docker_shell_cmd_list.extend(command)
return_code = subprocess.call(docker_shell_cmd_list)
if return_code != 0:
raise EnvironmentExecutionError(
__(
"error",
"controller.environment.driver.docker.run_container",
str(docker_shell_cmd_list)))
list_process_cmd = list(self.prefix)
list_process_cmd.extend(["ps", "-q", "-l"])
process = subprocess.Popen(list_process_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
if process.returncode > 0:
raise EnvironmentExecutionError(
__(
"error",
"controller.environment.driver.docker.run_container",
str(stderr)))
container_id = stdout.decode().strip()
except subprocess.CalledProcessError as e:
raise EnvironmentExecutionError(
__("error",
"controller.environment.driver.docker.run_container",
str(e)))
return return_code, container_id
# running daemon needed
def get_container(self, container_id):
try:
return self.client.containers.get(container_id)
except errors.NotFound:
raise EnvironmentContainerNotFound()
# running daemon needed
def list_containers(self,
all=False,
before=None,
filters=None,
limit=-1,
since=None):
return self.client.containers.list(all=all,
before=before,
filters=filters,
limit=limit,
since=since)
# running daemon needed
def stop_container(self, container_id):
try:
docker_container_stop_cmd = list(self.prefix)
docker_container_stop_cmd.extend(["stop", container_id])
process = subprocess.Popen(docker_container_stop_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
if process.returncode > 0:
raise EnvironmentExecutionError(
__("error",
"controller.environment.driver.docker.stop_container",
str(stderr)))
except subprocess.CalledProcessError as e:
raise EnvironmentExecutionError(
__("error",
"controller.environment.driver.docker.stop_container",
str(e)))
return True
# running daemon needed
def remove_container(self, container_id, force=False):
try:
docker_container_remove_cmd_list = list(self.prefix)
if force:
docker_container_remove_cmd_list.extend(
["rm", "-f", container_id])
else:
docker_container_remove_cmd_list.extend(["rm", container_id])
process = subprocess.Popen(docker_container_remove_cmd_list,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
if process.returncode > 0:
raise EnvironmentExecutionError(
__(
"error",
"controller.environment.driver.docker.remove_container",
str(stderr)))
except subprocess.CalledProcessError as e:
raise EnvironmentExecutionError(
__("error",
"controller.environment.driver.docker.remove_container",
str(e)))
return True
# running daemon needed
def log_container(self, container_id, filepath, api=False, follow=True):
"""Log capture at a particular point `docker logs`. Can also use `--follow` for real time logs
Parameters
----------
container_id : str
Docker container id
filepath : str
Filepath to store log file
api : bool
True to use the docker python api
follow : bool
Tail the output
Returns
-------
return_code : str
Process return code for the container
logs : str
Output logs read into a string format
"""
# TODO: Fix function to better accomodate all logs in the same way
if api: # calling the docker client via the API
with open(filepath, "wb") as log_file:
for line in self.client.containers.get(container_id).logs(
stream=True):
log_file.write(to_bytes(line.strip() + os.linesep))
else:
command = list(self.prefix)
if follow:
command.extend(["logs", "--follow", str(container_id)])
else:
command.extend(["logs", str(container_id)])
process = subprocess.Popen(command,
stdout=subprocess.PIPE,
universal_newlines=True)
with open(filepath, "wb") as log_file:
while True:
output = process.stdout.readline()
if output == "" and process.poll() is not None:
break
if output:
printable_output = output.strip().replace("\x08", " ")
log_file.write(to_bytes(printable_output + os.linesep))
return_code = process.poll()
with open(filepath, "rb") as log_file:
logs = log_file.read()
if type(logs) != str: # handle for python 3x
logs = logs.decode("utf-8")
return return_code, logs
# running daemon needed
def stop_remove_containers_by_term(self, term, force=False):
"""Stops and removes containers by term
"""
# TODO: split out the find containers function from stop / remove
try:
running_docker_container_cmd_list = list(self.prefix)
running_docker_container_cmd_list.extend([
"ps", "-a", "|", "grep",
"'%s'" % term, "|", "awk '{print $1}'"
])
running_docker_container_cmd_str = str(
" ".join(running_docker_container_cmd_list))
process = subprocess.Popen(running_docker_container_cmd_str,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out_list_cmd, err_list_cmd = process.communicate()
if process.returncode > 0:
raise EnvironmentExecutionError(
__(
"error",
"controller.environment.driver.docker.stop_remove_containers_by_term",
str(err_list_cmd)))
# checking for running container id before stopping any
if out_list_cmd:
docker_container_stop_cmd_list = list(self.prefix)
docker_container_stop_cmd_list = docker_container_stop_cmd_list + \
["stop", "$("] + running_docker_container_cmd_list + \
[")"]
docker_container_stop_cmd_str = str(
" ".join(docker_container_stop_cmd_list))
process = subprocess.Popen(docker_container_stop_cmd_str,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
_, _ = process.communicate()
# rechecking for container id after stopping them to ensure no errors
process = subprocess.Popen(running_docker_container_cmd_str,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out_list_cmd, err_list_cmd = process.communicate()
if process.returncode > 0:
raise EnvironmentExecutionError(
__(
"error",
"controller.environment.driver.docker.stop_remove_containers_by_term",
str(err_list_cmd)))
if out_list_cmd:
docker_container_remove_cmd_list = list(self.prefix)
if force:
docker_container_remove_cmd_list = docker_container_remove_cmd_list + \
["rm", "-f", "$("] + running_docker_container_cmd_list + \
[")"]
else:
docker_container_remove_cmd_list = docker_container_remove_cmd_list + \
["rm", "$("] + running_docker_container_cmd_list + \
[")"]
docker_container_remove_cmd_str = str(
" ".join(docker_container_remove_cmd_list))
process = subprocess.Popen(docker_container_remove_cmd_str,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
_, err_list_cmd = process.communicate()
if process.returncode > 0:
raise EnvironmentExecutionError(
__(
"error",
"controller.environment.driver.docker.stop_remove_containers_by_term",
str(err_list_cmd)))
except subprocess.CalledProcessError as e:
raise EnvironmentExecutionError(
__(
"error",
"controller.environment.driver.docker.stop_remove_containers_by_term",
str(e)))
return True
def create_requirements_file(self, package_manager="pip"):
"""Create python requirements txt file for the project
Parameters
----------
package_manager : str, optional
the package manager being used during the snapshot creation
Returns
-------
str
absolute filepath for requirements file
Raises
------
EnvironmentRequirementsCreateError
error in running package manager command to extract environment requirements
"""
if package_manager == "pip":
try:
requirements_filepath = os.path.join(self.root,
"datmorequirements.txt")
outfile_requirements = open(requirements_filepath, "wb")
process = subprocess.Popen(["pip", "freeze"],
cwd=self.root,
stdout=outfile_requirements,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
if process.returncode > 0:
raise EnvironmentRequirementsCreateError(
__("error",
"controller.environment.requirements.create",
str(stderr)))
except Exception as e:
raise EnvironmentRequirementsCreateError(
__("error", "controller.environment.requirements.create",
str(e)))
if not os.path.isfile(requirements_filepath):
return None
return requirements_filepath
else:
raise EnvironmentRequirementsCreateError(
__("error", "controller.environment.requirements.create",
"no such package manager"))
@staticmethod
def create_default_definition(directory, language="python3"):
language_dockerfile = "%sDockerfile" % language
default_dockerfile_filepath = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "templates",
language_dockerfile)
destination_dockerfile = os.path.join(directory, "Dockerfile")
with open(default_dockerfile_filepath, "rb") as input_file:
with open(destination_dockerfile, "wb") as output_file:
for line in input_file:
if to_bytes(os.linesep) in line:
output_file.write(line.strip() + to_bytes(os.linesep))
else:
output_file.write(line.strip())
return destination_dockerfile
def get_default_definition_filename(self):
return "Dockerfile"
def get_datmo_definition_filenames(self):
return ["hardware_info"]
def get_hardware_info(self):
# Extract hardware info of the container (currently taking from system platform)
# TODO: extract hardware information directly from the container
(system, node, release, version, machine, processor) = platform.uname()
return {
'system': system,
'node': node,
'release': release,
'version': version,
'machine': machine,
'processor': processor
}
@staticmethod
def create_datmo_definition(input_definition_path,
output_definition_path,
workspace=None):
"""
Creates a datmo dockerfiles to run at the output path specified
"""
datmo_base_dockerfile_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "templates",
"baseDockerfile")
# Combine dockerfiles
with open(input_definition_path, "rb") as input_file:
with open(datmo_base_dockerfile_path, "rb") as datmo_base_file:
with open(output_definition_path, "wb") as output_file:
for line in input_file:
bool_workspace_update = (to_bytes('FROM datmo/')
in line.strip() and workspace)
if to_bytes("\n") in line and bool_workspace_update:
updated_line = line.strip() + to_bytes(
"-%s%s" % (workspace, os.linesep))
elif to_bytes(os.linesep) in line:
updated_line = line.strip() + to_bytes(os.linesep)
else:
updated_line = line.strip()
output_file.write(updated_line)
for line in datmo_base_file:
if to_bytes("\n") in line:
output_file.write(line.strip() +
to_bytes(os.linesep))
else:
output_file.write(line.strip())
return True
class BugZoo(object):
"""
Used to interact with and manage a local BugZoo installation.
"""
def __init__(self,
path=None,
base_url_docker='unix:///var/run/docker.sock') -> None:
"""
Creates a new BugZoo installation manager.
Args:
path: the absolute path of a BugZoo installation on this machine.
If unspecified, the value of the environmental variable
:code:`BUGZOO_PATH` will be used, unless unspecified, in
which case :code:`./${HOME}/.bugzoo` will be used instead.
"""
# TODO support windows
if path is None:
default_path = os.path.join(os.environ['HOME'], '.bugzoo')
path = os.environ.get('BUGZOO_PATH', default_path)
self.__path = path
logger.debug("using BugZoo directory: %s", path)
logger.debug("preparing BugZoo directory")
if not os.path.exists(self.path):
os.makedirs(self.path)
if not os.path.exists(self.coverage_path):
os.makedirs(self.coverage_path)
logger.debug("prepared BugZoo directory")
logger.debug("connecting to Docker at %s", base_url_docker)
self.__base_url_docker = base_url_docker
try:
self.__docker = DockerClient(base_url=base_url_docker, timeout=120)
assert self.__docker.ping()
except (docker.errors.APIError, AssertionError):
logger.exception("failed to connect to Docker")
raise # FIXME add DockerConnectionFailure
logger.debug("connected to Docker")
logger.debug("Docker version: %s", self.__docker.version())
logger.debug("Docker server info: %s", self.__docker.info())
self.__mgr_build = BuildManager(self.__docker)
self.__bugs = BugManager(self)
self.__tools = ToolManager(self)
self.__sources = SourceManager(self)
self.__containers = ContainerManager(self)
self.__files = FileManager(self.__bugs, self.__containers)
self.__coverage = CoverageManager(self)
def shutdown(self) -> None:
logger.info("Shutting down daemon...")
self.__containers.clear()
logger.info("Shut down daemon")
@property
def docker(self) -> DockerClient:
"""
The Docker client used by this server.
"""
return self.__docker
@property
def base_url_docker(self) -> str:
"""
The base URL of the Docker server to which BugZoo is connected.
"""
return self.__base_url_docker
@property
def path(self) -> str:
"""
The absolute path to the local installation of BugZoo.
"""
return self.__path
@property
def coverage_path(self) -> str:
"""
The absolute path to the directory used to store cached coverage
information for bugs provided by this BugZoo installation.
"""
return os.path.join(self.path, "coverage")
def rescan(self):
self.__sources.scan()
@property
def build(self) -> BuildManager:
return self.__mgr_build
@property
def coverage(self) -> CoverageManager:
"""
The coverage manager is used to compute line coverage information.
"""
return self.__coverage
@property
def files(self) -> FileManager:
"""
Provides access to the file system within running containers.
"""
return self.__files
@property
def sources(self) -> SourceManager:
"""
The sources registered with this BugZoo installation.
"""
return self.__sources
@property
def tools(self) -> ToolManager:
"""
The tools registered with this BugZoo installation.
"""
return self.__tools
@property
def bugs(self) -> BugManager:
"""
The bugs registered with this BugZoo installation.
"""
return self.__bugs
@property
def containers(self) -> ContainerManager:
"""
The containers that are running on this server.
"""
return self.__containers
def __init__(self, client: docker.DockerClient, domains):
super().__init__(client, domains)
self.addresses = []
self.domains = domains
for manager in client.info()['Swarm']['RemoteManagers']:
self.addresses.append(manager['Addr'].split(':')[0])
