def execute_image(app, docker_timeout=15, metric_interval=15, metric_warmup=2):
task = image_type(app)
(execution_ready,
failure_msg) = task.does_task_pass_pre_execution_checks(app)
if not execution_ready:
app['logger'].critical(failure_msg)
return
task.before_container_hook(app)
image = image_version(app)
app['logger'].info("Creating Docker container from image {}".format(image))
biobox = create_container(app)
id_ = biobox['Id']
app['logger'].info("Starting Docker container {}".format(id_))
docker.client(docker_timeout).start(id_)
metrics = cgroup.collect_runtime_metrics(id_, metric_interval,
metric_warmup)
app['logger'].info("Docker container {} finished".format(id_))
fs.create_runtime_metric_file(app, metrics)
copy_output_files(app)
def create_container(image, config, directories, task = "default", docker_args = {}):
"""
Returns a new biobox Docker container created from the given image name. The
container is not started. Networking will be enabled by default until an issue
with Docker stats collection is resolved - docker/docker-py#1195.
Keyword arguments:
image -- name of a docker image, may optionally include sha256
config -- biobox configuration as specified by the biobox.yaml format
directories -- dictionary of host directories locations with the keys:
output -- REQUIRED location of output destination directory
metadata -- OPTIONAL location of metadata destination directory
task -- biobox container task to execute, defaults to "default".
docker_args -- Optional cgroup data passed to the docker daemon. See the
docker documentation for a list of available values
"""
volumes = prepare_volumes(config, directories.get('output'), directories.get('metadata'))
docker_args['volumes'] = list(map(vol.get_host_path, volumes))
docker_args['network_disabled'] = False
host_config = {'binds' : volumes}
if 'mem_limit' in docker_args:
host_config['mem_limit'] = docker_args['mem_limit']
del docker_args['mem_limit']
docker_args['host_config'] = util.client().create_host_config(**host_config)
return util.client().create_container(image, task, **docker_args)
def test_run_container():
app = app_helper.setup_app_state('quast', 'execute')
id_ = run.create_container(app)['Id']
docker.client().start(id_)
docker.client().wait(id_)
nose.assert_equal(container.did_exit_succcessfully(id_), True)
image_helper.clean_up_container(id_)
def create_container(image,
config,
directories,
task="default",
docker_args={}):
"""
Returns a new biobox Docker container created from the given image name. The
container is not started. Networking will be enabled by default until an issue
with Docker stats collection is resolved - docker/docker-py#1195.
Keyword arguments:
image -- name of a docker image, may optionally include sha256
config -- biobox configuration as specified by the biobox.yaml format
directories -- dictionary of host directories locations with the keys:
output -- REQUIRED location of output destination directory
metadata -- OPTIONAL location of metadata destination directory
task -- biobox container task to execute, defaults to "default".
docker_args -- Optional cgroup data passed to the docker daemon. See the
docker documentation for a list of available values
"""
volumes = prepare_volumes(config, directories.get('output'),
directories.get('metadata'))
docker_args['volumes'] = list(map(vol.get_host_path, volumes))
docker_args['network_disabled'] = False
host_config = {'binds': volumes}
if 'mem_limit' in docker_args:
host_config['mem_limit'] = docker_args['mem_limit']
del docker_args['mem_limit']
docker_args['host_config'] = util.client().create_host_config(
**host_config)
return util.client().create_container(image, task, **docker_args)
def create_tty(image, tty, volumes=[]):
command = ""
return docker.client().create_container(
image,
command,
stdin_open=True,
tty=tty,
entrypoint='/bin/bash',
volumes=list(map(vol.get_host_path, volumes)),
host_config=docker.client().create_host_config(binds=volumes))
def create_tty(image, tty, volumes = []):
command = ""
return docker.client().create_container(
image,
command,
stdin_open = True,
tty = tty,
entrypoint = '/bin/bash',
volumes = list(map(vol.get_host_path, volumes)),
host_config = docker.client().create_host_config(binds=volumes))
def test_executing_container():
out_dir = tempfile.mkdtemp()
cnt = exe.create_container(IMAGE, biobox_cfg(hlp.short_read_fastq()),
{"output": out_dir}, "default",
{"detach": False})
id_ = cnt['Id']
util.client().start(id_)
util.client().wait(id_)
assert funcy.get_in(util.client().inspect_container(id_),
['State', 'ExitCode']) == 0
assert os.path.isfile(os.path.join(out_dir, 'contigs.fa'))
hlp.clean_up_container(id_)
def test_executing_container():
out_dir = tempfile.mkdtemp()
cnt = exe.create_container(
IMAGE,
biobox_cfg(hlp.short_read_fastq()),
{"output" : out_dir},
"default",
{"detach" : False})
id_ = cnt['Id']
util.client().start(id_)
util.client().wait(id_)
assert funcy.get_in(util.client().inspect_container(id_), ['State', 'ExitCode']) == 0
assert os.path.isfile(os.path.join(out_dir, 'contigs.fa'))
hlp.clean_up_container(id_)
def list_of_local_images():
"""
Returns a set containing all local docker image tags and digests.
"""
images = util.client().images()
tags_and_digests = list(map(get_image_tags, images)) + list(map(get_image_digests, images))
return set(reduce(lambda acc, x: acc + x, tags_and_digests, []))
def test_collect_metric_with_dead_container():
id_ = hlp.dummy_container(2)
stream = util.client().stats(id_, decode = True, stream = True)
time.sleep(1)
cgroup.collect_metric(stream)
time.sleep(1)
assert cgroup.collect_metric(stream) == None
hlp.clean_up_container(id_)
def test_collect_metric_with_dead_container():
id_ = hlp.dummy_container(2)
stream = util.client().stats(id_, decode=True, stream=True)
time.sleep(1)
cgroup.collect_metric(stream)
time.sleep(1)
assert cgroup.collect_metric(stream) == None
hlp.clean_up_container(id_)
def get_image(name):
"""
Fetches the Docker image if it is not present locally.
"""
if not is_image_available_locally(name):
output = util.client().pull(name)
if "error" in output:
raise biobox.exception.NoImageFound(name)
return True
def dummy_container(arg, wait = True, docker_args = {}):
client = util.client()
container = client.create_container(
image = 'alpine:3.3',
command = '/bin/sleep {}'.format(str(arg)),
**{'network_disabled' : False})
id_ = container['Id']
client.start(id_)
if wait:
client.wait(id_)
return id_
def collect_runtime_metrics(container_id, interval=15, warmup=1):
"""
Collects cgroup runtime metrics from the specified container at the given
per-second intervals.
"""
stream = util.client().stats(container_id, decode=True, stream=True)
time.sleep(warmup)
stats = [next(stream)]
while ctn.is_running(container_id):
time.sleep(1)
entry = collect_metric(stream)
# Remove this when docker/docker-py#1195 is fixed
if not entry:
pass
else:
# Save the cgroup entry if it is greater than given time interval
if time_diff_in_seconds(stats[-1]['read'],
entry['read']) > interval:
stats.append(entry)
return stats
def metadata_lookup(path, container_id):
"""
Look up metadata about the docker container for the given path
"""
return funcy.get_in(util.client().inspect_container(container_id), path)
def test_collect_metric_with_running_container():
id_ = hlp.dummy_container(10, False)
stream = util.client().stats(id_, decode = True, stream = True)
assert 'read' in cgroup.collect_metric(stream)
util.client().kill(id_)
hlp.clean_up_container(id_)
def run(container):
docker.client().start(container)
docker.client().wait(container)
def login(container):
docker.client().start(container)
pty.PseudoTerminal(docker.client(), container).start()
docker.client().stop(container)
def remove(container):
"""
Removal of a container
NOTE: This method is not tested due to circle ci limitations
"""
docker.client().remove_container(container, v=True)
def test_is_container_running():
id_ = hlp.dummy_container(10, False)
time.sleep(0.1)
assert ctn.is_running(id_)
util.client().kill(id_)
hlp.clean_up_container(id_)
def remove(container):
"""
Removal of a container
NOTE: This method is not tested due to circle ci limitations
"""
docker.client().remove_container(container, v=True)
def run(container):
docker.client().start(container)
docker.client().wait(container)
def login(container):
docker.client().start(container)
pty.PseudoTerminal(docker.client(), container).start()
docker.client().stop(container)
def test_collect_metric_with_running_container():
id_ = hlp.dummy_container(10, False)
stream = util.client().stats(id_, decode=True, stream=True)
assert 'read' in cgroup.collect_metric(stream)
util.client().kill(id_)
hlp.clean_up_container(id_)
def clean_up_container(id_):
if not "CIRCLECI" in os.environ:
util.client().remove_container(id_)
