def execute_image(app, docker_timeout=15, metric_interval=15, metric_warmup=2): task = image_type(app) (execution_ready, failure_msg) = task.does_task_pass_pre_execution_checks(app) if not execution_ready: app['logger'].critical(failure_msg) return task.before_container_hook(app) image = image_version(app) app['logger'].info("Creating Docker container from image {}".format(image)) biobox = create_container(app) id_ = biobox['Id'] app['logger'].info("Starting Docker container {}".format(id_)) docker.client(docker_timeout).start(id_) metrics = cgroup.collect_runtime_metrics(id_, metric_interval, metric_warmup) app['logger'].info("Docker container {} finished".format(id_)) fs.create_runtime_metric_file(app, metrics) copy_output_files(app)
def create_container(image, config, directories, task = "default", docker_args = {}): """ Returns a new biobox Docker container created from the given image name. The container is not started. Networking will be enabled by default until an issue with Docker stats collection is resolved - docker/docker-py#1195. Keyword arguments: image -- name of a docker image, may optionally include sha256 config -- biobox configuration as specified by the biobox.yaml format directories -- dictionary of host directories locations with the keys: output -- REQUIRED location of output destination directory metadata -- OPTIONAL location of metadata destination directory task -- biobox container task to execute, defaults to "default". docker_args -- Optional cgroup data passed to the docker daemon. See the docker documentation for a list of available values """ volumes = prepare_volumes(config, directories.get('output'), directories.get('metadata')) docker_args['volumes'] = list(map(vol.get_host_path, volumes)) docker_args['network_disabled'] = False host_config = {'binds' : volumes} if 'mem_limit' in docker_args: host_config['mem_limit'] = docker_args['mem_limit'] del docker_args['mem_limit'] docker_args['host_config'] = util.client().create_host_config(**host_config) return util.client().create_container(image, task, **docker_args)
def test_run_container(): app = app_helper.setup_app_state('quast', 'execute') id_ = run.create_container(app)['Id'] docker.client().start(id_) docker.client().wait(id_) nose.assert_equal(container.did_exit_succcessfully(id_), True) image_helper.clean_up_container(id_)
def create_container(image, config, directories, task="default", docker_args={}): """ Returns a new biobox Docker container created from the given image name. The container is not started. Networking will be enabled by default until an issue with Docker stats collection is resolved - docker/docker-py#1195. Keyword arguments: image -- name of a docker image, may optionally include sha256 config -- biobox configuration as specified by the biobox.yaml format directories -- dictionary of host directories locations with the keys: output -- REQUIRED location of output destination directory metadata -- OPTIONAL location of metadata destination directory task -- biobox container task to execute, defaults to "default". docker_args -- Optional cgroup data passed to the docker daemon. See the docker documentation for a list of available values """ volumes = prepare_volumes(config, directories.get('output'), directories.get('metadata')) docker_args['volumes'] = list(map(vol.get_host_path, volumes)) docker_args['network_disabled'] = False host_config = {'binds': volumes} if 'mem_limit' in docker_args: host_config['mem_limit'] = docker_args['mem_limit'] del docker_args['mem_limit'] docker_args['host_config'] = util.client().create_host_config( **host_config) return util.client().create_container(image, task, **docker_args)
def create_tty(image, tty, volumes=[]): command = "" return docker.client().create_container( image, command, stdin_open=True, tty=tty, entrypoint='/bin/bash', volumes=list(map(vol.get_host_path, volumes)), host_config=docker.client().create_host_config(binds=volumes))
def create_tty(image, tty, volumes = []): command = "" return docker.client().create_container( image, command, stdin_open = True, tty = tty, entrypoint = '/bin/bash', volumes = list(map(vol.get_host_path, volumes)), host_config = docker.client().create_host_config(binds=volumes))
def test_executing_container(): out_dir = tempfile.mkdtemp() cnt = exe.create_container(IMAGE, biobox_cfg(hlp.short_read_fastq()), {"output": out_dir}, "default", {"detach": False}) id_ = cnt['Id'] util.client().start(id_) util.client().wait(id_) assert funcy.get_in(util.client().inspect_container(id_), ['State', 'ExitCode']) == 0 assert os.path.isfile(os.path.join(out_dir, 'contigs.fa')) hlp.clean_up_container(id_)
def test_executing_container(): out_dir = tempfile.mkdtemp() cnt = exe.create_container( IMAGE, biobox_cfg(hlp.short_read_fastq()), {"output" : out_dir}, "default", {"detach" : False}) id_ = cnt['Id'] util.client().start(id_) util.client().wait(id_) assert funcy.get_in(util.client().inspect_container(id_), ['State', 'ExitCode']) == 0 assert os.path.isfile(os.path.join(out_dir, 'contigs.fa')) hlp.clean_up_container(id_)
def list_of_local_images(): """ Returns a set containing all local docker image tags and digests. """ images = util.client().images() tags_and_digests = list(map(get_image_tags, images)) + list(map(get_image_digests, images)) return set(reduce(lambda acc, x: acc + x, tags_and_digests, []))
def test_collect_metric_with_dead_container(): id_ = hlp.dummy_container(2) stream = util.client().stats(id_, decode = True, stream = True) time.sleep(1) cgroup.collect_metric(stream) time.sleep(1) assert cgroup.collect_metric(stream) == None hlp.clean_up_container(id_)
def test_collect_metric_with_dead_container(): id_ = hlp.dummy_container(2) stream = util.client().stats(id_, decode=True, stream=True) time.sleep(1) cgroup.collect_metric(stream) time.sleep(1) assert cgroup.collect_metric(stream) == None hlp.clean_up_container(id_)
def get_image(name): """ Fetches the Docker image if it is not present locally. """ if not is_image_available_locally(name): output = util.client().pull(name) if "error" in output: raise biobox.exception.NoImageFound(name) return True
def dummy_container(arg, wait = True, docker_args = {}): client = util.client() container = client.create_container( image = 'alpine:3.3', command = '/bin/sleep {}'.format(str(arg)), **{'network_disabled' : False}) id_ = container['Id'] client.start(id_) if wait: client.wait(id_) return id_
def collect_runtime_metrics(container_id, interval=15, warmup=1): """ Collects cgroup runtime metrics from the specified container at the given per-second intervals. """ stream = util.client().stats(container_id, decode=True, stream=True) time.sleep(warmup) stats = [next(stream)] while ctn.is_running(container_id): time.sleep(1) entry = collect_metric(stream) # Remove this when docker/docker-py#1195 is fixed if not entry: pass else: # Save the cgroup entry if it is greater than given time interval if time_diff_in_seconds(stats[-1]['read'], entry['read']) > interval: stats.append(entry) return stats
def metadata_lookup(path, container_id): """ Look up metadata about the docker container for the given path """ return funcy.get_in(util.client().inspect_container(container_id), path)
def test_collect_metric_with_running_container(): id_ = hlp.dummy_container(10, False) stream = util.client().stats(id_, decode = True, stream = True) assert 'read' in cgroup.collect_metric(stream) util.client().kill(id_) hlp.clean_up_container(id_)
def run(container): docker.client().start(container) docker.client().wait(container)
def login(container): docker.client().start(container) pty.PseudoTerminal(docker.client(), container).start() docker.client().stop(container)
def remove(container): """ Removal of a container NOTE: This method is not tested due to circle ci limitations """ docker.client().remove_container(container, v=True)
def test_is_container_running(): id_ = hlp.dummy_container(10, False) time.sleep(0.1) assert ctn.is_running(id_) util.client().kill(id_) hlp.clean_up_container(id_)
def test_collect_metric_with_running_container(): id_ = hlp.dummy_container(10, False) stream = util.client().stats(id_, decode=True, stream=True) assert 'read' in cgroup.collect_metric(stream) util.client().kill(id_) hlp.clean_up_container(id_)
def clean_up_container(id_): if not "CIRCLECI" in os.environ: util.client().remove_container(id_)