async def query_started(self) -> None: channel = Docker().events.subscribe() while channel: event = await channel.get() if event['Type'] == 'container' and event['status'] == 'start': container = Docker().containers.container(event['id']) await container.show() await self.handle(container)
def __init__( self, rabbitmq_port: int, ) -> None: self._rabbitmq_port = rabbitmq_port self._running = False self._docker = Docker()
async def list_containers(): """ Retrieve the list of local images being built by repo2docker. Images are built in a Docker container. """ async with Docker() as docker: r2d_containers = await docker.containers.list( filters=json.dumps({"label": ["repo2docker.ref"]})) containers = [{ "repo": container["Labels"]["repo2docker.repo"], "ref": container["Labels"]["repo2docker.ref"], "image_name": container["Labels"]["repo2docker.build"], "display_name": container["Labels"]["tljh_repo2docker.display_name"], "mem_limit": container["Labels"]["tljh_repo2docker.mem_limit"], "cpu_limit": container["Labels"]["tljh_repo2docker.cpu_limit"], "status": "building", } for container in r2d_containers if "repo2docker.build" in container["Labels"]] return containers
async def service_start_(self, service_id): service_id = str(service_id) docker = Docker() try: service = self.application['services'][service_id] if not service.get('image_pulled'): logger.debug('Image pull started') auth = service['image'].get('auth') repository = service['image']['repository'] + \ service['image']['name'] if \ service['image'].get('repository') else \ service['image']['name'] if auth: logger.debug('Pulling {} with auth {}'.format( repository, auth)) image = await docker.images.pull(repository, auth=auth) else: logger.debug('Pulling {}'.format(repository)) image = await docker.images.pull(repository) logger.debug('Image pull ended') service['image_pulled'] = True logger.debug('Start service: {}'.format(service['name'])) container = await docker.containers.create_or_replace( name=service['name'], config=service['container']) self.application['services'][service_id][ 'container_id'] = container._id logger.debug('service started ({}): container {}'.format( service['name'], container._id)) await container.start() return True except IndexError as e: logger.exception(e) # See error locally raise # Return back RPC error finally: await docker.close()
async def _pull_image(self): docker_image = f"{config.DOCKER_REGISTRY}/{self.task.image['name']}:{self.task.image['tag']}" log.debug( "PULLING IMAGE %s as %s with pwd %s", docker_image, config.DOCKER_USER, config.DOCKER_PASSWORD, ) async with Docker() as docker_client: await self._post_messages( LogType.LOG, f"[sidecar]Pulling {self.task.image['name']}:{self.task.image['tag']}...", ) await docker_client.images.pull( docker_image, auth={ "username": config.DOCKER_USER, "password": config.DOCKER_PASSWORD, }, ) # get integration version image_cfg = await docker_client.images.inspect(docker_image) # NOTE: old services did not have that label if "io.simcore.integration-version" in image_cfg["Config"][ "Labels"]: self.integration_version = version.parse( json.loads(image_cfg["Config"]["Labels"] ["io.simcore.integration-version"]) ["integration-version"])
async def docker(loop): client = Docker() try: yield client finally: await client.close()
async def services_log(self): if not self.application.get('services'): logger.debug('No services to log') return docker = Docker() try: for service_id, service in self.application['services'].items(): if not service.get('container_id'): continue # Container not yet started container = await docker.containers.get(service['container_id'] ) logs = await container.log(stdout=True, stderr=True, details=True, since=self.last_logs) await self.device_log('\n'.join(logs), service_id=service_id) except Exception as e: logger.exception(e) finally: self.last_logs = datetime.now().timestamp() await docker.close() await asyncio.sleep(LOG_INTERVAL) await self.scheduler.spawn(self.services_log())
async def set_limits(self): """ Set the user environment limits if they are defined in the image """ imagename = self.user_options.get("image") async with Docker() as docker: image = await docker.images.inspect(imagename) mem_limit = image["ContainerConfig"]["Labels"].get( "tljh_repo2docker.mem_limit", None) cpu_limit = image["ContainerConfig"]["Labels"].get( "tljh_repo2docker.cpu_limit", None) # override the spawner limits if defined in the image if mem_limit: self.mem_limit = mem_limit if cpu_limit: self.cpu_limit = float(cpu_limit) if self.cpu_limit: self.extra_host_config.update({ "cpu_period": CPU_PERIOD, "cpu_quota": int(float(CPU_PERIOD) * self.cpu_limit), })
def test_execute(): client = Docker.local_client() conf = { 'name': 'aio-1', 'image': 'gliderlabs/alpine:3.1', 'command': ['sleep', '60'] } with pytest.raises(NotFound): yield from client.containers.get('aio-1') container_id = yield from client.containers.create(**conf) assert container_id started = yield from client.containers.start(container_id) assert started # the real test starts now exec_id = yield from client.executors.create(container_id, cmd=[ 'echo', 'FOOBAR' ]) yield from client.executors.start(exec_id) results = yield from client.executors.inspect(exec_id) assert results['exit_code'] == 0, 'should been 0' # end of the test stoped = yield from client.containers.stop(container_id) assert stoped, 'Should be stoped' deleted = yield from client.containers.delete(container_id) assert deleted, 'Should be deleted'
def test_delete(): client = Docker.local_client() images = yield from client.images.items() for image in images: if '<none>:<none>' in image['repo_tags']: deleted = yield from client.images.delete(image['id'], force=True) assert deleted, 'cannot remove image'
def events(bot): docker = Docker() events = docker.events events.saferun() stream = events.listen() while True: el = yield from stream.get() yield from bot.post("#cron", "`{}`".format(str(el)))
async def delete(self): data = self.get_json_body() name = data["name"] async with Docker() as docker: try: await docker.images.delete(name) except DockerError as e: raise web.HTTPError(e.status, e.message) self.set_status(200) self.finish(json.dumps({"status": "ok"}))
def test_conflict(): client = Docker.local_client() conf = { 'name': 'aio-1', 'image': 'gliderlabs/alpine:3.1', 'command': ['sleep', '60'] } with pytest.raises(NotFound): yield from client.containers.get('aio-1') container_id = yield from client.containers.create(**conf) assert container_id data_1 = yield from client.containers.get('aio-1') data_2 = yield from client.containers.get(container_id) assert data_1 == data_2, 'Fetch by name or id must returns same data' assert data_1['id'] == container_id # Cannot create container with the same name with pytest.raises(ConflictError): yield from client.containers.create(**conf) c1 = containers = yield from client.containers.items(status='all') assert {'id': container_id} in containers, 'Must match by id' c2 = containers = yield from client.containers.items(status='running') assert {'id': container_id} not in containers, 'Must not be running' c3 = containers = yield from client.containers.items(status='exited') assert {'id': container_id} in containers, 'Must be exited' started = yield from client.containers.start(container_id) assert started started = yield from client.containers.start(container_id) assert not started # Cannot delete running container with pytest.raises(ConflictError): yield from client.containers.delete(container_id) stoped = yield from client.containers.stop(container_id) assert stoped, 'Should be stoped' stoped = yield from client.containers.stop(container_id) assert not stoped, 'Should be already stoped' deleted = yield from client.containers.delete(container_id) assert deleted, 'Should be deleted' deleted = yield from client.containers.delete(container_id) assert not deleted, 'Should be already deleted'
async def test_delete_environment(app, minimal_repo, image_name): name, ref = image_name.split(":") await add_environment(app, repo=minimal_repo, name=name, ref=ref) await wait_for_image(image_name=image_name) r = await remove_environment(app, image_name=image_name) assert r.status_code == 200 # make sure the image does not exist anymore docker = Docker() with pytest.raises(DockerError): await docker.images.inspect(image_name) await docker.close()
async def wait_for_image(*, image_name): """wait until an image is built""" count, retries = 0, 60 * 10 image = None async with Docker() as docker: while count < retries: await asyncio.sleep(1) try: image = await docker.images.inspect(image_name) except DockerError: count += 1 continue else: break return image
async def application_start_(self): docker = Docker() try: for service_id in self.application['services']: await self.service_start_(service_id) return True except (DockerError, ValueError) as e: await self.device_log('{}'.format(e)) except Exception as e: logger.exception(e) await self.device_log('{}'.format(e)) finally: await docker.close()
async def get(self, name): self.set_header("Content-Type", "text/event-stream") self.set_header("Cache-Control", "no-cache") async with Docker() as docker: containers = await docker.containers.list( filters=json.dumps({"label": [f"repo2docker.build={name}"]})) if not containers: raise web.HTTPError(404, f"No logs for image: {name}") async for line in containers[0].log(stdout=True, stderr=True, follow=True): await self._emit({"phase": "log", "message": line}) await self._emit({"phase": "built", "message": "built"})
def test_pull(): client = Docker.local_client() ref = 'gliderlabs/alpine:2.6' pulled = yield from client.dockerhub.pull(ref) assert pulled, 'Should download gliderlabs/alpine:2.6' # is it present locally? image = yield from client.images.inspect(ref) assert image # start a new container with this image container_id = yield from client.containers.create(**{ 'name': 'aio-2', 'image': 'gliderlabs/alpine:2.6', 'command': ['sleep', '60'] }) assert container_id started = yield from client.containers.start(container_id) assert started pulled = yield from client.dockerhub.pull(ref) assert pulled, 'Should still download gliderlabs/alpine:2.6' with pytest.raises(ConflictError): destroyed = yield from client.images.delete(ref) assert not destroyed, 'Should not allow destroy' stoped = yield from client.containers.stop(container_id) assert stoped, 'Should stop container' removed = yield from client.containers.remove(container_id) assert removed, 'Should remove container' images = yield from client.images.items() assert {'repo_tag': ref} in images, 'Should be present' destroyed = yield from client.images.delete(ref, force=True) assert destroyed, 'Should be destroyed' images = yield from client.images.items() assert {'repo_tag': ref} not in images, 'Should be absent'
async def service_stop_(self, service_id): service_id = str(service_id) docker = Docker() try: service = self.application['services'][service_id] logger.debug('Stop service: {}'.format(service['name'])) if not self.application['services'][service_id].get('container_id'): logger.warning( 'service stopped ({}): no container'.format(service['name'])) return 'no container' container = await docker.containers.get( self.application['services'][service_id]['container_id']) await container.stop() return True except IndexError as e: logger.exception(e) # See error locally raise # Return back RPC error finally: await docker.close()
async def service_start_(self, service_id): service_id = str(service_id) service = self.application['services'][service_id] docker = Docker() try: await self.service_pull_(service_id) logger.debug('Start service: {}'.format(service['name'])) container = await docker.containers.create_or_replace( name=service['name'], config=service['container']) self.application['services'][service_id][ 'container_id'] = container._id logger.debug('service started ({}): container {}'.format( service['name'], container._id)) await container.start() return True except IndexError as e: logger.exception(e) # See error locally raise # Return back RPC error finally: await docker.close()
async def list_images(): """ Retrieve local images built by repo2docker """ async with Docker() as docker: r2d_images = await docker.images.list(filters=json.dumps({ "dangling": ["false"], "label": ["repo2docker.ref"] })) images = [{ "repo": image["Labels"]["repo2docker.repo"], "ref": image["Labels"]["repo2docker.ref"], "image_name": image["Labels"]["tljh_repo2docker.image_name"], "display_name": image["Labels"]["tljh_repo2docker.display_name"], "mem_limit": image["Labels"]["tljh_repo2docker.mem_limit"], "cpu_limit": image["Labels"]["tljh_repo2docker.cpu_limit"], "status": "built", } for image in r2d_images if "tljh_repo2docker.image_name" in image["Labels"]] return images
async def service_status_(self, service_id): service_id = str(service_id) # JSON keys are always strings docker = Docker() begin = self.loop.time() try: logger.debug('Service status for {}'.format( self.application['services'][service_id]['name'])) if not self.application['services'][service_id].get('container_id'): return 'starting' container = await docker.containers.get( self.application['services'][service_id]['container_id']) data = await container.show() logger.debug('Service status took {}'.format( self.loop.time() - begin)) return data['State']['Status'] except IndexError: raise RPCError('Service not found') except Exception as e: logger.exception(e) finally: await docker.close()
async def service_pull_(self, service_id): service_id = str(service_id) docker = Docker() try: service = self.application['services'][service_id] if not service.get('image_pulled'): logger.debug('Image pull started') auth = service['image'].get('auth') name = service['image'].get('name') if auth: logger.debug('Pulling {} with auth {}'.format( name, auth)) image = await docker.images.pull(name, auth=auth) else: logger.debug('Pulling {}'.format(name)) image = await docker.images.pull(name) logger.debug('Image pull ended') service['image_pulled'] = True return True finally: await docker.close()
async def _pull_image( self, ) -> Tuple[SimcoreServiceSettingsLabel, Version]: docker_image = f"{config.DOCKER_REGISTRY}/{self.task.image['name']}:{self.task.image['tag']}" async with Docker() as docker_client: await self._post_messages( LogType.LOG, f"[sidecar]Pulling {self.task.image['name']}:{self.task.image['tag']}...", ) await docker_client.images.pull( docker_image, auth={ "username": config.DOCKER_USER, "password": config.DOCKER_PASSWORD, }, ) # get integration version image_cfg = await docker_client.images.inspect(docker_image) # NOTE: old services did not have that label integration_version = version.parse("0.0.0") if "io.simcore.integration-version" in image_cfg["Config"][ "Labels"]: integration_version = version.parse( json.loads(image_cfg["Config"]["Labels"] ["io.simcore.integration-version"]) ["integration-version"]) # get service settings service_settings_labels = SimcoreServiceSettingsLabel.parse_raw( image_cfg["Config"]["Labels"].get("simcore.service.settings", "[]")) log.debug( "found following service settings: %s", pformat(service_settings_labels), ) await self._post_messages( LogType.LOG, f"[sidecar]Pulled {self.task.image['name']}:{self.task.image['tag']}", ) return (service_settings_labels, integration_version)
def test_pull(): client = Docker.local_client() images = yield from client.images.items()
async def docker(): client = Docker() await yield_(client) await client.close()
def __init__(self): super(ContainerService, self).__init__() self._containers = weakref.WeakValueDictionary() self._docker = Docker() self._database = Service.resolve("moxie.cores.database.DatabaseService")
class ContainerService(Service): """ This provides an interface to run container jobs somewhere off in the ether somewhere. """ identifier = "moxie.cores.container.ContainerService" def __init__(self): super(ContainerService, self).__init__() self._containers = weakref.WeakValueDictionary() self._docker = Docker() self._database = Service.resolve("moxie.cores.database.DatabaseService") def _check_container(self, name): job = yield from self._database.job.get(name) # Check if active if job is None: raise ValueError("Sorry, that's not something you can kill") @asyncio.coroutine def events(self, name): return (yield from self._docker.events) @asyncio.coroutine def pull(self, name): return (yield from self._docker.pull(name)) def _purge_cache(self, name): if name in self._containers: self._containers.pop(name) @asyncio.coroutine def delete(self, name): yield from self._check_container(name) try: obj = yield from self.get(name) except ValueError: return self._purge_cache(name) yield from obj.delete() @asyncio.coroutine def create(self, config, **kwargs): return (yield from self._docker.containers.create(config, **kwargs)) @asyncio.coroutine def start(self, name, config, **kwargs): yield from self._check_container(name) obj = yield from self.get(name) return (yield from obj.start(config, **kwargs)) @asyncio.coroutine def kill(self, name, *args, **kwargs): yield from self._check_container(name) obj = yield from self.get(name) return (yield from obj.kill(*args, **kwargs)) @asyncio.coroutine def get(self, name): yield from self._check_container(name) if name in self._containers: obj = self._containers[name] try: yield from obj.show() # update cache return obj except ValueError: self._purge_cache(name) container = yield from self._docker.containers.get(name) self._containers[name] = container return container @asyncio.coroutine def list(self, **kwargs): containers = yield from self._docker.containers.list(**kwargs) return containers @asyncio.coroutine def __call__(self): pass
async def _run_container(self): start_time = time.perf_counter() docker_image = f"{config.DOCKER_REGISTRY}/{self.task.image['name']}:{self.task.image['tag']}" container_config = await self._create_container_config(docker_image) # volume paths for car container (w/o prefix) result = "FAILURE" log_processor_task = None try: async with Docker() as docker_client: await self._post_messages( LogType.LOG, f"[sidecar]Running {self.task.image['name']}:{self.task.image['tag']}...", ) container = await docker_client.containers.create( config=container_config) log_processor_task = await self._start_monitoring_container( container) # start the container await container.start() # indicate container is started await self.rabbit_mq.post_instrumentation_message({ "metrics": "service_started", "user_id": self.user_id, "project_id": self.task.project_id, "service_uuid": self.task.node_id, "service_type": "COMPUTATIONAL", "service_key": self.task.image["name"], "service_tag": self.task.image["tag"], }) # wait until the container finished, either success or fail or timeout container_data = await container.show() TIME_TO_NEXT_PERDIODIC_CHECK_SECS = 2 while container_data["State"]["Running"]: await asyncio.sleep(TIME_TO_NEXT_PERDIODIC_CHECK_SECS) # reload container data container_data = await container.show() if ((time.perf_counter() - start_time) > config.SERVICES_TIMEOUT_SECONDS and config.SERVICES_TIMEOUT_SECONDS > 0): log.error( "Running container timed-out after %ss and will be stopped now\nlogs: %s", config.SERVICES_TIMEOUT_SECONDS, container.log(stdout=True, stderr=True), ) await container.stop() break # reload container data to check the error code with latest info container_data = await container.show() if container_data["State"]["ExitCode"] > 0: logs = await container.log(stdout=True, stderr=True, tail=10) exc = exceptions.SidecarException( f"{docker_image} completed with error code {container_data['State']['ExitCode']}:\n {container_data['State']['Error']}\n:Last logs:\n{logs}" ) # clean up the container await container.delete(force=True) raise exc # clean up the container await container.delete(force=True) # ensure progress 1.0 is sent await self._post_messages(LogType.PROGRESS, "1.0") result = "SUCCESS" log.info("%s completed with successfully!", docker_image) except DockerContainerError: log.exception( "Error while running %s with parameters %s", docker_image, container_config, ) raise except DockerError: log.exception( "Unknown error while trying to run %s with parameters %s", docker_image, container_config, ) raise except asyncio.CancelledError: log.warning("Container run was cancelled") raise finally: stop_time = time.perf_counter() log.info("Running %s took %sseconds", docker_image, stop_time - start_time) # stop monitoring logs now if log_processor_task: log_processor_task.cancel() await log_processor_task # instrumentation await self.rabbit_mq.post_instrumentation_message({ "metrics": "service_stopped", "user_id": self.user_id, "project_id": self.task.project_id, "service_uuid": self.task.node_id, "service_type": "COMPUTATIONAL", "service_key": self.task.image["name"], "service_tag": self.task.image["tag"], "result": result, })
def test_search(): # TODO mock result client = Docker.local_client() info = yield from client.dockerhub.search('ubuntu') assert info[0].name == 'ubuntu'
class ContainerService(Service): """ This provides an interface to run container jobs somewhere off in the ether somewhere. """ identifier = "moxie.cores.container.ContainerService" def __init__(self): super(ContainerService, self).__init__() self._containers = weakref.WeakValueDictionary() self._docker = Docker() self._database = Service.resolve( "moxie.cores.database.DatabaseService") def _check_container(self, name): job = yield from self._database.job.get(name) # Check if active if job is None: raise ValueError("Sorry, that's not something you can kill") @asyncio.coroutine def events(self, name): return (yield from self._docker.events) @asyncio.coroutine def pull(self, name): return (yield from self._docker.pull(name)) def _purge_cache(self, name): if name in self._containers: self._containers.pop(name) @asyncio.coroutine def delete(self, name): yield from self._check_container(name) try: obj = yield from self.get(name) except ValueError: return self._purge_cache(name) yield from obj.delete() @asyncio.coroutine def create(self, config, **kwargs): return (yield from self._docker.containers.create(config, **kwargs)) @asyncio.coroutine def start(self, name, config, **kwargs): yield from self._check_container(name) obj = yield from self.get(name) return (yield from obj.start(config, **kwargs)) @asyncio.coroutine def kill(self, name, *args, **kwargs): yield from self._check_container(name) obj = yield from self.get(name) return (yield from obj.kill(*args, **kwargs)) @asyncio.coroutine def get(self, name): yield from self._check_container(name) if name in self._containers: obj = self._containers[name] try: yield from obj.show() # update cache return obj except ValueError: self._purge_cache(name) container = yield from self._docker.containers.get(name) self._containers[name] = container return container @asyncio.coroutine def list(self, **kwargs): containers = yield from self._docker.containers.list(**kwargs) return containers @asyncio.coroutine def __call__(self): pass
def test_version(): client = Docker.local_client() version = yield from client.version() assert client.api.version <= version['api_version'], 'api versions should be equals'
def test_ping(): client = Docker.local_client() assert (yield from client.ping())
def test_info(): client = Docker.local_client() info = yield from client.info() assert 'containers' in info, 'must have containers key'
async def docker(loop): client = Docker() yield client await client.close()
async def list_images(): client = Docker() client.images.list()
async def start(self) -> None: self.docker = Docker()
def __init__(self): super(ContainerService, self).__init__() self._containers = weakref.WeakValueDictionary() self._docker = Docker() self._database = Service.resolve( "moxie.cores.database.DatabaseService")
async def build_image(repo, ref, name="", memory=None, cpu=None): """ Build an image given a repo, ref and limits """ ref = ref or "master" if len(ref) >= 40: ref = ref[:7] # default to the repo name if no name specified # and sanitize the name of the docker image name = name or urlparse(repo).path.strip("/") name = name.lower().replace("/", "-") image_name = f"{name}:{ref}" # memory is specified in GB memory = f"{memory}G" if memory else "" cpu = cpu or "" # add extra labels to set additional image properties labels = [ f"LABEL tljh_repo2docker.display_name={name}", f"LABEL tljh_repo2docker.image_name={image_name}", f"LABEL tljh_repo2docker.mem_limit={memory}", f"LABEL tljh_repo2docker.cpu_limit={cpu}", ] cmd = [ "jupyter-repo2docker", "--ref", ref, "--user-name", "jovyan", "--user-id", "1100", "--no-run", "--image-name", image_name, "--appendix", "\n".join(labels), repo, ] async with Docker() as docker: await docker.containers.run( config={ "Cmd": cmd, "Image": "jupyter/repo2docker:master", "Labels": { "repo2docker.repo": repo, "repo2docker.ref": ref, "repo2docker.build": image_name, "tljh_repo2docker.display_name": name, "tljh_repo2docker.mem_limit": memory, "tljh_repo2docker.cpu_limit": cpu, }, "Volumes": { "/var/run/docker.sock": {"bind": "/var/run/docker.sock", "mode": "rw",} }, "HostConfig": {"Binds": ["/var/run/docker.sock:/var/run/docker.sock"],}, "Tty": False, "AttachStdout": False, "AttachStderr": False, "OpenStdin": False, } )
async def remove_docker_image(image_name): async with Docker() as docker: try: await docker.images.delete(image_name, force=True) except DockerError: pass
async def run(self, command: List[str]) -> TaskOutputData: await self._publish_sidecar_state(RunningState.STARTED) await self._publish_sidecar_log( f"Starting task for {self.service_key}:{self.service_version} on {socket.gethostname()}..." ) settings = Settings.create_from_envs() run_id = f"{uuid4()}" async with Docker() as docker_client, TaskSharedVolumes( Path(f"{settings.SIDECAR_COMP_SERVICES_SHARED_FOLDER}/{run_id}" )) as task_volumes: # PRE-PROCESSING await pull_image( docker_client, self.docker_auth, self.service_key, self.service_version, self._publish_sidecar_log, ) integration_version = await get_integration_version( docker_client, self.docker_auth, self.service_key, self.service_version) computational_shared_data_mount_point = ( await get_computational_shared_data_mount_point(docker_client)) config = await create_container_config( docker_registry=self.docker_auth.server_address, service_key=self.service_key, service_version=self.service_version, command=command, comp_volume_mount_point= f"{computational_shared_data_mount_point}/{run_id}", boot_mode=self.boot_mode, task_max_resources=self.task_max_resources, ) await self._write_input_data(task_volumes, integration_version) # PROCESSING async with managed_container(docker_client, config) as container: async with managed_monitor_container_log_task( container=container, service_key=self.service_key, service_version=self.service_version, progress_pub=self.task_publishers.progress, logs_pub=self.task_publishers.logs, integration_version=integration_version, task_volumes=task_volumes, log_file_url=self.log_file_url, log_publishing_cb=self._publish_sidecar_log, ): await container.start() await self._publish_sidecar_log( f"Container started as '{container.id}' on {socket.gethostname()}..." ) # wait until the container finished, either success or fail or timeout while (container_data := await container.show())["State"]["Running"]: await asyncio.sleep(CONTAINER_WAIT_TIME_SECS) if container_data["State"]["ExitCode"] > os.EX_OK: await self._publish_sidecar_state( RunningState.FAILED, msg= f"error while running container '{container.id}' for '{self.service_key}:{self.service_version}'", ) raise ServiceRunError( service_key=self.service_key, service_version=self.service_version, container_id=container.id, exit_code=container_data["State"]["ExitCode"], service_logs=await container.log(stdout=True, stderr=True, tail=20), ) await self._publish_sidecar_log( "Container ran successfully.") # POST-PROCESSING results = await self._retrieve_output_data(task_volumes, integration_version) await self._publish_sidecar_log("Task completed successfully.") return results