def _create_file_result(self, *, interface, reader): output_file = Path(safe_join("/output/", interface.relative_path)) try: with TemporaryDirectory() as tmpdir: temp_file = Path(safe_join(tmpdir, "output.json")) get_file(container=reader, src=output_file, dest=temp_file) with open(temp_file, "rb") as file: result = json.loads( file.read().decode("utf-8"), parse_constant=lambda x: None, # Removes -inf, inf and NaN ) except NotFound: raise ComponentException(f"File {output_file} was not produced") except JSONDecodeError: raise ComponentException( f"The file produced at {output_file} is not valid json") try: civ = interface.create_instance(value=result) except ValidationError: raise ComponentException( f"The file produced at {output_file} is not valid") return civ
def _execute_container(self) -> None: with stop( self._client.containers.run( image=self._exec_image_sha256, volumes={ self._input_volume_name: { "bind": "/input/", "mode": "ro" }, self._output_volume_name: { "bind": "/output/", "mode": "rw", }, }, name=self._execution_container_name, detach=True, labels=self._labels, environment={ "NVIDIA_VISIBLE_DEVICES": settings.COMPONENTS_NVIDIA_VISIBLE_DEVICES }, **self._run_kwargs, )) as c: container_state = c.wait() exit_code = int(container_state["StatusCode"]) if exit_code == 137: raise ComponentException( "The container was killed as it exceeded the memory limit " f"of {self._run_kwargs['mem_limit']}.") elif exit_code != 0: raise ComponentException(user_error(self.stderr))
def _create_images_result(self, *, interface): base_dir = Path( safe_join(self._output_directory, interface.relative_path)) output_files = [f for f in base_dir.glob("*") if f.is_file()] if not output_files: raise ComponentException(f"{interface.relative_path} is empty") importer_result = import_images( input_directory=base_dir, builders=[image_builder_mhd, image_builder_tiff], recurse_subdirectories=False, ) if len(importer_result.new_images) == 0: raise ComponentException( f"No images imported from {interface.relative_path}") elif len(importer_result.new_images) > 1: raise ComponentException( f"Only 1 image should be produced in {interface.relative_path}, " f"we found {len(importer_result.new_images)}") try: civ = interface.create_instance( image=next(iter(importer_result.new_images))) except ValidationError: raise ComponentException( f"The image produced in {interface.relative_path} is not valid" ) return civ
def _create_json_result(self, *, interface): output_file = Path( safe_join(self._output_directory, interface.relative_path)) if (output_file.is_symlink() or not output_file.is_file() or not output_file.exists()): raise ComponentException( f"File {interface.relative_path} was not produced") try: with open(output_file, "rb") as f: result = json.loads( f.read().decode("utf-8"), parse_constant=lambda x: None, # Removes -inf, inf and NaN ) except JSONDecodeError: raise ComponentException( f"The file produced at {interface.relative_path} is not valid json" ) try: civ = interface.create_instance(value=result) except ValidationError: raise ComponentException( f"The file produced at {interface.relative_path} is not valid") return civ
def _handle_container_exit(self, *, container_exit_codes): if container_exit_codes.get(self._main_container_name) == 0: # Job's a good un return elif container_exit_codes.get(self._main_container_name) == 137: raise ComponentException( "The container was killed as it exceeded the memory limit " f"of {self._memory_limit}g.") elif container_exit_codes.get(self._timeout_container_name) == 0: raise ComponentException("Time limit exceeded") else: raise ComponentException(user_error(self.stderr))
def _create_file_result(self, *, interface, reader): output_file = Path(safe_join("/output/", interface.relative_path)) try: with TemporaryDirectory() as tmpdir: temp_file = Path(safe_join(tmpdir, interface.relative_path)) get_file(container=reader, src=output_file, dest=temp_file) with open(temp_file, "rb") as f: civ = interface.create_instance(fileobj=f) except NotFound: raise ComponentException(f"File {output_file} was not produced") except ValidationError: raise ComponentException( f"The file produced at {output_file} is not valid") return civ
def _create_file_result(self, *, interface): output_file = Path( safe_join(self._output_directory, interface.relative_path)) if (output_file.is_symlink() or not output_file.is_file() or not output_file.exists()): raise ComponentException( f"File {interface.relative_path} was not produced") try: with open(output_file, "rb") as f: civ = interface.create_instance(fileobj=f) except ValidationError: raise ComponentException( f"The file produced at {interface.relative_path} is not valid") return civ
def _create_images_result(self, *, interface, reader): base_dir = Path(safe_join("/output/", interface.relative_path)) found_files = reader.exec_run(f"find {base_dir} -type f") if found_files.exit_code != 0: raise ComponentException(f"Error listing {base_dir}") output_files = [ base_dir / Path(f) for f in found_files.output.decode().splitlines() ] if not output_files: raise ComponentException(f"{base_dir} is empty") with TemporaryDirectory() as tmpdir: for file in output_files: temp_file = Path(safe_join(tmpdir, file.relative_to(base_dir))) temp_file.parent.mkdir(parents=True, exist_ok=True) get_file(container=reader, src=file, dest=temp_file) importer_result = import_images( input_directory=tmpdir, builders=[image_builder_mhd, image_builder_tiff], ) if len(importer_result.new_images) == 0: raise ComponentException(f"No images imported from {base_dir}") elif len(importer_result.new_images) > 1: raise ComponentException( f"Only 1 image should be produced in {base_dir}, " f"we found {len(importer_result.new_images)}") try: civ = interface.create_instance( image=next(iter(importer_result.new_images))) except ValidationError: raise ComponentException( f"The image produced in {base_dir} is not valid") return civ
def start(self) -> None: """ Starts the service for this session, ensuring that the ``workstation_image`` is ready to be used and that ``WORKSTATIONS_MAXIMUM_SESSIONS`` has not been reached in this region. Raises ------ ComponentException If the service cannot be started. """ try: if not self.workstation_image.ready: raise ComponentException("Workstation image was not ready") if ( Session.objects.all() .filter( status__in=[Session.RUNNING, Session.STARTED], region=self.region, ) .count() >= settings.WORKSTATIONS_MAXIMUM_SESSIONS ): raise ComponentException("Too many sessions are running") self.service.start( http_port=self.workstation_image.http_port, websocket_port=self.workstation_image.websocket_port, hostname=self.hostname, environment=self.environment, ) self.update_status(status=self.STARTED) except Exception: self.update_status(status=self.FAILED) raise
def _copy_input_files(self, *, input_civs, input_prefixes): for civ in input_civs: prefix = self._input_directory if str(civ.pk) in input_prefixes: prefix = safe_join(prefix, input_prefixes[str(civ.pk)]) dest = Path(safe_join(prefix, civ.relative_path)) # We know that the dest is within the prefix as # safe_join is used, so ok to create the parents here dest.parent.mkdir(exist_ok=True, parents=True) if civ.decompress: try: safe_extract(src=civ.input_file, dest=dest.parent) except Exception as e: raise ComponentException( "Could not extract input zip file") from e else: with civ.input_file.open("rb") as fs, open(dest, "wb") as fd: for chunk in fs.chunks(): fd.write(chunk)