def _copy_output_files(self, *, container, base_dir: Path): output_files = { base_dir / Path(f) for f in container.exec_run(f"find {base_dir} -type f") .output.decode() .splitlines() } if not output_files: raise ValueError("Output directory is empty") job = SubmissionToAnnotationSetJob.objects.get(pk=self._job_id) annotationset = AnnotationSet.objects.create( creator=job.submission.creator, base=job.base, submission=job.submission, kind=AnnotationSet.PREDICTION, ) if self.__was_unzipped: with TemporaryDirectory() as tmpdir: input_files = set() for file in output_files: tmpfile = Path( safe_join(tmpdir, file.relative_to(base_dir)) ) tmpfile.parent.mkdir(parents=True, exist_ok=True) with open(tmpfile, "wb") as outfile: infile = get_file(container=container, src=file) buffer = True while buffer: buffer = infile.read(1024) outfile.write(buffer) input_files.add(tmpfile) importer_result = import_images(files=input_files,) annotationset.images.add(*importer_result.new_images) else: if not len(output_files) == 1: raise RuntimeError("This submission has too many files.") f = get_file(container=container, src=output_files.pop()) annotationset.labels = process_csv_file(f) annotationset.save()
def _create_file_result(self, reader, job): output_file = Path(self.output_path) try: file = get_file(container=reader, src=output_file) except NotFound: raise ComponentException( f"The evaluation or algorithm failed for an unknown reason as " f"file {self.output_path} was not produced. Please contact the " f"organisers for assistance.") if self.save_in_object_store: civ = ComponentInterfaceValue.objects.create(interface=self) try: civ.file = File(file, name=str(output_file.name)) civ.full_clean() civ.save() except ValidationError: raise ComponentException("Invalid filetype.") else: try: result = json.loads( file.read().decode(), parse_constant=lambda x: None, # Removes -inf, inf and NaN ) except JSONDecodeError as e: raise ComponentException( f"Could not decode json file: {e.msg}") civ = ComponentInterfaceValue.objects.create(interface=self, value=result) job.outputs.add(civ)
def _create_json_result(self, reader, job): try: result = get_file(container=reader, src=Path(self.output_path)) except NotFound: # The container exited without error, but no results file was # produced. This shouldn't happen, but does with poorly programmed # evaluation containers. raise ComponentException( "The evaluation failed for an unknown reason as no results " "file was produced. Please contact the organisers for " "assistance." ) try: result = json.loads( result.read().decode(), parse_constant=lambda x: None, # Removes -inf, inf and NaN ) except JSONDecodeError as e: raise ComponentException(f"Could not decode results file: {e.msg}") civ = ComponentInterfaceValue.objects.create( interface=self, value=result ) job.outputs.add(civ)
def _copy_output_files(self, *, container, base_dir: Path): found_files = container.exec_run(f"find {base_dir} -type f") if found_files.exit_code != 0: logger.warning(f"Error listing {base_dir}") return output_files = [ base_dir / Path(f) for f in found_files.output.decode().splitlines() ] if not output_files: logger.warning("Output directory is empty") return with TemporaryDirectory() as tmpdir: input_files = set() for file in output_files: tmpfile = Path(safe_join(tmpdir, file.relative_to(base_dir))) tmpfile.parent.mkdir(parents=True, exist_ok=True) with open(tmpfile, "wb") as outfile: infile = get_file(container=container, src=file) buffer = True while buffer: buffer = infile.read(1024) outfile.write(buffer) input_files.add(tmpfile) importer_result = import_images( files=input_files, builders=[image_builder_mhd, image_builder_tiff], ) default_output_interface = ComponentInterface.objects.get( slug=DEFAULT_OUTPUT_INTERFACE_SLUG ) job = Job.objects.get(pk=self._job_id) for image in importer_result.new_images: civ = ComponentInterfaceValue.objects.create( interface=default_output_interface, image=image ) job.outputs.add(civ)
def _create_images_result(self, *, reader, job): # TODO JM in the future this will be a file, not a directory base_dir = Path(self.output_path) found_files = reader.exec_run(f"find {base_dir} -type f") if found_files.exit_code != 0: logger.warning(f"Error listing {base_dir}") return output_files = [ base_dir / Path(f) for f in found_files.output.decode().splitlines() ] if not output_files: logger.warning("Output directory is empty") return with TemporaryDirectory() as tmpdir: input_files = set() for file in output_files: temp_file = Path(safe_join(tmpdir, file.relative_to(base_dir))) temp_file.parent.mkdir(parents=True, exist_ok=True) with open(temp_file, "wb") as outfile: infile = get_file(container=reader, src=file) buffer = True while buffer: buffer = infile.read(1024) outfile.write(buffer) input_files.add(temp_file) importer_result = import_images( files=input_files, builders=[image_builder_mhd, image_builder_tiff], ) for image in importer_result.new_images: civ = ComponentInterfaceValue.objects.create( interface=self, image=image ) job.outputs.add(civ)