def _copy_output_files(self, *, container, base_dir: Path):
output_files = {
base_dir / Path(f)
for f in container.exec_run(f"find {base_dir} -type f")
.output.decode()
.splitlines()
}
if not output_files:
raise ValueError("Output directory is empty")
job = SubmissionToAnnotationSetJob.objects.get(pk=self._job_id)
annotationset = AnnotationSet.objects.create(
creator=job.submission.creator,
base=job.base,
submission=job.submission,
kind=AnnotationSet.PREDICTION,
)
if self.__was_unzipped:
with TemporaryDirectory() as tmpdir:
input_files = set()
for file in output_files:
tmpfile = Path(
safe_join(tmpdir, file.relative_to(base_dir))
)
tmpfile.parent.mkdir(parents=True, exist_ok=True)
with open(tmpfile, "wb") as outfile:
infile = get_file(container=container, src=file)
buffer = True
while buffer:
buffer = infile.read(1024)
outfile.write(buffer)
input_files.add(tmpfile)
importer_result = import_images(files=input_files,)
annotationset.images.add(*importer_result.new_images)
else:
if not len(output_files) == 1:
raise RuntimeError("This submission has too many files.")
f = get_file(container=container, src=output_files.pop())
annotationset.labels = process_csv_file(f)
annotationset.save()
def _create_file_result(self, reader, job):
output_file = Path(self.output_path)
try:
file = get_file(container=reader, src=output_file)
except NotFound:
raise ComponentException(
f"The evaluation or algorithm failed for an unknown reason as "
f"file {self.output_path} was not produced. Please contact the "
f"organisers for assistance.")
if self.save_in_object_store:
civ = ComponentInterfaceValue.objects.create(interface=self)
try:
civ.file = File(file, name=str(output_file.name))
civ.full_clean()
civ.save()
except ValidationError:
raise ComponentException("Invalid filetype.")
else:
try:
result = json.loads(
file.read().decode(),
parse_constant=lambda x: None, # Removes -inf, inf and NaN
)
except JSONDecodeError as e:
raise ComponentException(
f"Could not decode json file: {e.msg}")
civ = ComponentInterfaceValue.objects.create(interface=self,
value=result)
job.outputs.add(civ)
def _create_json_result(self, reader, job):
try:
result = get_file(container=reader, src=Path(self.output_path))
except NotFound:
# The container exited without error, but no results file was
# produced. This shouldn't happen, but does with poorly programmed
# evaluation containers.
raise ComponentException(
"The evaluation failed for an unknown reason as no results "
"file was produced. Please contact the organisers for "
"assistance."
)
try:
result = json.loads(
result.read().decode(),
parse_constant=lambda x: None, # Removes -inf, inf and NaN
)
except JSONDecodeError as e:
raise ComponentException(f"Could not decode results file: {e.msg}")
civ = ComponentInterfaceValue.objects.create(
interface=self, value=result
)
job.outputs.add(civ)
def _copy_output_files(self, *, container, base_dir: Path):
found_files = container.exec_run(f"find {base_dir} -type f")
if found_files.exit_code != 0:
logger.warning(f"Error listing {base_dir}")
return
output_files = [
base_dir / Path(f)
for f in found_files.output.decode().splitlines()
]
if not output_files:
logger.warning("Output directory is empty")
return
with TemporaryDirectory() as tmpdir:
input_files = set()
for file in output_files:
tmpfile = Path(safe_join(tmpdir, file.relative_to(base_dir)))
tmpfile.parent.mkdir(parents=True, exist_ok=True)
with open(tmpfile, "wb") as outfile:
infile = get_file(container=container, src=file)
buffer = True
while buffer:
buffer = infile.read(1024)
outfile.write(buffer)
input_files.add(tmpfile)
importer_result = import_images(
files=input_files,
builders=[image_builder_mhd, image_builder_tiff],
)
default_output_interface = ComponentInterface.objects.get(
slug=DEFAULT_OUTPUT_INTERFACE_SLUG
)
job = Job.objects.get(pk=self._job_id)
for image in importer_result.new_images:
civ = ComponentInterfaceValue.objects.create(
interface=default_output_interface, image=image
)
job.outputs.add(civ)
def _create_images_result(self, *, reader, job):
# TODO JM in the future this will be a file, not a directory
base_dir = Path(self.output_path)
found_files = reader.exec_run(f"find {base_dir} -type f")
if found_files.exit_code != 0:
logger.warning(f"Error listing {base_dir}")
return
output_files = [
base_dir / Path(f)
for f in found_files.output.decode().splitlines()
]
if not output_files:
logger.warning("Output directory is empty")
return
with TemporaryDirectory() as tmpdir:
input_files = set()
for file in output_files:
temp_file = Path(safe_join(tmpdir, file.relative_to(base_dir)))
temp_file.parent.mkdir(parents=True, exist_ok=True)
with open(temp_file, "wb") as outfile:
infile = get_file(container=reader, src=file)
buffer = True
while buffer:
buffer = infile.read(1024)
outfile.write(buffer)
input_files.add(temp_file)
importer_result = import_images(
files=input_files,
builders=[image_builder_mhd, image_builder_tiff],
)
for image in importer_result.new_images:
civ = ComponentInterfaceValue.objects.create(
interface=self, image=image
)
job.outputs.add(civ)