def form_valid(self, form):
def create_upload(image_files):
upload_session = RawImageUploadSession.objects.create(
creator=self.request.user)
upload_session.user_uploads.set(image_files)
return upload_session.pk
job = Job.objects.create(
creator=self.request.user,
algorithm_image=self.algorithm.latest_ready_image,
)
# TODO AUG2021 JM permission management should be done in 1 place
# The execution for jobs over the API or non-sessions needs
# to be cleaned up. See callers of `execute_jobs`.
job.viewer_groups.add(self.algorithm.editors_group)
assign_perm("algorithms.view_logs", self.algorithm.editors_group, job)
upload_pks = {}
civs = []
interfaces = {ci.slug: ci for ci in self.algorithm.inputs.all()}
for slug, value in form.cleaned_data.items():
ci = interfaces[slug]
if ci.kind in InterfaceKind.interface_type_image():
if value:
# create civ without image, image will be added when import completes
civ = ComponentInterfaceValue.objects.create(interface=ci)
civs.append(civ)
upload_pks[civ.pk] = create_upload(value)
elif ci.kind in InterfaceKind.interface_type_file():
civ = ComponentInterfaceValue.objects.create(interface=ci)
value.copy_object(to_field=civ.file)
civ.full_clean()
civ.save()
value.delete()
civs.append(civ)
else:
civ = ci.create_instance(value=value)
civs.append(civ)
job.inputs.add(*civs)
job.run_job(upload_pks=upload_pks)
return HttpResponseRedirect(
reverse(
"algorithms:job-experiment-detail",
kwargs={
"slug": self.kwargs["slug"],
"pk": job.pk
},
))
def field_for_interface(i: InterfaceKind.InterfaceKindChoices):
fields = {}
for kind in InterfaceKind.interface_type_annotation():
fields[kind] = JSONField
for kind in (InterfaceKind.interface_type_image() +
InterfaceKind.interface_type_file()):
fields[kind] = UploadedAjaxFileList
fields.update({
InterfaceKind.InterfaceKindChoices.BOOL: BooleanField,
InterfaceKind.InterfaceKindChoices.STRING: CharField,
InterfaceKind.InterfaceKindChoices.INTEGER: IntegerField,
InterfaceKind.InterfaceKindChoices.FLOAT: FloatField,
})
return fields[i]
def __init__(
self,
*,
kind: InterfaceKind.InterfaceKindChoices,
schema: Dict,
initial=None,
user=None,
required=None,
help_text="",
):
field_type = InterfaceKind.get_default_field(kind=kind)
kwargs = {"required": required}
if initial is not None:
kwargs["initial"] = initial
if kind in InterfaceKind.interface_type_image():
kwargs["widget"] = UserUploadMultipleWidget()
kwargs["queryset"] = get_objects_for_user(
user, "uploads.change_userupload", accept_global_perms=False
).filter(status=UserUpload.StatusChoices.COMPLETED)
extra_help = IMAGE_UPLOAD_HELP_TEXT
elif kind in InterfaceKind.interface_type_file():
kwargs["widget"] = UserUploadSingleWidget(
allowed_file_types=InterfaceKind.get_file_mimetypes(kind=kind)
)
kwargs["queryset"] = get_objects_for_user(
user, "uploads.change_userupload", accept_global_perms=False
).filter(status=UserUpload.StatusChoices.COMPLETED)
extra_help = f"{file_upload_text} .{kind.lower()}"
elif kind in InterfaceKind.interface_type_json():
default_schema = {
**INTERFACE_VALUE_SCHEMA,
"anyOf": [{"$ref": f"#/definitions/{kind}"}],
}
if field_type == forms.JSONField:
kwargs["widget"] = JSONEditorWidget(schema=default_schema)
kwargs["validators"] = [
JSONValidator(schema=default_schema),
JSONValidator(schema=schema),
]
extra_help = ""
else:
raise RuntimeError(f"Unknown kind {kind}")
self._field = field_type(
help_text=_join_with_br(help_text, extra_help), **kwargs
)
def validate(self, attrs):
interface = attrs["interface"]
if interface.kind in InterfaceKind.interface_type_image():
if not attrs.get("image") and not attrs.get("upload_session"):
raise serializers.ValidationError(
f"upload_session or image are required for interface "
f"kind {interface.kind}"
)
if attrs.get("image") and attrs.get("upload_session"):
raise serializers.ValidationError(
"Only one of image or upload_session should be set"
)
if not attrs.get("upload_session"):
# Instances without an image are never valid, this will be checked
# later, but for now check everything else. DRF 3.0 dropped calling
# full_clean on instances, so we need to do it ourselves.
instance = ComponentInterfaceValue(
**{k: v for k, v in attrs.items() if k != "upload_session"}
)
instance.full_clean()
return attrs
class AddCasesForm(UploadRawImagesForm):
interface = ModelChoiceField(queryset=ComponentInterface.objects.filter(
kind__in=InterfaceKind.interface_type_image()))
def save(self, *args, **kwargs):
self._linked_task.kwargs.update(
{"interface_pk": self.cleaned_data["interface"].pk})
return super().save(*args, **kwargs)
def __init__(
self,
*,
kind: InterfaceKind.InterfaceKindChoices,
initial=None,
user=None,
help_text="",
):
field_type = field_for_interface(kind)
# bool can't be required
kwargs = {
"required": (kind != InterfaceKind.InterfaceKindChoices.BOOL),
}
extra_help = ""
if initial is not None:
kwargs["initial"] = initial
if kind in InterfaceKind.interface_type_annotation():
kwargs["widget"] = JSONEditorWidget(
schema=ANSWER_TYPE_SCHEMA["definitions"][kind])
if kind in InterfaceKind.interface_type_file():
kwargs["widget"] = uploader.AjaxUploadWidget(multifile=False,
auto_commit=False)
kwargs["validators"] = [
ExtensionValidator(allowed_extensions=(f".{kind.lower()}", ))
]
extra_help = f"{file_upload_text} .{kind.lower()}"
if kind in InterfaceKind.interface_type_image():
kwargs["widget"] = uploader.AjaxUploadWidget(multifile=True,
auto_commit=False)
extra_help = IMAGE_UPLOAD_HELP_TEXT
self._field = field_type(help_text=_join_with_br(
help_text, extra_help),
**kwargs)
if user:
self._field.widget.user = user
def test_relative_path_file_ending(kind):
if kind in InterfaceKind.interface_type_json():
good_suffix = "json"
else:
good_suffix = kind.lower()
i = ComponentInterfaceFactory(
kind=kind,
relative_path=f"foo/bar.{good_suffix}",
store_in_database=False,
)
i.full_clean()
i.relative_path = "foo/bar"
with pytest.raises(ValidationError):
i.full_clean()
class AddCasesForm(UploadRawImagesForm):
interface = ModelChoiceField(
queryset=ComponentInterface.objects.filter(
kind__in=InterfaceKind.interface_type_image()),
help_text=format_lazy(
('See the <a href="{}">list of interfaces</a> for more '
"information about each interface. "
"Please contact support if your desired output is missing."),
reverse_lazy("algorithms:component-interface-list"),
),
)
def save(self, *args, **kwargs):
self._linked_task.kwargs.update(
{"interface_pk": self.cleaned_data["interface"].pk})
return super().save(*args, **kwargs)
def test_all_interfaces_covered():
assert {str(i) for i in InterfaceKindChoices} == {
*InterfaceKind.interface_type_image(),
*InterfaceKind.interface_type_file(),
*InterfaceKind.interface_type_json(),
}
def test_all_interfaces_in_schema():
for i in InterfaceKind.interface_type_json():
assert str(i) in INTERFACE_VALUE_SCHEMA["definitions"]
# For multi job inputs we add uuid prefixes, so check that the relative
# path does not contain a UUID
i = ComponentInterfaceFactory(
relative_path=f"{uuid.uuid4()}/whatever.json",
kind=InterfaceKindChoices.ANY,
)
with pytest.raises(ValidationError) as e:
i.full_clean()
assert str(e.value) == "{'relative_path': ['Enter a valid value.']}"
@pytest.mark.django_db
@pytest.mark.parametrize(
"kind",
(
*InterfaceKind.interface_type_file(),
*InterfaceKind.interface_type_json(),
),
)
def test_relative_path_file_ending(kind):
if kind in InterfaceKind.interface_type_json():
good_suffix = "json"
else:
good_suffix = kind.lower()
i = ComponentInterfaceFactory(
kind=kind,
relative_path=f"foo/bar.{good_suffix}",
store_in_database=False,
)
i.full_clean()
def form_valid(self, form): # noqa: C901
def create_upload(image_files):
upload_session = RawImageUploadSession.objects.create(
creator=self.request.user)
upload_session.user_uploads.set(image_files)
return upload_session.pk
upload_pks = {}
civ_pks_to_remove = set()
civ_pks_to_add = set()
for slug, value in form.cleaned_data.items():
if value is None:
continue
ci = ComponentInterface.objects.get(slug=slug)
civ = self.archive_item.values.filter(interface=ci).first()
if civ:
if civ.value == value:
continue
civ_pks_to_remove.add(civ.pk)
if ci.kind in InterfaceKind.interface_type_image():
if value:
civ = ComponentInterfaceValue.objects.create(interface=ci)
civ_pks_to_add.add(civ.pk)
upload_pks[civ.pk] = create_upload(value)
elif ci.kind in InterfaceKind.interface_type_file():
civ = ComponentInterfaceValue.objects.create(interface=ci)
value.copy_object(to_field=civ.file)
civ.full_clean()
civ.save()
value.delete()
civ_pks_to_add.add(civ.pk)
else:
civ = ci.create_instance(value=value)
civ_pks_to_add.add(civ.pk)
tasks = update_archive_item_values.signature(
kwargs={
"archive_item_pk": self.archive_item.pk,
"civ_pks_to_add": list(civ_pks_to_add),
"civ_pks_to_remove": list(civ_pks_to_remove),
},
immutable=True,
)
if len(upload_pks) > 0:
image_tasks = group(
chain(
build_images.signature(
kwargs={"upload_session_pk": upload_pk}),
add_images_to_component_interface_value.signature(
kwargs={
"component_interface_value_pk": civ_pk,
"upload_session_pk": upload_pk,
},
immutable=True,
),
) for civ_pk, upload_pk in upload_pks.items())
tasks = chord(image_tasks, tasks)
on_commit(tasks.apply_async)
return HttpResponseRedirect(
reverse(
"archives:items-list",
kwargs={"slug": self.kwargs["slug"]},
))
def form_valid(self, form):
def create_upload(image_files):
raw_files = []
upload_session = RawImageUploadSession.objects.create(
creator=self.request.user)
for image_file in image_files:
raw_files.append(
RawImageFile(
upload_session=upload_session,
filename=image_file.name,
staged_file_id=image_file.uuid,
))
RawImageFile.objects.bulk_create(list(raw_files))
return upload_session.pk
job = Job.objects.create(
creator=self.request.user,
algorithm_image=self.algorithm.latest_ready_image,
)
job.viewer_groups.add(self.algorithm.editors_group)
upload_pks = {}
civs = []
interfaces = {ci.slug: ci for ci in self.algorithm.inputs.all()}
for slug, value in form.cleaned_data.items():
ci = interfaces[slug]
if ci.kind in InterfaceKind.interface_type_image():
# create civ without image, image will be added when import completes
civ = ComponentInterfaceValue.objects.create(interface=ci)
civs.append(civ)
upload_pks[civ.pk] = create_upload(value)
elif ci.kind in InterfaceKind.interface_type_file():
# should be a single file
civ = ComponentInterfaceValue.objects.create(interface=ci)
name = get_valid_filename(value[0].name)
with value[0].open() as f:
civ.file = File(f, name=name)
civ.save()
civs.append(civ)
else:
civ = ComponentInterfaceValue.objects.create(interface=ci,
value=value)
civs.append(civ)
job.inputs.add(*civs)
run_job = run_algorithm_job_for_inputs.signature(
kwargs={
"job_pk": job.pk,
"upload_pks": upload_pks
},
immutable=True,
)
on_commit(run_job.apply_async)
return HttpResponseRedirect(
reverse(
"algorithms:job-experiment-detail",
kwargs={
"slug": self.kwargs["slug"],
"pk": job.pk
},
))
def test_algorithm_multiple_inputs(client, algorithm_io_image, settings,
component_interfaces):
# Override the celery settings
settings.task_eager_propagates = (True, )
settings.task_always_eager = (True, )
creator = UserFactory()
assert Job.objects.count() == 0
# Create the algorithm image
algorithm_container, sha256 = algorithm_io_image
alg = AlgorithmImageFactory(image__from_path=algorithm_container,
image_sha256=sha256,
ready=True)
alg.algorithm.add_editor(creator)
alg.algorithm.inputs.set(ComponentInterface.objects.all())
# create the job
job = Job.objects.create(creator=creator, algorithm_image=alg)
expected = []
for ci in ComponentInterface.objects.all():
if ci.kind in InterfaceKind.interface_type_image():
image_file = ImageFileFactory(
file__from_path=Path(__file__).parent / "resources" /
"input_file.tif")
job.inputs.add(
ComponentInterfaceValueFactory(interface=ci,
image=image_file.image,
file=None))
expected.append("file")
elif ci.kind in InterfaceKind.interface_type_file():
job.inputs.add(
ComponentInterfaceValueFactory(
interface=ci,
file__from_path=Path(__file__).parent / "resources" /
"test.json",
))
expected.append("json")
else:
job.inputs.add(
ComponentInterfaceValueFactory(interface=ci,
value="test",
file=None))
expected.append("test")
# Nested on_commits created by these tasks
with capture_on_commit_callbacks(execute=True):
with capture_on_commit_callbacks(execute=True):
run_algorithm_job_for_inputs(job_pk=job.pk, upload_pks=[])
job = Job.objects.get()
assert job.status == job.SUCCESS
assert {x[0]
for x in job.input_files} == set(job.outputs.first().value.keys())
assert sorted(
map(
lambda x: x if x != {} else "json",
job.outputs.first().value.values(),
)) == sorted(expected)
# setup
interface = ComponentInterfaceFactory(kind=kind)
civ = {"interface": interface.slug}
# test
serializer = ComponentInterfaceValuePostSerializer(data=civ)
# verify
assert not serializer.is_valid()
assert "JSON does not fulfill schema: None is not of type" in str(
serializer.errors["__all__"][0])
@pytest.mark.django_db
@pytest.mark.parametrize("kind,", (InterfaceKind.interface_type_image()))
def test_civ_post_image_or_upload_required_validation(kind):
# setup
interface = ComponentInterfaceFactory(kind=kind)
civ = {"interface": interface.slug}
# test
serializer = ComponentInterfaceValuePostSerializer(data=civ)
# verify
assert not serializer.is_valid()
assert (f"upload_session or image are required for interface kind {kind}"
in serializer.errors["non_field_errors"])
def test_algorithm_multiple_inputs(client, algorithm_io_image, settings,
component_interfaces):
# Override the celery settings
settings.task_eager_propagates = (True, )
settings.task_always_eager = (True, )
creator = UserFactory()
assert Job.objects.count() == 0
# Create the algorithm image
algorithm_container, sha256 = algorithm_io_image
alg = AlgorithmImageFactory(image__from_path=algorithm_container,
image_sha256=sha256,
ready=True)
alg.algorithm.add_editor(creator)
alg.algorithm.inputs.set(ComponentInterface.objects.all())
alg.algorithm.outputs.set(
[ComponentInterface.objects.get(slug="results-json-file")])
# create the job
job = Job.objects.create(creator=creator, algorithm_image=alg)
expected = []
for ci in ComponentInterface.objects.exclude(
kind=InterfaceKindChoices.ZIP):
if ci.kind in InterfaceKind.interface_type_image():
image_file = ImageFileFactory(
file__from_path=Path(__file__).parent / "resources" /
"input_file.tif")
job.inputs.add(
ComponentInterfaceValueFactory(interface=ci,
image=image_file.image))
expected.append("file")
elif ci.kind in InterfaceKind.interface_type_file():
civ = ComponentInterfaceValueFactory(interface=ci)
civ.file.save("test", File(BytesIO(b"")))
civ.save()
job.inputs.add(civ)
expected.append("file")
else:
job.inputs.add(
ComponentInterfaceValueFactory(interface=ci, value="test"))
expected.append("test")
with capture_on_commit_callbacks() as callbacks:
run_algorithm_job_for_inputs(job_pk=job.pk, upload_pks=[])
recurse_callbacks(callbacks=callbacks)
job.refresh_from_db()
assert job.error_message == ""
assert job.status == job.SUCCESS
# Remove fake value for score
output_dict = job.outputs.first().value
output_dict.pop("score")
assert {f"/input/{x.relative_path}"
for x in job.inputs.all()} == set(output_dict.keys())
assert sorted(
map(
lambda x: x if x != {} else "json",
output_dict.values(),
)) == sorted(expected)