def form_valid(self, form): def create_upload(image_files): upload_session = RawImageUploadSession.objects.create( creator=self.request.user) upload_session.user_uploads.set(image_files) return upload_session.pk job = Job.objects.create( creator=self.request.user, algorithm_image=self.algorithm.latest_ready_image, ) # TODO AUG2021 JM permission management should be done in 1 place # The execution for jobs over the API or non-sessions needs # to be cleaned up. See callers of `execute_jobs`. job.viewer_groups.add(self.algorithm.editors_group) assign_perm("algorithms.view_logs", self.algorithm.editors_group, job) upload_pks = {} civs = [] interfaces = {ci.slug: ci for ci in self.algorithm.inputs.all()} for slug, value in form.cleaned_data.items(): ci = interfaces[slug] if ci.kind in InterfaceKind.interface_type_image(): if value: # create civ without image, image will be added when import completes civ = ComponentInterfaceValue.objects.create(interface=ci) civs.append(civ) upload_pks[civ.pk] = create_upload(value) elif ci.kind in InterfaceKind.interface_type_file(): civ = ComponentInterfaceValue.objects.create(interface=ci) value.copy_object(to_field=civ.file) civ.full_clean() civ.save() value.delete() civs.append(civ) else: civ = ci.create_instance(value=value) civs.append(civ) job.inputs.add(*civs) job.run_job(upload_pks=upload_pks) return HttpResponseRedirect( reverse( "algorithms:job-experiment-detail", kwargs={ "slug": self.kwargs["slug"], "pk": job.pk }, ))
def field_for_interface(i: InterfaceKind.InterfaceKindChoices): fields = {} for kind in InterfaceKind.interface_type_annotation(): fields[kind] = JSONField for kind in (InterfaceKind.interface_type_image() + InterfaceKind.interface_type_file()): fields[kind] = UploadedAjaxFileList fields.update({ InterfaceKind.InterfaceKindChoices.BOOL: BooleanField, InterfaceKind.InterfaceKindChoices.STRING: CharField, InterfaceKind.InterfaceKindChoices.INTEGER: IntegerField, InterfaceKind.InterfaceKindChoices.FLOAT: FloatField, }) return fields[i]
def __init__( self, *, kind: InterfaceKind.InterfaceKindChoices, schema: Dict, initial=None, user=None, required=None, help_text="", ): field_type = InterfaceKind.get_default_field(kind=kind) kwargs = {"required": required} if initial is not None: kwargs["initial"] = initial if kind in InterfaceKind.interface_type_image(): kwargs["widget"] = UserUploadMultipleWidget() kwargs["queryset"] = get_objects_for_user( user, "uploads.change_userupload", accept_global_perms=False ).filter(status=UserUpload.StatusChoices.COMPLETED) extra_help = IMAGE_UPLOAD_HELP_TEXT elif kind in InterfaceKind.interface_type_file(): kwargs["widget"] = UserUploadSingleWidget( allowed_file_types=InterfaceKind.get_file_mimetypes(kind=kind) ) kwargs["queryset"] = get_objects_for_user( user, "uploads.change_userupload", accept_global_perms=False ).filter(status=UserUpload.StatusChoices.COMPLETED) extra_help = f"{file_upload_text} .{kind.lower()}" elif kind in InterfaceKind.interface_type_json(): default_schema = { **INTERFACE_VALUE_SCHEMA, "anyOf": [{"$ref": f"#/definitions/{kind}"}], } if field_type == forms.JSONField: kwargs["widget"] = JSONEditorWidget(schema=default_schema) kwargs["validators"] = [ JSONValidator(schema=default_schema), JSONValidator(schema=schema), ] extra_help = "" else: raise RuntimeError(f"Unknown kind {kind}") self._field = field_type( help_text=_join_with_br(help_text, extra_help), **kwargs )
def validate(self, attrs): interface = attrs["interface"] if interface.kind in InterfaceKind.interface_type_image(): if not attrs.get("image") and not attrs.get("upload_session"): raise serializers.ValidationError( f"upload_session or image are required for interface " f"kind {interface.kind}" ) if attrs.get("image") and attrs.get("upload_session"): raise serializers.ValidationError( "Only one of image or upload_session should be set" ) if not attrs.get("upload_session"): # Instances without an image are never valid, this will be checked # later, but for now check everything else. DRF 3.0 dropped calling # full_clean on instances, so we need to do it ourselves. instance = ComponentInterfaceValue( **{k: v for k, v in attrs.items() if k != "upload_session"} ) instance.full_clean() return attrs
class AddCasesForm(UploadRawImagesForm): interface = ModelChoiceField(queryset=ComponentInterface.objects.filter( kind__in=InterfaceKind.interface_type_image())) def save(self, *args, **kwargs): self._linked_task.kwargs.update( {"interface_pk": self.cleaned_data["interface"].pk}) return super().save(*args, **kwargs)
def __init__( self, *, kind: InterfaceKind.InterfaceKindChoices, initial=None, user=None, help_text="", ): field_type = field_for_interface(kind) # bool can't be required kwargs = { "required": (kind != InterfaceKind.InterfaceKindChoices.BOOL), } extra_help = "" if initial is not None: kwargs["initial"] = initial if kind in InterfaceKind.interface_type_annotation(): kwargs["widget"] = JSONEditorWidget( schema=ANSWER_TYPE_SCHEMA["definitions"][kind]) if kind in InterfaceKind.interface_type_file(): kwargs["widget"] = uploader.AjaxUploadWidget(multifile=False, auto_commit=False) kwargs["validators"] = [ ExtensionValidator(allowed_extensions=(f".{kind.lower()}", )) ] extra_help = f"{file_upload_text} .{kind.lower()}" if kind in InterfaceKind.interface_type_image(): kwargs["widget"] = uploader.AjaxUploadWidget(multifile=True, auto_commit=False) extra_help = IMAGE_UPLOAD_HELP_TEXT self._field = field_type(help_text=_join_with_br( help_text, extra_help), **kwargs) if user: self._field.widget.user = user
def test_relative_path_file_ending(kind): if kind in InterfaceKind.interface_type_json(): good_suffix = "json" else: good_suffix = kind.lower() i = ComponentInterfaceFactory( kind=kind, relative_path=f"foo/bar.{good_suffix}", store_in_database=False, ) i.full_clean() i.relative_path = "foo/bar" with pytest.raises(ValidationError): i.full_clean()
class AddCasesForm(UploadRawImagesForm): interface = ModelChoiceField( queryset=ComponentInterface.objects.filter( kind__in=InterfaceKind.interface_type_image()), help_text=format_lazy( ('See the <a href="{}">list of interfaces</a> for more ' "information about each interface. " "Please contact support if your desired output is missing."), reverse_lazy("algorithms:component-interface-list"), ), ) def save(self, *args, **kwargs): self._linked_task.kwargs.update( {"interface_pk": self.cleaned_data["interface"].pk}) return super().save(*args, **kwargs)
def test_all_interfaces_covered(): assert {str(i) for i in InterfaceKindChoices} == { *InterfaceKind.interface_type_image(), *InterfaceKind.interface_type_file(), *InterfaceKind.interface_type_json(), }
def test_all_interfaces_in_schema(): for i in InterfaceKind.interface_type_json(): assert str(i) in INTERFACE_VALUE_SCHEMA["definitions"]
# For multi job inputs we add uuid prefixes, so check that the relative # path does not contain a UUID i = ComponentInterfaceFactory( relative_path=f"{uuid.uuid4()}/whatever.json", kind=InterfaceKindChoices.ANY, ) with pytest.raises(ValidationError) as e: i.full_clean() assert str(e.value) == "{'relative_path': ['Enter a valid value.']}" @pytest.mark.django_db @pytest.mark.parametrize( "kind", ( *InterfaceKind.interface_type_file(), *InterfaceKind.interface_type_json(), ), ) def test_relative_path_file_ending(kind): if kind in InterfaceKind.interface_type_json(): good_suffix = "json" else: good_suffix = kind.lower() i = ComponentInterfaceFactory( kind=kind, relative_path=f"foo/bar.{good_suffix}", store_in_database=False, ) i.full_clean()
def form_valid(self, form): # noqa: C901 def create_upload(image_files): upload_session = RawImageUploadSession.objects.create( creator=self.request.user) upload_session.user_uploads.set(image_files) return upload_session.pk upload_pks = {} civ_pks_to_remove = set() civ_pks_to_add = set() for slug, value in form.cleaned_data.items(): if value is None: continue ci = ComponentInterface.objects.get(slug=slug) civ = self.archive_item.values.filter(interface=ci).first() if civ: if civ.value == value: continue civ_pks_to_remove.add(civ.pk) if ci.kind in InterfaceKind.interface_type_image(): if value: civ = ComponentInterfaceValue.objects.create(interface=ci) civ_pks_to_add.add(civ.pk) upload_pks[civ.pk] = create_upload(value) elif ci.kind in InterfaceKind.interface_type_file(): civ = ComponentInterfaceValue.objects.create(interface=ci) value.copy_object(to_field=civ.file) civ.full_clean() civ.save() value.delete() civ_pks_to_add.add(civ.pk) else: civ = ci.create_instance(value=value) civ_pks_to_add.add(civ.pk) tasks = update_archive_item_values.signature( kwargs={ "archive_item_pk": self.archive_item.pk, "civ_pks_to_add": list(civ_pks_to_add), "civ_pks_to_remove": list(civ_pks_to_remove), }, immutable=True, ) if len(upload_pks) > 0: image_tasks = group( chain( build_images.signature( kwargs={"upload_session_pk": upload_pk}), add_images_to_component_interface_value.signature( kwargs={ "component_interface_value_pk": civ_pk, "upload_session_pk": upload_pk, }, immutable=True, ), ) for civ_pk, upload_pk in upload_pks.items()) tasks = chord(image_tasks, tasks) on_commit(tasks.apply_async) return HttpResponseRedirect( reverse( "archives:items-list", kwargs={"slug": self.kwargs["slug"]}, ))
def form_valid(self, form): def create_upload(image_files): raw_files = [] upload_session = RawImageUploadSession.objects.create( creator=self.request.user) for image_file in image_files: raw_files.append( RawImageFile( upload_session=upload_session, filename=image_file.name, staged_file_id=image_file.uuid, )) RawImageFile.objects.bulk_create(list(raw_files)) return upload_session.pk job = Job.objects.create( creator=self.request.user, algorithm_image=self.algorithm.latest_ready_image, ) job.viewer_groups.add(self.algorithm.editors_group) upload_pks = {} civs = [] interfaces = {ci.slug: ci for ci in self.algorithm.inputs.all()} for slug, value in form.cleaned_data.items(): ci = interfaces[slug] if ci.kind in InterfaceKind.interface_type_image(): # create civ without image, image will be added when import completes civ = ComponentInterfaceValue.objects.create(interface=ci) civs.append(civ) upload_pks[civ.pk] = create_upload(value) elif ci.kind in InterfaceKind.interface_type_file(): # should be a single file civ = ComponentInterfaceValue.objects.create(interface=ci) name = get_valid_filename(value[0].name) with value[0].open() as f: civ.file = File(f, name=name) civ.save() civs.append(civ) else: civ = ComponentInterfaceValue.objects.create(interface=ci, value=value) civs.append(civ) job.inputs.add(*civs) run_job = run_algorithm_job_for_inputs.signature( kwargs={ "job_pk": job.pk, "upload_pks": upload_pks }, immutable=True, ) on_commit(run_job.apply_async) return HttpResponseRedirect( reverse( "algorithms:job-experiment-detail", kwargs={ "slug": self.kwargs["slug"], "pk": job.pk }, ))
def test_algorithm_multiple_inputs(client, algorithm_io_image, settings, component_interfaces): # Override the celery settings settings.task_eager_propagates = (True, ) settings.task_always_eager = (True, ) creator = UserFactory() assert Job.objects.count() == 0 # Create the algorithm image algorithm_container, sha256 = algorithm_io_image alg = AlgorithmImageFactory(image__from_path=algorithm_container, image_sha256=sha256, ready=True) alg.algorithm.add_editor(creator) alg.algorithm.inputs.set(ComponentInterface.objects.all()) # create the job job = Job.objects.create(creator=creator, algorithm_image=alg) expected = [] for ci in ComponentInterface.objects.all(): if ci.kind in InterfaceKind.interface_type_image(): image_file = ImageFileFactory( file__from_path=Path(__file__).parent / "resources" / "input_file.tif") job.inputs.add( ComponentInterfaceValueFactory(interface=ci, image=image_file.image, file=None)) expected.append("file") elif ci.kind in InterfaceKind.interface_type_file(): job.inputs.add( ComponentInterfaceValueFactory( interface=ci, file__from_path=Path(__file__).parent / "resources" / "test.json", )) expected.append("json") else: job.inputs.add( ComponentInterfaceValueFactory(interface=ci, value="test", file=None)) expected.append("test") # Nested on_commits created by these tasks with capture_on_commit_callbacks(execute=True): with capture_on_commit_callbacks(execute=True): run_algorithm_job_for_inputs(job_pk=job.pk, upload_pks=[]) job = Job.objects.get() assert job.status == job.SUCCESS assert {x[0] for x in job.input_files} == set(job.outputs.first().value.keys()) assert sorted( map( lambda x: x if x != {} else "json", job.outputs.first().value.values(), )) == sorted(expected)
# setup interface = ComponentInterfaceFactory(kind=kind) civ = {"interface": interface.slug} # test serializer = ComponentInterfaceValuePostSerializer(data=civ) # verify assert not serializer.is_valid() assert "JSON does not fulfill schema: None is not of type" in str( serializer.errors["__all__"][0]) @pytest.mark.django_db @pytest.mark.parametrize("kind,", (InterfaceKind.interface_type_image())) def test_civ_post_image_or_upload_required_validation(kind): # setup interface = ComponentInterfaceFactory(kind=kind) civ = {"interface": interface.slug} # test serializer = ComponentInterfaceValuePostSerializer(data=civ) # verify assert not serializer.is_valid() assert (f"upload_session or image are required for interface kind {kind}" in serializer.errors["non_field_errors"])
def test_algorithm_multiple_inputs(client, algorithm_io_image, settings, component_interfaces): # Override the celery settings settings.task_eager_propagates = (True, ) settings.task_always_eager = (True, ) creator = UserFactory() assert Job.objects.count() == 0 # Create the algorithm image algorithm_container, sha256 = algorithm_io_image alg = AlgorithmImageFactory(image__from_path=algorithm_container, image_sha256=sha256, ready=True) alg.algorithm.add_editor(creator) alg.algorithm.inputs.set(ComponentInterface.objects.all()) alg.algorithm.outputs.set( [ComponentInterface.objects.get(slug="results-json-file")]) # create the job job = Job.objects.create(creator=creator, algorithm_image=alg) expected = [] for ci in ComponentInterface.objects.exclude( kind=InterfaceKindChoices.ZIP): if ci.kind in InterfaceKind.interface_type_image(): image_file = ImageFileFactory( file__from_path=Path(__file__).parent / "resources" / "input_file.tif") job.inputs.add( ComponentInterfaceValueFactory(interface=ci, image=image_file.image)) expected.append("file") elif ci.kind in InterfaceKind.interface_type_file(): civ = ComponentInterfaceValueFactory(interface=ci) civ.file.save("test", File(BytesIO(b""))) civ.save() job.inputs.add(civ) expected.append("file") else: job.inputs.add( ComponentInterfaceValueFactory(interface=ci, value="test")) expected.append("test") with capture_on_commit_callbacks() as callbacks: run_algorithm_job_for_inputs(job_pk=job.pk, upload_pks=[]) recurse_callbacks(callbacks=callbacks) job.refresh_from_db() assert job.error_message == "" assert job.status == job.SUCCESS # Remove fake value for score output_dict = job.outputs.first().value output_dict.pop("score") assert {f"/input/{x.relative_path}" for x in job.inputs.all()} == set(output_dict.keys()) assert sorted( map( lambda x: x if x != {} else "json", output_dict.values(), )) == sorted(expected)