def test_identical_when_drivers_unmatched(self): driver1 = CodeResourceRevision() driver2 = CodeResourceRevision() user = User() m1 = Method(revision_name='A', driver=driver1, user=user) for i in range(2): inp = m1.inputs.create(dataset_name='a_in_{}'.format(i), dataset_idx=i + 1) inp.transformationinput = inp for i in range(3): out = m1.outputs.create(dataset_name='a_out_{}'.format(i), dataset_idx=i + 1) out.transformationoutput = out m2 = Method(revision_name='B', driver=driver2, user=user) for i in range(2): inp = m2.inputs.create(dataset_name='b_in_{}'.format(i), dataset_idx=i + 1) inp.transformationinput = inp for i in range(3): out = m2.outputs.create(dataset_name='b_in_{}'.format(i), dataset_idx=i + 1) out.transformationoutput = out self.assertFalse(m1.is_identical(m2))
def test_copy_io_from_parent_with_structure(self): cdt = CompoundDatatype() min_row = 1 max_row = 100 structure = XputStructure(compounddatatype=cdt, min_row=min_row, max_row=max_row) parent = self.create_parent() def get_structure(xput_self): if xput_self.dataset_idx == 1: return structure # noinspection PyUnresolvedReferences raise XputStructure.DoesNotExist TransformationXput.structure = property(get_structure) expected_inputs = {inp.dataset_idx for inp in parent.inputs} expected_outputs = {out.dataset_idx for out in parent.outputs} foo = Method(revision_parent=parent) foo.copy_io_from_parent() self.assertEqual(expected_inputs, {inp.dataset_idx for inp in foo.inputs}) self.assertEqual(expected_outputs, {out.dataset_idx for out in foo.outputs}) # noinspection PyUnresolvedReferences create_args = XputStructure.objects.create.call_args_list # @UndefinedVariable self.assertEqual(2, len(create_args)) _args, kwargs = create_args[0] self.assertEqual(100, kwargs['max_row'])
def create_valid_pipeline(self): p = Pipeline(family=PipelineFamily()) self.add_inputs(p, self.create_input(datatypes.STR_PK, dataset_idx=1)) m = Method() m.method = m self.add_inputs(m, self.create_input(datatypes.STR_PK, dataset_idx=1)) self.add_outputs(m, self.create_output(datatypes.STR_PK, dataset_idx=1)) step1 = PipelineStep(pipeline=p, transformation=m, step_num=1) p.steps.add(step1) cable = PipelineStepInputCable(pipelinestep=step1, source_step=0, source=p.inputs.all()[0], dest=m.inputs.all()[0]) cable.pipelinestepinputcable = cable step1.cables_in.add(cable) outcable = PipelineOutputCable(pipeline=p, output_idx=1, source_step=1, source=m.outputs.all()[0], output_cdt=m.outputs.all()[0].get_cdt()) p.outcables.add(outcable) yield p
def test_with_family_str(self): """ expect "Method revision name and family name" """ family = MethodFamily(name="Example") method = Method(revision_name="rounded", revision_number=3, family=family) self.assertEqual(str(method), "Example:3 (rounded)")
def add_step(self, pipeline): prev_step = pipeline.steps[-1] m = Method() m.method = m self.add_inputs(m, self.create_input(datatypes.STR_PK, dataset_idx=1)) self.add_outputs(m, self.create_output(datatypes.STR_PK, dataset_idx=1)) step = PipelineStep(pipeline=pipeline, transformation=m, step_num=prev_step.step_num + 1) pipeline.steps.add(step) cable = PipelineStepInputCable( pipelinestep=step, source_step=prev_step.step_num, source=prev_step.transformation.outputs[0], dest=m.inputs[0]) cable.pipelinestepinputcable = cable step.cables_in.add(cable) outcable = PipelineOutputCable(pipeline=pipeline, output_idx=step.step_num, source_step=step.step_num, source=m.outputs[0], output_cdt=m.outputs[0].get_cdt()) pipeline.outcables.add(outcable) return step
def test_no_outputs_checkOutputIndices_good(self): """Test output index check, one well-indexed output case.""" driver = CodeResourceRevision(coderesource=CodeResource()) foo = Method(driver=driver, family=MethodFamily()) foo.check_output_indices() foo.clean()
def setUp(self): patcher = mocked_relations(Method, MethodDependency, Transformation) patcher.start() self.addCleanup(patcher.stop) driver = CodeResourceRevision( coderesource=CodeResource(filename='driver.py')) self.method = Method(driver=driver, family=MethodFamily()) self.dependency = self.add_dependency('helper.py')
def test_without_family_str(self): """ Test unicode representation when family is unset. """ nofamily = Method(revision_name="foo") self.assertEqual(str(nofamily), "[family unset]:None (foo)")
def test_copy_io_from_no_parent(self): foo = Method() foo.copy_io_from_parent() self.assertEqual(set(), {inp.dataset_idx for inp in foo.inputs}) self.assertEqual(set(), {out.dataset_idx for out in foo.outputs})
def test_pipeline_many_valid_steps_clean(self): """Test step index check, well-indexed multi-step case.""" p = Pipeline(family=PipelineFamily()) self.add_inputs(p, TransformationInput(dataset_idx=1)) m = Method() self.add_inputs(m, TransformationInput(dataset_idx=1)) p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=2)) p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=1)) p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=3)) p.clean()
def test_one_valid_output_checkOutputIndices_good(self): """Test output index check, one well-indexed output case.""" driver = CodeResourceRevision(coderesource=CodeResource()) foo = Method(driver=driver, family=MethodFamily()) out = foo.outputs.create(dataset_idx=1) out.transformationoutput = out foo.check_output_indices() foo.clean()
def test_many_valid_outputs_scrambled_checkOutputIndices_good(self): """Test output index check, well-indexed multi-output (scrambled order) case.""" driver = CodeResourceRevision(coderesource=CodeResource()) foo = Method(driver=driver, family=MethodFamily()) for i in (3, 1, 2): out = foo.outputs.create(dataset_idx=i) out.transformationoutput = out foo.check_output_indices() foo.clean()
def test_copy_io_from_parent(self): parent = self.create_parent() expected_inputs = {inp.dataset_idx for inp in parent.inputs} expected_outputs = {out.dataset_idx for out in parent.outputs} foo = Method(revision_parent=parent) foo.copy_io_from_parent() self.assertEqual(expected_inputs, {inp.dataset_idx for inp in foo.inputs}) self.assertEqual(expected_outputs, {out.dataset_idx for out in foo.outputs})
def test_pipeline_many_invalid_steps_clean(self): """Test step index check, badly-indexed multi-step case.""" p = Pipeline(family=PipelineFamily()) self.add_inputs(p, TransformationInput(dataset_idx=1)) m = Method() self.add_inputs(m, TransformationInput(dataset_idx=1)) p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=1)) p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=4)) p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=5)) self.assertRaisesRegexp( ValidationError, "Steps are not consecutively numbered starting from 1", p.clean)
def create_parent(self): parent = Method() parent.inputs = MockSet(name='parent.inputs', model=TransformationInput) parent.outputs = MockSet(name='parent.outputs', model=TransformationOutput) for i in range(2): inp = parent.inputs.create(dataset_idx=i + 1) inp.transformationinput = inp for i in range(3): out = parent.outputs.create(dataset_idx=i + 1) out.transformationoutput = out return parent
def test_no_inputs_checkInputIndices_good(self): """ Method with no inputs defined should have check_input_indices() return with no exception. """ driver = CodeResourceRevision(coderesource=CodeResource()) foo = Method(driver=driver, family=MethodFamily()) # check_input_indices() should not raise a ValidationError foo.check_input_indices() foo.clean()
def test_single_valid_input_checkInputIndices_good(self): """ Method with a single, 1-indexed input should have check_input_indices() return with no exception. """ driver = CodeResourceRevision(coderesource=CodeResource()) foo = Method(driver=driver, family=MethodFamily()) inp = foo.inputs.create(dataset_idx=1) inp.transformationinput = inp # check_input_indices() should not raise a ValidationError foo.check_input_indices() foo.clean()
def test_many_ordered_valid_inputs_checkInputIndices_good(self): """ Test check_input_indices on a method with several inputs, correctly indexed and in order. """ driver = CodeResourceRevision(coderesource=CodeResource()) foo = Method(driver=driver, family=MethodFamily()) for i in range(3): inp = foo.inputs.create(dataset_idx=i + 1) inp.transformationinput = inp # check_input_indices() should not raise a ValidationError foo.check_input_indices() foo.clean()
def setUp(self): super(MethodViewMockTests, self).setUp() patcher = mocked_relations(KiveUser, MethodFamily, Method, CodeResource, CodeResourceRevision, CompoundDatatype, ContainerFamily, Container, Transformation, TransformationInput, TransformationOutput, User, Group) patcher.start() self.addCleanup(patcher.stop) # noinspection PyUnresolvedReferences patcher = patch.object(MethodFamily._meta, 'default_manager', MethodFamily.objects) patcher.start() self.addCleanup(patcher.stop) self.client = self.create_client() self.dev_group = Group(pk=groups.DEVELOPERS_PK) self.everyone = Group(pk=groups.EVERYONE_PK) Group.objects.add(self.dev_group, self.everyone) self.user = kive_user() self.user.groups.add(self.dev_group) self.other_user = User(pk=5) self.method_family = MethodFamily(pk='99', user=self.user) MethodFamily.objects.add(self.method_family) self.driver = CodeResourceRevision(user=self.user) self.driver.pk = 1337 # needed for viewing a method self.driver.coderesource = CodeResource() self.method = Method(pk='199', user=self.user) self.method.driver = self.driver self.method.family = self.method_family Method.objects.add(self.method) KiveUser.objects.add(KiveUser(pk=users.KIVE_USER_PK))
def test_one_invalid_output_checkOutputIndices_bad(self): """Test output index check, one badly-indexed output case.""" driver = CodeResourceRevision(coderesource=CodeResource()) foo = Method(driver=driver, family=MethodFamily()) out = foo.outputs.create(dataset_idx=4) out.transformationoutput = out self.assertRaisesRegexp( ValidationError, "Outputs are not consecutively numbered starting from 1", foo.check_output_indices) self.assertRaisesRegexp( ValidationError, "Outputs are not consecutively numbered starting from 1", foo.clean)
def test_pipeline_oneStep_outcable_references_invalid_output_clean(self): """Bad output cabling, request output not belonging to requested step""" with self.create_valid_pipeline() as p: unrelated_output = self.create_output(datatypes.STR_PK, dataset_idx=3) m2 = Method() m2.method = m2 unrelated_output.transformation = m2 outcable = p.outcables[0] outcable.source = unrelated_output self.assertRaisesRegexp( ValidationError, 'Transformation at step 1 does not produce output ".*"', outcable.clean) self.assertRaisesRegexp( ValidationError, 'Transformation at step 1 does not produce output ".*"', p.clean)
def test_many_nonconsective_inputs_scrambled_checkInputIndices_bad(self): """Test input index check, badly-indexed multi-input case.""" driver = CodeResourceRevision(coderesource=CodeResource()) foo = Method(driver=driver, family=MethodFamily()) for i in (2, 6, 1): inp = foo.inputs.create(dataset_idx=i) inp.transformationinput = inp self.assertRaisesRegexp( ValidationError, "Inputs are not consecutively numbered starting from 1", foo.check_input_indices) self.assertRaisesRegexp( ValidationError, "Inputs are not consecutively numbered starting from 1", foo.clean)
def test_one_invalid_input_checkInputIndices_bad(self): """ Test input index check, one badly-indexed input case. """ driver = CodeResourceRevision(coderesource=CodeResource()) foo = Method(driver=driver, family=MethodFamily()) inp = foo.inputs.create(dataset_idx=4) inp.transformationinput = inp # check_input_indices() should raise a ValidationError self.assertRaisesRegexp( ValidationError, "Inputs are not consecutively numbered starting from 1", foo.check_input_indices) self.assertRaisesRegexp( ValidationError, "Inputs are not consecutively numbered starting from 1", foo.clean)
def test_pipeline_manySteps_outcable_references_invalid_output_clean(self): """Bad output cabling, chained-step pipeline: request output not belonging to requested step""" with self.create_valid_pipeline() as p: self.add_step(p) outcable1, outcable2 = p.outcables.all() unrelated_output = self.create_output(datatypes.STR_PK, dataset_idx=3) m3 = Method() m3.method = m3 unrelated_output.transformation = m3 outcable2.source = unrelated_output self.assertEquals(outcable1.clean(), None) self.assertRaisesRegexp( ValidationError, 'Transformation at step 2 does not produce output ".*"', outcable2.clean) self.assertRaisesRegexp( ValidationError, 'Transformation at step 2 does not produce output ".*"', p.clean)
def test_pipeline_oneStep_invalid_cabling_incorrect_cdt_clean(self): """Bad cabling: input is of wrong CompoundDatatype.""" p = Pipeline(family=PipelineFamily()) self.add_inputs(p, self.create_input(datatypes.INT_PK, dataset_idx=1)) m = Method() self.add_inputs(m, self.create_input(datatypes.STR_PK, dataset_idx=1)) step1 = PipelineStep(pipeline=p, transformation=m, step_num=1) p.steps.add(step1) cable = PipelineStepInputCable(pipelinestep=step1, source_step=0, source=p.inputs.all()[0], dest=m.inputs.all()[0]) cable.pipelinestepinputcable = cable step1.cables_in.add(cable) cable.clean() self.assertRaisesRegexp( ValidationError, 'Custom wiring required for cable "{}"'.format(cable), cable.clean_and_completely_wired)
def test_identical_self(self): """A Method should be identical to itself.""" m = Method(driver=CodeResourceRevision(), user=User()) self.assertTrue(m.is_identical(m))
def create_method_from_forms(family_form, method_form, dep_forms, input_forms, output_forms, creating_user, family=None, parent_method=None): """ Given Forms representing the MethodFamily, Method, inputs, and outputs, create a Method. Warning: this routine has side effects (it can mod its arguments): If an error occurs: return None and update the forms with errors. else: return the new method and the forms are returned without modification. """ # This assures that not both family_form and family are None. assert family is not None or family_form is not None for dep_form in dep_forms: assert isinstance(dep_form, MethodDependencyForm) or dep_form is None # Retrieve the CodeResource revision as driver. try: coderesource_revision = CodeResourceRevision.objects.get( pk=method_form.cleaned_data['driver_revisions']) except CodeResourceRevision.DoesNotExist: coderesource_revision = None # Retrieve the Container. try: container = Container.objects.get( pk=method_form.cleaned_data['container']) except Container.DoesNotExist: container = None new_method = None try: # Note how the except blocks re-raise their exception: that is to terminate # this transaction. with transaction.atomic(): if family is None: try: family = family_form.save() family.grant_from_json( method_form.cleaned_data["permissions"]) family.full_clean() except ValidationError as e: family_form.add_error(None, e) raise e new_method = Method( family=family, revision_name=method_form.cleaned_data['revision_name'], revision_desc=method_form.cleaned_data['revision_desc'], revision_parent=parent_method, driver=coderesource_revision, container=container, reusable=method_form.cleaned_data['reusable'], user=creating_user, threads=method_form.cleaned_data["threads"], memory=method_form.cleaned_data["memory"]) new_method.save() new_method.grant_from_json(method_form.cleaned_data["permissions"]) # Bind dependencies. for i in range(len(dep_forms)): if dep_forms[i] is None: continue try: on_revision = CodeResourceRevision.objects.get( pk=dep_forms[i].cleaned_data["revisions"]) dependency = MethodDependency( method=new_method, requirement=on_revision, path=dep_forms[i].cleaned_data["path"], filename=dep_forms[i].cleaned_data["filename"]) dependency.full_clean() dependency.save() except ValidationError as e: dep_forms[i].add_error(None, e) raise e # Attempt to make in/outputs. num_outputs = len(output_forms) if num_outputs == 0: method_form.add_error(None, "You must specify at least one output.") raise ValidationError("You must specify at least one output.") for xput_type in ("in", "out"): curr_forms = input_forms if xput_type == "out": curr_forms = output_forms for form_tuple in curr_forms: t_form = form_tuple[0] xs_form = form_tuple[1] dataset_name = t_form.cleaned_data["dataset_name"] cdt_id = xs_form.cleaned_data["compounddatatype"] if dataset_name == '' and cdt_id == '': # ignore blank form continue my_compound_datatype = None min_row = None max_row = None if cdt_id != '__raw__': try: my_compound_datatype = CompoundDatatype.objects.get( pk=cdt_id) min_row = xs_form.cleaned_data["min_row"] max_row = xs_form.cleaned_data["max_row"] except (ValueError, CompoundDatatype.DoesNotExist) as e: xs_form.add_error("compounddatatype", e) raise e curr_xput = new_method.create_xput( dataset_name=dataset_name, compounddatatype=my_compound_datatype, row_limits=(min_row, max_row), input=(xput_type == "in"), clean=False) if cdt_id != "__raw__": try: curr_xput.structure.clean() except ValidationError as e: xs_form.add_error(None, e) raise e try: curr_xput.clean() except ValidationError as e: t_form.add_error(None, e) raise e try: new_method.complete_clean() except ValidationError as e: method_form.add_error(None, e) raise e except ValidationError: return None return new_method
def test_display_name(self): method = Method(revision_number=1, revision_name='Example') self.assertEqual(method.display_name, '1: Example')
def test_display_name_without_revision_name(self): method = Method(revision_number=1) self.assertEqual(method.display_name, '1: ')