def test_identical_when_drivers_unmatched(self):
driver1 = CodeResourceRevision()
driver2 = CodeResourceRevision()
user = User()
m1 = Method(revision_name='A', driver=driver1, user=user)
for i in range(2):
inp = m1.inputs.create(dataset_name='a_in_{}'.format(i),
dataset_idx=i + 1)
inp.transformationinput = inp
for i in range(3):
out = m1.outputs.create(dataset_name='a_out_{}'.format(i),
dataset_idx=i + 1)
out.transformationoutput = out
m2 = Method(revision_name='B', driver=driver2, user=user)
for i in range(2):
inp = m2.inputs.create(dataset_name='b_in_{}'.format(i),
dataset_idx=i + 1)
inp.transformationinput = inp
for i in range(3):
out = m2.outputs.create(dataset_name='b_in_{}'.format(i),
dataset_idx=i + 1)
out.transformationoutput = out
self.assertFalse(m1.is_identical(m2))
def test_copy_io_from_parent_with_structure(self):
cdt = CompoundDatatype()
min_row = 1
max_row = 100
structure = XputStructure(compounddatatype=cdt,
min_row=min_row,
max_row=max_row)
parent = self.create_parent()
def get_structure(xput_self):
if xput_self.dataset_idx == 1:
return structure
# noinspection PyUnresolvedReferences
raise XputStructure.DoesNotExist
TransformationXput.structure = property(get_structure)
expected_inputs = {inp.dataset_idx for inp in parent.inputs}
expected_outputs = {out.dataset_idx for out in parent.outputs}
foo = Method(revision_parent=parent)
foo.copy_io_from_parent()
self.assertEqual(expected_inputs,
{inp.dataset_idx for inp in foo.inputs})
self.assertEqual(expected_outputs,
{out.dataset_idx for out in foo.outputs})
# noinspection PyUnresolvedReferences
create_args = XputStructure.objects.create.call_args_list # @UndefinedVariable
self.assertEqual(2, len(create_args))
_args, kwargs = create_args[0]
self.assertEqual(100, kwargs['max_row'])
def create_valid_pipeline(self):
p = Pipeline(family=PipelineFamily())
self.add_inputs(p, self.create_input(datatypes.STR_PK, dataset_idx=1))
m = Method()
m.method = m
self.add_inputs(m, self.create_input(datatypes.STR_PK, dataset_idx=1))
self.add_outputs(m, self.create_output(datatypes.STR_PK,
dataset_idx=1))
step1 = PipelineStep(pipeline=p, transformation=m, step_num=1)
p.steps.add(step1)
cable = PipelineStepInputCable(pipelinestep=step1,
source_step=0,
source=p.inputs.all()[0],
dest=m.inputs.all()[0])
cable.pipelinestepinputcable = cable
step1.cables_in.add(cable)
outcable = PipelineOutputCable(pipeline=p,
output_idx=1,
source_step=1,
source=m.outputs.all()[0],
output_cdt=m.outputs.all()[0].get_cdt())
p.outcables.add(outcable)
yield p
def test_with_family_str(self):
""" expect "Method revision name and family name" """
family = MethodFamily(name="Example")
method = Method(revision_name="rounded", revision_number=3, family=family)
self.assertEqual(str(method),
"Example:3 (rounded)")
def add_step(self, pipeline):
prev_step = pipeline.steps[-1]
m = Method()
m.method = m
self.add_inputs(m, self.create_input(datatypes.STR_PK, dataset_idx=1))
self.add_outputs(m, self.create_output(datatypes.STR_PK,
dataset_idx=1))
step = PipelineStep(pipeline=pipeline,
transformation=m,
step_num=prev_step.step_num + 1)
pipeline.steps.add(step)
cable = PipelineStepInputCable(
pipelinestep=step,
source_step=prev_step.step_num,
source=prev_step.transformation.outputs[0],
dest=m.inputs[0])
cable.pipelinestepinputcable = cable
step.cables_in.add(cable)
outcable = PipelineOutputCable(pipeline=pipeline,
output_idx=step.step_num,
source_step=step.step_num,
source=m.outputs[0],
output_cdt=m.outputs[0].get_cdt())
pipeline.outcables.add(outcable)
return step
def test_no_outputs_checkOutputIndices_good(self):
"""Test output index check, one well-indexed output case."""
driver = CodeResourceRevision(coderesource=CodeResource())
foo = Method(driver=driver, family=MethodFamily())
foo.check_output_indices()
foo.clean()
def setUp(self):
patcher = mocked_relations(Method, MethodDependency, Transformation)
patcher.start()
self.addCleanup(patcher.stop)
driver = CodeResourceRevision(
coderesource=CodeResource(filename='driver.py'))
self.method = Method(driver=driver, family=MethodFamily())
self.dependency = self.add_dependency('helper.py')
def test_without_family_str(self):
"""
Test unicode representation when family is unset.
"""
nofamily = Method(revision_name="foo")
self.assertEqual(str(nofamily),
"[family unset]:None (foo)")
def test_copy_io_from_no_parent(self):
foo = Method()
foo.copy_io_from_parent()
self.assertEqual(set(),
{inp.dataset_idx for inp in foo.inputs})
self.assertEqual(set(),
{out.dataset_idx for out in foo.outputs})
def test_pipeline_many_valid_steps_clean(self):
"""Test step index check, well-indexed multi-step case."""
p = Pipeline(family=PipelineFamily())
self.add_inputs(p, TransformationInput(dataset_idx=1))
m = Method()
self.add_inputs(m, TransformationInput(dataset_idx=1))
p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=2))
p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=1))
p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=3))
p.clean()
def test_one_valid_output_checkOutputIndices_good(self):
"""Test output index check, one well-indexed output case."""
driver = CodeResourceRevision(coderesource=CodeResource())
foo = Method(driver=driver, family=MethodFamily())
out = foo.outputs.create(dataset_idx=1)
out.transformationoutput = out
foo.check_output_indices()
foo.clean()
def test_many_valid_outputs_scrambled_checkOutputIndices_good(self):
"""Test output index check, well-indexed multi-output (scrambled order) case."""
driver = CodeResourceRevision(coderesource=CodeResource())
foo = Method(driver=driver, family=MethodFamily())
for i in (3, 1, 2):
out = foo.outputs.create(dataset_idx=i)
out.transformationoutput = out
foo.check_output_indices()
foo.clean()
def test_copy_io_from_parent(self):
parent = self.create_parent()
expected_inputs = {inp.dataset_idx for inp in parent.inputs}
expected_outputs = {out.dataset_idx for out in parent.outputs}
foo = Method(revision_parent=parent)
foo.copy_io_from_parent()
self.assertEqual(expected_inputs,
{inp.dataset_idx for inp in foo.inputs})
self.assertEqual(expected_outputs,
{out.dataset_idx for out in foo.outputs})
def test_pipeline_many_invalid_steps_clean(self):
"""Test step index check, badly-indexed multi-step case."""
p = Pipeline(family=PipelineFamily())
self.add_inputs(p, TransformationInput(dataset_idx=1))
m = Method()
self.add_inputs(m, TransformationInput(dataset_idx=1))
p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=1))
p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=4))
p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=5))
self.assertRaisesRegexp(
ValidationError,
"Steps are not consecutively numbered starting from 1", p.clean)
def create_parent(self):
parent = Method()
parent.inputs = MockSet(name='parent.inputs',
model=TransformationInput)
parent.outputs = MockSet(name='parent.outputs',
model=TransformationOutput)
for i in range(2):
inp = parent.inputs.create(dataset_idx=i + 1)
inp.transformationinput = inp
for i in range(3):
out = parent.outputs.create(dataset_idx=i + 1)
out.transformationoutput = out
return parent
def test_no_inputs_checkInputIndices_good(self):
"""
Method with no inputs defined should have
check_input_indices() return with no exception.
"""
driver = CodeResourceRevision(coderesource=CodeResource())
foo = Method(driver=driver, family=MethodFamily())
# check_input_indices() should not raise a ValidationError
foo.check_input_indices()
foo.clean()
def test_single_valid_input_checkInputIndices_good(self):
"""
Method with a single, 1-indexed input should have
check_input_indices() return with no exception.
"""
driver = CodeResourceRevision(coderesource=CodeResource())
foo = Method(driver=driver, family=MethodFamily())
inp = foo.inputs.create(dataset_idx=1)
inp.transformationinput = inp
# check_input_indices() should not raise a ValidationError
foo.check_input_indices()
foo.clean()
def test_many_ordered_valid_inputs_checkInputIndices_good(self):
"""
Test check_input_indices on a method with several inputs,
correctly indexed and in order.
"""
driver = CodeResourceRevision(coderesource=CodeResource())
foo = Method(driver=driver, family=MethodFamily())
for i in range(3):
inp = foo.inputs.create(dataset_idx=i + 1)
inp.transformationinput = inp
# check_input_indices() should not raise a ValidationError
foo.check_input_indices()
foo.clean()
def setUp(self):
super(MethodViewMockTests, self).setUp()
patcher = mocked_relations(KiveUser,
MethodFamily,
Method,
CodeResource,
CodeResourceRevision,
CompoundDatatype,
ContainerFamily,
Container,
Transformation,
TransformationInput,
TransformationOutput,
User,
Group)
patcher.start()
self.addCleanup(patcher.stop)
# noinspection PyUnresolvedReferences
patcher = patch.object(MethodFamily._meta,
'default_manager',
MethodFamily.objects)
patcher.start()
self.addCleanup(patcher.stop)
self.client = self.create_client()
self.dev_group = Group(pk=groups.DEVELOPERS_PK)
self.everyone = Group(pk=groups.EVERYONE_PK)
Group.objects.add(self.dev_group, self.everyone)
self.user = kive_user()
self.user.groups.add(self.dev_group)
self.other_user = User(pk=5)
self.method_family = MethodFamily(pk='99',
user=self.user)
MethodFamily.objects.add(self.method_family)
self.driver = CodeResourceRevision(user=self.user)
self.driver.pk = 1337 # needed for viewing a method
self.driver.coderesource = CodeResource()
self.method = Method(pk='199', user=self.user)
self.method.driver = self.driver
self.method.family = self.method_family
Method.objects.add(self.method)
KiveUser.objects.add(KiveUser(pk=users.KIVE_USER_PK))
def test_one_invalid_output_checkOutputIndices_bad(self):
"""Test output index check, one badly-indexed output case."""
driver = CodeResourceRevision(coderesource=CodeResource())
foo = Method(driver=driver, family=MethodFamily())
out = foo.outputs.create(dataset_idx=4)
out.transformationoutput = out
self.assertRaisesRegexp(
ValidationError,
"Outputs are not consecutively numbered starting from 1",
foo.check_output_indices)
self.assertRaisesRegexp(
ValidationError,
"Outputs are not consecutively numbered starting from 1",
foo.clean)
def test_pipeline_oneStep_outcable_references_invalid_output_clean(self):
"""Bad output cabling, request output not belonging to requested step"""
with self.create_valid_pipeline() as p:
unrelated_output = self.create_output(datatypes.STR_PK,
dataset_idx=3)
m2 = Method()
m2.method = m2
unrelated_output.transformation = m2
outcable = p.outcables[0]
outcable.source = unrelated_output
self.assertRaisesRegexp(
ValidationError,
'Transformation at step 1 does not produce output ".*"',
outcable.clean)
self.assertRaisesRegexp(
ValidationError,
'Transformation at step 1 does not produce output ".*"',
p.clean)
def test_many_nonconsective_inputs_scrambled_checkInputIndices_bad(self):
"""Test input index check, badly-indexed multi-input case."""
driver = CodeResourceRevision(coderesource=CodeResource())
foo = Method(driver=driver, family=MethodFamily())
for i in (2, 6, 1):
inp = foo.inputs.create(dataset_idx=i)
inp.transformationinput = inp
self.assertRaisesRegexp(
ValidationError,
"Inputs are not consecutively numbered starting from 1",
foo.check_input_indices)
self.assertRaisesRegexp(
ValidationError,
"Inputs are not consecutively numbered starting from 1",
foo.clean)
def test_one_invalid_input_checkInputIndices_bad(self):
"""
Test input index check, one badly-indexed input case.
"""
driver = CodeResourceRevision(coderesource=CodeResource())
foo = Method(driver=driver, family=MethodFamily())
inp = foo.inputs.create(dataset_idx=4)
inp.transformationinput = inp
# check_input_indices() should raise a ValidationError
self.assertRaisesRegexp(
ValidationError,
"Inputs are not consecutively numbered starting from 1",
foo.check_input_indices)
self.assertRaisesRegexp(
ValidationError,
"Inputs are not consecutively numbered starting from 1",
foo.clean)
def test_pipeline_manySteps_outcable_references_invalid_output_clean(self):
"""Bad output cabling, chained-step pipeline: request output not belonging to requested step"""
with self.create_valid_pipeline() as p:
self.add_step(p)
outcable1, outcable2 = p.outcables.all()
unrelated_output = self.create_output(datatypes.STR_PK,
dataset_idx=3)
m3 = Method()
m3.method = m3
unrelated_output.transformation = m3
outcable2.source = unrelated_output
self.assertEquals(outcable1.clean(), None)
self.assertRaisesRegexp(
ValidationError,
'Transformation at step 2 does not produce output ".*"',
outcable2.clean)
self.assertRaisesRegexp(
ValidationError,
'Transformation at step 2 does not produce output ".*"',
p.clean)
def test_pipeline_oneStep_invalid_cabling_incorrect_cdt_clean(self):
"""Bad cabling: input is of wrong CompoundDatatype."""
p = Pipeline(family=PipelineFamily())
self.add_inputs(p, self.create_input(datatypes.INT_PK, dataset_idx=1))
m = Method()
self.add_inputs(m, self.create_input(datatypes.STR_PK, dataset_idx=1))
step1 = PipelineStep(pipeline=p, transformation=m, step_num=1)
p.steps.add(step1)
cable = PipelineStepInputCable(pipelinestep=step1,
source_step=0,
source=p.inputs.all()[0],
dest=m.inputs.all()[0])
cable.pipelinestepinputcable = cable
step1.cables_in.add(cable)
cable.clean()
self.assertRaisesRegexp(
ValidationError,
'Custom wiring required for cable "{}"'.format(cable),
cable.clean_and_completely_wired)
def test_identical_self(self):
"""A Method should be identical to itself."""
m = Method(driver=CodeResourceRevision(), user=User())
self.assertTrue(m.is_identical(m))
def create_method_from_forms(family_form,
method_form,
dep_forms,
input_forms,
output_forms,
creating_user,
family=None,
parent_method=None):
"""
Given Forms representing the MethodFamily, Method, inputs, and outputs, create a Method.
Warning: this routine has side effects (it can mod its arguments):
If an error occurs:
return None and update the forms with errors.
else:
return the new method and the forms are returned without modification.
"""
# This assures that not both family_form and family are None.
assert family is not None or family_form is not None
for dep_form in dep_forms:
assert isinstance(dep_form, MethodDependencyForm) or dep_form is None
# Retrieve the CodeResource revision as driver.
try:
coderesource_revision = CodeResourceRevision.objects.get(
pk=method_form.cleaned_data['driver_revisions'])
except CodeResourceRevision.DoesNotExist:
coderesource_revision = None
# Retrieve the Container.
try:
container = Container.objects.get(
pk=method_form.cleaned_data['container'])
except Container.DoesNotExist:
container = None
new_method = None
try:
# Note how the except blocks re-raise their exception: that is to terminate
# this transaction.
with transaction.atomic():
if family is None:
try:
family = family_form.save()
family.grant_from_json(
method_form.cleaned_data["permissions"])
family.full_clean()
except ValidationError as e:
family_form.add_error(None, e)
raise e
new_method = Method(
family=family,
revision_name=method_form.cleaned_data['revision_name'],
revision_desc=method_form.cleaned_data['revision_desc'],
revision_parent=parent_method,
driver=coderesource_revision,
container=container,
reusable=method_form.cleaned_data['reusable'],
user=creating_user,
threads=method_form.cleaned_data["threads"],
memory=method_form.cleaned_data["memory"])
new_method.save()
new_method.grant_from_json(method_form.cleaned_data["permissions"])
# Bind dependencies.
for i in range(len(dep_forms)):
if dep_forms[i] is None:
continue
try:
on_revision = CodeResourceRevision.objects.get(
pk=dep_forms[i].cleaned_data["revisions"])
dependency = MethodDependency(
method=new_method,
requirement=on_revision,
path=dep_forms[i].cleaned_data["path"],
filename=dep_forms[i].cleaned_data["filename"])
dependency.full_clean()
dependency.save()
except ValidationError as e:
dep_forms[i].add_error(None, e)
raise e
# Attempt to make in/outputs.
num_outputs = len(output_forms)
if num_outputs == 0:
method_form.add_error(None,
"You must specify at least one output.")
raise ValidationError("You must specify at least one output.")
for xput_type in ("in", "out"):
curr_forms = input_forms
if xput_type == "out":
curr_forms = output_forms
for form_tuple in curr_forms:
t_form = form_tuple[0]
xs_form = form_tuple[1]
dataset_name = t_form.cleaned_data["dataset_name"]
cdt_id = xs_form.cleaned_data["compounddatatype"]
if dataset_name == '' and cdt_id == '':
# ignore blank form
continue
my_compound_datatype = None
min_row = None
max_row = None
if cdt_id != '__raw__':
try:
my_compound_datatype = CompoundDatatype.objects.get(
pk=cdt_id)
min_row = xs_form.cleaned_data["min_row"]
max_row = xs_form.cleaned_data["max_row"]
except (ValueError,
CompoundDatatype.DoesNotExist) as e:
xs_form.add_error("compounddatatype", e)
raise e
curr_xput = new_method.create_xput(
dataset_name=dataset_name,
compounddatatype=my_compound_datatype,
row_limits=(min_row, max_row),
input=(xput_type == "in"),
clean=False)
if cdt_id != "__raw__":
try:
curr_xput.structure.clean()
except ValidationError as e:
xs_form.add_error(None, e)
raise e
try:
curr_xput.clean()
except ValidationError as e:
t_form.add_error(None, e)
raise e
try:
new_method.complete_clean()
except ValidationError as e:
method_form.add_error(None, e)
raise e
except ValidationError:
return None
return new_method
def test_display_name(self):
method = Method(revision_number=1, revision_name='Example')
self.assertEqual(method.display_name, '1: Example')
def test_display_name_without_revision_name(self):
method = Method(revision_number=1)
self.assertEqual(method.display_name, '1: ')