def test_copy_io_from_parent_with_structure(self):
cdt = CompoundDatatype()
min_row = 1
max_row = 100
structure = XputStructure(compounddatatype=cdt,
min_row=min_row,
max_row=max_row)
parent = self.create_parent()
def get_structure(xput_self):
if xput_self.dataset_idx == 1:
return structure
# noinspection PyUnresolvedReferences
raise XputStructure.DoesNotExist
TransformationXput.structure = property(get_structure)
expected_inputs = {inp.dataset_idx for inp in parent.inputs}
expected_outputs = {out.dataset_idx for out in parent.outputs}
foo = Method(revision_parent=parent)
foo.copy_io_from_parent()
self.assertEqual(expected_inputs,
{inp.dataset_idx for inp in foo.inputs})
self.assertEqual(expected_outputs,
{out.dataset_idx for out in foo.outputs})
# noinspection PyUnresolvedReferences
create_args = XputStructure.objects.create.call_args_list # @UndefinedVariable
self.assertEqual(2, len(create_args))
_args, kwargs = create_args[0]
self.assertEqual(100, kwargs['max_row'])
def test_identical_when_drivers_unmatched(self):
driver1 = CodeResourceRevision()
driver2 = CodeResourceRevision()
user = User()
m1 = Method(revision_name='A', driver=driver1, user=user)
for i in range(2):
inp = m1.inputs.create(dataset_name='a_in_{}'.format(i),
dataset_idx=i + 1)
inp.transformationinput = inp
for i in range(3):
out = m1.outputs.create(dataset_name='a_out_{}'.format(i),
dataset_idx=i + 1)
out.transformationoutput = out
m2 = Method(revision_name='B', driver=driver2, user=user)
for i in range(2):
inp = m2.inputs.create(dataset_name='b_in_{}'.format(i),
dataset_idx=i + 1)
inp.transformationinput = inp
for i in range(3):
out = m2.outputs.create(dataset_name='b_in_{}'.format(i),
dataset_idx=i + 1)
out.transformationoutput = out
self.assertFalse(m1.is_identical(m2))
def create_valid_pipeline(self):
p = Pipeline(family=PipelineFamily())
self.add_inputs(p, self.create_input(datatypes.STR_PK, dataset_idx=1))
m = Method()
m.method = m
self.add_inputs(m, self.create_input(datatypes.STR_PK, dataset_idx=1))
self.add_outputs(m, self.create_output(datatypes.STR_PK,
dataset_idx=1))
step1 = PipelineStep(pipeline=p, transformation=m, step_num=1)
p.steps.add(step1)
cable = PipelineStepInputCable(pipelinestep=step1,
source_step=0,
source=p.inputs.all()[0],
dest=m.inputs.all()[0])
cable.pipelinestepinputcable = cable
step1.cables_in.add(cable)
outcable = PipelineOutputCable(pipeline=p,
output_idx=1,
source_step=1,
source=m.outputs.all()[0],
output_cdt=m.outputs.all()[0].get_cdt())
p.outcables.add(outcable)
yield p
def add_step(self, pipeline):
prev_step = pipeline.steps[-1]
m = Method()
m.method = m
self.add_inputs(m, self.create_input(datatypes.STR_PK, dataset_idx=1))
self.add_outputs(m, self.create_output(datatypes.STR_PK,
dataset_idx=1))
step = PipelineStep(pipeline=pipeline,
transformation=m,
step_num=prev_step.step_num + 1)
pipeline.steps.add(step)
cable = PipelineStepInputCable(
pipelinestep=step,
source_step=prev_step.step_num,
source=prev_step.transformation.outputs[0],
dest=m.inputs[0])
cable.pipelinestepinputcable = cable
step.cables_in.add(cable)
outcable = PipelineOutputCable(pipeline=pipeline,
output_idx=step.step_num,
source_step=step.step_num,
source=m.outputs[0],
output_cdt=m.outputs[0].get_cdt())
pipeline.outcables.add(outcable)
return step
def test_copy_io_from_no_parent(self):
foo = Method()
foo.copy_io_from_parent()
self.assertEqual(set(),
{inp.dataset_idx for inp in foo.inputs})
self.assertEqual(set(),
{out.dataset_idx for out in foo.outputs})
def setUp(self):
patcher = mocked_relations(Method, MethodDependency, Transformation)
patcher.start()
self.addCleanup(patcher.stop)
driver = CodeResourceRevision(
coderesource=CodeResource(filename='driver.py'))
self.method = Method(driver=driver, family=MethodFamily())
self.dependency = self.add_dependency('helper.py')
def test_copy_io_from_parent(self):
parent = self.create_parent()
expected_inputs = {inp.dataset_idx for inp in parent.inputs}
expected_outputs = {out.dataset_idx for out in parent.outputs}
foo = Method(revision_parent=parent)
foo.copy_io_from_parent()
self.assertEqual(expected_inputs,
{inp.dataset_idx for inp in foo.inputs})
self.assertEqual(expected_outputs,
{out.dataset_idx for out in foo.outputs})
def create_parent(self):
parent = Method()
parent.inputs = MockSet(name='parent.inputs',
model=TransformationInput)
parent.outputs = MockSet(name='parent.outputs',
model=TransformationOutput)
for i in range(2):
inp = parent.inputs.create(dataset_idx=i + 1)
inp.transformationinput = inp
for i in range(3):
out = parent.outputs.create(dataset_idx=i + 1)
out.transformationoutput = out
return parent
def test_with_family_str(self):
""" expect "Method revision name and family name" """
family = MethodFamily(name="Example")
method = Method(revision_name="rounded", revision_number=3, family=family)
self.assertEqual(str(method),
"Example:3 (rounded)")
def test_without_family_str(self):
"""
Test unicode representation when family is unset.
"""
nofamily = Method(revision_name="foo")
self.assertEqual(str(nofamily),
"[family unset]:None (foo)")
def test_pipeline_oneStep_outcable_references_invalid_output_clean(self):
"""Bad output cabling, request output not belonging to requested step"""
with self.create_valid_pipeline() as p:
unrelated_output = self.create_output(datatypes.STR_PK,
dataset_idx=3)
m2 = Method()
m2.method = m2
unrelated_output.transformation = m2
outcable = p.outcables[0]
outcable.source = unrelated_output
self.assertRaisesRegexp(
ValidationError,
'Transformation at step 1 does not produce output ".*"',
outcable.clean)
self.assertRaisesRegexp(
ValidationError,
'Transformation at step 1 does not produce output ".*"',
p.clean)
def test_pipeline_many_valid_steps_clean(self):
"""Test step index check, well-indexed multi-step case."""
p = Pipeline(family=PipelineFamily())
self.add_inputs(p, TransformationInput(dataset_idx=1))
m = Method()
self.add_inputs(m, TransformationInput(dataset_idx=1))
p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=2))
p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=1))
p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=3))
p.clean()
def test_pipeline_many_invalid_steps_clean(self):
"""Test step index check, badly-indexed multi-step case."""
p = Pipeline(family=PipelineFamily())
self.add_inputs(p, TransformationInput(dataset_idx=1))
m = Method()
self.add_inputs(m, TransformationInput(dataset_idx=1))
p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=1))
p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=4))
p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=5))
self.assertRaisesRegexp(
ValidationError,
"Steps are not consecutively numbered starting from 1", p.clean)
def test_pipeline_manySteps_outcable_references_invalid_output_clean(self):
"""Bad output cabling, chained-step pipeline: request output not belonging to requested step"""
with self.create_valid_pipeline() as p:
self.add_step(p)
outcable1, outcable2 = p.outcables.all()
unrelated_output = self.create_output(datatypes.STR_PK,
dataset_idx=3)
m3 = Method()
m3.method = m3
unrelated_output.transformation = m3
outcable2.source = unrelated_output
self.assertEquals(outcable1.clean(), None)
self.assertRaisesRegexp(
ValidationError,
'Transformation at step 2 does not produce output ".*"',
outcable2.clean)
self.assertRaisesRegexp(
ValidationError,
'Transformation at step 2 does not produce output ".*"',
p.clean)
def test_no_outputs_checkOutputIndices_good(self):
"""Test output index check, one well-indexed output case."""
driver = CodeResourceRevision(coderesource=CodeResource())
foo = Method(driver=driver, family=MethodFamily())
foo.check_output_indices()
foo.clean()
def test_one_valid_output_checkOutputIndices_good(self):
"""Test output index check, one well-indexed output case."""
driver = CodeResourceRevision(coderesource=CodeResource())
foo = Method(driver=driver, family=MethodFamily())
out = foo.outputs.create(dataset_idx=1)
out.transformationoutput = out
foo.check_output_indices()
foo.clean()
def test_many_valid_outputs_scrambled_checkOutputIndices_good(self):
"""Test output index check, well-indexed multi-output (scrambled order) case."""
driver = CodeResourceRevision(coderesource=CodeResource())
foo = Method(driver=driver, family=MethodFamily())
for i in (3, 1, 2):
out = foo.outputs.create(dataset_idx=i)
out.transformationoutput = out
foo.check_output_indices()
foo.clean()
def setUp(self):
super(MethodViewMockTests, self).setUp()
patcher = mocked_relations(KiveUser,
MethodFamily,
Method,
CodeResource,
CodeResourceRevision,
CompoundDatatype,
ContainerFamily,
Container,
Transformation,
TransformationInput,
TransformationOutput,
User,
Group)
patcher.start()
self.addCleanup(patcher.stop)
# noinspection PyUnresolvedReferences
patcher = patch.object(MethodFamily._meta,
'default_manager',
MethodFamily.objects)
patcher.start()
self.addCleanup(patcher.stop)
self.client = self.create_client()
self.dev_group = Group(pk=groups.DEVELOPERS_PK)
self.everyone = Group(pk=groups.EVERYONE_PK)
Group.objects.add(self.dev_group, self.everyone)
self.user = kive_user()
self.user.groups.add(self.dev_group)
self.other_user = User(pk=5)
self.method_family = MethodFamily(pk='99',
user=self.user)
MethodFamily.objects.add(self.method_family)
self.driver = CodeResourceRevision(user=self.user)
self.driver.pk = 1337 # needed for viewing a method
self.driver.coderesource = CodeResource()
self.method = Method(pk='199', user=self.user)
self.method.driver = self.driver
self.method.family = self.method_family
Method.objects.add(self.method)
KiveUser.objects.add(KiveUser(pk=users.KIVE_USER_PK))
def test_one_invalid_output_checkOutputIndices_bad(self):
"""Test output index check, one badly-indexed output case."""
driver = CodeResourceRevision(coderesource=CodeResource())
foo = Method(driver=driver, family=MethodFamily())
out = foo.outputs.create(dataset_idx=4)
out.transformationoutput = out
self.assertRaisesRegexp(
ValidationError,
"Outputs are not consecutively numbered starting from 1",
foo.check_output_indices)
self.assertRaisesRegexp(
ValidationError,
"Outputs are not consecutively numbered starting from 1",
foo.clean)
def test_no_inputs_checkInputIndices_good(self):
"""
Method with no inputs defined should have
check_input_indices() return with no exception.
"""
driver = CodeResourceRevision(coderesource=CodeResource())
foo = Method(driver=driver, family=MethodFamily())
# check_input_indices() should not raise a ValidationError
foo.check_input_indices()
foo.clean()
def test_many_nonconsective_inputs_scrambled_checkInputIndices_bad(self):
"""Test input index check, badly-indexed multi-input case."""
driver = CodeResourceRevision(coderesource=CodeResource())
foo = Method(driver=driver, family=MethodFamily())
for i in (2, 6, 1):
inp = foo.inputs.create(dataset_idx=i)
inp.transformationinput = inp
self.assertRaisesRegexp(
ValidationError,
"Inputs are not consecutively numbered starting from 1",
foo.check_input_indices)
self.assertRaisesRegexp(
ValidationError,
"Inputs are not consecutively numbered starting from 1",
foo.clean)
def __init__(self, *args, **kwargs):
super(MethodSerializer, self).__init__(*args, **kwargs)
request = self.context.get("request")
if request is not None:
curr_user = request.user
# Set the querysets of the related model fields.
revision_parent_field = self.fields["revision_parent"]
revision_parent_field.queryset = Method.filter_by_user(curr_user)
family_field = self.fields["family"]
family_field.queryset = MethodFamily.filter_by_user(curr_user)
driver_field = self.fields["driver"]
driver_field.queryset = CodeResourceRevision.filter_by_user(
curr_user)
docker_field = self.fields["docker_image"]
docker_field.queryset = DockerImage.filter_by_user(curr_user)
def test_single_valid_input_checkInputIndices_good(self):
"""
Method with a single, 1-indexed input should have
check_input_indices() return with no exception.
"""
driver = CodeResourceRevision(coderesource=CodeResource())
foo = Method(driver=driver, family=MethodFamily())
inp = foo.inputs.create(dataset_idx=1)
inp.transformationinput = inp
# check_input_indices() should not raise a ValidationError
foo.check_input_indices()
foo.clean()
def test_one_invalid_input_checkInputIndices_bad(self):
"""
Test input index check, one badly-indexed input case.
"""
driver = CodeResourceRevision(coderesource=CodeResource())
foo = Method(driver=driver, family=MethodFamily())
inp = foo.inputs.create(dataset_idx=4)
inp.transformationinput = inp
# check_input_indices() should raise a ValidationError
self.assertRaisesRegexp(
ValidationError,
"Inputs are not consecutively numbered starting from 1",
foo.check_input_indices)
self.assertRaisesRegexp(
ValidationError,
"Inputs are not consecutively numbered starting from 1",
foo.clean)
def test_many_ordered_valid_inputs_checkInputIndices_good(self):
"""
Test check_input_indices on a method with several inputs,
correctly indexed and in order.
"""
driver = CodeResourceRevision(coderesource=CodeResource())
foo = Method(driver=driver, family=MethodFamily())
for i in range(3):
inp = foo.inputs.create(dataset_idx=i + 1)
inp.transformationinput = inp
# check_input_indices() should not raise a ValidationError
foo.check_input_indices()
foo.clean()
def test_pipeline_oneStep_invalid_cabling_incorrect_cdt_clean(self):
"""Bad cabling: input is of wrong CompoundDatatype."""
p = Pipeline(family=PipelineFamily())
self.add_inputs(p, self.create_input(datatypes.INT_PK, dataset_idx=1))
m = Method()
self.add_inputs(m, self.create_input(datatypes.STR_PK, dataset_idx=1))
step1 = PipelineStep(pipeline=p, transformation=m, step_num=1)
p.steps.add(step1)
cable = PipelineStepInputCable(pipelinestep=step1,
source_step=0,
source=p.inputs.all()[0],
dest=m.inputs.all()[0])
cable.pipelinestepinputcable = cable
step1.cables_in.add(cable)
cable.clean()
self.assertRaisesRegexp(
ValidationError,
'Custom wiring required for cable "{}"'.format(cable),
cable.clean_and_completely_wired)
def create_method_from_forms(family_form, method_form, dep_forms, input_forms, output_forms, creating_user,
family=None, parent_method=None):
"""
Given Forms representing the MethodFamily, Method, inputs, and outputs, create a Method.
Warning: this routine has side effects (it can mod its arguments):
If an error occurs:
return None and update the forms with errors.
else:
return the new method and the forms are returned without modification.
"""
# This assures that not both family_form and family are None.
assert family is not None or family_form is not None
for dep_form in dep_forms:
assert isinstance(dep_form, MethodDependencyForm) or dep_form is None
# Retrieve the CodeResource revision as driver.
try:
coderesource_revision = CodeResourceRevision.objects.get(
pk=method_form.cleaned_data['driver_revisions'])
except CodeResourceRevision.DoesNotExist:
coderesource_revision = None
# Retrieve the Container.
try:
container = Container.objects.get(
pk=method_form.cleaned_data['container'])
except Container.DoesNotExist:
container = None
new_method = None
try:
# Note how the except blocks re-raise their exception: that is to terminate
# this transaction.
with transaction.atomic():
if family is None:
try:
family = family_form.save()
family.grant_from_json(method_form.cleaned_data["permissions"])
family.full_clean()
except ValidationError as e:
family_form.add_error(None, e)
raise e
new_method = Method(
family=family,
revision_name=method_form.cleaned_data['revision_name'],
revision_desc=method_form.cleaned_data['revision_desc'],
revision_parent=parent_method,
driver=coderesource_revision,
container=container,
reusable=method_form.cleaned_data['reusable'],
user=creating_user,
threads=method_form.cleaned_data["threads"],
memory=method_form.cleaned_data["memory"]
)
new_method.save()
new_method.grant_from_json(method_form.cleaned_data["permissions"])
# Bind dependencies.
for i in range(len(dep_forms)):
if dep_forms[i] is None:
continue
try:
on_revision = CodeResourceRevision.objects.get(
pk=dep_forms[i].cleaned_data["revisions"])
dependency = MethodDependency(
method=new_method,
requirement=on_revision,
path=dep_forms[i].cleaned_data["path"],
filename=dep_forms[i].cleaned_data["filename"]
)
dependency.full_clean()
dependency.save()
except ValidationError as e:
dep_forms[i].add_error(None, e)
raise e
# Attempt to make in/outputs.
num_outputs = len(output_forms)
if num_outputs == 0:
method_form.add_error(None, "You must specify at least one output.")
raise ValidationError("You must specify at least one output.")
for xput_type in ("in", "out"):
curr_forms = input_forms
if xput_type == "out":
curr_forms = output_forms
for form_tuple in curr_forms:
t_form = form_tuple[0]
xs_form = form_tuple[1]
dataset_name = t_form.cleaned_data["dataset_name"]
cdt_id = xs_form.cleaned_data["compounddatatype"]
if dataset_name == '' and cdt_id == '':
# ignore blank form
continue
my_compound_datatype = None
min_row = None
max_row = None
if cdt_id != '__raw__':
try:
my_compound_datatype = CompoundDatatype.objects.get(pk=cdt_id)
min_row = xs_form.cleaned_data["min_row"]
max_row = xs_form.cleaned_data["max_row"]
except (ValueError, CompoundDatatype.DoesNotExist) as e:
xs_form.add_error("compounddatatype", e)
raise e
curr_xput = new_method.create_xput(
dataset_name=dataset_name,
compounddatatype=my_compound_datatype,
row_limits=(min_row, max_row),
input=(xput_type == "in"),
clean=False
)
if cdt_id != "__raw__":
try:
curr_xput.structure.clean()
except ValidationError as e:
xs_form.add_error(None, e)
raise e
try:
curr_xput.clean()
except ValidationError as e:
t_form.add_error(None, e)
raise e
try:
new_method.complete_clean()
except ValidationError as e:
method_form.add_error(None, e)
raise e
except ValidationError:
return None
return new_method
def method_revise(request, pk):
"""
Add a revision of an existing Method. revision_parent is defined by the
previous version.
"""
t = loader.get_template('method/method.html')
c = {}
creating_user = request.user
# Retrieve the most recent member of this Method's family.
parent_method = Method.check_accessible(pk, creating_user)
family = parent_method.family
# Retrieve the most recent revision of the corresponding CR.
parent_revision = parent_method.driver
if not parent_revision:
this_code_resource = None
all_revisions = []
else:
this_code_resource = parent_revision.coderesource
# Filter the available revisions by user.
all_revisions = CodeResourceRevision.filter_by_user(
creating_user,
queryset=this_code_resource.revisions.all()).order_by('-revision_DateTime')
parent_container = parent_method.container
if not parent_container:
this_container_family = None
all_containers = []
else:
this_container_family = parent_container.family
all_containers = Container.filter_by_user(
creating_user,
queryset=this_container_family.containers.all())
if request.method == 'POST':
# Because there is no CodeResource specified, the second value is of type MethodReviseForm.
family_form, method_revise_form,\
dep_forms, input_form_tuples,\
output_form_tuples, _ = create_method_forms(request.POST, creating_user, family=family)
if not _method_forms_check_valid(family_form, method_revise_form, dep_forms,
input_form_tuples, output_form_tuples):
# Bail out now if there are any problems.
c.update(
{
'coderesource': this_code_resource,
'containerfamily': this_container_family,
'method_revise_form': method_revise_form,
'dep_forms': dep_forms,
'input_forms': input_form_tuples,
'output_forms': output_form_tuples,
'family': family,
'family_form': family_form,
'parent': parent_method,
'docker_default': DockerImage.DEFAULT_IMAGE
})
return HttpResponse(t.render(c, request))
# Next, attempt to build the Method and add it to family.
create_method_from_forms(
family_form, method_revise_form, dep_forms, input_form_tuples, output_form_tuples, creating_user,
family=family, parent_method=parent_method
)
if _method_forms_check_valid(family_form, method_revise_form, dep_forms,
input_form_tuples, output_form_tuples):
# Success!
return HttpResponseRedirect('/methods/{}'.format(family.pk))
else:
# Initialize forms with values of parent Method.
family_form = MethodFamilyForm({"name": family.name, "description": family.description})
parent_users_allowed = [x.username for x in parent_method.users_allowed.all()]
parent_groups_allowed = [x.name for x in parent_method.groups_allowed.all()]
method_revise_form = MethodReviseForm(
initial={
"revision_desc": parent_method.revision_desc,
"driver_revisions": parent_revision and parent_revision.pk,
"docker_image": parent_method.docker_image_id,
"reusable": parent_method.reusable,
"threads": parent_method.threads,
"memory": parent_method.memory,
"permissions": [parent_users_allowed, parent_groups_allowed]
})
dependencies = parent_method.dependencies.all()
dep_forms = []
for i, dependency in enumerate(dependencies):
its_crv = dependency.requirement
its_cr = its_crv.coderesource
dep_form = MethodDependencyForm(
user=creating_user,
auto_id='id_%s_'+str(i),
initial={
'coderesource': its_cr.pk,
'revisions': its_crv.pk,
'path': dependency.path,
'filename': dependency.filename
}
)
dep_forms.append(dep_form)
# If the parent Method has no dependencies, add a blank form.
if len(dep_forms) == 0:
dep_forms.append(MethodDependencyForm(user=creating_user, auto_id='id_%s_0'))
xput_forms = []
inputs = parent_method.inputs.order_by("dataset_idx")
outputs = parent_method.outputs.order_by("dataset_idx")
for xput_type, xputs in (("in", inputs), ("out", outputs)):
forms = []
for xput in xputs:
tx_form = TransformationXputForm(auto_id='id_%s_{}_{}'.format(xput_type, len(forms)),
initial={'dataset_name': xput.dataset_name,
'dataset_idx': xput.dataset_idx})
if xput.has_structure:
structure = xput.structure
xs_form = XputStructureForm(user=creating_user,
auto_id='id_%s_{}_{}'.format(xput_type, len(forms)),
initial={'compounddatatype': structure.compounddatatype.id,
'min_row': structure.min_row,
'max_row': structure.max_row})
else:
xs_form = XputStructureForm(user=creating_user,
auto_id='id_%s_{}_{}'.format(xput_type, len(forms)),
initial={'compounddatatype': '__raw__'})
forms.append((tx_form, xs_form))
xput_forms.append(forms)
input_form_tuples, output_form_tuples = xput_forms
# if previous Method has no inputs, provide blank forms
if len(input_form_tuples) == 0:
tx_form = TransformationXputForm(auto_id='id_%s_in_0')
xs_form = XputStructureForm(user=creating_user, auto_id='id_%s_in_0')
input_form_tuples.append((tx_form, xs_form))
method_revise_form.fields['driver_revisions'].widget.choices = [
(str(x.id), '{}: {}'.format(x.revision_number, x.revision_name)) for x in all_revisions
]
method_revise_form.fields['container'].widget.choices = [
(str(x.id), x.tag) for x in all_containers]
c.update(
{
'coderesource': this_code_resource,
'containerfamily': this_container_family,
'method_form': method_revise_form,
'dep_forms': dep_forms,
'input_forms': input_form_tuples,
'output_forms': output_form_tuples,
'family': family,
'family_form': family_form,
'parent': parent_method,
'docker_default': DockerImage.DEFAULT_IMAGE
}
)
return HttpResponse(t.render(c, request))
def method_view(request, pk):
"""
View a Method or edit its metadata/permissions.
"""
method = Method.check_accessible(pk, request.user)
addable_users, addable_groups = method.other_users_groups()
addable_users, addable_groups = method.family.intersect_permissions(
addable_users,
addable_groups)
if method.revision_parent is not None:
addable_users, addable_groups = (
method.revision_parent.intersect_permissions(addable_users,
addable_groups))
if method.driver is not None:
addable_users, addable_groups = method.driver.intersect_permissions(
addable_users,
addable_groups)
if method.docker_image is not None:
addable_users, addable_groups = (
method.docker_image.intersect_permissions(addable_users,
addable_groups))
for dep in method.dependencies.all():
addable_users, addable_groups = dep.requirement.intersect_permissions(addable_users, addable_groups)
for xput in itertools.chain(method.inputs.all(), method.outputs.all()):
xput_cdt = xput.get_cdt()
if xput_cdt is not None:
addable_users, addable_groups = xput_cdt.intersect_permissions(addable_users, addable_groups)
if request.method == 'POST':
# We are attempting to update the Method's metadata/permissions.
method_form = MethodDetailsForm(
request.POST,
addable_users=addable_users,
addable_groups=addable_groups,
instance=method
)
if method_form.is_valid():
try:
method.revision_name = method_form.cleaned_data["revision_name"]
method.revision_desc = method_form.cleaned_data["revision_desc"]
method.save()
method.grant_from_json(method_form.cleaned_data["permissions"])
method.clean()
# Success -- go back to the CodeResource page.
return HttpResponseRedirect('/methods/{}'.format(method.family.pk))
except (AttributeError, ValidationError, ValueError) as e:
LOGGER.exception(e.message)
method_form.add_error(None, e)
else:
method_form = MethodDetailsForm(
addable_users=addable_users,
addable_groups=addable_groups,
initial={
"revision_name": method.revision_name,
"revision_desc": method.revision_desc
}
)
t = loader.get_template("method/method_view.html")
c = {
"method": method,
"method_form": method_form,
"is_owner": method.user == request.user,
"is_admin": admin_check(request.user)
}
return HttpResponse(t.render(c, request))
def method_revise(request, pk):
"""
Add a revision of an existing Method. revision_parent is defined by the
previous version.
"""
t = loader.get_template('method/method.html')
c = {}
creating_user = request.user
# Retrieve the most recent member of this Method's family.
parent_method = Method.check_accessible(pk, creating_user)
family = parent_method.family
# Retrieve the most recent revision of the corresponding CR.
parent_revision = parent_method.driver
if not parent_revision:
this_code_resource = None
all_revisions = []
else:
this_code_resource = parent_revision.coderesource
# Filter the available revisions by user.
all_revisions = CodeResourceRevision.filter_by_user(
creating_user,
queryset=this_code_resource.revisions.all()).order_by(
'-revision_DateTime')
parent_container = parent_method.container
if not parent_container:
this_container_family = None
all_containers = []
else:
this_container_family = parent_container.family
all_containers = Container.filter_by_user(
creating_user, queryset=this_container_family.containers.all())
if request.method == 'POST':
# Because there is no CodeResource specified, the second value is of type MethodReviseForm.
family_form, method_revise_form,\
dep_forms, input_form_tuples,\
output_form_tuples, _ = create_method_forms(request.POST, creating_user, family=family)
if not _method_forms_check_valid(family_form, method_revise_form,
dep_forms, input_form_tuples,
output_form_tuples):
# Bail out now if there are any problems.
c.update({
'coderesource': this_code_resource,
'containerfamily': this_container_family,
'method_revise_form': method_revise_form,
'dep_forms': dep_forms,
'input_forms': input_form_tuples,
'output_forms': output_form_tuples,
'family': family,
'family_form': family_form,
'parent': parent_method,
'docker_default': DockerImage.DEFAULT_IMAGE
})
return HttpResponse(t.render(c, request))
# Next, attempt to build the Method and add it to family.
create_method_from_forms(family_form,
method_revise_form,
dep_forms,
input_form_tuples,
output_form_tuples,
creating_user,
family=family,
parent_method=parent_method)
if _method_forms_check_valid(family_form, method_revise_form,
dep_forms, input_form_tuples,
output_form_tuples):
# Success!
return HttpResponseRedirect('/methods/{}'.format(family.pk))
else:
# Initialize forms with values of parent Method.
family_form = MethodFamilyForm({
"name": family.name,
"description": family.description
})
parent_users_allowed = [
x.username for x in parent_method.users_allowed.all()
]
parent_groups_allowed = [
x.name for x in parent_method.groups_allowed.all()
]
method_revise_form = MethodReviseForm(
initial={
"revision_desc": parent_method.revision_desc,
"driver_revisions": parent_revision and parent_revision.pk,
"docker_image": parent_method.docker_image_id,
"reusable": parent_method.reusable,
"threads": parent_method.threads,
"memory": parent_method.memory,
"permissions": [parent_users_allowed, parent_groups_allowed]
})
dependencies = parent_method.dependencies.all()
dep_forms = []
for i, dependency in enumerate(dependencies):
its_crv = dependency.requirement
its_cr = its_crv.coderesource
dep_form = MethodDependencyForm(user=creating_user,
auto_id='id_%s_' + str(i),
initial={
'coderesource': its_cr.pk,
'revisions': its_crv.pk,
'path': dependency.path,
'filename': dependency.filename
})
dep_forms.append(dep_form)
# If the parent Method has no dependencies, add a blank form.
if len(dep_forms) == 0:
dep_forms.append(
MethodDependencyForm(user=creating_user, auto_id='id_%s_0'))
xput_forms = []
inputs = parent_method.inputs.order_by("dataset_idx")
outputs = parent_method.outputs.order_by("dataset_idx")
for xput_type, xputs in (("in", inputs), ("out", outputs)):
forms = []
for xput in xputs:
tx_form = TransformationXputForm(auto_id='id_%s_{}_{}'.format(
xput_type, len(forms)),
initial={
'dataset_name':
xput.dataset_name,
'dataset_idx':
xput.dataset_idx
})
if xput.has_structure:
structure = xput.structure
xs_form = XputStructureForm(
user=creating_user,
auto_id='id_%s_{}_{}'.format(xput_type, len(forms)),
initial={
'compounddatatype': structure.compounddatatype.id,
'min_row': structure.min_row,
'max_row': structure.max_row
})
else:
xs_form = XputStructureForm(
user=creating_user,
auto_id='id_%s_{}_{}'.format(xput_type, len(forms)),
initial={'compounddatatype': '__raw__'})
forms.append((tx_form, xs_form))
xput_forms.append(forms)
input_form_tuples, output_form_tuples = xput_forms
# if previous Method has no inputs, provide blank forms
if len(input_form_tuples) == 0:
tx_form = TransformationXputForm(auto_id='id_%s_in_0')
xs_form = XputStructureForm(user=creating_user,
auto_id='id_%s_in_0')
input_form_tuples.append((tx_form, xs_form))
method_revise_form.fields['driver_revisions'].widget.choices = [
(str(x.id), '{}: {}'.format(x.revision_number, x.revision_name))
for x in all_revisions
]
method_revise_form.fields['container'].widget.choices = [
(str(x.id), x.tag) for x in all_containers
]
c.update({
'coderesource': this_code_resource,
'containerfamily': this_container_family,
'method_form': method_revise_form,
'dep_forms': dep_forms,
'input_forms': input_form_tuples,
'output_forms': output_form_tuples,
'family': family,
'family_form': family_form,
'parent': parent_method,
'docker_default': DockerImage.DEFAULT_IMAGE
})
return HttpResponse(t.render(c, request))
def method_view(request, pk):
"""
View a Method or edit its metadata/permissions.
"""
method = Method.check_accessible(pk, request.user)
addable_users, addable_groups = method.other_users_groups()
addable_users, addable_groups = method.family.intersect_permissions(
addable_users, addable_groups)
if method.revision_parent is not None:
addable_users, addable_groups = (
method.revision_parent.intersect_permissions(
addable_users, addable_groups))
if method.driver is not None:
addable_users, addable_groups = method.driver.intersect_permissions(
addable_users, addable_groups)
if method.docker_image is not None:
addable_users, addable_groups = (
method.docker_image.intersect_permissions(addable_users,
addable_groups))
for dep in method.dependencies.all():
addable_users, addable_groups = dep.requirement.intersect_permissions(
addable_users, addable_groups)
for xput in itertools.chain(method.inputs.all(), method.outputs.all()):
xput_cdt = xput.get_cdt()
if xput_cdt is not None:
addable_users, addable_groups = xput_cdt.intersect_permissions(
addable_users, addable_groups)
if request.method == 'POST':
# We are attempting to update the Method's metadata/permissions.
method_form = MethodDetailsForm(request.POST,
addable_users=addable_users,
addable_groups=addable_groups,
instance=method)
if method_form.is_valid():
try:
method.revision_name = method_form.cleaned_data[
"revision_name"]
method.revision_desc = method_form.cleaned_data[
"revision_desc"]
method.save()
method.grant_from_json(method_form.cleaned_data["permissions"])
method.clean()
# Success -- go back to the CodeResource page.
return HttpResponseRedirect('/methods/{}'.format(
method.family.pk))
except (AttributeError, ValidationError, ValueError) as e:
LOGGER.exception(e.message)
method_form.add_error(None, e)
else:
method_form = MethodDetailsForm(addable_users=addable_users,
addable_groups=addable_groups,
initial={
"revision_name":
method.revision_name,
"revision_desc":
method.revision_desc
})
t = loader.get_template("method/method_view.html")
c = {
"method": method,
"method_form": method_form,
"is_owner": method.user == request.user,
"is_admin": admin_check(request.user)
}
return HttpResponse(t.render(c, request))
def create_method_from_forms(family_form,
method_form,
dep_forms,
input_forms,
output_forms,
creating_user,
family=None,
parent_method=None):
"""
Given Forms representing the MethodFamily, Method, inputs, and outputs, create a Method.
Warning: this routine has side effects (it can mod its arguments):
If an error occurs:
return None and update the forms with errors.
else:
return the new method and the forms are returned without modification.
"""
# This assures that not both family_form and family are None.
assert family is not None or family_form is not None
for dep_form in dep_forms:
assert isinstance(dep_form, MethodDependencyForm) or dep_form is None
# Retrieve the CodeResource revision as driver.
try:
coderesource_revision = CodeResourceRevision.objects.get(
pk=method_form.cleaned_data['driver_revisions'])
except CodeResourceRevision.DoesNotExist:
coderesource_revision = None
# Retrieve the Container.
try:
container = Container.objects.get(
pk=method_form.cleaned_data['container'])
except Container.DoesNotExist:
container = None
new_method = None
try:
# Note how the except blocks re-raise their exception: that is to terminate
# this transaction.
with transaction.atomic():
if family is None:
try:
family = family_form.save()
family.grant_from_json(
method_form.cleaned_data["permissions"])
family.full_clean()
except ValidationError as e:
family_form.add_error(None, e)
raise e
new_method = Method(
family=family,
revision_name=method_form.cleaned_data['revision_name'],
revision_desc=method_form.cleaned_data['revision_desc'],
revision_parent=parent_method,
driver=coderesource_revision,
container=container,
reusable=method_form.cleaned_data['reusable'],
user=creating_user,
threads=method_form.cleaned_data["threads"],
memory=method_form.cleaned_data["memory"])
new_method.save()
new_method.grant_from_json(method_form.cleaned_data["permissions"])
# Bind dependencies.
for i in range(len(dep_forms)):
if dep_forms[i] is None:
continue
try:
on_revision = CodeResourceRevision.objects.get(
pk=dep_forms[i].cleaned_data["revisions"])
dependency = MethodDependency(
method=new_method,
requirement=on_revision,
path=dep_forms[i].cleaned_data["path"],
filename=dep_forms[i].cleaned_data["filename"])
dependency.full_clean()
dependency.save()
except ValidationError as e:
dep_forms[i].add_error(None, e)
raise e
# Attempt to make in/outputs.
num_outputs = len(output_forms)
if num_outputs == 0:
method_form.add_error(None,
"You must specify at least one output.")
raise ValidationError("You must specify at least one output.")
for xput_type in ("in", "out"):
curr_forms = input_forms
if xput_type == "out":
curr_forms = output_forms
for form_tuple in curr_forms:
t_form = form_tuple[0]
xs_form = form_tuple[1]
dataset_name = t_form.cleaned_data["dataset_name"]
cdt_id = xs_form.cleaned_data["compounddatatype"]
if dataset_name == '' and cdt_id == '':
# ignore blank form
continue
my_compound_datatype = None
min_row = None
max_row = None
if cdt_id != '__raw__':
try:
my_compound_datatype = CompoundDatatype.objects.get(
pk=cdt_id)
min_row = xs_form.cleaned_data["min_row"]
max_row = xs_form.cleaned_data["max_row"]
except (ValueError,
CompoundDatatype.DoesNotExist) as e:
xs_form.add_error("compounddatatype", e)
raise e
curr_xput = new_method.create_xput(
dataset_name=dataset_name,
compounddatatype=my_compound_datatype,
row_limits=(min_row, max_row),
input=(xput_type == "in"),
clean=False)
if cdt_id != "__raw__":
try:
curr_xput.structure.clean()
except ValidationError as e:
xs_form.add_error(None, e)
raise e
try:
curr_xput.clean()
except ValidationError as e:
t_form.add_error(None, e)
raise e
try:
new_method.complete_clean()
except ValidationError as e:
method_form.add_error(None, e)
raise e
except ValidationError:
return None
return new_method
def test_display_name_without_revision_name(self):
method = Method(revision_number=1)
self.assertEqual(method.display_name, '1: ')
