def setUp(self):
super(BackendTest, self).setUp()
self.p = Process(slug='test-processor',
name='Test Process',
contributor=self.contributor,
type='data:test',
version=1)
self.p.save()
self.d = Data(slug='test-data',
name='Test Data',
contributor=self.contributor,
process=self.p)
self.d.save()
class BackendTest(TestCase):
def setUp(self):
u = get_user_model().objects.create_superuser('test', '*****@*****.**', 'test')
self.p = Process(slug='test-processor',
name='Test Process',
contributor=u,
type='data:test',
version=1)
self.p.save()
self.d = Data(slug='test-data',
name='Test Data',
contributor=u,
process=self.p)
self.d.save()
def tearDown(self):
for data in Data.objects.all():
data_dir = os.path.join(settings.FLOW_EXECUTOR['DATA_DIR'], str(data.id))
shutil.rmtree(data_dir, ignore_errors=True)
def test_manager(self):
manager.communicate(verbosity=0)
def test_dtlbash(self):
self.p.slug = 'test-processor-dtlbash'
self.p.run = {'script': """
gen-info \"Test processor info\"
gen-warning \"Test processor warning\"
echo '{"proc.info": "foo"}'
"""}
self.p.save()
self.d.slug = 'test-data-dtlbash'
self.d.process = self.p
self.d.save()
self.d = Data(id=self.d.id)
def setUp(self):
super().setUp()
self.process = Process(
output_schema=[
{'name': 'test_file', 'type': 'basic:file:', 'required': False},
{'name': 'file_list', 'type': 'list:basic:file:', 'required': False}
]
)
self.data = Data(
pk=13,
process=self.process,
output={'test_file': {'file': 'test_file.tmp'}}
)
def setUp(self):
u = get_user_model().objects.create_superuser('test', '*****@*****.**', 'test')
self.p = Process(slug='test-processor',
name='Test Process',
contributor=u,
type='data:test',
version=1)
self.p.save()
self.d = Data(slug='test-data',
name='Test Data',
contributor=u,
process=self.p)
self.d.save()
class BackendTest(TestCase):
def setUp(self):
super(BackendTest, self).setUp()
self.p = Process(slug='test-processor',
name='Test Process',
contributor=self.contributor,
type='data:test',
version=1)
self.p.save()
self.d = Data(slug='test-data',
name='Test Data',
contributor=self.contributor,
process=self.p)
self.d.save()
def test_manager(self):
manager.communicate(verbosity=0)
def test_dtlbash(self):
self.p.slug = 'test-processor-dtlbash'
self.p.run = {
'script':
"""
gen-info \"Test processor info\"
gen-warning \"Test processor warning\"
echo '{"proc.info": "foo"}'
"""
}
self.p.save()
self.d.slug = 'test-data-dtlbash'
self.d.process = self.p
self.d.save()
self.d = Data(id=self.d.id)
def test_checksum_consistency(self):
process = Process(version="1.0.0", slug="my-process")
data = Data()
data.input = {"tss": 0, "genome": "HG19"}
checksum = get_data_checksum(data.input, process.slug, process.version)
self.assertEqual(
checksum,
"ca322c2bb48b58eea3946e624fe6cfdc53c2cc12478465b6f0ca2d722e280c4c")
data.input = {"genome": "HG19", "tss": 0}
checksum = get_data_checksum(data.input, process.slug, process.version)
self.assertEqual(
checksum,
"ca322c2bb48b58eea3946e624fe6cfdc53c2cc12478465b6f0ca2d722e280c4c")
def test_checksum_consistency(self):
process = Process(version='1.0.0', slug='my-process')
data = Data()
data.input = {'tss': 0, 'genome': 'HG19'}
checksum = get_data_checksum(data.input, process.slug, process.version)
self.assertEqual(
checksum,
'ca322c2bb48b58eea3946e624fe6cfdc53c2cc12478465b6f0ca2d722e280c4c')
data.input = {'genome': 'HG19', 'tss': 0}
checksum = get_data_checksum(data.input, process.slug, process.version)
self.assertEqual(
checksum,
'ca322c2bb48b58eea3946e624fe6cfdc53c2cc12478465b6f0ca2d722e280c4c')
def migrate_data(self, data):
contributor = self.get_contributor(data["author_id"])
# DESCRIPTOR SCHEMA ############################################
ds_fields = []
ds_fields.extend(data.get("static_schema", []))
ds_fields.extend(data.get("var_template", []))
ds_fields.sort(key=lambda d: d["name"])
ds_fields_dumped = json.dumps(ds_fields)
if ds_fields_dumped in self.descriptor_schema_index:
descriptor_schema = self.descriptor_schema_index[ds_fields_dumped]
else:
descriptor_schema = DescriptorSchema(schema=ds_fields)
descriptor_schema.name = "data_{}_descriptor".format(data["_id"])
descriptor_schema.contributor = contributor
descriptor_schema.save()
self.descriptor_schema_index[ds_fields_dumped] = descriptor_schema
descriptor = {}
descriptor.update(data.get("static", {}))
descriptor.update(data.get("var", {}))
# PROCESS ######################################################
if "processor_version" not in data:
data["processor_version"] = "0.0.0"
process_slug = self.process_slug(data["processor_name"])
process_version = data["processor_version"]
try:
process = Process.objects.get(slug=process_slug, version=process_version)
except Process.DoesNotExist:
latest = Process.objects.filter(slug=process_slug).order_by("-version").first()
if latest:
process = Process()
process.name = latest.name
process.slug = latest.slug
process.category = latest.category
process.description = latest.description
process.contributor = latest.contributor
process.version = process_version
process.type = data["type"]
process.output_schema = data["output_schema"]
process.input_schema = data.get("input_schema", {})
process.persistence = self.persistence_dict[data["persistence"]]
process.run["script"] = 'gen-require common\ngen-error "Depricated process, use the latest version."'
# XXX
# process.created =
# process.modified =
process.save()
# copy permissions from latest process
for user, perms in get_users_with_perms(latest, attach_perms=True).iteritems():
for perm in perms:
assign_perm(perm, user, process)
for group, perms in get_groups_with_perms(latest, attach_perms=True).iteritems():
for perm in perms:
assign_perm(perm, group, process)
else:
# Create dummy processor if there is no other version
dummy_name = "Dummy processor of type {}".format(data["type"])
try:
process = Process.objects.get(name=dummy_name)
except Process.DoesNotExist:
process = Process.objects.create(
name=dummy_name,
slug="non-existent",
contributor=get_user_model().objects.filter(is_superuser=True).first(),
type=data["type"],
category="data:non-existent",
run={"script": {'gen-require common\ngen-error "This processor is not intendent to be run."'}},
)
# DATA #########################################################
new = Data()
new.name = data.get("static", {}).get("name", "")
if len(new.name) > 100:
self.long_names.append(new.name)
new.name = new.name[:97] + "..."
new.status = self.status_dict[data["status"]]
new.process = process
new.contributor = contributor
new.input = data["input"] if "input" in data else {}
new.output = data["output"]
new.descriptor_schema = descriptor_schema
new.descriptor = descriptor
new.checksum = data.get("checksum", "")
# XXX: Django will change this on create
new.created = data["date_created"]
# XXX: Django will change this on save
new.modified = data["date_modified"]
if "date_start" in data and "date_finish" in data:
new.started = data["date_start"]
new.finished = data["date_finish"]
elif "date_finish" in data:
new.started = data["date_finish"]
new.finished = data["date_finish"]
elif "date_start" in data:
new.started = data["date_start"]
new.finished = data["date_start"]
else:
new.started = datetime.fromtimestamp(0)
new.finished = datetime.fromtimestamp(0)
new.save()
for case_id in data["case_ids"]:
try:
collection = Collection.objects.get(pk=self.id_mapping["collection"][str(case_id)])
except KeyError:
self.missing_collections.add(str(case_id))
continue
collection.data.add(new)
for field_schema, fields, path in iterate_fields(data["output"], data["output_schema"], ""):
if "type" in field_schema and field_schema["type"].startswith("basic:json:"):
self.storage_index[fields[field_schema["name"]]] = {"id": new.pk, "path": path}
self.migrate_permissions(new, data)
self.id_mapping["data"][str(data["_id"])] = new.pk
# DESCRIPTOR SCHEMA PERMISSIONS ################################
for user in get_users_with_perms(new):
assign_perm("view_descriptorschema", user, obj=descriptor_schema)
for group in get_groups_with_perms(new):
assign_perm("view_descriptorschema", group, obj=descriptor_schema)
def migrate_process(self, process):
new = Process()
new.name = process["label"]
new.slug = self.process_slug(process["name"])
new.version = process["version"]
new.type = process["type"]
new.description = process.get("description", "")
new.contributor = self.get_contributor(process["author_id"])
new.category = process.get("category", "")
# XXX: Django will change this on create
new.created = process["date_created"]
# XXX: Django will change this on save
new.modified = process["date_modified"]
new.output_schema = process["output_schema"]
new.input_schema = process.get("input_schema", {})
new.persistence = self.persistence_dict[process["persistence"]]
new.run["script"] = process["run"]["bash"]
new.save()
self.migrate_permissions(new, process)
self.id_mapping["process"][str(process["_id"])] = new.pk
def migrate_data(self, data):
"""Migrate data."""
contributor = self.get_contributor(data[u'author_id'])
# DESCRIPTOR SCHEMA ############################################
ds_fields = []
ds_fields.extend(data.get(u'static_schema', []))
ds_fields.extend(data.get(u'var_template', []))
ds_fields.sort(key=lambda d: d[u'name'])
ds_fields_dumped = json.dumps(ds_fields)
if ds_fields_dumped in self.descriptor_schema_index:
descriptor_schema = self.descriptor_schema_index[ds_fields_dumped]
else:
descriptor_schema = DescriptorSchema(schema=ds_fields)
descriptor_schema.name = 'data_{}_descriptor'.format(data[u'_id'])
descriptor_schema.contributor = contributor
descriptor_schema.save()
self.descriptor_schema_index[ds_fields_dumped] = descriptor_schema
descriptor = {}
descriptor.update(data.get(u'static', {}))
descriptor.update(data.get(u'var', {}))
# PROCESS ######################################################
if u'processor_version' not in data:
data[u'processor_version'] = '0.0.0'
process_slug = self.process_slug(data[u'processor_name'])
process_version = data[u'processor_version']
try:
process = Process.objects.get(slug=process_slug, version=process_version)
except Process.DoesNotExist:
latest = Process.objects.filter(slug=process_slug).order_by('-version').first()
if latest:
process = Process()
process.name = latest.name
process.slug = latest.slug
process.category = latest.category
process.description = latest.description
process.contributor = latest.contributor
process.version = process_version
process.type = data[u'type']
process.output_schema = data[u'output_schema']
process.input_schema = data.get(u'input_schema', {})
process.persistence = self.persistence_dict[data[u'persistence']]
process.run['script'] = 'gen-require common\ngen-error "Depricated process, use the latest version."' # noqa pylint: disable=unsubscriptable-object
# XXX
# process.created =
# process.modified =
process.save()
# copy permissions from latest process
for user, perms in six.iteritems(get_users_with_perms(latest, attach_perms=True)):
for perm in perms:
assign_perm(perm, user, process)
for group, perms in six.iteritems(get_groups_with_perms(latest, attach_perms=True)):
for perm in perms:
assign_perm(perm, group, process)
else:
# Create dummy processor if there is no other version
dummy_name = 'Dummy processor of type {}'.format(data[u'type'])
try:
process = Process.objects.get(name=dummy_name)
except Process.DoesNotExist:
process = Process.objects.create(
name=dummy_name,
slug='non-existent',
contributor=get_user_model().objects.filter(is_superuser=True).first(),
type=data[u'type'],
category='data:non-existent',
run={'script': {'gen-require common\ngen-error "This processor is not intendent to be run."'}},
)
# DATA #########################################################
new = Data()
new.name = data.get(u'static', {}).get(u'name', '')
if len(new.name) > 100:
self.long_names.append(new.name)
new.name = new.name[:97] + '...'
new.status = self.status_dict[data[u'status']]
new.process = process
new.contributor = contributor
new.input = data[u'input'] if u'input' in data else {}
new.output = data[u'output']
new.descriptor_schema = descriptor_schema
new.descriptor = descriptor
new.checksum = data.get(u'checksum', '')
# XXX: Django will change this on create
new.created = data[u'date_created']
# XXX: Django will change this on save
new.modified = data[u'date_modified']
if u'date_start' in data and u'date_finish' in data:
new.started = data[u'date_start']
new.finished = data[u'date_finish']
elif u'date_finish' in data:
new.started = data[u'date_finish']
new.finished = data[u'date_finish']
elif u'date_start' in data:
new.started = data[u'date_start']
new.finished = data[u'date_start']
else:
new.started = datetime.fromtimestamp(0)
new.finished = datetime.fromtimestamp(0)
new.save()
for case_id in data[u'case_ids']:
try:
collection = Collection.objects.get(pk=self.id_mapping[u'collection'][str(case_id)])
except KeyError:
self.missing_collections.add(str(case_id))
continue
collection.data.add(new)
for field_schema, fields, path in iterate_fields(data[u'output'], data[u'output_schema'], ''):
if 'type' in field_schema and field_schema['type'].startswith('basic:json:'):
self.storage_index[fields[field_schema['name']]] = {
'id': new.pk,
'path': path,
}
self.migrate_permissions(new, data)
self.id_mapping['data'][str(data[u'_id'])] = new.pk
# DESCRIPTOR SCHEMA PERMISSIONS ################################
for user in get_users_with_perms(new):
assign_perm('view_descriptorschema', user, obj=descriptor_schema)
for group in get_groups_with_perms(new):
assign_perm('view_descriptorschema', group, obj=descriptor_schema)
def migrate_process(self, process):
"""Migrate processes."""
new = Process()
new.name = process[u'label']
new.slug = self.process_slug(process[u'name'])
new.version = process[u'version']
new.type = process[u'type']
new.description = process.get(u'description', '')
new.contributor = self.get_contributor(process['author_id'])
new.category = process.get(u'category', '')
# XXX: Django will change this on create
new.created = process[u'date_created']
# XXX: Django will change this on save
new.modified = process[u'date_modified']
new.output_schema = process[u'output_schema']
new.input_schema = process.get(u'input_schema', {})
new.persistence = self.persistence_dict[process[u'persistence']]
new.run['script'] = process[u'run'][u'bash'] # pylint: disable=unsubscriptable-object
new.save()
self.migrate_permissions(new, process)
self.id_mapping['process'][str(process[u'_id'])] = new.pk
def migrate_data(self, data):
"""Migrate data."""
contributor = self.get_contributor(data[u'author_id'])
# DESCRIPTOR SCHEMA ############################################
ds_fields = []
ds_fields.extend(data.get(u'static_schema', []))
ds_fields.extend(data.get(u'var_template', []))
ds_fields.sort(key=lambda d: d[u'name'])
ds_fields_dumped = json.dumps(ds_fields)
if ds_fields_dumped in self.descriptor_schema_index:
descriptor_schema = self.descriptor_schema_index[ds_fields_dumped]
else:
descriptor_schema = DescriptorSchema(schema=ds_fields)
descriptor_schema.name = 'data_{}_descriptor'.format(data[u'_id'])
descriptor_schema.contributor = contributor
descriptor_schema.save()
self.descriptor_schema_index[ds_fields_dumped] = descriptor_schema
descriptor = {}
descriptor.update(data.get(u'static', {}))
descriptor.update(data.get(u'var', {}))
# PROCESS ######################################################
if u'processor_version' not in data:
data[u'processor_version'] = '0.0.0'
process_slug = self.process_slug(data[u'processor_name'])
process_version = data[u'processor_version']
try:
process = Process.objects.get(slug=process_slug,
version=process_version)
except Process.DoesNotExist:
latest = Process.objects.filter(
slug=process_slug).order_by('-version').first()
if latest:
process = Process()
process.name = latest.name
process.slug = latest.slug
process.category = latest.category
process.description = latest.description
process.contributor = latest.contributor
process.version = process_version
process.type = data[u'type']
process.output_schema = data[u'output_schema']
process.input_schema = data.get(u'input_schema', {})
process.persistence = self.persistence_dict[
data[u'persistence']]
process.run[
'script'] = 'gen-require common\ngen-error "Depricated process, use the latest version."' # noqa pylint: disable=unsubscriptable-object
# XXX
# process.created =
# process.modified =
process.save()
# copy permissions from latest process
for user, perms in six.iteritems(
get_users_with_perms(latest, attach_perms=True)):
for perm in perms:
assign_perm(perm, user, process)
for group, perms in six.iteritems(
get_groups_with_perms(latest, attach_perms=True)):
for perm in perms:
assign_perm(perm, group, process)
else:
# Create dummy processor if there is no other version
dummy_name = 'Dummy processor of type {}'.format(data[u'type'])
try:
process = Process.objects.get(name=dummy_name)
except Process.DoesNotExist:
process = Process.objects.create(
name=dummy_name,
slug='non-existent',
contributor=get_user_model().objects.filter(
is_superuser=True).first(),
type=data[u'type'],
category='data:non-existent',
run={
'script': {
'gen-require common\ngen-error "This processor is not intendent to be run."'
}
},
)
# DATA #########################################################
new = Data()
new.name = data.get(u'static', {}).get(u'name', '')
if len(new.name) > 100:
self.long_names.append(new.name)
new.name = new.name[:97] + '...'
new.status = self.status_dict[data[u'status']]
new.process = process
new.contributor = contributor
new.input = data[u'input'] if u'input' in data else {}
new.output = data[u'output']
new.descriptor_schema = descriptor_schema
new.descriptor = descriptor
new.checksum = data.get(u'checksum', '')
# XXX: Django will change this on create
new.created = data[u'date_created']
# XXX: Django will change this on save
new.modified = data[u'date_modified']
if u'date_start' in data and u'date_finish' in data:
new.started = data[u'date_start']
new.finished = data[u'date_finish']
elif u'date_finish' in data:
new.started = data[u'date_finish']
new.finished = data[u'date_finish']
elif u'date_start' in data:
new.started = data[u'date_start']
new.finished = data[u'date_start']
else:
new.started = datetime.fromtimestamp(0)
new.finished = datetime.fromtimestamp(0)
new.save()
for case_id in data[u'case_ids']:
try:
collection = Collection.objects.get(
pk=self.id_mapping[u'collection'][str(case_id)])
except KeyError:
self.missing_collections.add(str(case_id))
continue
collection.data.add(new)
for field_schema, fields, path in iterate_fields(
data[u'output'], data[u'output_schema'], ''):
if 'type' in field_schema and field_schema['type'].startswith(
'basic:json:'):
self.storage_index[fields[field_schema['name']]] = {
'id': new.pk,
'path': path,
}
self.migrate_permissions(new, data)
self.id_mapping['data'][str(data[u'_id'])] = new.pk
# DESCRIPTOR SCHEMA PERMISSIONS ################################
for user in get_users_with_perms(new):
assign_perm('view_descriptorschema', user, obj=descriptor_schema)
for group in get_groups_with_perms(new):
assign_perm('view_descriptorschema', group, obj=descriptor_schema)
def migrate_process(self, process):
"""Migrate processes."""
new = Process()
new.name = process[u'label']
new.slug = self.process_slug(process[u'name'])
new.version = process[u'version']
new.type = process[u'type']
new.description = process.get(u'description', '')
new.contributor = self.get_contributor(process['author_id'])
new.category = process.get(u'category', '')
# XXX: Django will change this on create
new.created = process[u'date_created']
# XXX: Django will change this on save
new.modified = process[u'date_modified']
new.output_schema = process[u'output_schema']
new.input_schema = process.get(u'input_schema', {})
new.persistence = self.persistence_dict[process[u'persistence']]
new.run['script'] = process[u'run'][u'bash'] # pylint: disable=unsubscriptable-object
new.save()
self.migrate_permissions(new, process)
self.id_mapping['process'][str(process[u'_id'])] = new.pk