def setUp(self): super().setUp() self.process = Process( output_schema=[ {'name': 'test_file', 'type': 'basic:file:', 'required': False}, {'name': 'file_list', 'type': 'list:basic:file:', 'required': False} ] ) self.data = Data( pk=13, process=self.process, output={'test_file': {'file': 'test_file.tmp'}} )
def test_checksum_consistency(self): process = Process(version='1.0.0', slug='my-process') data = Data() data.input = {'tss': 0, 'genome': 'HG19'} checksum = get_data_checksum(data.input, process.slug, process.version) self.assertEqual( checksum, 'ca322c2bb48b58eea3946e624fe6cfdc53c2cc12478465b6f0ca2d722e280c4c') data.input = {'genome': 'HG19', 'tss': 0} checksum = get_data_checksum(data.input, process.slug, process.version) self.assertEqual( checksum, 'ca322c2bb48b58eea3946e624fe6cfdc53c2cc12478465b6f0ca2d722e280c4c')
def setUp(self): super(BackendTest, self).setUp() self.p = Process(slug='test-processor', name='Test Process', contributor=self.contributor, type='data:test', version=1) self.p.save() self.d = Data(slug='test-data', name='Test Data', contributor=self.contributor, process=self.p) self.d.save()
def test_dtlbash(self): self.p.slug = 'test-processor-dtlbash' self.p.run = { 'script': """ gen-info \"Test processor info\" gen-warning \"Test processor warning\" echo '{"proc.info": "foo"}' """ } self.p.save() self.d.slug = 'test-data-dtlbash' self.d.process = self.p self.d.save() self.d = Data(id=self.d.id)
def migrate_data(self, data): """Migrate data.""" contributor = self.get_contributor(data[u'author_id']) # DESCRIPTOR SCHEMA ############################################ ds_fields = [] ds_fields.extend(data.get(u'static_schema', [])) ds_fields.extend(data.get(u'var_template', [])) ds_fields.sort(key=lambda d: d[u'name']) ds_fields_dumped = json.dumps(ds_fields) if ds_fields_dumped in self.descriptor_schema_index: descriptor_schema = self.descriptor_schema_index[ds_fields_dumped] else: descriptor_schema = DescriptorSchema(schema=ds_fields) descriptor_schema.name = 'data_{}_descriptor'.format(data[u'_id']) descriptor_schema.contributor = contributor descriptor_schema.save() self.descriptor_schema_index[ds_fields_dumped] = descriptor_schema descriptor = {} descriptor.update(data.get(u'static', {})) descriptor.update(data.get(u'var', {})) # PROCESS ###################################################### if u'processor_version' not in data: data[u'processor_version'] = '0.0.0' process_slug = self.process_slug(data[u'processor_name']) process_version = data[u'processor_version'] try: process = Process.objects.get(slug=process_slug, version=process_version) except Process.DoesNotExist: latest = Process.objects.filter( slug=process_slug).order_by('-version').first() if latest: process = Process() process.name = latest.name process.slug = latest.slug process.category = latest.category process.description = latest.description process.contributor = latest.contributor process.version = process_version process.type = data[u'type'] process.output_schema = data[u'output_schema'] process.input_schema = data.get(u'input_schema', {}) process.persistence = self.persistence_dict[ data[u'persistence']] process.run[ 'script'] = 'gen-require common\ngen-error "Depricated process, use the latest version."' # noqa pylint: disable=unsubscriptable-object # XXX # process.created = # process.modified = process.save() # copy permissions from latest process for user, perms in six.iteritems( get_users_with_perms(latest, attach_perms=True)): for perm in perms: assign_perm(perm, user, process) for group, perms in six.iteritems( get_groups_with_perms(latest, attach_perms=True)): for perm in perms: assign_perm(perm, group, process) else: # Create dummy processor if there is no other version dummy_name = 'Dummy processor of type {}'.format(data[u'type']) try: process = Process.objects.get(name=dummy_name) except Process.DoesNotExist: process = Process.objects.create( name=dummy_name, slug='non-existent', contributor=get_user_model().objects.filter( is_superuser=True).first(), type=data[u'type'], category='data:non-existent', run={ 'script': { 'gen-require common\ngen-error "This processor is not intendent to be run."' } }, ) # DATA ######################################################### new = Data() new.name = data.get(u'static', {}).get(u'name', '') if len(new.name) > 100: self.long_names.append(new.name) new.name = new.name[:97] + '...' new.status = self.status_dict[data[u'status']] new.process = process new.contributor = contributor new.input = data[u'input'] if u'input' in data else {} new.output = data[u'output'] new.descriptor_schema = descriptor_schema new.descriptor = descriptor new.checksum = data.get(u'checksum', '') # XXX: Django will change this on create new.created = data[u'date_created'] # XXX: Django will change this on save new.modified = data[u'date_modified'] if u'date_start' in data and u'date_finish' in data: new.started = data[u'date_start'] new.finished = data[u'date_finish'] elif u'date_finish' in data: new.started = data[u'date_finish'] new.finished = data[u'date_finish'] elif u'date_start' in data: new.started = data[u'date_start'] new.finished = data[u'date_start'] else: new.started = datetime.fromtimestamp(0) new.finished = datetime.fromtimestamp(0) new.save() for case_id in data[u'case_ids']: try: collection = Collection.objects.get( pk=self.id_mapping[u'collection'][str(case_id)]) except KeyError: self.missing_collections.add(str(case_id)) continue collection.data.add(new) for field_schema, fields, path in iterate_fields( data[u'output'], data[u'output_schema'], ''): if 'type' in field_schema and field_schema['type'].startswith( 'basic:json:'): self.storage_index[fields[field_schema['name']]] = { 'id': new.pk, 'path': path, } self.migrate_permissions(new, data) self.id_mapping['data'][str(data[u'_id'])] = new.pk # DESCRIPTOR SCHEMA PERMISSIONS ################################ for user in get_users_with_perms(new): assign_perm('view_descriptorschema', user, obj=descriptor_schema) for group in get_groups_with_perms(new): assign_perm('view_descriptorschema', group, obj=descriptor_schema)