def duplicate(self, contributor=None): """Duplicate (make a copy).""" duplicate = Collection.objects.get(id=self.id) duplicate.pk = None duplicate.slug = None duplicate.name = 'Copy of {}'.format(self.name) duplicate.duplicated = now() if contributor: duplicate.contributor = contributor duplicate.save(force_insert=True) assign_contributor_permissions(duplicate) # Fields to inherit from original data object. duplicate.created = self.created duplicate.save() # Duplicate collection's entities. entities = get_objects_for_user(contributor, 'view_entity', self.entity_set.all()) # pylint: disable=no-member duplicated_entities = entities.duplicate(contributor=contributor) duplicate.entity_set.add(*duplicated_entities) # Add duplicated data objects to collection. for duplicated_entity in duplicate.entity_set.all(): duplicate.data.add(*duplicated_entity.data.all()) return duplicate
def perform_create(self, serializer): """Create a resource.""" with transaction.atomic(): instance = serializer.save() # Assign all permissions to the object contributor. assign_contributor_permissions(instance)
def perform_create(self, serializer): """Create a resource.""" process = serializer.validated_data.get('process') if not process.is_active: raise exceptions.ParseError( 'Process retired (id: {}, slug: {}/{}).'.format( process.id, process.slug, process.version)) with transaction.atomic(): instance = serializer.save() assign_contributor_permissions(instance) # Entity is added to the collection only when it is # created - when it only contains 1 Data object. entities = Entity.objects.annotate(num_data=Count('data')).filter( data=instance, num_data=1) # Assign data object to all specified collections. collection_pks = self.request.data.get('collections', []) for collection in Collection.objects.filter(pk__in=collection_pks): collection.data.add(instance) copy_permissions(collection, instance) # Add entities to which data belongs to the collection. for entity in entities: entity.collections.add(collection) copy_permissions(collection, entity)
def duplicate(self, contributor=None): """Duplicate (make a copy).""" duplicate = Collection.objects.get(id=self.id) duplicate.pk = None duplicate.slug = None duplicate.name = 'Copy of {}'.format(self.name) duplicate.duplicated = now() if contributor: duplicate.contributor = contributor duplicate.save(force_insert=True) assign_contributor_permissions(duplicate) # Fields to inherit from original data object. duplicate.created = self.created duplicate.save() # Duplicate collection's entities. entities = get_objects_for_user(contributor, 'view_entity', self.entity_set.all()) # pylint: disable=no-member duplicated_entities = entities.duplicate(contributor=contributor) duplicate.entity_set.add(*duplicated_entities) # Add duplicated data objects to collection. for duplicated_entity in duplicate.entity_set.all(): duplicate.data.add(*duplicated_entity.data.all()) return duplicate
def perform_create(self, serializer): """Create a resource.""" with transaction.atomic(): instance = serializer.save() # Assign all permissions to the object contributor. assign_contributor_permissions(instance)
def perform_create(self, serializer): """Create a resource.""" process = serializer.validated_data.get('process') if not process.is_active: raise exceptions.ParseError( 'Process retired (id: {}, slug: {}/{}).'.format(process.id, process.slug, process.version) ) with transaction.atomic(): instance = serializer.save() assign_contributor_permissions(instance) # Entity is added to the collection only when it is # created - when it only contains 1 Data object. entities = Entity.objects.annotate(num_data=Count('data')).filter(data=instance, num_data=1) # Assign data object to all specified collections. collection_pks = self.request.data.get('collections', []) for collection in Collection.objects.filter(pk__in=collection_pks): collection.data.add(instance) copy_permissions(collection, instance) # Add entities to which data belongs to the collection. for entity in entities: entity.collections.add(collection) copy_permissions(collection, entity)
def create(self, subprocess_parent=None, **kwargs): """Create new object with the given kwargs.""" obj = super().create(**kwargs) # Data dependencies obj.save_dependencies(obj.input, obj.process.input_schema) if subprocess_parent: DataDependency.objects.create( parent=subprocess_parent, child=obj, kind=DataDependency.KIND_SUBPROCESS, ) # Data was from a workflow / spawned process if not obj.in_container(): copy_permissions(subprocess_parent, obj) # Entity, Collection assignment entity_operation = self._handle_entity(obj) self._handle_collection(obj, entity_operation=entity_operation) # Assign contributor permission only if Data is not in the container. if not obj.in_container(): assign_contributor_permissions(obj) return obj
def duplicate(self, contributor=None, inherit_entity=False, inherit_collection=False): """Duplicate (make a copy).""" if self.status not in [self.STATUS_DONE, self.STATUS_ERROR]: raise ValidationError('Data object must have done or error status to be duplicated') duplicate = Data.objects.get(id=self.id) duplicate.pk = None duplicate.slug = None duplicate.name = 'Copy of {}'.format(self.name) duplicate.duplicated = now() if contributor: duplicate.contributor = contributor duplicate.entity = None if inherit_entity: if not contributor.has_perm('add_entity', self.entity): raise ValidationError("You do not have add permission on entity {}.".format(self.entity)) duplicate.entity = self.entity duplicate.collection = None if inherit_collection: if not contributor.has_perm('add_collection', self.collection): raise ValidationError("You do not have add permission on collection {}.".format(self.collection)) duplicate.collection = self.collection duplicate._perform_save(force_insert=True) # pylint: disable=protected-access # Override fields that are automatically set on create. duplicate.created = self.created duplicate._perform_save() # pylint: disable=protected-access if self.location: self.location.data.add(duplicate) # pylint: disable=no-member duplicate.storages.set(self.storages.all()) # pylint: disable=no-member for migration in self.migration_history.order_by('created'): # pylint: disable=no-member migration.pk = None migration.data = duplicate migration.save(force_insert=True) # Inherit existing child dependencies. DataDependency.objects.bulk_create([ DataDependency(child=duplicate, parent=dependency.parent, kind=dependency.kind) for dependency in DataDependency.objects.filter(child=self) ]) # Inherit existing parent dependencies. DataDependency.objects.bulk_create([ DataDependency(child=dependency.child, parent=duplicate, kind=dependency.kind) for dependency in DataDependency.objects.filter(parent=self) ]) # Permissions assign_contributor_permissions(duplicate) copy_permissions(duplicate.entity, duplicate) copy_permissions(duplicate.collection, duplicate) return duplicate
def perform_create(self, serializer): """Create a resource.""" with transaction.atomic(): instance = serializer.save() # Assign all permissions to the object contributor. if hasattr(instance, "permission_group") and not instance.in_container(): assign_contributor_permissions(instance)
def set_descriptor_owner(apps, schema_editor): user_model = get_user_model() DescriptorSchema = apps.get_model("flow", "DescriptorSchema") for descriptor in DescriptorSchema.objects.all(): # In migrations contributor is not a real user instance, so we # have to get it. user = user_model.objects.get(pk=descriptor.contributor.pk) assign_contributor_permissions(descriptor, user)
def set_processes_owner(apps, schema_editor): user_model = get_user_model() Process = apps.get_model("flow", "Process") for process in Process.objects.all(): # In migrations contributor is not a real user instance, so we # have to get it. user = user_model.objects.get(pk=process.contributor.pk) assign_contributor_permissions(process, user)
def test_delete_different_user(self): resp = self._delete(self.relation_group.pk, user=self.user) self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND) self.assertTrue(Relation.objects.filter(pk=self.relation_group.pk).exists()) assign_contributor_permissions(self.collection, self.user) resp = self._delete(self.relation_group.pk, user=self.contributor) self.assertEqual(resp.status_code, status.HTTP_204_NO_CONTENT) self.assertFalse(Relation.objects.filter(pk=self.relation_group.pk).exists())
def setUp(self): self.group = Group.objects.create(name='Test group') self.resource_name = 'collection' self.viewset = DescriptorSchemaViewSet super().setUp() self.descriptor_schema = DescriptorSchema.objects.create(contributor=self.contributor) assign_contributor_permissions(self.descriptor_schema)
def test_delete_different_user(self): resp = self._delete(self.relation_group.pk, user=self.user) self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND) self.assertTrue(Relation.objects.filter(pk=self.relation_group.pk).exists()) assign_contributor_permissions(self.collection, self.user) resp = self._delete(self.relation_group.pk, user=self.contributor) self.assertEqual(resp.status_code, status.HTTP_204_NO_CONTENT) self.assertFalse(Relation.objects.filter(pk=self.relation_group.pk).exists())
def setUp(self): self.viewset = RelationViewSet self.resource_name = 'relation' super().setUp() # Load fixtures with relation types flow_config = apps.get_app_config('flow') call_command('loaddata', os.path.join(flow_config.path, 'tests', 'fixtures', 'relationtypes'), verbosity=0) self.collection = Collection.objects.create(name='Test collection', contributor=self.contributor) self.collection_2 = Collection.objects.create(name='Second collection', contributor=self.contributor) self.entity_1 = Entity.objects.create(name='First entity', contributor=self.contributor) self.entity_2 = Entity.objects.create(name='Second entity', contributor=self.contributor) self.entity_3 = Entity.objects.create(name='Third entity', contributor=self.contributor) self.entity_4 = Entity.objects.create(name='Fourth entity', contributor=self.contributor) assign_contributor_permissions(self.collection, self.contributor) assign_contributor_permissions(self.collection_2, self.contributor) rel_type_group = RelationType.objects.get(name='group') rel_type_series = RelationType.objects.get(name='series') self.relation_group = Relation.objects.create( contributor=self.contributor, collection=self.collection, type=rel_type_group, category='replicates', ) self.group_partiton_1 = RelationPartition.objects.create(relation=self.relation_group, entity=self.entity_1) self.group_partiton_2 = RelationPartition.objects.create(relation=self.relation_group, entity=self.entity_2) self.relation_series = Relation.objects.create( contributor=self.contributor, collection=self.collection_2, type=rel_type_series, category='time-series', unit=Relation.UNIT_HOUR, ) self.series_partiton_1 = RelationPartition.objects.create( relation=self.relation_series, entity=self.entity_1, label='beginning', position=0 ) self.series_partiton_2 = RelationPartition.objects.create( relation=self.relation_series, entity=self.entity_2, label='beginning', position=0 ) self.series_partiton_3 = RelationPartition.objects.create( relation=self.relation_series, entity=self.entity_3, label='end', position=1 ) self.series_partiton_4 = RelationPartition.objects.create( relation=self.relation_series, entity=self.entity_4, label='end', position=1 ) assign_perm('view_relation', self.contributor, self.relation_group) assign_perm('view_relation', self.contributor, self.relation_series)
def setUp(self): self.viewset = RelationViewSet self.resource_name = 'relation' super().setUp() # Load fixtures with relation types flow_config = apps.get_app_config('flow') call_command('loaddata', os.path.join(flow_config.path, 'tests', 'fixtures', 'relationtypes'), verbosity=0) self.collection = Collection.objects.create(name='Test collection', contributor=self.contributor) self.collection_2 = Collection.objects.create(name='Second collection', contributor=self.contributor) self.entity_1 = Entity.objects.create(name='First entity', contributor=self.contributor) self.entity_2 = Entity.objects.create(name='Second entity', contributor=self.contributor) self.entity_3 = Entity.objects.create(name='Third entity', contributor=self.contributor) self.entity_4 = Entity.objects.create(name='Fourth entity', contributor=self.contributor) assign_contributor_permissions(self.collection, self.contributor) assign_contributor_permissions(self.collection_2, self.contributor) rel_type_group = RelationType.objects.get(name='group') rel_type_series = RelationType.objects.get(name='series') self.relation_group = Relation.objects.create( contributor=self.contributor, collection=self.collection, type=rel_type_group, category='replicates', ) self.group_partiton_1 = RelationPartition.objects.create(relation=self.relation_group, entity=self.entity_1) self.group_partiton_2 = RelationPartition.objects.create(relation=self.relation_group, entity=self.entity_2) self.relation_series = Relation.objects.create( contributor=self.contributor, collection=self.collection_2, type=rel_type_series, category='time-series', unit=Relation.UNIT_HOUR, ) self.series_partiton_1 = RelationPartition.objects.create( relation=self.relation_series, entity=self.entity_1, label='beginning', position=0 ) self.series_partiton_2 = RelationPartition.objects.create( relation=self.relation_series, entity=self.entity_2, label='beginning', position=0 ) self.series_partiton_3 = RelationPartition.objects.create( relation=self.relation_series, entity=self.entity_3, label='end', position=1 ) self.series_partiton_4 = RelationPartition.objects.create( relation=self.relation_series, entity=self.entity_4, label='end', position=1 ) assign_perm('view_relation', self.contributor, self.relation_group) assign_perm('view_relation', self.contributor, self.relation_series)
def test_update_different_user(self): data = {'collection': self.collection_2.pk} resp = self._patch(self.relation_group.pk, data, user=self.user) self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND) self.relation_group.refresh_from_db() self.assertEqual(self.relation_group.collection, self.collection) assign_contributor_permissions(self.collection, self.user) resp = self._patch(self.relation_group.pk, data, user=self.user) self.assertEqual(resp.status_code, status.HTTP_200_OK) self.relation_group.refresh_from_db() self.assertEqual(self.relation_group.collection, self.collection_2)
def test_update_different_user(self): data = {'collection': self.collection_2.pk} resp = self._patch(self.relation_group.pk, data, user=self.user) self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND) self.relation_group.refresh_from_db() self.assertEqual(self.relation_group.collection, self.collection) assign_contributor_permissions(self.collection, self.user) resp = self._patch(self.relation_group.pk, data, user=self.user) self.assertEqual(resp.status_code, status.HTTP_200_OK) self.relation_group.refresh_from_db() self.assertEqual(self.relation_group.collection, self.collection_2)
def register_descriptors(self, descriptor_schemas, user, force=False, verbosity=1): """Read and register descriptors.""" log_descriptors = [] for descriptor_schema in descriptor_schemas: for schema, _, _ in iterate_schema({}, descriptor_schema.get('schema', {})): if not schema['type'][-1].endswith(':'): schema['type'] += ':' if 'schema' not in descriptor_schema: descriptor_schema['schema'] = [] if not self.valid(descriptor_schema, DESCRIPTOR_SCHEMA): continue slug = descriptor_schema['slug'] version = descriptor_schema.get('version', '0.0.0') int_version = convert_version_string_to_int(version, VERSION_NUMBER_BITS) # `latest version` is returned as `int` so it has to be compared to `int_version` latest_version = DescriptorSchema.objects.filter(slug=slug).aggregate(Max('version'))['version__max'] if latest_version is not None and latest_version > int_version: self.stderr.write("Skip descriptor schema {}: newer version installed".format(slug)) continue previous_descriptor_qs = DescriptorSchema.objects.filter(slug=slug) if previous_descriptor_qs.exists(): previous_descriptor = previous_descriptor_qs.latest() else: previous_descriptor = None descriptor_query = DescriptorSchema.objects.filter(slug=slug, version=version) if descriptor_query.exists(): if not force: if verbosity > 0: self.stdout.write("Skip descriptor schema {}: same version installed".format(slug)) continue descriptor_query.update(**descriptor_schema) log_descriptors.append("Updated {}".format(slug)) else: descriptor = DescriptorSchema.objects.create(contributor=user, **descriptor_schema) assign_contributor_permissions(descriptor) if previous_descriptor: copy_permissions(previous_descriptor, descriptor) log_descriptors.append("Inserted {}".format(slug)) if log_descriptors and verbosity > 0: self.stdout.write("Descriptor schemas Updates:") for log in log_descriptors: self.stdout.write(" {}".format(log))
def duplicate(self, contributor=None): """Duplicate (make a copy).""" if self.status not in [self.STATUS_DONE, self.STATUS_ERROR]: raise ValidationError('Data object must have done or error status to be duplicated') duplicate = Data.objects.get(id=self.id) duplicate.pk = None duplicate.slug = None duplicate.name = 'Copy of {}'.format(self.name) duplicate.duplicated = now() if contributor: duplicate.contributor = contributor duplicate._perform_save(force_insert=True) # pylint: disable=protected-access assign_contributor_permissions(duplicate) # Override fields that are automatically set on create. duplicate.created = self.created duplicate._perform_save() # pylint: disable=protected-access if self.location: self.location.data.add(duplicate) # pylint: disable=no-member duplicate.storages.set(self.storages.all()) # pylint: disable=no-member for migration in self.migration_history.order_by('created'): # pylint: disable=no-member migration.pk = None migration.data = duplicate migration.save(force_insert=True) # Inherit existing child dependencies. DataDependency.objects.bulk_create([ DataDependency(child=duplicate, parent=dependency.parent, kind=dependency.kind) for dependency in DataDependency.objects.filter(child=self) ]) # Inherit existing parent dependencies. DataDependency.objects.bulk_create([ DataDependency(child=dependency.child, parent=duplicate, kind=dependency.kind) for dependency in DataDependency.objects.filter(parent=self) ]) return duplicate
def test_delete_different_user(self): # No view permissions, authenticated. resp = self._delete(self.relation_group.pk, user=self.user) self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND) self.assertTrue( Relation.objects.filter(pk=self.relation_group.pk).exists()) # No edit permissions, authenticated. assign_perm("view_collection", self.user, self.collection) resp = self._delete(self.relation_group.pk, user=self.user) self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN) self.assertTrue( Relation.objects.filter(pk=self.relation_group.pk).exists()) # Edit & view permissions, authenticated. assign_contributor_permissions(self.collection, self.user) resp = self._delete(self.relation_group.pk, user=self.contributor) self.assertEqual(resp.status_code, status.HTTP_204_NO_CONTENT) self.assertFalse( Relation.objects.filter(pk=self.relation_group.pk).exists())
def test_delete_different_user(self): # No view permissions: anonymous user has view permission: remove it. self.collection.set_permission(Permission.NONE, AnonymousUser()) resp = self._delete(self.relation_group.pk, user=self.user) self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND) self.assertTrue( Relation.objects.filter(pk=self.relation_group.pk).exists()) # No edit permissions, authenticated. self.collection.set_permission(Permission.VIEW, self.user) resp = self._delete(self.relation_group.pk, user=self.user) self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN) self.assertTrue( Relation.objects.filter(pk=self.relation_group.pk).exists()) # Edit & view permissions, authenticated. assign_contributor_permissions(self.collection, self.user) resp = self._delete(self.relation_group.pk, user=self.contributor) self.assertEqual(resp.status_code, status.HTTP_204_NO_CONTENT) self.assertFalse( Relation.objects.filter(pk=self.relation_group.pk).exists())
def create_entity(self): """Create entity if `flow_collection` is defined in process. Following rules applies for adding `Data` object to `Entity`: * Only add `Data object` to `Entity` if process has defined `flow_collwection` field * Add object to existing `Entity`, if all parents that are part of it (but not necessary all parents), are part of the same `Entity` * If parents belong to different `Entities` or do not belong to any `Entity`, create new `Entity` """ ds_slug = self.process.flow_collection # pylint: disable=no-member if ds_slug: entity_query = Entity.objects.filter( data__in=self.parents.all()).distinct() # pylint: disable=no-member if entity_query.count() == 1: entity = entity_query.first() copy_permissions(entity, self) else: descriptor_schema = DescriptorSchema.objects.filter( slug=ds_slug).latest() entity = Entity.objects.create( contributor=self.contributor, descriptor_schema=descriptor_schema, name=self.name, tags=self.tags, ) assign_contributor_permissions(entity) entity.data.add(self) # Inherite collections from entity. for collection in entity.collections.all(): collection.data.add(self)
def test_update_different_user(self): # No view permission. data = {"collection": {"id": self.collection_2.pk}} resp = self._patch(self.relation_group.pk, data, user=self.user) self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND) self.relation_group.refresh_from_db() self.assertEqual(self.relation_group.collection, self.collection) # No edit permission. assign_perm("view_collection", self.user, self.collection) resp = self._patch(self.relation_group.pk, data, user=self.user) self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN) self.relation_group.refresh_from_db() self.assertEqual(self.relation_group.collection, self.collection) # All permissions. assign_contributor_permissions(self.collection, self.user) assign_contributor_permissions(self.collection_2, self.user) resp = self._patch(self.relation_group.pk, data, user=self.user) self.assertEqual(resp.status_code, status.HTTP_200_OK) self.relation_group.refresh_from_db() self.assertEqual(self.relation_group.collection, self.collection_2)
def perform_create(self, serializer): """Create a resource.""" with transaction.atomic(): instance = serializer.save() assign_contributor_permissions(instance) # Entity is added to the collection only when it is # created - when it only contains 1 Data object. entities = Entity.objects.annotate(num_data=Count('data')).filter( data=instance, num_data=1) # Assign data object to all specified collections. collection_pks = self.request.data.get('collections', []) for collection in Collection.objects.filter(pk__in=collection_pks): collection.data.add(instance) copy_permissions(collection, instance) # Add entities to which data belongs to the collection. for entity in entities: entity.collections.add(collection) copy_permissions(collection, entity)
def test_update_different_user(self): # No view permission: anonymous user has view permission: remove it. self.collection.set_permission(Permission.NONE, AnonymousUser()) data = {"collection": {"id": self.collection_2.pk}} resp = self._patch(self.relation_group.pk, data, user=self.user) self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND) self.relation_group.refresh_from_db() self.assertEqual(self.relation_group.collection, self.collection) # No edit permission. self.collection.set_permission(Permission.VIEW, self.user) resp = self._patch(self.relation_group.pk, data, user=self.user) self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN) self.relation_group.refresh_from_db() self.assertEqual(self.relation_group.collection, self.collection) # All permissions. assign_contributor_permissions(self.collection, self.user) assign_contributor_permissions(self.collection_2, self.user) resp = self._patch(self.relation_group.pk, data, user=self.user) self.assertEqual(resp.status_code, status.HTTP_200_OK) self.relation_group.refresh_from_db() self.assertEqual(self.relation_group.collection, self.collection_2)
def duplicate(self, contributor=None, inherit_collection=False): """Duplicate (make a copy).""" duplicate = Entity.objects.get(id=self.id) duplicate.pk = None duplicate.slug = None duplicate.name = "Copy of {}".format(self.name) duplicate.duplicated = now() if contributor: duplicate.contributor = contributor duplicate.collection = None if inherit_collection: if not contributor.has_perm("edit_collection", self.collection): raise ValidationError( "You do not have edit permission on collection {}.".format( self.collection)) duplicate.collection = self.collection duplicate.save(force_insert=True) assign_contributor_permissions(duplicate) # Override fields that are automatically set on create. duplicate.created = self.created duplicate.save() # Duplicate entity's data objects. data = get_objects_for_user(contributor, "view_data", self.data.all()) duplicated_data = data.duplicate(contributor, inherit_collection=inherit_collection) duplicate.data.add(*duplicated_data) # Permissions assign_contributor_permissions(duplicate) copy_permissions(duplicate.collection, duplicate) return duplicate
def duplicate(self, contributor=None, inherit_collections=False): """Duplicate (make a copy).""" duplicate = Entity.objects.get(id=self.id) duplicate.pk = None duplicate.slug = None duplicate.name = 'Copy of {}'.format(self.name) duplicate.duplicated = now() if contributor: duplicate.contributor = contributor duplicate.save(force_insert=True) assign_contributor_permissions(duplicate) # Override fields that are automatically set on create. duplicate.created = self.created duplicate.save() # Duplicate entity's data objects. data = get_objects_for_user(contributor, 'view_data', self.data.all()) # pylint: disable=no-member duplicated_data = data.duplicate(contributor) duplicate.data.add(*duplicated_data) if inherit_collections: collections = get_objects_for_user( contributor, 'add_collection', self.collections.all() # pylint: disable=no-member ) for collection in collections: collection.entity_set.add(duplicate) copy_permissions(collection, duplicate) collection.data.add(*duplicated_data) for datum in duplicated_data: copy_permissions(collection, datum) return duplicate
def register_processes(self, process_schemas, user, force=False, verbosity=1): """Read and register processors.""" log_processors = [] log_templates = [] for p in process_schemas: # TODO: Remove this when all processes are migrated to the # new syntax. if 'flow_collection' in p: if 'entity' in p: self.stderr.write( "Skip processor {}: only one of 'flow_collection' and 'entity' fields " "allowed".format(p['slug']) ) continue p['entity'] = {'type': p.pop('flow_collection')} if p['type'][-1] != ':': p['type'] += ':' if 'category' in p and not p['category'].endswith(':'): p['category'] += ':' for field in ['input', 'output']: for schema, _, _ in iterate_schema({}, p[field] if field in p else {}): if not schema['type'][-1].endswith(':'): schema['type'] += ':' # TODO: Check if schemas validate with our JSON meta schema and Processor model docs. if not self.valid(p, PROCESSOR_SCHEMA): continue if 'entity' in p: if 'type' not in p['entity']: self.stderr.write( "Skip process {}: 'entity.type' required if 'entity' defined".format(p['slug']) ) continue p['entity_type'] = p['entity']['type'] p['entity_descriptor_schema'] = p['entity'].get('descriptor_schema', p['entity_type']) p['entity_input'] = p['entity'].get('input', None) p.pop('entity') if not DescriptorSchema.objects.filter(slug=p['entity_descriptor_schema']).exists(): self.stderr.write( "Skip processor {}: Unknown descriptor schema '{}' used in 'entity' " "field.".format(p['slug'], p['entity_descriptor_schema']) ) continue if 'persistence' in p: persistence_mapping = { 'RAW': Process.PERSISTENCE_RAW, 'CACHED': Process.PERSISTENCE_CACHED, 'TEMP': Process.PERSISTENCE_TEMP, } p['persistence'] = persistence_mapping[p['persistence']] if 'scheduling_class' in p: scheduling_class_mapping = { 'interactive': Process.SCHEDULING_CLASS_INTERACTIVE, 'batch': Process.SCHEDULING_CLASS_BATCH } p['scheduling_class'] = scheduling_class_mapping[p['scheduling_class']] if 'input' in p: p['input_schema'] = p.pop('input') if 'output' in p: p['output_schema'] = p.pop('output') slug = p['slug'] if 'run' in p: # Set default language to 'bash' if not set. p['run'].setdefault('language', 'bash') # Transform output schema using the execution engine. try: execution_engine = manager.get_execution_engine(p['run']['language']) extra_output_schema = execution_engine.get_output_schema(p) if extra_output_schema: p.setdefault('output_schema', []).extend(extra_output_schema) except InvalidEngineError: self.stderr.write("Skip processor {}: execution engine '{}' not supported".format( slug, p['run']['language'] )) continue # Validate if container image is allowed based on the configured pattern. # NOTE: This validation happens here and is not deferred to executors because the idea # is that this will be moved to a "container" requirement independent of the # executor. if hasattr(settings, 'FLOW_CONTAINER_VALIDATE_IMAGE'): try: container_image = dict_dot(p, 'requirements.executor.docker.image') if not re.match(settings.FLOW_CONTAINER_VALIDATE_IMAGE, container_image): self.stderr.write("Skip processor {}: container image does not match '{}'".format( slug, settings.FLOW_CONTAINER_VALIDATE_IMAGE, )) continue except KeyError: pass version = p['version'] int_version = convert_version_string_to_int(version, VERSION_NUMBER_BITS) # `latest version` is returned as `int` so it has to be compared to `int_version` latest_version = Process.objects.filter(slug=slug).aggregate(Max('version'))['version__max'] if latest_version is not None and latest_version > int_version: self.stderr.write("Skip processor {}: newer version installed".format(slug)) continue previous_process_qs = Process.objects.filter(slug=slug) if previous_process_qs.exists(): previous_process = previous_process_qs.latest() else: previous_process = None process_query = Process.objects.filter(slug=slug, version=version) if process_query.exists(): if not force: if verbosity > 0: self.stdout.write("Skip processor {}: same version installed".format(slug)) continue process_query.update(**p) log_processors.append("Updated {}".format(slug)) else: process = Process.objects.create(contributor=user, **p) assign_contributor_permissions(process) if previous_process: copy_permissions(previous_process, process) log_processors.append("Inserted {}".format(slug)) if verbosity > 0: if log_processors: self.stdout.write("Processor Updates:") for log in log_processors: self.stdout.write(" {}".format(log)) if log_templates: self.stdout.write("Default Template Updates:") for log in log_templates: self.stdout.write(" {}".format(log))
def register_processes(self, process_schemas, user, force=False, verbosity=1): """Read and register processors.""" log_processors = [] log_templates = [] for p in process_schemas: if p['type'][-1] != ':': p['type'] += ':' if 'category' in p and not p['category'].endswith(':'): p['category'] += ':' for field in ['input', 'output']: for schema, _, _ in iterate_schema( {}, p[field] if field in p else {}): if not schema['type'][-1].endswith(':'): schema['type'] += ':' # TODO: Check if schemas validate with our JSON meta schema and Processor model docs. if not self.valid(p, PROCESSOR_SCHEMA): continue if 'persistence' in p: persistence_mapping = { 'RAW': Process.PERSISTENCE_RAW, 'CACHED': Process.PERSISTENCE_CACHED, 'TEMP': Process.PERSISTENCE_TEMP, } p['persistence'] = persistence_mapping[p['persistence']] if 'scheduling_class' in p: scheduling_class_mapping = { 'interactive': Process.SCHEDULING_CLASS_INTERACTIVE, 'batch': Process.SCHEDULING_CLASS_BATCH } p['scheduling_class'] = scheduling_class_mapping[ p['scheduling_class']] if 'input' in p: p['input_schema'] = p.pop('input') if 'output' in p: p['output_schema'] = p.pop('output') slug = p['slug'] if 'run' in p: # Set default language to 'bash' if not set. p['run'].setdefault('language', 'bash') # Transform output schema using the execution engine. try: execution_engine = manager.get_execution_engine( p['run']['language']) extra_output_schema = execution_engine.get_output_schema(p) if extra_output_schema: p.setdefault('output_schema', []).extend(extra_output_schema) except InvalidEngineError: self.stderr.write( "Skip processor {}: execution engine '{}' not supported" .format(slug, p['run']['language'])) continue # Validate if container image is allowed based on the configured pattern. # NOTE: This validation happens here and is not deferred to executors because the idea # is that this will be moved to a "container" requirement independent of the # executor. if hasattr(settings, 'FLOW_CONTAINER_VALIDATE_IMAGE'): try: container_image = dict_dot( p, 'requirements.executor.docker.image') if not re.match(settings.FLOW_CONTAINER_VALIDATE_IMAGE, container_image): self.stderr.write( "Skip processor {}: container image does not match '{}'" .format( slug, settings.FLOW_CONTAINER_VALIDATE_IMAGE, )) continue except KeyError: pass version = p['version'] int_version = convert_version_string_to_int( version, VERSION_NUMBER_BITS) # `latest version` is returned as `int` so it has to be compared to `int_version` latest_version = Process.objects.filter(slug=slug).aggregate( Max('version'))['version__max'] if latest_version is not None and latest_version > int_version: self.stderr.write( "Skip processor {}: newer version installed".format(slug)) continue previous_process_qs = Process.objects.filter(slug=slug) if previous_process_qs.exists(): previous_process = previous_process_qs.latest() else: previous_process = None process_query = Process.objects.filter(slug=slug, version=version) if process_query.exists(): if not force: if verbosity > 0: self.stdout.write( "Skip processor {}: same version installed".format( slug)) continue process_query.update(**p) log_processors.append("Updated {}".format(slug)) else: process = Process.objects.create(contributor=user, **p) assign_contributor_permissions(process) if previous_process: copy_permissions(previous_process, process) log_processors.append("Inserted {}".format(slug)) if verbosity > 0: if log_processors: self.stdout.write("Processor Updates:") for log in log_processors: self.stdout.write(" {}".format(log)) if log_templates: self.stdout.write("Default Template Updates:") for log in log_templates: self.stdout.write(" {}".format(log))
def setUp(self): self.viewset = RelationViewSet self.resource_name = "relation" super().setUp() # Load fixtures with relation types flow_config = apps.get_app_config("flow") call_command( "loaddata", os.path.join(flow_config.path, "tests", "fixtures", "relationtypes"), verbosity=0, ) self.collection = Collection.objects.create( name="Test collection", contributor=self.contributor) self.collection_2 = Collection.objects.create( name="Second collection", contributor=self.contributor) self.entity_1 = Entity.objects.create(name="First entity", contributor=self.contributor) self.entity_2 = Entity.objects.create(name="Second entity", contributor=self.contributor) self.entity_3 = Entity.objects.create(name="Third entity", contributor=self.contributor) self.entity_4 = Entity.objects.create(name="Fourth entity", contributor=self.contributor) assign_contributor_permissions(self.collection, self.contributor) assign_contributor_permissions(self.collection_2, self.contributor) rel_type_group = RelationType.objects.get(name="group") rel_type_series = RelationType.objects.get(name="series") self.relation_group = Relation.objects.create( contributor=self.contributor, collection=self.collection, type=rel_type_group, category="replicates", ) self.group_partiton_1 = RelationPartition.objects.create( relation=self.relation_group, entity=self.entity_1) self.group_partiton_2 = RelationPartition.objects.create( relation=self.relation_group, entity=self.entity_2) self.relation_series = Relation.objects.create( contributor=self.contributor, collection=self.collection_2, type=rel_type_series, category="time-series", unit=Relation.UNIT_HOUR, ) self.series_partiton_1 = RelationPartition.objects.create( relation=self.relation_series, entity=self.entity_1, label="beginning", position=0, ) self.series_partiton_2 = RelationPartition.objects.create( relation=self.relation_series, entity=self.entity_2, label="beginning", position=0, ) self.series_partiton_3 = RelationPartition.objects.create( relation=self.relation_series, entity=self.entity_3, label="end", position=1) self.series_partiton_4 = RelationPartition.objects.create( relation=self.relation_series, entity=self.entity_4, label="end", position=1) assign_perm("view_relation", self.contributor, self.relation_group) assign_perm("view_relation", self.contributor, self.relation_series)
def register_descriptors(self, descriptor_schemas, user, force=False, verbosity=1): """Read and register descriptors.""" log_descriptors = [] for descriptor_schema in descriptor_schemas: for schema, _, _ in iterate_schema({}, descriptor_schema.get( "schema", {})): if not schema["type"][-1].endswith(":"): schema["type"] += ":" if "schema" not in descriptor_schema: descriptor_schema["schema"] = [] if not self.valid(descriptor_schema, DESCRIPTOR_SCHEMA): continue slug = descriptor_schema["slug"] version = descriptor_schema.get("version", "0.0.0") int_version = convert_version_string_to_int( version, VERSION_NUMBER_BITS) # `latest version` is returned as `int` so it has to be compared to `int_version` latest_version = DescriptorSchema.objects.filter( slug=slug).aggregate(Max("version"))["version__max"] if latest_version is not None and latest_version > int_version: self.stderr.write( "Skip descriptor schema {}: newer version installed". format(slug)) continue previous_descriptor_qs = DescriptorSchema.objects.filter(slug=slug) if previous_descriptor_qs.exists(): previous_descriptor = previous_descriptor_qs.latest() else: previous_descriptor = None descriptor_query = DescriptorSchema.objects.filter(slug=slug, version=version) if descriptor_query.exists(): if not force: if verbosity > 0: self.stdout.write( "Skip descriptor schema {}: same version installed" .format(slug)) continue descriptor_query.update(**descriptor_schema) log_descriptors.append("Updated {}".format(slug)) else: descriptor = DescriptorSchema.objects.create( contributor=user, **descriptor_schema) assign_contributor_permissions(descriptor) if previous_descriptor: copy_permissions(previous_descriptor, descriptor) log_descriptors.append("Inserted {}".format(slug)) if log_descriptors and verbosity > 0: self.stdout.write("Descriptor schemas Updates:") for log in log_descriptors: self.stdout.write(" {}".format(log))
def register_processes(self, process_schemas, user, force=False, verbosity=1): """Read and register processors.""" log_processors = [] log_templates = [] for p in process_schemas: # TODO: Remove this when all processes are migrated to the # new syntax. if "flow_collection" in p: if "entity" in p: self.stderr.write( "Skip processor {}: only one of 'flow_collection' and 'entity' fields " "allowed".format(p["slug"])) continue p["entity"] = {"type": p.pop("flow_collection")} if p["type"][-1] != ":": p["type"] += ":" if "category" in p and not p["category"].endswith(":"): p["category"] += ":" for field in ["input", "output"]: for schema, _, _ in iterate_schema( {}, p[field] if field in p else {}): if not schema["type"][-1].endswith(":"): schema["type"] += ":" # TODO: Check if schemas validate with our JSON meta schema and Processor model docs. if not self.valid(p, PROCESSOR_SCHEMA): continue if "entity" in p: if "type" not in p["entity"]: self.stderr.write( "Skip process {}: 'entity.type' required if 'entity' defined" .format(p["slug"])) continue if "input" in p["entity"] and p["entity"].get( "always_create", False): self.stderr.write( "Skip process {}: 'entity.input' will not be considered if 'entity.always_create' " "is set to true.".format(p["slug"])) continue p["entity_type"] = p["entity"]["type"] p["entity_descriptor_schema"] = p["entity"].get( "descriptor_schema", p["entity_type"]) p["entity_input"] = p["entity"].get("input", None) p["entity_always_create"] = p["entity"].get( "always_create", False) p.pop("entity") if not DescriptorSchema.objects.filter( slug=p["entity_descriptor_schema"]).exists(): self.stderr.write( "Skip processor {}: Unknown descriptor schema '{}' used in 'entity' " "field.".format(p["slug"], p["entity_descriptor_schema"])) continue if "persistence" in p: persistence_mapping = { "RAW": Process.PERSISTENCE_RAW, "CACHED": Process.PERSISTENCE_CACHED, "TEMP": Process.PERSISTENCE_TEMP, } p["persistence"] = persistence_mapping[p["persistence"]] if "scheduling_class" in p: scheduling_class_mapping = { "interactive": Process.SCHEDULING_CLASS_INTERACTIVE, "batch": Process.SCHEDULING_CLASS_BATCH, } p["scheduling_class"] = scheduling_class_mapping[ p["scheduling_class"]] if "input" in p: p["input_schema"] = p.pop("input") if "output" in p: p["output_schema"] = p.pop("output") slug = p["slug"] if "run" in p: # Set default language to 'bash' if not set. p["run"].setdefault("language", "bash") # Transform output schema using the execution engine. try: execution_engine = manager.get_execution_engine( p["run"]["language"]) extra_output_schema = execution_engine.get_output_schema(p) if extra_output_schema: p.setdefault("output_schema", []).extend(extra_output_schema) except InvalidEngineError: self.stderr.write( "Skip processor {}: execution engine '{}' not supported" .format(slug, p["run"]["language"])) continue # Validate if container image is allowed based on the configured pattern. # NOTE: This validation happens here and is not deferred to executors because the idea # is that this will be moved to a "container" requirement independent of the # executor. if hasattr(settings, "FLOW_CONTAINER_VALIDATE_IMAGE"): try: container_image = dict_dot( p, "requirements.executor.docker.image") if not re.match(settings.FLOW_CONTAINER_VALIDATE_IMAGE, container_image): self.stderr.write( "Skip processor {}: container image does not match '{}'" .format( slug, settings.FLOW_CONTAINER_VALIDATE_IMAGE, )) continue except KeyError: pass version = p["version"] int_version = convert_version_string_to_int( version, VERSION_NUMBER_BITS) # `latest version` is returned as `int` so it has to be compared to `int_version` latest_version = Process.objects.filter(slug=slug).aggregate( Max("version"))["version__max"] if latest_version is not None and latest_version > int_version: self.stderr.write( "Skip processor {}: newer version installed".format(slug)) continue previous_process_qs = Process.objects.filter(slug=slug) if previous_process_qs.exists(): previous_process = previous_process_qs.latest() else: previous_process = None process_query = Process.objects.filter(slug=slug, version=version) if process_query.exists(): if not force: if verbosity > 0: self.stdout.write( "Skip processor {}: same version installed".format( slug)) continue process_query.update(**p) log_processors.append("Updated {}".format(slug)) else: process = Process.objects.create(contributor=user, **p) assign_contributor_permissions(process) if previous_process: copy_permissions(previous_process, process) log_processors.append("Inserted {}".format(slug)) if verbosity > 0: if log_processors: self.stdout.write("Processor Updates:") for log in log_processors: self.stdout.write(" {}".format(log)) if log_templates: self.stdout.write("Default Template Updates:") for log in log_templates: self.stdout.write(" {}".format(log))
def register_processes(self, process_schemas, user, force=False, verbosity=1): """Read and register processors.""" log_processors = [] log_templates = [] for p in process_schemas: if p['type'][-1] != ':': p['type'] += ':' if 'category' in p and not p['category'].endswith(':'): p['category'] += ':' # get `data_name` from `static` if 'static' in p: for schema, _, _ in iterate_schema({}, p['static']): if schema['name'] == 'name' and 'default' in schema: p['data_name'] = schema['default'] # support backward compatibility # TODO: update .yml files and remove if 'slug' not in p: p['slug'] = slugify(p.pop('name').replace(':', '-')) p['name'] = p.pop('label') p.pop('var', None) p.pop('static', None) for field in ['input', 'output', 'var', 'static']: for schema, _, _ in iterate_schema( {}, p[field] if field in p else {}): if not schema['type'][-1].endswith(':'): schema['type'] += ':' # TODO: Check if schemas validate with our JSON meta schema and Processor model docs. if not self.valid(p, PROCESSOR_SCHEMA): continue if 'persistence' in p: persistence_mapping = { 'RAW': Process.PERSISTENCE_RAW, 'CACHED': Process.PERSISTENCE_CACHED, 'TEMP': Process.PERSISTENCE_TEMP, } p['persistence'] = persistence_mapping[p['persistence']] if 'scheduling_class' in p: scheduling_class_mapping = { 'interactive': Process.SCHEDULING_CLASS_INTERACTIVE, 'batch': Process.SCHEDULING_CLASS_BATCH } p['scheduling_class'] = scheduling_class_mapping[ p['scheduling_class']] if 'input' in p: p['input_schema'] = p.pop('input') if 'output' in p: p['output_schema'] = p.pop('output') slug = p['slug'] if 'run' in p: # Set default language to 'bash' if not set. p['run'].setdefault('language', 'bash') # Transform output schema using the execution engine. try: execution_engine = manager.get_execution_engine( p['run']['language']) extra_output_schema = execution_engine.get_output_schema(p) if extra_output_schema: p.setdefault('output_schema', []).extend(extra_output_schema) except InvalidEngineError: self.stderr.write( "Skip processor {}: execution engine '{}' not supported" .format(slug, p['run']['language'])) continue version = p['version'] int_version = convert_version_string_to_int( version, VERSION_NUMBER_BITS) # `latest version` is returned as `int` so it has to be compared to `int_version` latest_version = Process.objects.filter(slug=slug).aggregate( Max('version'))['version__max'] if latest_version is not None and latest_version > int_version: self.stderr.write( "Skip processor {}: newer version installed".format(slug)) continue previous_process_qs = Process.objects.filter(slug=slug) if previous_process_qs.exists(): previous_process = previous_process_qs.latest() else: previous_process = None process_query = Process.objects.filter(slug=slug, version=version) if process_query.exists(): if not force: if verbosity > 0: self.stdout.write( "Skip processor {}: same version installed".format( slug)) continue process_query.update(**p) log_processors.append("Updated {}".format(slug)) else: process = Process.objects.create(contributor=user, **p) assign_contributor_permissions(process) if previous_process: copy_permissions(previous_process, process) log_processors.append("Inserted {}".format(slug)) if verbosity > 0: if len(log_processors) > 0: self.stdout.write("Processor Updates:") for log in log_processors: self.stdout.write(" {}".format(log)) if len(log_templates) > 0: self.stdout.write("Default Template Updates:") for log in log_templates: self.stdout.write(" {}".format(log))
def copy_objects(objects, contributor, name_prefix, obj_processor=None): """Make a copy of given queryset. Shallow copy given queryset and set contributor to the given value, prepend name with the prefix, set slug to a unique value, and set ``duplicated`` date to the current time. Special attention is paid to keep the ``created`` date to its original value. If ``obj_processor`` function is given, each object is passed to it and the return value is used instead of it. :param objects: A queryset to be copied. :type objects: `~resolwe.flow.models.base.BaseQuerySet` :param contributor: A Django user that will be assigned to copied objects as contributor. :type contributor: `~django.contrib.auth.models.User` :param str name_prefix: A prefix that will be prepend to the name of copied objects. """ first = objects.first() if not first: return objects name_max_length = first._meta.get_field("name").max_length model = first._meta.model new_objects = [] for obj in objects: new_obj = deepcopy(obj) new_obj.pk = None new_obj.slug = None new_obj.contributor = contributor new_obj.name = "{} {}".format(name_prefix, obj.name) new_obj._container_attributes = dict() if len(new_obj.name) > name_max_length: new_obj.name = "{}...".format(new_obj.name[:name_max_length - 3]) if obj_processor: new_obj = obj_processor(new_obj) new_objects.append(new_obj) try: # Add another atomic block to avoid corupting the main one. with transaction.atomic(): model.objects.bulk_create(new_objects) # Send the bulk create custom signal, avoid circular import. from resolwe.flow.signals import post_duplicate post_duplicate.send(sender=model, instances=new_objects, old_instances=objects) except IntegrityError: # Probably a slug collision occured, try to create objects one by one. for obj in new_objects: obj.slug = None # Call the parent method to skip pre-processing and validation. models.Model.save(obj) object_permission_group = dict() not_in_container = list() for old, new in zip(objects, new_objects): new.created = old.created new.duplicated = timezone.now() # Deal with permissions. When object is in container fix the pointer # to permission_group object. # When object is not in container new PermissionGroup proxy object must # be created, assigned to new object and permissions copied from old # object to new one. if getattr(new, "collection_id", None) or getattr( new, "entity_id", None): new.permission_group = new.topmost_container.permission_group else: not_in_container.append((new, old)) object_permission_group[new] = PermissionGroup() PermissionGroup.objects.bulk_create(object_permission_group.values()) for new, old in not_in_container: new.permission_group = object_permission_group[new] copy_permissions(old, new) assign_contributor_permissions(new, contributor) model.objects.bulk_update(new_objects, ["created", "duplicated", "permission_group"]) return new_objects
def create_entity(self): """Create entity if `flow_collection` is defined in process. Following rules applies for adding `Data` object to `Entity`: * Only add `Data object` to `Entity` if process has defined `flow_collection` field * Add object to existing `Entity`, if all parents that are part of it (but not necessary all parents), are part of the same `Entity` * If parents belong to different `Entities` or do not belong to any `Entity`, create new `Entity` """ entity_type = self.process.entity_type # pylint: disable=no-member entity_descriptor_schema = self.process.entity_descriptor_schema # pylint: disable=no-member entity_input = self.process.entity_input # pylint: disable=no-member if entity_type: data_filter = {} if entity_input: input_id = dict_dot(self.input, entity_input, default=lambda: None) if input_id is None: logger.warning("Skipping creation of entity due to missing input.") return if isinstance(input_id, int): data_filter['data__pk'] = input_id elif isinstance(input_id, list): data_filter['data__pk__in'] = input_id else: raise ValueError( "Cannot create entity due to invalid value of field {}.".format(entity_input) ) else: data_filter['data__in'] = self.parents.all() # pylint: disable=no-member entity_query = Entity.objects.filter(type=entity_type, **data_filter).distinct() entity_count = entity_query.count() if entity_count == 0: descriptor_schema = DescriptorSchema.objects.filter( slug=entity_descriptor_schema ).latest() entity = Entity.objects.create( contributor=self.contributor, descriptor_schema=descriptor_schema, type=entity_type, name=self.name, tags=self.tags, ) assign_contributor_permissions(entity) elif entity_count == 1: entity = entity_query.first() copy_permissions(entity, self) else: logger.info("Skipping creation of entity due to multiple entities found.") entity = None if entity: entity.data.add(self) # Inherit collections from entity. for collection in entity.collections.all(): collection.data.add(self)
def perform_create(self, serializer): """Create a relation.""" with transaction.atomic(): instance = serializer.save() assign_contributor_permissions(instance)
def create_entity(self): """Create entity if `flow_collection` is defined in process. Following rules applies for adding `Data` object to `Entity`: * Only add `Data object` to `Entity` if process has defined `flow_collwection` field * Add object to existing `Entity`, if all parents that are part of it (but not necessary all parents), are part of the same `Entity` * If parents belong to different `Entities` or do not belong to any `Entity`, create new `Entity` """ entity_type = self.process.entity_type # pylint: disable=no-member entity_descriptor_schema = self.process.entity_descriptor_schema # pylint: disable=no-member entity_input = self.process.entity_input # pylint: disable=no-member if entity_type: data_filter = {} if entity_input: input_id = dict_dot(self.input, entity_input, default=lambda: None) if input_id is None: logger.warning( "Skipping creation of entity due to missing input.") return if isinstance(input_id, int): data_filter['data__pk'] = input_id elif isinstance(input_id, list): data_filter['data__pk__in'] = input_id else: raise ValueError( "Cannot create entity due to invalid value of field {}." .format(entity_input)) else: data_filter['data__in'] = self.parents.all() # pylint: disable=no-member entity_query = Entity.objects.filter(type=entity_type, **data_filter).distinct() entity_count = entity_query.count() if entity_count == 0: descriptor_schema = DescriptorSchema.objects.filter( slug=entity_descriptor_schema).latest() entity = Entity.objects.create( contributor=self.contributor, descriptor_schema=descriptor_schema, type=entity_type, name=self.name, tags=self.tags, ) assign_contributor_permissions(entity) elif entity_count == 1: entity = entity_query.first() copy_permissions(entity, self) else: logger.info( "Skipping creation of entity due to multiple entities found." ) entity = None if entity: entity.data.add(self) # Inherite collections from entity. for collection in entity.collections.all(): collection.data.add(self)
def setUp(self): # Force reading anonymous user from the database for every test. resolwe.permissions.models.ANONYMOUS_USER = None self.viewset = RelationViewSet self.resource_name = "relation" super().setUp() # Load fixtures with relation types flow_config = apps.get_app_config("flow") call_command( "loaddata", os.path.join(flow_config.path, "tests", "fixtures", "relationtypes"), verbosity=0, ) self.collection = Collection.objects.create( name="Test collection", contributor=self.contributor) self.collection_2 = Collection.objects.create( name="Second collection", contributor=self.contributor) self.entity_1 = Entity.objects.create(name="First entity", contributor=self.contributor) self.entity_2 = Entity.objects.create(name="Second entity", contributor=self.contributor) self.entity_3 = Entity.objects.create(name="Third entity", contributor=self.contributor) self.entity_4 = Entity.objects.create(name="Fourth entity", contributor=self.contributor) assign_contributor_permissions(self.collection, self.contributor) assign_contributor_permissions(self.collection_2, self.contributor) self.rel_type_group = RelationType.objects.get(name="group") self.rel_type_series = RelationType.objects.get(name="series") self.relation_group = Relation.objects.create( contributor=self.contributor, collection=self.collection, type=self.rel_type_group, category="replicates", ) self.group_partiton_1 = RelationPartition.objects.create( relation=self.relation_group, entity=self.entity_1) self.group_partiton_2 = RelationPartition.objects.create( relation=self.relation_group, entity=self.entity_2) self.relation_series = Relation.objects.create( contributor=self.contributor, collection=self.collection_2, type=self.rel_type_series, category="time-series", unit=Relation.UNIT_HOUR, ) self.series_partiton_1 = RelationPartition.objects.create( relation=self.relation_series, entity=self.entity_1, label="beginning", position=0, ) self.series_partiton_2 = RelationPartition.objects.create( relation=self.relation_series, entity=self.entity_2, label="beginning", position=0, ) self.series_partiton_3 = RelationPartition.objects.create( relation=self.relation_series, entity=self.entity_3, label="end", position=1) self.series_partiton_4 = RelationPartition.objects.create( relation=self.relation_series, entity=self.entity_4, label="end", position=1) self.collection.set_permission(Permission.VIEW, AnonymousUser())
def _handle_entity(obj): """Create entity if `entity.type` is defined in process. Following rules applies for adding `Data` object to `Entity`: * Only add `Data object` to `Entity` if process has defined `entity.type` field * Create new entity if parents do not belong to any `Entity` * Add object to existing `Entity`, if all parents that are part of it (but not necessary all parents), are part of the same `Entity` * If parents belong to different `Entities` don't do anything """ entity_type = obj.process.entity_type entity_descriptor_schema = obj.process.entity_descriptor_schema entity_input = obj.process.entity_input entity_always_create = obj.process.entity_always_create operation = HandleEntityOperation.PASS if entity_type: data_filter = {} if entity_input: input_id = dict_dot(obj.input, entity_input, default=lambda: None) if input_id is None: logger.warning( "Skipping creation of entity due to missing input.") return if isinstance(input_id, int): data_filter["data__pk"] = input_id elif isinstance(input_id, list): data_filter["data__pk__in"] = input_id else: raise ValueError( "Cannot create entity due to invalid value of field {}." .format(entity_input)) else: data_filter["data__in"] = obj.parents.all() entity_query = Entity.objects.filter(type=entity_type, **data_filter).distinct() entity_count = entity_query.count() if entity_count == 0 or entity_always_create: descriptor_schema = DescriptorSchema.objects.filter( slug=entity_descriptor_schema).latest() entity = Entity.objects.create( contributor=obj.contributor, descriptor_schema=descriptor_schema, type=entity_type, name=obj.name, tags=obj.tags, ) assign_contributor_permissions(entity) operation = HandleEntityOperation.CREATE elif entity_count == 1: entity = entity_query.first() obj.tags = entity.tags copy_permissions(entity, obj) operation = HandleEntityOperation.ADD else: logger.info( "Skipping creation of entity due to multiple entities found." ) entity = None if entity: obj.entity = entity obj.save() return operation
def register_descriptors(self, descriptor_schemas, user, force=False, verbosity=1): """Read and register descriptors.""" log_descriptors = [] for descriptor_schema in descriptor_schemas: for field in ['var', 'schema']: for schema, _, _ in iterate_schema({}, descriptor_schema.get( field, {})): if not schema['type'][-1].endswith(':'): schema['type'] += ':' # support backward compatibility # TODO: update .yml files and remove if 'slug' not in descriptor_schema: descriptor_schema['slug'] = slugify( descriptor_schema.pop('name').replace(':', '-')) descriptor_schema['name'] = descriptor_schema.pop('label') if 'schema' not in descriptor_schema: descriptor_schema['schema'] = [] if 'static' in descriptor_schema: descriptor_schema['schema'].extend( descriptor_schema.pop('static')) if 'var' in descriptor_schema: descriptor_schema['schema'].extend( descriptor_schema.pop('var')) if not self.valid(descriptor_schema, DESCRIPTOR_SCHEMA): continue slug = descriptor_schema['slug'] version = descriptor_schema.get('version', '0.0.0') int_version = convert_version_string_to_int( version, VERSION_NUMBER_BITS) # `latest version` is returned as `int` so it has to be compared to `int_version` latest_version = DescriptorSchema.objects.filter( slug=slug).aggregate(Max('version'))['version__max'] if latest_version is not None and latest_version > int_version: self.stderr.write( "Skip descriptor schema {}: newer version installed". format(slug)) continue previous_descriptor_qs = DescriptorSchema.objects.filter(slug=slug) if previous_descriptor_qs.exists(): previous_descriptor = previous_descriptor_qs.latest() else: previous_descriptor = None descriptor_query = DescriptorSchema.objects.filter(slug=slug, version=version) if descriptor_query.exists(): if not force: if verbosity > 0: self.stdout.write( "Skip descriptor schema {}: same version installed" .format(slug)) continue descriptor_query.update(**descriptor_schema) log_descriptors.append("Updated {}".format(slug)) else: descriptor = DescriptorSchema.objects.create( contributor=user, **descriptor_schema) assign_contributor_permissions(descriptor) if previous_descriptor: copy_permissions(previous_descriptor, descriptor) log_descriptors.append("Inserted {}".format(slug)) if len(log_descriptors) > 0 and verbosity > 0: self.stdout.write("Descriptor schemas Updates:") for log in log_descriptors: self.stdout.write(" {}".format(log))