def duplicate(self, contributor=None):
"""Duplicate (make a copy)."""
duplicate = Collection.objects.get(id=self.id)
duplicate.pk = None
duplicate.slug = None
duplicate.name = 'Copy of {}'.format(self.name)
duplicate.duplicated = now()
if contributor:
duplicate.contributor = contributor
duplicate.save(force_insert=True)
assign_contributor_permissions(duplicate)
# Fields to inherit from original data object.
duplicate.created = self.created
duplicate.save()
# Duplicate collection's entities.
entities = get_objects_for_user(contributor, 'view_entity',
self.entity_set.all()) # pylint: disable=no-member
duplicated_entities = entities.duplicate(contributor=contributor)
duplicate.entity_set.add(*duplicated_entities)
# Add duplicated data objects to collection.
for duplicated_entity in duplicate.entity_set.all():
duplicate.data.add(*duplicated_entity.data.all())
return duplicate
def perform_create(self, serializer):
"""Create a resource."""
with transaction.atomic():
instance = serializer.save()
# Assign all permissions to the object contributor.
assign_contributor_permissions(instance)
def perform_create(self, serializer):
"""Create a resource."""
process = serializer.validated_data.get('process')
if not process.is_active:
raise exceptions.ParseError(
'Process retired (id: {}, slug: {}/{}).'.format(
process.id, process.slug, process.version))
with transaction.atomic():
instance = serializer.save()
assign_contributor_permissions(instance)
# Entity is added to the collection only when it is
# created - when it only contains 1 Data object.
entities = Entity.objects.annotate(num_data=Count('data')).filter(
data=instance, num_data=1)
# Assign data object to all specified collections.
collection_pks = self.request.data.get('collections', [])
for collection in Collection.objects.filter(pk__in=collection_pks):
collection.data.add(instance)
copy_permissions(collection, instance)
# Add entities to which data belongs to the collection.
for entity in entities:
entity.collections.add(collection)
copy_permissions(collection, entity)
def duplicate(self, contributor=None):
"""Duplicate (make a copy)."""
duplicate = Collection.objects.get(id=self.id)
duplicate.pk = None
duplicate.slug = None
duplicate.name = 'Copy of {}'.format(self.name)
duplicate.duplicated = now()
if contributor:
duplicate.contributor = contributor
duplicate.save(force_insert=True)
assign_contributor_permissions(duplicate)
# Fields to inherit from original data object.
duplicate.created = self.created
duplicate.save()
# Duplicate collection's entities.
entities = get_objects_for_user(contributor, 'view_entity', self.entity_set.all()) # pylint: disable=no-member
duplicated_entities = entities.duplicate(contributor=contributor)
duplicate.entity_set.add(*duplicated_entities)
# Add duplicated data objects to collection.
for duplicated_entity in duplicate.entity_set.all():
duplicate.data.add(*duplicated_entity.data.all())
return duplicate
def perform_create(self, serializer):
"""Create a resource."""
with transaction.atomic():
instance = serializer.save()
# Assign all permissions to the object contributor.
assign_contributor_permissions(instance)
def perform_create(self, serializer):
"""Create a resource."""
process = serializer.validated_data.get('process')
if not process.is_active:
raise exceptions.ParseError(
'Process retired (id: {}, slug: {}/{}).'.format(process.id, process.slug, process.version)
)
with transaction.atomic():
instance = serializer.save()
assign_contributor_permissions(instance)
# Entity is added to the collection only when it is
# created - when it only contains 1 Data object.
entities = Entity.objects.annotate(num_data=Count('data')).filter(data=instance, num_data=1)
# Assign data object to all specified collections.
collection_pks = self.request.data.get('collections', [])
for collection in Collection.objects.filter(pk__in=collection_pks):
collection.data.add(instance)
copy_permissions(collection, instance)
# Add entities to which data belongs to the collection.
for entity in entities:
entity.collections.add(collection)
copy_permissions(collection, entity)
def create(self, subprocess_parent=None, **kwargs):
"""Create new object with the given kwargs."""
obj = super().create(**kwargs)
# Data dependencies
obj.save_dependencies(obj.input, obj.process.input_schema)
if subprocess_parent:
DataDependency.objects.create(
parent=subprocess_parent,
child=obj,
kind=DataDependency.KIND_SUBPROCESS,
)
# Data was from a workflow / spawned process
if not obj.in_container():
copy_permissions(subprocess_parent, obj)
# Entity, Collection assignment
entity_operation = self._handle_entity(obj)
self._handle_collection(obj, entity_operation=entity_operation)
# Assign contributor permission only if Data is not in the container.
if not obj.in_container():
assign_contributor_permissions(obj)
return obj
def duplicate(self, contributor=None, inherit_entity=False, inherit_collection=False):
"""Duplicate (make a copy)."""
if self.status not in [self.STATUS_DONE, self.STATUS_ERROR]:
raise ValidationError('Data object must have done or error status to be duplicated')
duplicate = Data.objects.get(id=self.id)
duplicate.pk = None
duplicate.slug = None
duplicate.name = 'Copy of {}'.format(self.name)
duplicate.duplicated = now()
if contributor:
duplicate.contributor = contributor
duplicate.entity = None
if inherit_entity:
if not contributor.has_perm('add_entity', self.entity):
raise ValidationError("You do not have add permission on entity {}.".format(self.entity))
duplicate.entity = self.entity
duplicate.collection = None
if inherit_collection:
if not contributor.has_perm('add_collection', self.collection):
raise ValidationError("You do not have add permission on collection {}.".format(self.collection))
duplicate.collection = self.collection
duplicate._perform_save(force_insert=True) # pylint: disable=protected-access
# Override fields that are automatically set on create.
duplicate.created = self.created
duplicate._perform_save() # pylint: disable=protected-access
if self.location:
self.location.data.add(duplicate) # pylint: disable=no-member
duplicate.storages.set(self.storages.all()) # pylint: disable=no-member
for migration in self.migration_history.order_by('created'): # pylint: disable=no-member
migration.pk = None
migration.data = duplicate
migration.save(force_insert=True)
# Inherit existing child dependencies.
DataDependency.objects.bulk_create([
DataDependency(child=duplicate, parent=dependency.parent, kind=dependency.kind)
for dependency in DataDependency.objects.filter(child=self)
])
# Inherit existing parent dependencies.
DataDependency.objects.bulk_create([
DataDependency(child=dependency.child, parent=duplicate, kind=dependency.kind)
for dependency in DataDependency.objects.filter(parent=self)
])
# Permissions
assign_contributor_permissions(duplicate)
copy_permissions(duplicate.entity, duplicate)
copy_permissions(duplicate.collection, duplicate)
return duplicate
def perform_create(self, serializer):
"""Create a resource."""
with transaction.atomic():
instance = serializer.save()
# Assign all permissions to the object contributor.
if hasattr(instance,
"permission_group") and not instance.in_container():
assign_contributor_permissions(instance)
def set_descriptor_owner(apps, schema_editor):
user_model = get_user_model()
DescriptorSchema = apps.get_model("flow", "DescriptorSchema")
for descriptor in DescriptorSchema.objects.all():
# In migrations contributor is not a real user instance, so we
# have to get it.
user = user_model.objects.get(pk=descriptor.contributor.pk)
assign_contributor_permissions(descriptor, user)
def set_processes_owner(apps, schema_editor):
user_model = get_user_model()
Process = apps.get_model("flow", "Process")
for process in Process.objects.all():
# In migrations contributor is not a real user instance, so we
# have to get it.
user = user_model.objects.get(pk=process.contributor.pk)
assign_contributor_permissions(process, user)
def test_delete_different_user(self):
resp = self._delete(self.relation_group.pk, user=self.user)
self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND)
self.assertTrue(Relation.objects.filter(pk=self.relation_group.pk).exists())
assign_contributor_permissions(self.collection, self.user)
resp = self._delete(self.relation_group.pk, user=self.contributor)
self.assertEqual(resp.status_code, status.HTTP_204_NO_CONTENT)
self.assertFalse(Relation.objects.filter(pk=self.relation_group.pk).exists())
def setUp(self):
self.group = Group.objects.create(name='Test group')
self.resource_name = 'collection'
self.viewset = DescriptorSchemaViewSet
super().setUp()
self.descriptor_schema = DescriptorSchema.objects.create(contributor=self.contributor)
assign_contributor_permissions(self.descriptor_schema)
def test_delete_different_user(self):
resp = self._delete(self.relation_group.pk, user=self.user)
self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND)
self.assertTrue(Relation.objects.filter(pk=self.relation_group.pk).exists())
assign_contributor_permissions(self.collection, self.user)
resp = self._delete(self.relation_group.pk, user=self.contributor)
self.assertEqual(resp.status_code, status.HTTP_204_NO_CONTENT)
self.assertFalse(Relation.objects.filter(pk=self.relation_group.pk).exists())
def setUp(self):
self.viewset = RelationViewSet
self.resource_name = 'relation'
super().setUp()
# Load fixtures with relation types
flow_config = apps.get_app_config('flow')
call_command('loaddata', os.path.join(flow_config.path, 'tests', 'fixtures', 'relationtypes'), verbosity=0)
self.collection = Collection.objects.create(name='Test collection', contributor=self.contributor)
self.collection_2 = Collection.objects.create(name='Second collection', contributor=self.contributor)
self.entity_1 = Entity.objects.create(name='First entity', contributor=self.contributor)
self.entity_2 = Entity.objects.create(name='Second entity', contributor=self.contributor)
self.entity_3 = Entity.objects.create(name='Third entity', contributor=self.contributor)
self.entity_4 = Entity.objects.create(name='Fourth entity', contributor=self.contributor)
assign_contributor_permissions(self.collection, self.contributor)
assign_contributor_permissions(self.collection_2, self.contributor)
rel_type_group = RelationType.objects.get(name='group')
rel_type_series = RelationType.objects.get(name='series')
self.relation_group = Relation.objects.create(
contributor=self.contributor,
collection=self.collection,
type=rel_type_group,
category='replicates',
)
self.group_partiton_1 = RelationPartition.objects.create(relation=self.relation_group, entity=self.entity_1)
self.group_partiton_2 = RelationPartition.objects.create(relation=self.relation_group, entity=self.entity_2)
self.relation_series = Relation.objects.create(
contributor=self.contributor,
collection=self.collection_2,
type=rel_type_series,
category='time-series',
unit=Relation.UNIT_HOUR,
)
self.series_partiton_1 = RelationPartition.objects.create(
relation=self.relation_series, entity=self.entity_1, label='beginning', position=0
)
self.series_partiton_2 = RelationPartition.objects.create(
relation=self.relation_series, entity=self.entity_2, label='beginning', position=0
)
self.series_partiton_3 = RelationPartition.objects.create(
relation=self.relation_series, entity=self.entity_3, label='end', position=1
)
self.series_partiton_4 = RelationPartition.objects.create(
relation=self.relation_series, entity=self.entity_4, label='end', position=1
)
assign_perm('view_relation', self.contributor, self.relation_group)
assign_perm('view_relation', self.contributor, self.relation_series)
def setUp(self):
self.viewset = RelationViewSet
self.resource_name = 'relation'
super().setUp()
# Load fixtures with relation types
flow_config = apps.get_app_config('flow')
call_command('loaddata', os.path.join(flow_config.path, 'tests', 'fixtures', 'relationtypes'), verbosity=0)
self.collection = Collection.objects.create(name='Test collection', contributor=self.contributor)
self.collection_2 = Collection.objects.create(name='Second collection', contributor=self.contributor)
self.entity_1 = Entity.objects.create(name='First entity', contributor=self.contributor)
self.entity_2 = Entity.objects.create(name='Second entity', contributor=self.contributor)
self.entity_3 = Entity.objects.create(name='Third entity', contributor=self.contributor)
self.entity_4 = Entity.objects.create(name='Fourth entity', contributor=self.contributor)
assign_contributor_permissions(self.collection, self.contributor)
assign_contributor_permissions(self.collection_2, self.contributor)
rel_type_group = RelationType.objects.get(name='group')
rel_type_series = RelationType.objects.get(name='series')
self.relation_group = Relation.objects.create(
contributor=self.contributor,
collection=self.collection,
type=rel_type_group,
category='replicates',
)
self.group_partiton_1 = RelationPartition.objects.create(relation=self.relation_group, entity=self.entity_1)
self.group_partiton_2 = RelationPartition.objects.create(relation=self.relation_group, entity=self.entity_2)
self.relation_series = Relation.objects.create(
contributor=self.contributor,
collection=self.collection_2,
type=rel_type_series,
category='time-series',
unit=Relation.UNIT_HOUR,
)
self.series_partiton_1 = RelationPartition.objects.create(
relation=self.relation_series, entity=self.entity_1, label='beginning', position=0
)
self.series_partiton_2 = RelationPartition.objects.create(
relation=self.relation_series, entity=self.entity_2, label='beginning', position=0
)
self.series_partiton_3 = RelationPartition.objects.create(
relation=self.relation_series, entity=self.entity_3, label='end', position=1
)
self.series_partiton_4 = RelationPartition.objects.create(
relation=self.relation_series, entity=self.entity_4, label='end', position=1
)
assign_perm('view_relation', self.contributor, self.relation_group)
assign_perm('view_relation', self.contributor, self.relation_series)
def test_update_different_user(self):
data = {'collection': self.collection_2.pk}
resp = self._patch(self.relation_group.pk, data, user=self.user)
self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND)
self.relation_group.refresh_from_db()
self.assertEqual(self.relation_group.collection, self.collection)
assign_contributor_permissions(self.collection, self.user)
resp = self._patch(self.relation_group.pk, data, user=self.user)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.relation_group.refresh_from_db()
self.assertEqual(self.relation_group.collection, self.collection_2)
def test_update_different_user(self):
data = {'collection': self.collection_2.pk}
resp = self._patch(self.relation_group.pk, data, user=self.user)
self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND)
self.relation_group.refresh_from_db()
self.assertEqual(self.relation_group.collection, self.collection)
assign_contributor_permissions(self.collection, self.user)
resp = self._patch(self.relation_group.pk, data, user=self.user)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.relation_group.refresh_from_db()
self.assertEqual(self.relation_group.collection, self.collection_2)
def register_descriptors(self, descriptor_schemas, user, force=False, verbosity=1):
"""Read and register descriptors."""
log_descriptors = []
for descriptor_schema in descriptor_schemas:
for schema, _, _ in iterate_schema({}, descriptor_schema.get('schema', {})):
if not schema['type'][-1].endswith(':'):
schema['type'] += ':'
if 'schema' not in descriptor_schema:
descriptor_schema['schema'] = []
if not self.valid(descriptor_schema, DESCRIPTOR_SCHEMA):
continue
slug = descriptor_schema['slug']
version = descriptor_schema.get('version', '0.0.0')
int_version = convert_version_string_to_int(version, VERSION_NUMBER_BITS)
# `latest version` is returned as `int` so it has to be compared to `int_version`
latest_version = DescriptorSchema.objects.filter(slug=slug).aggregate(Max('version'))['version__max']
if latest_version is not None and latest_version > int_version:
self.stderr.write("Skip descriptor schema {}: newer version installed".format(slug))
continue
previous_descriptor_qs = DescriptorSchema.objects.filter(slug=slug)
if previous_descriptor_qs.exists():
previous_descriptor = previous_descriptor_qs.latest()
else:
previous_descriptor = None
descriptor_query = DescriptorSchema.objects.filter(slug=slug, version=version)
if descriptor_query.exists():
if not force:
if verbosity > 0:
self.stdout.write("Skip descriptor schema {}: same version installed".format(slug))
continue
descriptor_query.update(**descriptor_schema)
log_descriptors.append("Updated {}".format(slug))
else:
descriptor = DescriptorSchema.objects.create(contributor=user, **descriptor_schema)
assign_contributor_permissions(descriptor)
if previous_descriptor:
copy_permissions(previous_descriptor, descriptor)
log_descriptors.append("Inserted {}".format(slug))
if log_descriptors and verbosity > 0:
self.stdout.write("Descriptor schemas Updates:")
for log in log_descriptors:
self.stdout.write(" {}".format(log))
def duplicate(self, contributor=None):
"""Duplicate (make a copy)."""
if self.status not in [self.STATUS_DONE, self.STATUS_ERROR]:
raise ValidationError('Data object must have done or error status to be duplicated')
duplicate = Data.objects.get(id=self.id)
duplicate.pk = None
duplicate.slug = None
duplicate.name = 'Copy of {}'.format(self.name)
duplicate.duplicated = now()
if contributor:
duplicate.contributor = contributor
duplicate._perform_save(force_insert=True) # pylint: disable=protected-access
assign_contributor_permissions(duplicate)
# Override fields that are automatically set on create.
duplicate.created = self.created
duplicate._perform_save() # pylint: disable=protected-access
if self.location:
self.location.data.add(duplicate) # pylint: disable=no-member
duplicate.storages.set(self.storages.all()) # pylint: disable=no-member
for migration in self.migration_history.order_by('created'): # pylint: disable=no-member
migration.pk = None
migration.data = duplicate
migration.save(force_insert=True)
# Inherit existing child dependencies.
DataDependency.objects.bulk_create([
DataDependency(child=duplicate, parent=dependency.parent, kind=dependency.kind)
for dependency in DataDependency.objects.filter(child=self)
])
# Inherit existing parent dependencies.
DataDependency.objects.bulk_create([
DataDependency(child=dependency.child, parent=duplicate, kind=dependency.kind)
for dependency in DataDependency.objects.filter(parent=self)
])
return duplicate
def test_delete_different_user(self):
# No view permissions, authenticated.
resp = self._delete(self.relation_group.pk, user=self.user)
self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND)
self.assertTrue(
Relation.objects.filter(pk=self.relation_group.pk).exists())
# No edit permissions, authenticated.
assign_perm("view_collection", self.user, self.collection)
resp = self._delete(self.relation_group.pk, user=self.user)
self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
self.assertTrue(
Relation.objects.filter(pk=self.relation_group.pk).exists())
# Edit & view permissions, authenticated.
assign_contributor_permissions(self.collection, self.user)
resp = self._delete(self.relation_group.pk, user=self.contributor)
self.assertEqual(resp.status_code, status.HTTP_204_NO_CONTENT)
self.assertFalse(
Relation.objects.filter(pk=self.relation_group.pk).exists())
def test_delete_different_user(self):
# No view permissions: anonymous user has view permission: remove it.
self.collection.set_permission(Permission.NONE, AnonymousUser())
resp = self._delete(self.relation_group.pk, user=self.user)
self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND)
self.assertTrue(
Relation.objects.filter(pk=self.relation_group.pk).exists())
# No edit permissions, authenticated.
self.collection.set_permission(Permission.VIEW, self.user)
resp = self._delete(self.relation_group.pk, user=self.user)
self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
self.assertTrue(
Relation.objects.filter(pk=self.relation_group.pk).exists())
# Edit & view permissions, authenticated.
assign_contributor_permissions(self.collection, self.user)
resp = self._delete(self.relation_group.pk, user=self.contributor)
self.assertEqual(resp.status_code, status.HTTP_204_NO_CONTENT)
self.assertFalse(
Relation.objects.filter(pk=self.relation_group.pk).exists())
def create_entity(self):
"""Create entity if `flow_collection` is defined in process.
Following rules applies for adding `Data` object to `Entity`:
* Only add `Data object` to `Entity` if process has defined
`flow_collwection` field
* Add object to existing `Entity`, if all parents that are part
of it (but not necessary all parents), are part of the same
`Entity`
* If parents belong to different `Entities` or do not belong to
any `Entity`, create new `Entity`
"""
ds_slug = self.process.flow_collection # pylint: disable=no-member
if ds_slug:
entity_query = Entity.objects.filter(
data__in=self.parents.all()).distinct() # pylint: disable=no-member
if entity_query.count() == 1:
entity = entity_query.first()
copy_permissions(entity, self)
else:
descriptor_schema = DescriptorSchema.objects.filter(
slug=ds_slug).latest()
entity = Entity.objects.create(
contributor=self.contributor,
descriptor_schema=descriptor_schema,
name=self.name,
tags=self.tags,
)
assign_contributor_permissions(entity)
entity.data.add(self)
# Inherite collections from entity.
for collection in entity.collections.all():
collection.data.add(self)
def test_update_different_user(self):
# No view permission.
data = {"collection": {"id": self.collection_2.pk}}
resp = self._patch(self.relation_group.pk, data, user=self.user)
self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND)
self.relation_group.refresh_from_db()
self.assertEqual(self.relation_group.collection, self.collection)
# No edit permission.
assign_perm("view_collection", self.user, self.collection)
resp = self._patch(self.relation_group.pk, data, user=self.user)
self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
self.relation_group.refresh_from_db()
self.assertEqual(self.relation_group.collection, self.collection)
# All permissions.
assign_contributor_permissions(self.collection, self.user)
assign_contributor_permissions(self.collection_2, self.user)
resp = self._patch(self.relation_group.pk, data, user=self.user)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.relation_group.refresh_from_db()
self.assertEqual(self.relation_group.collection, self.collection_2)
def perform_create(self, serializer):
"""Create a resource."""
with transaction.atomic():
instance = serializer.save()
assign_contributor_permissions(instance)
# Entity is added to the collection only when it is
# created - when it only contains 1 Data object.
entities = Entity.objects.annotate(num_data=Count('data')).filter(
data=instance, num_data=1)
# Assign data object to all specified collections.
collection_pks = self.request.data.get('collections', [])
for collection in Collection.objects.filter(pk__in=collection_pks):
collection.data.add(instance)
copy_permissions(collection, instance)
# Add entities to which data belongs to the collection.
for entity in entities:
entity.collections.add(collection)
copy_permissions(collection, entity)
def test_update_different_user(self):
# No view permission: anonymous user has view permission: remove it.
self.collection.set_permission(Permission.NONE, AnonymousUser())
data = {"collection": {"id": self.collection_2.pk}}
resp = self._patch(self.relation_group.pk, data, user=self.user)
self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND)
self.relation_group.refresh_from_db()
self.assertEqual(self.relation_group.collection, self.collection)
# No edit permission.
self.collection.set_permission(Permission.VIEW, self.user)
resp = self._patch(self.relation_group.pk, data, user=self.user)
self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
self.relation_group.refresh_from_db()
self.assertEqual(self.relation_group.collection, self.collection)
# All permissions.
assign_contributor_permissions(self.collection, self.user)
assign_contributor_permissions(self.collection_2, self.user)
resp = self._patch(self.relation_group.pk, data, user=self.user)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.relation_group.refresh_from_db()
self.assertEqual(self.relation_group.collection, self.collection_2)
def duplicate(self, contributor=None, inherit_collection=False):
"""Duplicate (make a copy)."""
duplicate = Entity.objects.get(id=self.id)
duplicate.pk = None
duplicate.slug = None
duplicate.name = "Copy of {}".format(self.name)
duplicate.duplicated = now()
if contributor:
duplicate.contributor = contributor
duplicate.collection = None
if inherit_collection:
if not contributor.has_perm("edit_collection", self.collection):
raise ValidationError(
"You do not have edit permission on collection {}.".format(
self.collection))
duplicate.collection = self.collection
duplicate.save(force_insert=True)
assign_contributor_permissions(duplicate)
# Override fields that are automatically set on create.
duplicate.created = self.created
duplicate.save()
# Duplicate entity's data objects.
data = get_objects_for_user(contributor, "view_data", self.data.all())
duplicated_data = data.duplicate(contributor,
inherit_collection=inherit_collection)
duplicate.data.add(*duplicated_data)
# Permissions
assign_contributor_permissions(duplicate)
copy_permissions(duplicate.collection, duplicate)
return duplicate
def duplicate(self, contributor=None, inherit_collections=False):
"""Duplicate (make a copy)."""
duplicate = Entity.objects.get(id=self.id)
duplicate.pk = None
duplicate.slug = None
duplicate.name = 'Copy of {}'.format(self.name)
duplicate.duplicated = now()
if contributor:
duplicate.contributor = contributor
duplicate.save(force_insert=True)
assign_contributor_permissions(duplicate)
# Override fields that are automatically set on create.
duplicate.created = self.created
duplicate.save()
# Duplicate entity's data objects.
data = get_objects_for_user(contributor, 'view_data', self.data.all()) # pylint: disable=no-member
duplicated_data = data.duplicate(contributor)
duplicate.data.add(*duplicated_data)
if inherit_collections:
collections = get_objects_for_user(
contributor,
'add_collection',
self.collections.all() # pylint: disable=no-member
)
for collection in collections:
collection.entity_set.add(duplicate)
copy_permissions(collection, duplicate)
collection.data.add(*duplicated_data)
for datum in duplicated_data:
copy_permissions(collection, datum)
return duplicate
def register_processes(self, process_schemas, user, force=False, verbosity=1):
"""Read and register processors."""
log_processors = []
log_templates = []
for p in process_schemas:
# TODO: Remove this when all processes are migrated to the
# new syntax.
if 'flow_collection' in p:
if 'entity' in p:
self.stderr.write(
"Skip processor {}: only one of 'flow_collection' and 'entity' fields "
"allowed".format(p['slug'])
)
continue
p['entity'] = {'type': p.pop('flow_collection')}
if p['type'][-1] != ':':
p['type'] += ':'
if 'category' in p and not p['category'].endswith(':'):
p['category'] += ':'
for field in ['input', 'output']:
for schema, _, _ in iterate_schema({}, p[field] if field in p else {}):
if not schema['type'][-1].endswith(':'):
schema['type'] += ':'
# TODO: Check if schemas validate with our JSON meta schema and Processor model docs.
if not self.valid(p, PROCESSOR_SCHEMA):
continue
if 'entity' in p:
if 'type' not in p['entity']:
self.stderr.write(
"Skip process {}: 'entity.type' required if 'entity' defined".format(p['slug'])
)
continue
p['entity_type'] = p['entity']['type']
p['entity_descriptor_schema'] = p['entity'].get('descriptor_schema', p['entity_type'])
p['entity_input'] = p['entity'].get('input', None)
p.pop('entity')
if not DescriptorSchema.objects.filter(slug=p['entity_descriptor_schema']).exists():
self.stderr.write(
"Skip processor {}: Unknown descriptor schema '{}' used in 'entity' "
"field.".format(p['slug'], p['entity_descriptor_schema'])
)
continue
if 'persistence' in p:
persistence_mapping = {
'RAW': Process.PERSISTENCE_RAW,
'CACHED': Process.PERSISTENCE_CACHED,
'TEMP': Process.PERSISTENCE_TEMP,
}
p['persistence'] = persistence_mapping[p['persistence']]
if 'scheduling_class' in p:
scheduling_class_mapping = {
'interactive': Process.SCHEDULING_CLASS_INTERACTIVE,
'batch': Process.SCHEDULING_CLASS_BATCH
}
p['scheduling_class'] = scheduling_class_mapping[p['scheduling_class']]
if 'input' in p:
p['input_schema'] = p.pop('input')
if 'output' in p:
p['output_schema'] = p.pop('output')
slug = p['slug']
if 'run' in p:
# Set default language to 'bash' if not set.
p['run'].setdefault('language', 'bash')
# Transform output schema using the execution engine.
try:
execution_engine = manager.get_execution_engine(p['run']['language'])
extra_output_schema = execution_engine.get_output_schema(p)
if extra_output_schema:
p.setdefault('output_schema', []).extend(extra_output_schema)
except InvalidEngineError:
self.stderr.write("Skip processor {}: execution engine '{}' not supported".format(
slug, p['run']['language']
))
continue
# Validate if container image is allowed based on the configured pattern.
# NOTE: This validation happens here and is not deferred to executors because the idea
# is that this will be moved to a "container" requirement independent of the
# executor.
if hasattr(settings, 'FLOW_CONTAINER_VALIDATE_IMAGE'):
try:
container_image = dict_dot(p, 'requirements.executor.docker.image')
if not re.match(settings.FLOW_CONTAINER_VALIDATE_IMAGE, container_image):
self.stderr.write("Skip processor {}: container image does not match '{}'".format(
slug, settings.FLOW_CONTAINER_VALIDATE_IMAGE,
))
continue
except KeyError:
pass
version = p['version']
int_version = convert_version_string_to_int(version, VERSION_NUMBER_BITS)
# `latest version` is returned as `int` so it has to be compared to `int_version`
latest_version = Process.objects.filter(slug=slug).aggregate(Max('version'))['version__max']
if latest_version is not None and latest_version > int_version:
self.stderr.write("Skip processor {}: newer version installed".format(slug))
continue
previous_process_qs = Process.objects.filter(slug=slug)
if previous_process_qs.exists():
previous_process = previous_process_qs.latest()
else:
previous_process = None
process_query = Process.objects.filter(slug=slug, version=version)
if process_query.exists():
if not force:
if verbosity > 0:
self.stdout.write("Skip processor {}: same version installed".format(slug))
continue
process_query.update(**p)
log_processors.append("Updated {}".format(slug))
else:
process = Process.objects.create(contributor=user, **p)
assign_contributor_permissions(process)
if previous_process:
copy_permissions(previous_process, process)
log_processors.append("Inserted {}".format(slug))
if verbosity > 0:
if log_processors:
self.stdout.write("Processor Updates:")
for log in log_processors:
self.stdout.write(" {}".format(log))
if log_templates:
self.stdout.write("Default Template Updates:")
for log in log_templates:
self.stdout.write(" {}".format(log))
def register_processes(self,
process_schemas,
user,
force=False,
verbosity=1):
"""Read and register processors."""
log_processors = []
log_templates = []
for p in process_schemas:
if p['type'][-1] != ':':
p['type'] += ':'
if 'category' in p and not p['category'].endswith(':'):
p['category'] += ':'
for field in ['input', 'output']:
for schema, _, _ in iterate_schema(
{}, p[field] if field in p else {}):
if not schema['type'][-1].endswith(':'):
schema['type'] += ':'
# TODO: Check if schemas validate with our JSON meta schema and Processor model docs.
if not self.valid(p, PROCESSOR_SCHEMA):
continue
if 'persistence' in p:
persistence_mapping = {
'RAW': Process.PERSISTENCE_RAW,
'CACHED': Process.PERSISTENCE_CACHED,
'TEMP': Process.PERSISTENCE_TEMP,
}
p['persistence'] = persistence_mapping[p['persistence']]
if 'scheduling_class' in p:
scheduling_class_mapping = {
'interactive': Process.SCHEDULING_CLASS_INTERACTIVE,
'batch': Process.SCHEDULING_CLASS_BATCH
}
p['scheduling_class'] = scheduling_class_mapping[
p['scheduling_class']]
if 'input' in p:
p['input_schema'] = p.pop('input')
if 'output' in p:
p['output_schema'] = p.pop('output')
slug = p['slug']
if 'run' in p:
# Set default language to 'bash' if not set.
p['run'].setdefault('language', 'bash')
# Transform output schema using the execution engine.
try:
execution_engine = manager.get_execution_engine(
p['run']['language'])
extra_output_schema = execution_engine.get_output_schema(p)
if extra_output_schema:
p.setdefault('output_schema',
[]).extend(extra_output_schema)
except InvalidEngineError:
self.stderr.write(
"Skip processor {}: execution engine '{}' not supported"
.format(slug, p['run']['language']))
continue
# Validate if container image is allowed based on the configured pattern.
# NOTE: This validation happens here and is not deferred to executors because the idea
# is that this will be moved to a "container" requirement independent of the
# executor.
if hasattr(settings, 'FLOW_CONTAINER_VALIDATE_IMAGE'):
try:
container_image = dict_dot(
p, 'requirements.executor.docker.image')
if not re.match(settings.FLOW_CONTAINER_VALIDATE_IMAGE,
container_image):
self.stderr.write(
"Skip processor {}: container image does not match '{}'"
.format(
slug,
settings.FLOW_CONTAINER_VALIDATE_IMAGE,
))
continue
except KeyError:
pass
version = p['version']
int_version = convert_version_string_to_int(
version, VERSION_NUMBER_BITS)
# `latest version` is returned as `int` so it has to be compared to `int_version`
latest_version = Process.objects.filter(slug=slug).aggregate(
Max('version'))['version__max']
if latest_version is not None and latest_version > int_version:
self.stderr.write(
"Skip processor {}: newer version installed".format(slug))
continue
previous_process_qs = Process.objects.filter(slug=slug)
if previous_process_qs.exists():
previous_process = previous_process_qs.latest()
else:
previous_process = None
process_query = Process.objects.filter(slug=slug, version=version)
if process_query.exists():
if not force:
if verbosity > 0:
self.stdout.write(
"Skip processor {}: same version installed".format(
slug))
continue
process_query.update(**p)
log_processors.append("Updated {}".format(slug))
else:
process = Process.objects.create(contributor=user, **p)
assign_contributor_permissions(process)
if previous_process:
copy_permissions(previous_process, process)
log_processors.append("Inserted {}".format(slug))
if verbosity > 0:
if log_processors:
self.stdout.write("Processor Updates:")
for log in log_processors:
self.stdout.write(" {}".format(log))
if log_templates:
self.stdout.write("Default Template Updates:")
for log in log_templates:
self.stdout.write(" {}".format(log))
def setUp(self):
self.viewset = RelationViewSet
self.resource_name = "relation"
super().setUp()
# Load fixtures with relation types
flow_config = apps.get_app_config("flow")
call_command(
"loaddata",
os.path.join(flow_config.path, "tests", "fixtures",
"relationtypes"),
verbosity=0,
)
self.collection = Collection.objects.create(
name="Test collection", contributor=self.contributor)
self.collection_2 = Collection.objects.create(
name="Second collection", contributor=self.contributor)
self.entity_1 = Entity.objects.create(name="First entity",
contributor=self.contributor)
self.entity_2 = Entity.objects.create(name="Second entity",
contributor=self.contributor)
self.entity_3 = Entity.objects.create(name="Third entity",
contributor=self.contributor)
self.entity_4 = Entity.objects.create(name="Fourth entity",
contributor=self.contributor)
assign_contributor_permissions(self.collection, self.contributor)
assign_contributor_permissions(self.collection_2, self.contributor)
rel_type_group = RelationType.objects.get(name="group")
rel_type_series = RelationType.objects.get(name="series")
self.relation_group = Relation.objects.create(
contributor=self.contributor,
collection=self.collection,
type=rel_type_group,
category="replicates",
)
self.group_partiton_1 = RelationPartition.objects.create(
relation=self.relation_group, entity=self.entity_1)
self.group_partiton_2 = RelationPartition.objects.create(
relation=self.relation_group, entity=self.entity_2)
self.relation_series = Relation.objects.create(
contributor=self.contributor,
collection=self.collection_2,
type=rel_type_series,
category="time-series",
unit=Relation.UNIT_HOUR,
)
self.series_partiton_1 = RelationPartition.objects.create(
relation=self.relation_series,
entity=self.entity_1,
label="beginning",
position=0,
)
self.series_partiton_2 = RelationPartition.objects.create(
relation=self.relation_series,
entity=self.entity_2,
label="beginning",
position=0,
)
self.series_partiton_3 = RelationPartition.objects.create(
relation=self.relation_series,
entity=self.entity_3,
label="end",
position=1)
self.series_partiton_4 = RelationPartition.objects.create(
relation=self.relation_series,
entity=self.entity_4,
label="end",
position=1)
assign_perm("view_relation", self.contributor, self.relation_group)
assign_perm("view_relation", self.contributor, self.relation_series)
def register_descriptors(self,
descriptor_schemas,
user,
force=False,
verbosity=1):
"""Read and register descriptors."""
log_descriptors = []
for descriptor_schema in descriptor_schemas:
for schema, _, _ in iterate_schema({},
descriptor_schema.get(
"schema", {})):
if not schema["type"][-1].endswith(":"):
schema["type"] += ":"
if "schema" not in descriptor_schema:
descriptor_schema["schema"] = []
if not self.valid(descriptor_schema, DESCRIPTOR_SCHEMA):
continue
slug = descriptor_schema["slug"]
version = descriptor_schema.get("version", "0.0.0")
int_version = convert_version_string_to_int(
version, VERSION_NUMBER_BITS)
# `latest version` is returned as `int` so it has to be compared to `int_version`
latest_version = DescriptorSchema.objects.filter(
slug=slug).aggregate(Max("version"))["version__max"]
if latest_version is not None and latest_version > int_version:
self.stderr.write(
"Skip descriptor schema {}: newer version installed".
format(slug))
continue
previous_descriptor_qs = DescriptorSchema.objects.filter(slug=slug)
if previous_descriptor_qs.exists():
previous_descriptor = previous_descriptor_qs.latest()
else:
previous_descriptor = None
descriptor_query = DescriptorSchema.objects.filter(slug=slug,
version=version)
if descriptor_query.exists():
if not force:
if verbosity > 0:
self.stdout.write(
"Skip descriptor schema {}: same version installed"
.format(slug))
continue
descriptor_query.update(**descriptor_schema)
log_descriptors.append("Updated {}".format(slug))
else:
descriptor = DescriptorSchema.objects.create(
contributor=user, **descriptor_schema)
assign_contributor_permissions(descriptor)
if previous_descriptor:
copy_permissions(previous_descriptor, descriptor)
log_descriptors.append("Inserted {}".format(slug))
if log_descriptors and verbosity > 0:
self.stdout.write("Descriptor schemas Updates:")
for log in log_descriptors:
self.stdout.write(" {}".format(log))
def register_processes(self,
process_schemas,
user,
force=False,
verbosity=1):
"""Read and register processors."""
log_processors = []
log_templates = []
for p in process_schemas:
# TODO: Remove this when all processes are migrated to the
# new syntax.
if "flow_collection" in p:
if "entity" in p:
self.stderr.write(
"Skip processor {}: only one of 'flow_collection' and 'entity' fields "
"allowed".format(p["slug"]))
continue
p["entity"] = {"type": p.pop("flow_collection")}
if p["type"][-1] != ":":
p["type"] += ":"
if "category" in p and not p["category"].endswith(":"):
p["category"] += ":"
for field in ["input", "output"]:
for schema, _, _ in iterate_schema(
{}, p[field] if field in p else {}):
if not schema["type"][-1].endswith(":"):
schema["type"] += ":"
# TODO: Check if schemas validate with our JSON meta schema and Processor model docs.
if not self.valid(p, PROCESSOR_SCHEMA):
continue
if "entity" in p:
if "type" not in p["entity"]:
self.stderr.write(
"Skip process {}: 'entity.type' required if 'entity' defined"
.format(p["slug"]))
continue
if "input" in p["entity"] and p["entity"].get(
"always_create", False):
self.stderr.write(
"Skip process {}: 'entity.input' will not be considered if 'entity.always_create' "
"is set to true.".format(p["slug"]))
continue
p["entity_type"] = p["entity"]["type"]
p["entity_descriptor_schema"] = p["entity"].get(
"descriptor_schema", p["entity_type"])
p["entity_input"] = p["entity"].get("input", None)
p["entity_always_create"] = p["entity"].get(
"always_create", False)
p.pop("entity")
if not DescriptorSchema.objects.filter(
slug=p["entity_descriptor_schema"]).exists():
self.stderr.write(
"Skip processor {}: Unknown descriptor schema '{}' used in 'entity' "
"field.".format(p["slug"],
p["entity_descriptor_schema"]))
continue
if "persistence" in p:
persistence_mapping = {
"RAW": Process.PERSISTENCE_RAW,
"CACHED": Process.PERSISTENCE_CACHED,
"TEMP": Process.PERSISTENCE_TEMP,
}
p["persistence"] = persistence_mapping[p["persistence"]]
if "scheduling_class" in p:
scheduling_class_mapping = {
"interactive": Process.SCHEDULING_CLASS_INTERACTIVE,
"batch": Process.SCHEDULING_CLASS_BATCH,
}
p["scheduling_class"] = scheduling_class_mapping[
p["scheduling_class"]]
if "input" in p:
p["input_schema"] = p.pop("input")
if "output" in p:
p["output_schema"] = p.pop("output")
slug = p["slug"]
if "run" in p:
# Set default language to 'bash' if not set.
p["run"].setdefault("language", "bash")
# Transform output schema using the execution engine.
try:
execution_engine = manager.get_execution_engine(
p["run"]["language"])
extra_output_schema = execution_engine.get_output_schema(p)
if extra_output_schema:
p.setdefault("output_schema",
[]).extend(extra_output_schema)
except InvalidEngineError:
self.stderr.write(
"Skip processor {}: execution engine '{}' not supported"
.format(slug, p["run"]["language"]))
continue
# Validate if container image is allowed based on the configured pattern.
# NOTE: This validation happens here and is not deferred to executors because the idea
# is that this will be moved to a "container" requirement independent of the
# executor.
if hasattr(settings, "FLOW_CONTAINER_VALIDATE_IMAGE"):
try:
container_image = dict_dot(
p, "requirements.executor.docker.image")
if not re.match(settings.FLOW_CONTAINER_VALIDATE_IMAGE,
container_image):
self.stderr.write(
"Skip processor {}: container image does not match '{}'"
.format(
slug,
settings.FLOW_CONTAINER_VALIDATE_IMAGE,
))
continue
except KeyError:
pass
version = p["version"]
int_version = convert_version_string_to_int(
version, VERSION_NUMBER_BITS)
# `latest version` is returned as `int` so it has to be compared to `int_version`
latest_version = Process.objects.filter(slug=slug).aggregate(
Max("version"))["version__max"]
if latest_version is not None and latest_version > int_version:
self.stderr.write(
"Skip processor {}: newer version installed".format(slug))
continue
previous_process_qs = Process.objects.filter(slug=slug)
if previous_process_qs.exists():
previous_process = previous_process_qs.latest()
else:
previous_process = None
process_query = Process.objects.filter(slug=slug, version=version)
if process_query.exists():
if not force:
if verbosity > 0:
self.stdout.write(
"Skip processor {}: same version installed".format(
slug))
continue
process_query.update(**p)
log_processors.append("Updated {}".format(slug))
else:
process = Process.objects.create(contributor=user, **p)
assign_contributor_permissions(process)
if previous_process:
copy_permissions(previous_process, process)
log_processors.append("Inserted {}".format(slug))
if verbosity > 0:
if log_processors:
self.stdout.write("Processor Updates:")
for log in log_processors:
self.stdout.write(" {}".format(log))
if log_templates:
self.stdout.write("Default Template Updates:")
for log in log_templates:
self.stdout.write(" {}".format(log))
def register_processes(self,
process_schemas,
user,
force=False,
verbosity=1):
"""Read and register processors."""
log_processors = []
log_templates = []
for p in process_schemas:
if p['type'][-1] != ':':
p['type'] += ':'
if 'category' in p and not p['category'].endswith(':'):
p['category'] += ':'
# get `data_name` from `static`
if 'static' in p:
for schema, _, _ in iterate_schema({}, p['static']):
if schema['name'] == 'name' and 'default' in schema:
p['data_name'] = schema['default']
# support backward compatibility
# TODO: update .yml files and remove
if 'slug' not in p:
p['slug'] = slugify(p.pop('name').replace(':', '-'))
p['name'] = p.pop('label')
p.pop('var', None)
p.pop('static', None)
for field in ['input', 'output', 'var', 'static']:
for schema, _, _ in iterate_schema(
{}, p[field] if field in p else {}):
if not schema['type'][-1].endswith(':'):
schema['type'] += ':'
# TODO: Check if schemas validate with our JSON meta schema and Processor model docs.
if not self.valid(p, PROCESSOR_SCHEMA):
continue
if 'persistence' in p:
persistence_mapping = {
'RAW': Process.PERSISTENCE_RAW,
'CACHED': Process.PERSISTENCE_CACHED,
'TEMP': Process.PERSISTENCE_TEMP,
}
p['persistence'] = persistence_mapping[p['persistence']]
if 'scheduling_class' in p:
scheduling_class_mapping = {
'interactive': Process.SCHEDULING_CLASS_INTERACTIVE,
'batch': Process.SCHEDULING_CLASS_BATCH
}
p['scheduling_class'] = scheduling_class_mapping[
p['scheduling_class']]
if 'input' in p:
p['input_schema'] = p.pop('input')
if 'output' in p:
p['output_schema'] = p.pop('output')
slug = p['slug']
if 'run' in p:
# Set default language to 'bash' if not set.
p['run'].setdefault('language', 'bash')
# Transform output schema using the execution engine.
try:
execution_engine = manager.get_execution_engine(
p['run']['language'])
extra_output_schema = execution_engine.get_output_schema(p)
if extra_output_schema:
p.setdefault('output_schema',
[]).extend(extra_output_schema)
except InvalidEngineError:
self.stderr.write(
"Skip processor {}: execution engine '{}' not supported"
.format(slug, p['run']['language']))
continue
version = p['version']
int_version = convert_version_string_to_int(
version, VERSION_NUMBER_BITS)
# `latest version` is returned as `int` so it has to be compared to `int_version`
latest_version = Process.objects.filter(slug=slug).aggregate(
Max('version'))['version__max']
if latest_version is not None and latest_version > int_version:
self.stderr.write(
"Skip processor {}: newer version installed".format(slug))
continue
previous_process_qs = Process.objects.filter(slug=slug)
if previous_process_qs.exists():
previous_process = previous_process_qs.latest()
else:
previous_process = None
process_query = Process.objects.filter(slug=slug, version=version)
if process_query.exists():
if not force:
if verbosity > 0:
self.stdout.write(
"Skip processor {}: same version installed".format(
slug))
continue
process_query.update(**p)
log_processors.append("Updated {}".format(slug))
else:
process = Process.objects.create(contributor=user, **p)
assign_contributor_permissions(process)
if previous_process:
copy_permissions(previous_process, process)
log_processors.append("Inserted {}".format(slug))
if verbosity > 0:
if len(log_processors) > 0:
self.stdout.write("Processor Updates:")
for log in log_processors:
self.stdout.write(" {}".format(log))
if len(log_templates) > 0:
self.stdout.write("Default Template Updates:")
for log in log_templates:
self.stdout.write(" {}".format(log))
def copy_objects(objects, contributor, name_prefix, obj_processor=None):
"""Make a copy of given queryset.
Shallow copy given queryset and set contributor to the given value,
prepend name with the prefix, set slug to a unique value, and set
``duplicated`` date to the current time. Special attention is paid
to keep the ``created`` date to its original value.
If ``obj_processor`` function is given, each object is passed to it
and the return value is used instead of it.
:param objects: A queryset to be copied.
:type objects: `~resolwe.flow.models.base.BaseQuerySet`
:param contributor: A Django user that will be assigned to copied objects
as contributor.
:type contributor: `~django.contrib.auth.models.User`
:param str name_prefix: A prefix that will be prepend to the name of copied
objects.
"""
first = objects.first()
if not first:
return objects
name_max_length = first._meta.get_field("name").max_length
model = first._meta.model
new_objects = []
for obj in objects:
new_obj = deepcopy(obj)
new_obj.pk = None
new_obj.slug = None
new_obj.contributor = contributor
new_obj.name = "{} {}".format(name_prefix, obj.name)
new_obj._container_attributes = dict()
if len(new_obj.name) > name_max_length:
new_obj.name = "{}...".format(new_obj.name[:name_max_length - 3])
if obj_processor:
new_obj = obj_processor(new_obj)
new_objects.append(new_obj)
try:
# Add another atomic block to avoid corupting the main one.
with transaction.atomic():
model.objects.bulk_create(new_objects)
# Send the bulk create custom signal, avoid circular import.
from resolwe.flow.signals import post_duplicate
post_duplicate.send(sender=model,
instances=new_objects,
old_instances=objects)
except IntegrityError:
# Probably a slug collision occured, try to create objects one by one.
for obj in new_objects:
obj.slug = None
# Call the parent method to skip pre-processing and validation.
models.Model.save(obj)
object_permission_group = dict()
not_in_container = list()
for old, new in zip(objects, new_objects):
new.created = old.created
new.duplicated = timezone.now()
# Deal with permissions. When object is in container fix the pointer
# to permission_group object.
# When object is not in container new PermissionGroup proxy object must
# be created, assigned to new object and permissions copied from old
# object to new one.
if getattr(new, "collection_id", None) or getattr(
new, "entity_id", None):
new.permission_group = new.topmost_container.permission_group
else:
not_in_container.append((new, old))
object_permission_group[new] = PermissionGroup()
PermissionGroup.objects.bulk_create(object_permission_group.values())
for new, old in not_in_container:
new.permission_group = object_permission_group[new]
copy_permissions(old, new)
assign_contributor_permissions(new, contributor)
model.objects.bulk_update(new_objects,
["created", "duplicated", "permission_group"])
return new_objects
def create_entity(self):
"""Create entity if `flow_collection` is defined in process.
Following rules applies for adding `Data` object to `Entity`:
* Only add `Data object` to `Entity` if process has defined
`flow_collection` field
* Add object to existing `Entity`, if all parents that are part
of it (but not necessary all parents), are part of the same
`Entity`
* If parents belong to different `Entities` or do not belong to
any `Entity`, create new `Entity`
"""
entity_type = self.process.entity_type # pylint: disable=no-member
entity_descriptor_schema = self.process.entity_descriptor_schema # pylint: disable=no-member
entity_input = self.process.entity_input # pylint: disable=no-member
if entity_type:
data_filter = {}
if entity_input:
input_id = dict_dot(self.input, entity_input, default=lambda: None)
if input_id is None:
logger.warning("Skipping creation of entity due to missing input.")
return
if isinstance(input_id, int):
data_filter['data__pk'] = input_id
elif isinstance(input_id, list):
data_filter['data__pk__in'] = input_id
else:
raise ValueError(
"Cannot create entity due to invalid value of field {}.".format(entity_input)
)
else:
data_filter['data__in'] = self.parents.all() # pylint: disable=no-member
entity_query = Entity.objects.filter(type=entity_type, **data_filter).distinct()
entity_count = entity_query.count()
if entity_count == 0:
descriptor_schema = DescriptorSchema.objects.filter(
slug=entity_descriptor_schema
).latest()
entity = Entity.objects.create(
contributor=self.contributor,
descriptor_schema=descriptor_schema,
type=entity_type,
name=self.name,
tags=self.tags,
)
assign_contributor_permissions(entity)
elif entity_count == 1:
entity = entity_query.first()
copy_permissions(entity, self)
else:
logger.info("Skipping creation of entity due to multiple entities found.")
entity = None
if entity:
entity.data.add(self)
# Inherit collections from entity.
for collection in entity.collections.all():
collection.data.add(self)
def perform_create(self, serializer):
"""Create a relation."""
with transaction.atomic():
instance = serializer.save()
assign_contributor_permissions(instance)
def create_entity(self):
"""Create entity if `flow_collection` is defined in process.
Following rules applies for adding `Data` object to `Entity`:
* Only add `Data object` to `Entity` if process has defined
`flow_collwection` field
* Add object to existing `Entity`, if all parents that are part
of it (but not necessary all parents), are part of the same
`Entity`
* If parents belong to different `Entities` or do not belong to
any `Entity`, create new `Entity`
"""
entity_type = self.process.entity_type # pylint: disable=no-member
entity_descriptor_schema = self.process.entity_descriptor_schema # pylint: disable=no-member
entity_input = self.process.entity_input # pylint: disable=no-member
if entity_type:
data_filter = {}
if entity_input:
input_id = dict_dot(self.input,
entity_input,
default=lambda: None)
if input_id is None:
logger.warning(
"Skipping creation of entity due to missing input.")
return
if isinstance(input_id, int):
data_filter['data__pk'] = input_id
elif isinstance(input_id, list):
data_filter['data__pk__in'] = input_id
else:
raise ValueError(
"Cannot create entity due to invalid value of field {}."
.format(entity_input))
else:
data_filter['data__in'] = self.parents.all() # pylint: disable=no-member
entity_query = Entity.objects.filter(type=entity_type,
**data_filter).distinct()
entity_count = entity_query.count()
if entity_count == 0:
descriptor_schema = DescriptorSchema.objects.filter(
slug=entity_descriptor_schema).latest()
entity = Entity.objects.create(
contributor=self.contributor,
descriptor_schema=descriptor_schema,
type=entity_type,
name=self.name,
tags=self.tags,
)
assign_contributor_permissions(entity)
elif entity_count == 1:
entity = entity_query.first()
copy_permissions(entity, self)
else:
logger.info(
"Skipping creation of entity due to multiple entities found."
)
entity = None
if entity:
entity.data.add(self)
# Inherite collections from entity.
for collection in entity.collections.all():
collection.data.add(self)
def setUp(self):
# Force reading anonymous user from the database for every test.
resolwe.permissions.models.ANONYMOUS_USER = None
self.viewset = RelationViewSet
self.resource_name = "relation"
super().setUp()
# Load fixtures with relation types
flow_config = apps.get_app_config("flow")
call_command(
"loaddata",
os.path.join(flow_config.path, "tests", "fixtures",
"relationtypes"),
verbosity=0,
)
self.collection = Collection.objects.create(
name="Test collection", contributor=self.contributor)
self.collection_2 = Collection.objects.create(
name="Second collection", contributor=self.contributor)
self.entity_1 = Entity.objects.create(name="First entity",
contributor=self.contributor)
self.entity_2 = Entity.objects.create(name="Second entity",
contributor=self.contributor)
self.entity_3 = Entity.objects.create(name="Third entity",
contributor=self.contributor)
self.entity_4 = Entity.objects.create(name="Fourth entity",
contributor=self.contributor)
assign_contributor_permissions(self.collection, self.contributor)
assign_contributor_permissions(self.collection_2, self.contributor)
self.rel_type_group = RelationType.objects.get(name="group")
self.rel_type_series = RelationType.objects.get(name="series")
self.relation_group = Relation.objects.create(
contributor=self.contributor,
collection=self.collection,
type=self.rel_type_group,
category="replicates",
)
self.group_partiton_1 = RelationPartition.objects.create(
relation=self.relation_group, entity=self.entity_1)
self.group_partiton_2 = RelationPartition.objects.create(
relation=self.relation_group, entity=self.entity_2)
self.relation_series = Relation.objects.create(
contributor=self.contributor,
collection=self.collection_2,
type=self.rel_type_series,
category="time-series",
unit=Relation.UNIT_HOUR,
)
self.series_partiton_1 = RelationPartition.objects.create(
relation=self.relation_series,
entity=self.entity_1,
label="beginning",
position=0,
)
self.series_partiton_2 = RelationPartition.objects.create(
relation=self.relation_series,
entity=self.entity_2,
label="beginning",
position=0,
)
self.series_partiton_3 = RelationPartition.objects.create(
relation=self.relation_series,
entity=self.entity_3,
label="end",
position=1)
self.series_partiton_4 = RelationPartition.objects.create(
relation=self.relation_series,
entity=self.entity_4,
label="end",
position=1)
self.collection.set_permission(Permission.VIEW, AnonymousUser())
def _handle_entity(obj):
"""Create entity if `entity.type` is defined in process.
Following rules applies for adding `Data` object to `Entity`:
* Only add `Data object` to `Entity` if process has defined
`entity.type` field
* Create new entity if parents do not belong to any `Entity`
* Add object to existing `Entity`, if all parents that are part
of it (but not necessary all parents), are part of the same
`Entity`
* If parents belong to different `Entities` don't do anything
"""
entity_type = obj.process.entity_type
entity_descriptor_schema = obj.process.entity_descriptor_schema
entity_input = obj.process.entity_input
entity_always_create = obj.process.entity_always_create
operation = HandleEntityOperation.PASS
if entity_type:
data_filter = {}
if entity_input:
input_id = dict_dot(obj.input,
entity_input,
default=lambda: None)
if input_id is None:
logger.warning(
"Skipping creation of entity due to missing input.")
return
if isinstance(input_id, int):
data_filter["data__pk"] = input_id
elif isinstance(input_id, list):
data_filter["data__pk__in"] = input_id
else:
raise ValueError(
"Cannot create entity due to invalid value of field {}."
.format(entity_input))
else:
data_filter["data__in"] = obj.parents.all()
entity_query = Entity.objects.filter(type=entity_type,
**data_filter).distinct()
entity_count = entity_query.count()
if entity_count == 0 or entity_always_create:
descriptor_schema = DescriptorSchema.objects.filter(
slug=entity_descriptor_schema).latest()
entity = Entity.objects.create(
contributor=obj.contributor,
descriptor_schema=descriptor_schema,
type=entity_type,
name=obj.name,
tags=obj.tags,
)
assign_contributor_permissions(entity)
operation = HandleEntityOperation.CREATE
elif entity_count == 1:
entity = entity_query.first()
obj.tags = entity.tags
copy_permissions(entity, obj)
operation = HandleEntityOperation.ADD
else:
logger.info(
"Skipping creation of entity due to multiple entities found."
)
entity = None
if entity:
obj.entity = entity
obj.save()
return operation
def register_descriptors(self,
descriptor_schemas,
user,
force=False,
verbosity=1):
"""Read and register descriptors."""
log_descriptors = []
for descriptor_schema in descriptor_schemas:
for field in ['var', 'schema']:
for schema, _, _ in iterate_schema({},
descriptor_schema.get(
field, {})):
if not schema['type'][-1].endswith(':'):
schema['type'] += ':'
# support backward compatibility
# TODO: update .yml files and remove
if 'slug' not in descriptor_schema:
descriptor_schema['slug'] = slugify(
descriptor_schema.pop('name').replace(':', '-'))
descriptor_schema['name'] = descriptor_schema.pop('label')
if 'schema' not in descriptor_schema:
descriptor_schema['schema'] = []
if 'static' in descriptor_schema:
descriptor_schema['schema'].extend(
descriptor_schema.pop('static'))
if 'var' in descriptor_schema:
descriptor_schema['schema'].extend(
descriptor_schema.pop('var'))
if not self.valid(descriptor_schema, DESCRIPTOR_SCHEMA):
continue
slug = descriptor_schema['slug']
version = descriptor_schema.get('version', '0.0.0')
int_version = convert_version_string_to_int(
version, VERSION_NUMBER_BITS)
# `latest version` is returned as `int` so it has to be compared to `int_version`
latest_version = DescriptorSchema.objects.filter(
slug=slug).aggregate(Max('version'))['version__max']
if latest_version is not None and latest_version > int_version:
self.stderr.write(
"Skip descriptor schema {}: newer version installed".
format(slug))
continue
previous_descriptor_qs = DescriptorSchema.objects.filter(slug=slug)
if previous_descriptor_qs.exists():
previous_descriptor = previous_descriptor_qs.latest()
else:
previous_descriptor = None
descriptor_query = DescriptorSchema.objects.filter(slug=slug,
version=version)
if descriptor_query.exists():
if not force:
if verbosity > 0:
self.stdout.write(
"Skip descriptor schema {}: same version installed"
.format(slug))
continue
descriptor_query.update(**descriptor_schema)
log_descriptors.append("Updated {}".format(slug))
else:
descriptor = DescriptorSchema.objects.create(
contributor=user, **descriptor_schema)
assign_contributor_permissions(descriptor)
if previous_descriptor:
copy_permissions(previous_descriptor, descriptor)
log_descriptors.append("Inserted {}".format(slug))
if len(log_descriptors) > 0 and verbosity > 0:
self.stdout.write("Descriptor schemas Updates:")
for log in log_descriptors:
self.stdout.write(" {}".format(log))
