class ProcessViewSet(
ResolweCreateModelMixin,
mixins.RetrieveModelMixin,
mixins.ListModelMixin,
ResolwePermissionsMixin,
ResolweCheckSlugMixin,
viewsets.GenericViewSet,
):
"""API view for :class:`Process` objects."""
qs_permission_model = PermissionModel.objects.select_related("user", "group")
queryset = Process.objects.all().select_related("contributor")
serializer_class = ProcessSerializer
permission_classes = (get_permissions_class(),)
filterset_class = ProcessFilter
ordering_fields = ("id", "created", "modified", "name", "version")
ordering = ("id",)
def create(self, request, *args, **kwargs):
"""Only superusers can create new processes."""
if not request.user.is_superuser:
raise exceptions.NotFound
return super().create(request, *args, **kwargs)
def get_queryset(self):
"""Prefetch permissions for current user."""
return self.prefetch_current_user_permissions(self.queryset)
class DescriptorSchemaViewSet(mixins.RetrieveModelMixin, mixins.ListModelMixin,
ResolwePermissionsMixin,
viewsets.GenericViewSet):
"""API view for :class:`DescriptorSchema` objects."""
queryset = DescriptorSchema.objects.all().prefetch_related('contributor')
serializer_class = DescriptorSchemaSerializer
permission_classes = (get_permissions_class(), )
filter_class = DescriptorSchemaFilter
ordering_fields = ('id', 'created', 'modified', 'name', 'version')
ordering = ('id', )
class DescriptorSchemaViewSet(
mixins.RetrieveModelMixin,
mixins.ListModelMixin,
ResolwePermissionsMixin,
viewsets.GenericViewSet,
):
"""API view for :class:`DescriptorSchema` objects."""
queryset = DescriptorSchema.objects.all().select_related("contributor")
serializer_class = DescriptorSchemaSerializer
permission_classes = (get_permissions_class(),)
filterset_class = DescriptorSchemaFilter
ordering_fields = ("id", "created", "modified", "name", "version")
ordering = ("id",)
class ProcessViewSet(ResolweCreateModelMixin, mixins.RetrieveModelMixin,
mixins.ListModelMixin, ResolwePermissionsMixin,
ResolweCheckSlugMixin, viewsets.GenericViewSet):
"""API view for :class:`Process` objects."""
queryset = Process.objects.all().prefetch_related('contributor')
serializer_class = ProcessSerializer
permission_classes = (get_permissions_class(), )
filter_class = ProcessFilter
ordering_fields = ('id', 'created', 'modified', 'name', 'version')
ordering = ('id', )
def create(self, request, *args, **kwargs):
"""Only superusers can create new processes."""
if not request.user.is_superuser:
raise exceptions.NotFound
return super().create(request, *args, **kwargs)
class DataViewSet(ElasticSearchCombinedViewSet, ResolweCreateModelMixin,
mixins.RetrieveModelMixin, ResolweUpdateModelMixin,
mixins.DestroyModelMixin, ResolwePermissionsMixin,
ResolweCheckSlugMixin, viewsets.GenericViewSet):
"""API view for :class:`Data` objects."""
queryset = Data.objects.all().prefetch_related('process',
'descriptor_schema',
'contributor')
serializer_class = DataSerializer
permission_classes = (get_permissions_class(), )
document_class = DataDocument
filtering_fields = ('id', 'slug', 'version', 'name', 'created', 'modified',
'contributor', 'owners', 'status', 'process',
'process_type', 'type', 'process_name', 'tags',
'collection', 'parents', 'children', 'entity',
'started', 'finished', 'text')
filtering_map = {
'name': 'name.ngrams',
'contributor': 'contributor_id',
'owners': 'owner_ids',
'process_name': 'process_name.ngrams',
}
ordering_fields = ('id', 'created', 'modified', 'started', 'finished',
'name', 'contributor', 'process_name', 'process_type',
'type')
ordering_map = {
'name': 'name.raw',
'process_type': 'process_type.raw',
'type': 'type.raw',
'process_name': 'process_name.raw',
'contributor': 'contributor_sort',
}
ordering = '-created'
def get_always_allowed_arguments(self):
"""Return query arguments which are always allowed."""
return super().get_always_allowed_arguments() + [
'hydrate_data',
'hydrate_collections',
'hydrate_entities',
]
def custom_filter_tags(self, value, search):
"""Support tags query."""
if not isinstance(value, list):
value = value.split(',')
filters = [Q('match', **{'tags': item}) for item in value]
search = search.query('bool', must=filters)
return search
def custom_filter_text(self, value, search):
"""Support general query using the 'text' attribute."""
if isinstance(value, list):
value = ' '.join(value)
should = [
Q('match', slug={
'query': value,
'operator': 'and',
'boost': 10.0
}),
Q(
'match', **{
'slug.ngrams': {
'query': value,
'operator': 'and',
'boost': 5.0
}
}),
Q('match', name={
'query': value,
'operator': 'and',
'boost': 10.0
}),
Q(
'match', **{
'name.ngrams': {
'query': value,
'operator': 'and',
'boost': 5.0
}
}),
Q('match',
contributor_name={
'query': value,
'operator': 'and',
'boost': 5.0
}),
Q(
'match', **{
'contributor_name.ngrams': {
'query': value,
'operator': 'and',
'boost': 2.0
}
}),
Q('match',
owner_names={
'query': value,
'operator': 'and',
'boost': 5.0
}),
Q(
'match', **{
'owner_names.ngrams': {
'query': value,
'operator': 'and',
'boost': 2.0
}
}),
Q('match',
process_name={
'query': value,
'operator': 'and',
'boost': 5.0
}),
Q(
'match', **{
'process_name.ngrams': {
'query': value,
'operator': 'and',
'boost': 2.0
}
}),
Q('match',
status={
'query': value,
'operator': 'and',
'boost': 2.0
}),
Q('match', type={
'query': value,
'operator': 'and',
'boost': 2.0
}),
]
# Add registered text extensions.
for extension in composer.get_extensions(self):
if hasattr(extension, 'text_filter'):
should += extension.text_filter(value)
search = search.query('bool', should=should)
return search
def create(self, request, *args, **kwargs):
"""Create a resource."""
collections = request.data.get('collections', [])
# check that user has permissions on all collections that Data
# object will be added to
for collection_id in collections:
try:
collection = Collection.objects.get(pk=collection_id)
except Collection.DoesNotExist:
return Response(
{
'collections': [
'Invalid pk "{}" - object does not exist.'.format(
collection_id)
]
},
status=status.HTTP_400_BAD_REQUEST)
if not request.user.has_perm('add_collection', obj=collection):
if request.user.has_perm('view_collection', obj=collection):
raise exceptions.PermissionDenied(
"You don't have `ADD` permission on collection (id: {})."
.format(collection_id))
else:
raise exceptions.NotFound(
"Collection not found (id: {}).".format(collection_id))
self.define_contributor(request)
if kwargs.pop('get_or_create', False):
response = self.perform_get_or_create(request, *args, **kwargs)
if response:
return response
return super().create(request, *args, **kwargs)
@list_route(methods=['post'])
def get_or_create(self, request, *args, **kwargs):
"""Get ``Data`` object if similar already exists, otherwise create it."""
kwargs['get_or_create'] = True
return self.create(request, *args, **kwargs)
def perform_get_or_create(self, request, *args, **kwargs):
"""Perform "get_or_create" - return existing object if found."""
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
process = serializer.validated_data.get('process')
process_input = request.data.get('input', {})
fill_with_defaults(process_input, process.input_schema)
checksum = get_data_checksum(process_input, process.slug,
process.version)
data_qs = Data.objects.filter(
checksum=checksum,
process__persistence__in=[
Process.PERSISTENCE_CACHED, Process.PERSISTENCE_TEMP
],
)
data_qs = get_objects_for_user(request.user, 'view_data', data_qs)
if data_qs.exists():
data = data_qs.order_by('created').last()
serializer = self.get_serializer(data)
return Response(serializer.data)
def perform_create(self, serializer):
"""Create a resource."""
process = serializer.validated_data.get('process')
if not process.is_active:
raise exceptions.ParseError(
'Process retired (id: {}, slug: {}/{}).'.format(
process.id, process.slug, process.version))
with transaction.atomic():
instance = serializer.save()
assign_contributor_permissions(instance)
# Entity is added to the collection only when it is
# created - when it only contains 1 Data object.
entities = Entity.objects.annotate(num_data=Count('data')).filter(
data=instance, num_data=1)
# Assign data object to all specified collections.
collection_pks = self.request.data.get('collections', [])
for collection in Collection.objects.filter(pk__in=collection_pks):
collection.data.add(instance)
copy_permissions(collection, instance)
# Add entities to which data belongs to the collection.
for entity in entities:
entity.collections.add(collection)
copy_permissions(collection, entity)
class DataViewSet(ResolweCreateModelMixin, mixins.RetrieveModelMixin,
ResolweUpdateModelMixin, mixins.DestroyModelMixin,
mixins.ListModelMixin, ResolwePermissionsMixin,
ResolweCheckSlugMixin, viewsets.GenericViewSet):
"""API view for :class:`Data` objects."""
queryset = Data.objects.all().prefetch_related('process',
'descriptor_schema',
'contributor')
serializer_class = DataSerializer
permission_classes = (get_permissions_class(), )
filter_class = DataFilter
ordering_fields = ('id', 'created', 'modified', 'started', 'finished',
'name')
ordering = ('id', )
def create(self, request, *args, **kwargs):
"""Create a resource."""
collections = request.data.get('collections', [])
# check that user has permissions on all collections that Data
# object will be added to
for collection_id in collections:
try:
collection = Collection.objects.get(pk=collection_id)
except Collection.DoesNotExist:
return Response(
{
'collections': [
'Invalid pk "{}" - object does not exist.'.format(
collection_id)
]
},
status=status.HTTP_400_BAD_REQUEST)
if not request.user.has_perm('add_collection', obj=collection):
if request.user.has_perm('view_collection', obj=collection):
raise exceptions.PermissionDenied(
"You don't have `ADD` permission on collection (id: {})."
.format(collection_id))
else:
raise exceptions.NotFound(
"Collection not found (id: {}).".format(collection_id))
# translate processe's slug to id
process_slug = request.data.get('process', None)
process_query = Process.objects.filter(slug=process_slug)
process_query = get_objects_for_user(request.user, 'view_process',
process_query)
try:
process = process_query.latest()
except Process.DoesNotExist:
return Response(
{
'process': [
'Invalid process slug "{}" - object does not exist.'.
format(process_slug)
]
},
status=status.HTTP_400_BAD_REQUEST)
request.data['process'] = process.pk
# perform "get_or_create" if requested - return existing object
# if found
if kwargs.pop('get_or_create', False):
process_input = request.data.get('input', {})
# use default values if they are not given
for field_schema, fields, path in iterate_schema(
process_input, process.input_schema):
if 'default' in field_schema and field_schema[
'name'] not in fields:
dict_dot(process_input, path, field_schema['default'])
checksum = get_data_checksum(process_input, process.slug,
process.version)
data_qs = Data.objects.filter(
checksum=checksum,
process__persistence__in=[
Process.PERSISTENCE_CACHED, Process.PERSISTENCE_TEMP
],
)
data_qs = get_objects_for_user(request.user, 'view_data', data_qs)
if data_qs.exists():
data = data_qs.order_by('created').last()
serializer = self.get_serializer(data)
return Response(serializer.data)
# create the objects
resp = super(DataViewSet, self).create(request, *args, **kwargs)
# run manager
manager.communicate()
return resp
@list_route(methods=[u'post'])
def get_or_create(self, request, *args, **kwargs):
"""Get ``Data`` object if similar already exists, otherwise create it."""
kwargs['get_or_create'] = True
return self.create(request, *args, **kwargs)
def perform_create(self, serializer):
"""Create a resource."""
with transaction.atomic():
instance = serializer.save()
assign_contributor_permissions(instance)
# Entity is added to the collection only when it is
# created - when it only contains 1 Data object.
entities = Entity.objects.annotate(num_data=Count('data')).filter(
data=instance, num_data=1)
# Assign data object to all specified collections.
collection_pks = self.request.data.get('collections', [])
for collection in Collection.objects.filter(pk__in=collection_pks):
collection.data.add(instance)
copy_permissions(collection, instance)
# Add entities to which data belongs to the collection.
for entity in entities:
entity.collections.add(collection)
copy_permissions(collection, entity)
class DataViewSet(ElasticSearchCombinedViewSet, ResolweCreateModelMixin,
mixins.RetrieveModelMixin, ResolweUpdateModelMixin,
mixins.DestroyModelMixin, ResolwePermissionsMixin,
ResolweCheckSlugMixin, ParametersMixin,
viewsets.GenericViewSet):
"""API view for :class:`Data` objects."""
queryset = Data.objects.all().prefetch_related('process',
'descriptor_schema',
'contributor', 'collection',
'entity')
serializer_class = DataSerializer
permission_classes = (get_permissions_class(), )
document_class = DataDocument
filtering_fields = ('id', 'slug', 'version', 'name', 'created', 'modified',
'contributor', 'owners', 'status', 'process',
'process_type', 'type', 'process_name', 'tags',
'collection', 'entity', 'started', 'finished', 'text')
filtering_map = {
'name': 'name.raw',
'contributor': 'contributor_id',
'owners': 'owner_ids',
'process_name': 'process_name.ngrams',
}
ordering_fields = ('id', 'created', 'modified', 'started', 'finished',
'name', 'contributor', 'process_name', 'process_type',
'type')
ordering_map = {
'name': 'name.raw',
'process_type': 'process_type.raw',
'type': 'type.raw',
'process_name': 'process_name.raw',
'contributor': 'contributor_sort',
}
ordering = '-created'
def custom_filter_tags(self, value, search):
"""Support tags query."""
if not isinstance(value, list):
value = value.split(',')
filters = [Q('match', **{'tags': item}) for item in value]
search = search.query('bool', must=filters)
return search
def custom_filter_text(self, value, search):
"""Support general query using the 'text' attribute."""
if isinstance(value, list):
value = ' '.join(value)
should = [
Q('match', slug={
'query': value,
'operator': 'and',
'boost': 10.0
}),
Q(
'match', **{
'slug.ngrams': {
'query': value,
'operator': 'and',
'boost': 5.0
}
}),
Q('match', name={
'query': value,
'operator': 'and',
'boost': 10.0
}),
Q(
'match', **{
'name.ngrams': {
'query': value,
'operator': 'and',
'boost': 5.0
}
}),
Q('match',
contributor_name={
'query': value,
'operator': 'and',
'boost': 5.0
}),
Q(
'match', **{
'contributor_name.ngrams': {
'query': value,
'operator': 'and',
'boost': 2.0
}
}),
Q('match',
owner_names={
'query': value,
'operator': 'and',
'boost': 5.0
}),
Q(
'match', **{
'owner_names.ngrams': {
'query': value,
'operator': 'and',
'boost': 2.0
}
}),
Q('match',
process_name={
'query': value,
'operator': 'and',
'boost': 5.0
}),
Q(
'match', **{
'process_name.ngrams': {
'query': value,
'operator': 'and',
'boost': 2.0
}
}),
Q('match',
status={
'query': value,
'operator': 'and',
'boost': 2.0
}),
Q('match', type={
'query': value,
'operator': 'and',
'boost': 2.0
}),
]
# Add registered text extensions.
for extension in composer.get_extensions(self):
if hasattr(extension, 'text_filter'):
should += extension.text_filter(value)
search = search.query('bool', should=should)
return search
@action(detail=False, methods=['post'])
def duplicate(self, request, *args, **kwargs):
"""Duplicate (make copy of) ``Data`` objects."""
if not request.user.is_authenticated:
raise exceptions.NotFound
ids = self.get_ids(request.data)
queryset = get_objects_for_user(request.user, 'view_data',
Data.objects.filter(id__in=ids))
actual_ids = queryset.values_list('id', flat=True)
missing_ids = list(set(ids) - set(actual_ids))
if missing_ids:
raise exceptions.ParseError(
"Data objects with the following ids not found: {}".format(
', '.join(map(str, missing_ids))))
# TODO support ``inherit_collection``
duplicated = queryset.duplicate(contributor=request.user)
serializer = self.get_serializer(duplicated, many=True)
return Response(serializer.data)
@action(detail=False, methods=['post'])
def get_or_create(self, request, *args, **kwargs):
"""Get ``Data`` object if similar already exists, otherwise create it."""
response = self.perform_get_or_create(request, *args, **kwargs)
if response:
return response
return super().create(request, *args, **kwargs)
def perform_get_or_create(self, request, *args, **kwargs):
"""Perform "get_or_create" - return existing object if found."""
self.define_contributor(request)
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
process = serializer.validated_data.get('process')
process_input = request.data.get('input', {})
fill_with_defaults(process_input, process.input_schema)
checksum = get_data_checksum(process_input, process.slug,
process.version)
data_qs = Data.objects.filter(
checksum=checksum,
process__persistence__in=[
Process.PERSISTENCE_CACHED, Process.PERSISTENCE_TEMP
],
)
data_qs = get_objects_for_user(request.user, 'view_data', data_qs)
if data_qs.exists():
data = data_qs.order_by('created').last()
serializer = self.get_serializer(data)
return Response(serializer.data)
def _parents_children(self, request, queryset):
"""Process given queryset and return serialized objects."""
queryset = get_objects_for_user(request.user, 'view_data', queryset)
page = self.paginate_queryset(queryset)
if page is not None:
serializer = self.get_serializer(page, many=True)
return self.get_paginated_response(serializer.data)
serializer = self.get_serializer(queryset, many=True)
return Response(serializer.data)
@action(detail=True)
def parents(self, request, pk=None):
"""Return parents of the current data object."""
return self._parents_children(request, self.get_object().parents)
@action(detail=True)
def children(self, request, pk=None):
"""Return children of the current data object."""
return self._parents_children(request, self.get_object().children)
class CollectionViewSet(ResolweCreateModelMixin, mixins.RetrieveModelMixin,
ResolweUpdateModelMixin, mixins.DestroyModelMixin,
mixins.ListModelMixin, ResolwePermissionsMixin,
ResolweCheckSlugMixin, viewsets.GenericViewSet):
"""API view for :class:`Collection` objects."""
queryset = Collection.objects.all().prefetch_related(
'descriptor_schema', 'contributor',
Prefetch('data', queryset=Data.objects.all().order_by('id')))
serializer_class = CollectionSerializer
permission_classes = (get_permissions_class(), )
filter_class = CollectionFilter
ordering_fields = ('id', 'created', 'modified', 'name')
ordering = ('id', )
def set_content_permissions(self, user, obj, payload):
"""Apply permissions to data objects and entities in ``Collection``."""
for entity in obj.entity_set.all():
if user.has_perm('share_entity', entity):
update_permission(entity, payload)
# Data doesn't have "ADD" permission, so it has to be removed
payload = remove_permission(payload, 'add')
for data in obj.data.all():
if user.has_perm('share_data', data):
update_permission(data, payload)
def create(self, request, *args, **kwargs):
"""Only authenticated usesr can create new collections."""
if not request.user.is_authenticated:
raise exceptions.NotFound
return super(CollectionViewSet, self).create(request, *args, **kwargs)
def destroy(self, request, *args, **kwargs):
"""Destroy a model instance.
If ``delete_content`` flag is set in query parameters, also all
Data objects and Entities, on which user has ``EDIT``
permission, contained in collection will be deleted.
"""
obj = self.get_object()
user = request.user
if strtobool(request.query_params.get('delete_content', 'false')):
for entity in obj.entity_set.all():
if user.has_perm('edit_entity', entity):
entity.delete()
for data in obj.data.all():
if user.has_perm('edit_data', data):
data.delete()
return super(CollectionViewSet, self).destroy(request, *args, **kwargs) # pylint: disable=no-member
@detail_route(methods=[u'post'])
def add_data(self, request, pk=None):
"""Add data to collection."""
collection = self.get_object()
if 'ids' not in request.data:
return Response({"error": "`ids`parameter is required"},
status=status.HTTP_400_BAD_REQUEST)
missing = []
for data_id in request.data['ids']:
if not Data.objects.filter(pk=data_id).exists():
missing.append(data_id)
if missing:
return Response(
{
"error":
"Data objects with following ids are missing: {}".format(
', '.join(missing))
},
status=status.HTTP_400_BAD_REQUEST)
for data_id in request.data['ids']:
collection.data.add(data_id)
return Response()
@detail_route(methods=[u'post'])
def remove_data(self, request, pk=None):
"""Remove data from collection."""
collection = self.get_object()
if 'ids' not in request.data:
return Response({"error": "`ids`parameter is required"},
status=status.HTTP_400_BAD_REQUEST)
for data_id in request.data['ids']:
collection.data.remove(data_id)
return Response()
class CollectionViewSet(
ElasticSearchCombinedViewSet,
ResolweCreateModelMixin,
mixins.RetrieveModelMixin,
ResolweUpdateModelMixin,
mixins.DestroyModelMixin,
mixins.ListModelMixin,
ResolwePermissionsMixin,
ResolweCheckSlugMixin,
ParametersMixin,
viewsets.GenericViewSet,
):
"""API view for :class:`Collection` objects."""
queryset = Collection.objects.all().prefetch_related(
"descriptor_schema", "contributor"
)
serializer_class = CollectionSerializer
permission_classes = (get_permissions_class(),)
document_class = CollectionDocument
filtering_fields = (
"id",
"slug",
"name",
"name_contains",
"created",
"modified",
"contributor",
"contributor_name",
"owners",
"owners_name",
"text",
"tags",
)
filtering_map = {
"name": "name.raw",
"name_contains": "name.ngrams",
"contributor": "contributor_id",
"contributor_name": "contributor_name.ngrams",
"owners": "owner_ids",
"owners_name": "owner_names.ngrams",
}
ordering_fields = ("id", "created", "modified", "name", "contributor")
ordering_map = {
"name": "name.raw",
"contributor": "contributor_sort",
}
ordering = "id"
def custom_filter_tags(self, value, search):
"""Support tags query."""
if not isinstance(value, list):
value = value.split(",")
filters = [Q("match", **{"tags": item}) for item in value]
search = search.query("bool", must=filters)
return search
def custom_filter_text(self, value, search):
"""Support general query using the 'text' attribute."""
if isinstance(value, list):
value = " ".join(value)
should = [
Q("match", slug={"query": value, "operator": "and", "boost": 10.0}),
Q(
"match",
**{"slug.ngrams": {"query": value, "operator": "and", "boost": 5.0}},
),
Q("match", name={"query": value, "operator": "and", "boost": 10.0}),
Q(
"match",
**{"name.ngrams": {"query": value, "operator": "and", "boost": 5.0}},
),
Q(
"match",
contributor_name={"query": value, "operator": "and", "boost": 5.0},
),
Q(
"match",
**{
"contributor_name.ngrams": {
"query": value,
"operator": "and",
"boost": 2.0,
}
},
),
Q("match", owner_names={"query": value, "operator": "and", "boost": 5.0}),
Q(
"match",
**{
"owner_names.ngrams": {
"query": value,
"operator": "and",
"boost": 2.0,
}
},
),
Q("match", descriptor_data={"query": value, "operator": "and"}),
Q("match", description={"query": value, "operator": "and"}),
]
# Add registered text extensions.
for extension in composer.get_extensions(self):
if hasattr(extension, "text_filter"):
should += extension.text_filter(value)
search = search.query("bool", should=should)
return search
def set_content_permissions(self, user, obj, payload):
"""Apply permissions to data objects and entities in ``Collection``."""
for entity in obj.entity_set.all():
if user.has_perm("share_entity", entity):
update_permission(entity, payload)
for data in obj.data.all():
if user.has_perm("share_data", data):
update_permission(data, payload)
def create(self, request, *args, **kwargs):
"""Only authenticated users can create new collections."""
if not request.user.is_authenticated:
raise exceptions.NotFound
return super().create(request, *args, **kwargs)
@action(detail=False, methods=["post"])
def duplicate(self, request, *args, **kwargs):
"""Duplicate (make copy of) ``Collection`` models."""
if not request.user.is_authenticated:
raise exceptions.NotFound
ids = self.get_ids(request.data)
queryset = get_objects_for_user(
request.user, "view_collection", Collection.objects.filter(id__in=ids)
)
actual_ids = queryset.values_list("id", flat=True)
missing_ids = list(set(ids) - set(actual_ids))
if missing_ids:
raise exceptions.ParseError(
"Collections with the following ids not found: {}".format(
", ".join(map(str, missing_ids))
)
)
duplicated = queryset.duplicate(contributor=request.user)
serializer = self.get_serializer(duplicated, many=True)
return Response(serializer.data)
class DataViewSet(
ElasticSearchCombinedViewSet,
ResolweCreateModelMixin,
mixins.RetrieveModelMixin,
ResolweUpdateModelMixin,
mixins.DestroyModelMixin,
ResolwePermissionsMixin,
ResolweCheckSlugMixin,
ParametersMixin,
viewsets.GenericViewSet,
):
"""API view for :class:`Data` objects."""
queryset = Data.objects.all().prefetch_related("process",
"descriptor_schema",
"contributor", "collection",
"entity")
serializer_class = DataSerializer
permission_classes = (get_permissions_class(), )
document_class = DataDocument
filtering_fields = (
"id",
"slug",
"version",
"name",
"created",
"modified",
"contributor",
"owners",
"status",
"process",
"process_type",
"type",
"process_name",
"tags",
"collection",
"entity",
"started",
"finished",
"text",
"process_slug",
)
filtering_map = {
"name": "name.raw",
"contributor": "contributor_id",
"owners": "owner_ids",
"process_name": "process_name.ngrams",
}
ordering_fields = (
"id",
"created",
"modified",
"started",
"finished",
"name",
"contributor",
"process_name",
"process_type",
"type",
)
ordering_map = {
"name": "name.raw",
"process_type": "process_type.raw",
"type": "type.raw",
"process_name": "process_name.raw",
"contributor": "contributor_sort",
}
ordering = "-created"
def custom_filter_tags(self, value, search):
"""Support tags query."""
if not isinstance(value, list):
value = value.split(",")
filters = [Q("match", **{"tags": item}) for item in value]
search = search.query("bool", must=filters)
return search
def custom_filter_text(self, value, search):
"""Support general query using the 'text' attribute."""
if isinstance(value, list):
value = " ".join(value)
should = [
Q("match", slug={
"query": value,
"operator": "and",
"boost": 10.0
}),
Q(
"match",
**{
"slug.ngrams": {
"query": value,
"operator": "and",
"boost": 5.0
}
},
),
Q("match", name={
"query": value,
"operator": "and",
"boost": 10.0
}),
Q(
"match",
**{
"name.ngrams": {
"query": value,
"operator": "and",
"boost": 5.0
}
},
),
Q(
"match",
contributor_name={
"query": value,
"operator": "and",
"boost": 5.0
},
),
Q(
"match",
**{
"contributor_name.ngrams": {
"query": value,
"operator": "and",
"boost": 2.0,
}
},
),
Q("match",
owner_names={
"query": value,
"operator": "and",
"boost": 5.0
}),
Q(
"match",
**{
"owner_names.ngrams": {
"query": value,
"operator": "and",
"boost": 2.0,
}
},
),
Q("match",
process_name={
"query": value,
"operator": "and",
"boost": 5.0
}),
Q(
"match",
**{
"process_name.ngrams": {
"query": value,
"operator": "and",
"boost": 2.0,
}
},
),
Q("match",
status={
"query": value,
"operator": "and",
"boost": 2.0
}),
Q("match", type={
"query": value,
"operator": "and",
"boost": 2.0
}),
]
# Add registered text extensions.
for extension in composer.get_extensions(self):
if hasattr(extension, "text_filter"):
should += extension.text_filter(value)
search = search.query("bool", should=should)
return search
@action(detail=False, methods=["post"])
def duplicate(self, request, *args, **kwargs):
"""Duplicate (make copy of) ``Data`` objects."""
if not request.user.is_authenticated:
raise exceptions.NotFound
ids = self.get_ids(request.data)
queryset = get_objects_for_user(request.user, "view_data",
Data.objects.filter(id__in=ids))
actual_ids = queryset.values_list("id", flat=True)
missing_ids = list(set(ids) - set(actual_ids))
if missing_ids:
raise exceptions.ParseError(
"Data objects with the following ids not found: {}".format(
", ".join(map(str, missing_ids))))
# TODO support ``inherit_collection``
duplicated = queryset.duplicate(contributor=request.user)
serializer = self.get_serializer(duplicated, many=True)
return Response(serializer.data)
@action(detail=False, methods=["post"])
def get_or_create(self, request, *args, **kwargs):
"""Get ``Data`` object if similar already exists, otherwise create it."""
response = self.perform_get_or_create(request, *args, **kwargs)
if response:
return response
return super().create(request, *args, **kwargs)
def perform_get_or_create(self, request, *args, **kwargs):
"""Perform "get_or_create" - return existing object if found."""
self.define_contributor(request)
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
process = serializer.validated_data.get("process")
process_input = request.data.get("input", {})
fill_with_defaults(process_input, process.input_schema)
checksum = get_data_checksum(process_input, process.slug,
process.version)
data_qs = Data.objects.filter(
checksum=checksum,
process__persistence__in=[
Process.PERSISTENCE_CACHED,
Process.PERSISTENCE_TEMP,
],
)
data_qs = get_objects_for_user(request.user, "view_data", data_qs)
if data_qs.exists():
data = data_qs.order_by("created").last()
serializer = self.get_serializer(data)
return Response(serializer.data)
def _parents_children(self, request, queryset):
"""Process given queryset and return serialized objects."""
queryset = get_objects_for_user(request.user, "view_data", queryset)
page = self.paginate_queryset(queryset)
if page is not None:
serializer = self.get_serializer(page, many=True)
return self.get_paginated_response(serializer.data)
serializer = self.get_serializer(queryset, many=True)
return Response(serializer.data)
@action(detail=True)
def parents(self, request, pk=None):
"""Return parents of the current data object."""
return self._parents_children(request, self.get_object().parents)
@action(detail=True)
def children(self, request, pk=None):
"""Return children of the current data object."""
return self._parents_children(request, self.get_object().children)
class CollectionViewSet(ElasticSearchCombinedViewSet,
ResolweCreateModelMixin,
mixins.RetrieveModelMixin,
ResolweUpdateModelMixin,
mixins.DestroyModelMixin,
mixins.ListModelMixin,
ResolwePermissionsMixin,
ResolweCheckSlugMixin,
ParametersMixin,
viewsets.GenericViewSet):
"""API view for :class:`Collection` objects."""
queryset = Collection.objects.all().prefetch_related('descriptor_schema', 'contributor')
serializer_class = CollectionSerializer
permission_classes = (get_permissions_class(),)
document_class = CollectionDocument
filtering_fields = (
'id', 'slug', 'name', 'created', 'modified', 'contributor', 'owners', 'text', 'tags',
)
filtering_map = {
'name': 'name.raw',
'contributor': 'contributor_id',
'owners': 'owner_ids',
}
ordering_fields = ('id', 'created', 'modified', 'name', 'contributor')
ordering_map = {
'name': 'name.raw',
'contributor': 'contributor_sort',
}
ordering = 'id'
def custom_filter_tags(self, value, search):
"""Support tags query."""
if not isinstance(value, list):
value = value.split(',')
filters = [Q('match', **{'tags': item}) for item in value]
search = search.query('bool', must=filters)
return search
def custom_filter_text(self, value, search):
"""Support general query using the 'text' attribute."""
if isinstance(value, list):
value = ' '.join(value)
should = [
Q('match', slug={'query': value, 'operator': 'and', 'boost': 10.0}),
Q('match', **{'slug.ngrams': {'query': value, 'operator': 'and', 'boost': 5.0}}),
Q('match', name={'query': value, 'operator': 'and', 'boost': 10.0}),
Q('match', **{'name.ngrams': {'query': value, 'operator': 'and', 'boost': 5.0}}),
Q('match', contributor_name={'query': value, 'operator': 'and', 'boost': 5.0}),
Q('match', **{'contributor_name.ngrams': {'query': value, 'operator': 'and', 'boost': 2.0}}),
Q('match', owner_names={'query': value, 'operator': 'and', 'boost': 5.0}),
Q('match', **{'owner_names.ngrams': {'query': value, 'operator': 'and', 'boost': 2.0}}),
Q('match', descriptor_data={'query': value, 'operator': 'and'}),
]
# Add registered text extensions.
for extension in composer.get_extensions(self):
if hasattr(extension, 'text_filter'):
should += extension.text_filter(value)
search = search.query('bool', should=should)
return search
def set_content_permissions(self, user, obj, payload):
"""Apply permissions to data objects and entities in ``Collection``."""
for entity in obj.entity_set.all():
if user.has_perm('share_entity', entity):
update_permission(entity, payload)
# Data doesn't have "ADD" permission, so it has to be removed
payload = remove_permission(payload, 'add')
for data in obj.data.all():
if user.has_perm('share_data', data):
update_permission(data, payload)
def create(self, request, *args, **kwargs):
"""Only authenticated users can create new collections."""
if not request.user.is_authenticated:
raise exceptions.NotFound
return super().create(request, *args, **kwargs)
@action(detail=False, methods=['post'])
def duplicate(self, request, *args, **kwargs):
"""Duplicate (make copy of) ``Collection`` models."""
if not request.user.is_authenticated:
raise exceptions.NotFound
ids = self.get_ids(request.data)
queryset = get_objects_for_user(request.user, 'view_collection', Collection.objects.filter(id__in=ids))
actual_ids = queryset.values_list('id', flat=True)
missing_ids = list(set(ids) - set(actual_ids))
if missing_ids:
raise exceptions.ParseError(
"Collections with the following ids not found: {}".format(', '.join(map(str, missing_ids)))
)
duplicated = queryset.duplicate(contributor=request.user)
serializer = self.get_serializer(duplicated, many=True)
return Response(serializer.data)
class DataViewSet(
ResolweCreateModelMixin,
mixins.ListModelMixin,
mixins.RetrieveModelMixin,
ResolweUpdateModelMixin,
mixins.DestroyModelMixin,
ResolwePermissionsMixin,
ResolweCheckSlugMixin,
ParametersMixin,
viewsets.GenericViewSet,
):
"""API view for :class:`Data` objects."""
qs_collection_ds = DescriptorSchema.objects.select_related("contributor")
qs_collection = Collection.objects.select_related("contributor")
qs_collection = qs_collection.prefetch_related(
"data",
"entity_set",
Prefetch("descriptor_schema", queryset=qs_collection_ds),
)
qs_descriptor_schema = DescriptorSchema.objects.select_related(
"contributor")
qs_entity_col_ds = DescriptorSchema.objects.select_related("contributor")
qs_entity_col = Collection.objects.select_related("contributor")
qs_entity_col = qs_entity_col.prefetch_related(
"data",
"entity_set",
Prefetch("descriptor_schema", queryset=qs_entity_col_ds),
)
qs_entity_ds = DescriptorSchema.objects.select_related("contributor")
qs_entity = Entity.objects.select_related("contributor")
qs_entity = qs_entity.prefetch_related(
"data",
Prefetch("collection", queryset=qs_entity_col),
Prefetch("descriptor_schema", queryset=qs_entity_ds),
)
qs_process = Process.objects.select_related("contributor")
queryset = Data.objects.select_related("contributor").prefetch_related(
Prefetch("collection", queryset=qs_collection),
Prefetch("descriptor_schema", queryset=qs_descriptor_schema),
Prefetch("entity", queryset=qs_entity),
Prefetch("process", queryset=qs_process),
)
serializer_class = DataSerializer
filter_class = DataFilter
permission_classes = (get_permissions_class(), )
ordering_fields = (
"contributor",
"contributor__first_name",
"contributor__last_name",
"created",
"finished",
"id",
"modified",
"name",
"process__name",
"process__type",
"started",
)
ordering = "-created"
@action(detail=False, methods=["post"])
def duplicate(self, request, *args, **kwargs):
"""Duplicate (make copy of) ``Data`` objects."""
if not request.user.is_authenticated:
raise exceptions.NotFound
inherit_collection = request.data.get("inherit_collection", False)
ids = self.get_ids(request.data)
queryset = get_objects_for_user(request.user, "view_data",
Data.objects.filter(id__in=ids))
actual_ids = queryset.values_list("id", flat=True)
missing_ids = list(set(ids) - set(actual_ids))
if missing_ids:
raise exceptions.ParseError(
"Data objects with the following ids not found: {}".format(
", ".join(map(str, missing_ids))))
duplicated = queryset.duplicate(
contributor=request.user,
inherit_collection=inherit_collection,
)
serializer = self.get_serializer(duplicated, many=True)
return Response(serializer.data)
@action(detail=False, methods=["post"])
def get_or_create(self, request, *args, **kwargs):
"""Get ``Data`` object if similar already exists, otherwise create it."""
response = self.perform_get_or_create(request, *args, **kwargs)
if response:
return response
return super().create(request, *args, **kwargs)
def perform_get_or_create(self, request, *args, **kwargs):
"""Perform "get_or_create" - return existing object if found."""
self.define_contributor(request)
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
process = serializer.validated_data.get("process")
process_input = request.data.get("input", {})
fill_with_defaults(process_input, process.input_schema)
checksum = get_data_checksum(process_input, process.slug,
process.version)
data_qs = Data.objects.filter(
checksum=checksum,
process__persistence__in=[
Process.PERSISTENCE_CACHED,
Process.PERSISTENCE_TEMP,
],
)
data_qs = get_objects_for_user(request.user, "view_data", data_qs)
if data_qs.exists():
data = data_qs.order_by("created").last()
serializer = self.get_serializer(data)
return Response(serializer.data)
def _parents_children(self, request, queryset):
"""Process given queryset and return serialized objects."""
queryset = get_objects_for_user(request.user, "view_data", queryset)
page = self.paginate_queryset(queryset)
if page is not None:
serializer = self.get_serializer(page, many=True)
return self.get_paginated_response(serializer.data)
serializer = self.get_serializer(queryset, many=True)
return Response(serializer.data)
@action(detail=True)
def parents(self, request, pk=None):
"""Return parents of the current data object."""
return self._parents_children(request, self.get_object().parents)
@action(detail=True)
def children(self, request, pk=None):
"""Return children of the current data object."""
return self._parents_children(request, self.get_object().children)
@action(detail=False, methods=["post"])
def move_to_collection(self, request, *args, **kwargs):
"""Move data objects to destination collection."""
ids = self.get_ids(request.data)
dst_collection_id = self.get_id(request.data, "destination_collection")
dst_collection = get_collection_for_user(dst_collection_id,
request.user)
queryset = self._get_data(request.user, ids)
queryset.move_to_collection(dst_collection)
return Response()
def _get_data(self, user, ids):
"""Return data objects queryset based on provided ids."""
queryset = get_objects_for_user(user, "view_data",
Data.objects.filter(id__in=ids))
actual_ids = queryset.values_list("id", flat=True)
missing_ids = list(set(ids) - set(actual_ids))
if missing_ids:
raise exceptions.ParseError(
"Data objects with the following ids not found: {}".format(
", ".join(map(str, missing_ids))))
for data in queryset:
collection = data.collection
if collection and not user.has_perm("edit_collection",
obj=collection):
if user.is_authenticated:
raise exceptions.PermissionDenied()
else:
raise exceptions.NotFound()
return queryset
class BaseCollectionViewSet(
ResolweCreateModelMixin,
mixins.RetrieveModelMixin,
ResolweUpdateModelMixin,
mixins.DestroyModelMixin,
mixins.ListModelMixin,
ResolwePermissionsMixin,
ResolweCheckSlugMixin,
ParametersMixin,
viewsets.GenericViewSet,
):
"""Base API view for :class:`Collection` objects."""
qs_descriptor_schema = DescriptorSchema.objects.select_related(
"contributor")
qs_permission_model = PermissionModel.objects.select_related(
"user", "group")
queryset = Collection.objects.select_related(
"contributor").prefetch_related(
"data", Prefetch("descriptor_schema",
queryset=qs_descriptor_schema))
filter_class = CollectionFilter
permission_classes = (get_permissions_class(), )
ordering_fields = (
"contributor",
"contributor__first_name",
"contributor__last_name",
"created",
"id",
"modified",
"name",
)
ordering = "id"
def get_queryset(self):
"""Prefetch permissions for current user."""
return self.prefetch_current_user_permissions(self.queryset)
def create(self, request, *args, **kwargs):
"""Only authenticated users can create new collections."""
if not request.user.is_authenticated:
raise exceptions.NotFound
return super().create(request, *args, **kwargs)
@action(detail=False, methods=["post"])
def duplicate(self, request, *args, **kwargs):
"""Duplicate (make copy of) ``Collection`` models."""
if not request.user.is_authenticated:
raise exceptions.NotFound
ids = self.get_ids(request.data)
queryset = Collection.objects.filter(id__in=ids).filter_for_user(
request.user, Permission.VIEW)
actual_ids = queryset.values_list("id", flat=True)
missing_ids = list(set(ids) - set(actual_ids))
if missing_ids:
raise exceptions.ParseError(
"Collections with the following ids not found: {}".format(
", ".join(map(str, missing_ids))))
duplicated = queryset.duplicate(contributor=request.user)
serializer = self.get_serializer(duplicated, many=True)
return Response(serializer.data)
class CollectionViewSet(ElasticSearchCombinedViewSet, ResolweCreateModelMixin,
mixins.RetrieveModelMixin, ResolweUpdateModelMixin,
mixins.DestroyModelMixin, mixins.ListModelMixin,
ResolwePermissionsMixin, ResolweCheckSlugMixin,
viewsets.GenericViewSet):
"""API view for :class:`Collection` objects."""
queryset = Collection.objects.all().prefetch_related(
'descriptor_schema', 'contributor',
Prefetch('data', queryset=Data.objects.all().order_by('id')))
serializer_class = CollectionSerializer
permission_classes = (get_permissions_class(), )
document_class = CollectionDocument
filtering_fields = (
'id',
'slug',
'name',
'created',
'modified',
'contributor',
'owners',
'text',
'tags',
)
filtering_map = {
'name': 'name.ngrams',
'contributor': 'contributor_id',
'owners': 'owner_ids',
}
ordering_fields = ('id', 'created', 'modified', 'name', 'contributor')
ordering_map = {
'name': 'name.raw',
'contributor': 'contributor_sort',
}
ordering = 'id'
def custom_filter_tags(self, value, search):
"""Support tags query."""
if not isinstance(value, list):
value = value.split(',')
filters = [Q('match', **{'tags': item}) for item in value]
search = search.query('bool', must=filters)
return search
def get_always_allowed_arguments(self):
"""Return query arguments which are always allowed."""
return super().get_always_allowed_arguments() + [
'hydrate_data',
]
def custom_filter_text(self, value, search):
"""Support general query using the 'text' attribute."""
if isinstance(value, list):
value = ' '.join(value)
should = [
Q('match', slug={
'query': value,
'operator': 'and',
'boost': 10.0
}),
Q(
'match', **{
'slug.ngrams': {
'query': value,
'operator': 'and',
'boost': 5.0
}
}),
Q('match', name={
'query': value,
'operator': 'and',
'boost': 10.0
}),
Q(
'match', **{
'name.ngrams': {
'query': value,
'operator': 'and',
'boost': 5.0
}
}),
Q('match',
contributor_name={
'query': value,
'operator': 'and',
'boost': 5.0
}),
Q(
'match', **{
'contributor_name.ngrams': {
'query': value,
'operator': 'and',
'boost': 2.0
}
}),
Q('match',
owner_names={
'query': value,
'operator': 'and',
'boost': 5.0
}),
Q(
'match', **{
'owner_names.ngrams': {
'query': value,
'operator': 'and',
'boost': 2.0
}
}),
Q('match', descriptor_data={
'query': value,
'operator': 'and'
}),
]
# Add registered text extensions.
for extension in composer.get_extensions(self):
if hasattr(extension, 'text_filter'):
should += extension.text_filter(value)
search = search.query('bool', should=should)
return search
def set_content_permissions(self, user, obj, payload):
"""Apply permissions to data objects and entities in ``Collection``."""
for entity in obj.entity_set.all():
if user.has_perm('share_entity', entity):
update_permission(entity, payload)
# Data doesn't have "ADD" permission, so it has to be removed
payload = remove_permission(payload, 'add')
for data in obj.data.all():
if user.has_perm('share_data', data):
update_permission(data, payload)
def create(self, request, *args, **kwargs):
"""Only authenticated usesr can create new collections."""
if not request.user.is_authenticated:
raise exceptions.NotFound
return super().create(request, *args, **kwargs)
def destroy(self, request, *args, **kwargs):
"""Destroy a model instance.
If ``delete_content`` flag is set in query parameters, also all
Data objects and Entities, on which user has ``EDIT``
permission, contained in collection will be deleted.
"""
obj = self.get_object()
user = request.user
if strtobool(request.query_params.get('delete_content', 'false')):
for entity in obj.entity_set.all():
if user.has_perm('edit_entity', entity):
entity.delete()
for data in obj.data.all():
if user.has_perm('edit_data', data):
data.delete()
return super().destroy(request, *args, **kwargs) # pylint: disable=no-member
@detail_route(methods=['post'])
def add_data(self, request, pk=None):
"""Add data to collection."""
collection = self.get_object()
if 'ids' not in request.data:
return Response({"error": "`ids`parameter is required"},
status=status.HTTP_400_BAD_REQUEST)
missing = []
for data_id in request.data['ids']:
if not Data.objects.filter(pk=data_id).exists():
missing.append(data_id)
if missing:
return Response(
{
"error":
"Data objects with following ids are missing: {}".format(
', '.join(missing))
},
status=status.HTTP_400_BAD_REQUEST)
for data_id in request.data['ids']:
collection.data.add(data_id)
return Response()
@detail_route(methods=['post'])
def remove_data(self, request, pk=None):
"""Remove data from collection."""
collection = self.get_object()
if 'ids' not in request.data:
return Response({"error": "`ids`parameter is required"},
status=status.HTTP_400_BAD_REQUEST)
for data_id in request.data['ids']:
collection.data.remove(data_id)
return Response()
class BaseCollectionViewSet(
ResolweCreateModelMixin,
mixins.RetrieveModelMixin,
ResolweUpdateModelMixin,
mixins.DestroyModelMixin,
mixins.ListModelMixin,
ResolwePermissionsMixin,
ResolweCheckSlugMixin,
ParametersMixin,
viewsets.GenericViewSet,
):
"""Base API view for :class:`Collection` objects."""
qs_descriptor_schema = DescriptorSchema.objects.select_related("contributor")
queryset = Collection.objects.select_related("contributor").prefetch_related(
"data",
Prefetch("descriptor_schema", queryset=qs_descriptor_schema),
)
filter_class = CollectionFilter
permission_classes = (get_permissions_class(),)
ordering_fields = (
"contributor",
"contributor__first_name",
"contributor__last_name",
"created",
"id",
"modified",
"name",
)
ordering = "id"
def set_content_permissions(self, user, obj, payload):
"""Apply permissions to data objects and entities in ``Collection``."""
for entity in obj.entity_set.all():
if user.has_perm("share_entity", entity):
update_permission(entity, payload)
for data in obj.data.all():
if user.has_perm("share_data", data):
update_permission(data, payload)
def create(self, request, *args, **kwargs):
"""Only authenticated users can create new collections."""
if not request.user.is_authenticated:
raise exceptions.NotFound
return super().create(request, *args, **kwargs)
@action(detail=False, methods=["post"])
def duplicate(self, request, *args, **kwargs):
"""Duplicate (make copy of) ``Collection`` models."""
if not request.user.is_authenticated:
raise exceptions.NotFound
ids = self.get_ids(request.data)
queryset = get_objects_for_user(
request.user, "view_collection", Collection.objects.filter(id__in=ids)
)
actual_ids = queryset.values_list("id", flat=True)
missing_ids = list(set(ids) - set(actual_ids))
if missing_ids:
raise exceptions.ParseError(
"Collections with the following ids not found: {}".format(
", ".join(map(str, missing_ids))
)
)
duplicated = queryset.duplicate(contributor=request.user)
serializer = self.get_serializer(duplicated, many=True)
return Response(serializer.data)
