def _filter_datasets_by_permissions(datasets, user):
"""
Filter out datasets that the user is not allowed to even know about
@param datasets: django queryset
@param user: request.user
@return: queryset with filter applied
"""
visibility_filter = Q(published=True)
if datasets.model is ReferenceDataset:
if user.has_perm(
dataset_type_to_manage_unpublished_permission_codename(
DataSetType.REFERENCE)):
visibility_filter |= Q(published=False)
if datasets.model is DataSet:
if user.has_perm(
dataset_type_to_manage_unpublished_permission_codename(
DataSetType.MASTER)):
visibility_filter |= Q(published=False, type=DataSetType.MASTER)
if user.has_perm(
dataset_type_to_manage_unpublished_permission_codename(
DataSetType.DATACUT)):
visibility_filter |= Q(published=False, type=DataSetType.DATACUT)
if datasets.model is VisualisationCatalogueItem:
if user.has_perm(
dataset_type_to_manage_unpublished_permission_codename(
DataSetType.VISUALISATION)):
visibility_filter |= Q(published=False)
datasets = datasets.filter(visibility_filter)
return datasets
def test_dataset_type_to_manage_unpublished_permission_codename():
assert (dataset_type_to_manage_unpublished_permission_codename(0) ==
'datasets.manage_unpublished_reference_datasets')
assert (dataset_type_to_manage_unpublished_permission_codename(
DataSet.TYPE_DATA_CUT) ==
'datasets.manage_unpublished_datacut_datasets')
assert (dataset_type_to_manage_unpublished_permission_codename(
DataSet.TYPE_MASTER_DATASET) ==
'datasets.manage_unpublished_master_datasets')
def filter_datasets(
datasets: Union[ReferenceDataset, DataSet],
query,
source,
use=None,
user=None,
form=None,
):
search = SearchVector('name', 'short_description', config='english')
search_query = SearchQuery(query, config='english')
dataset_filter = Q(published=True)
if user:
if datasets.model is ReferenceDataset:
reference_type = DataSetType.REFERENCE.value
reference_perm = dataset_type_to_manage_unpublished_permission_codename(
reference_type)
if user.has_perm(reference_perm):
dataset_filter |= Q(published=False)
if datasets.model is DataSet:
master_type, datacut_type = (
DataSetType.MASTER.value,
DataSetType.DATACUT.value,
)
master_perm = dataset_type_to_manage_unpublished_permission_codename(
master_type)
datacut_perm = dataset_type_to_manage_unpublished_permission_codename(
datacut_type)
if user.has_perm(master_perm) and (not use
or str(master_type) in use):
dataset_filter |= Q(published=False, type=master_type)
if user.has_perm(datacut_perm) and (not use
or str(datacut_type) in use):
dataset_filter |= Q(published=False, type=datacut_type)
datasets = datasets.filter(dataset_filter).annotate(search=search,
search_rank=SearchRank(
search,
search_query))
if query:
datasets = datasets.filter(search=query)
if source:
datasets = datasets.filter(source_tags__in=source)
if use:
datasets = datasets.filter(type__in=use)
return datasets
def get_object(self, queryset=None):
dataset_uuid = self.kwargs['dataset_uuid']
dataset = None
try:
dataset = ReferenceDataset.objects.live().get(uuid=dataset_uuid)
except ReferenceDataset.DoesNotExist:
try:
dataset = DataSet.objects.live().get(id=dataset_uuid)
except DataSet.DoesNotExist:
try:
dataset = VisualisationCatalogueItem.objects.live().get(
id=dataset_uuid)
except VisualisationCatalogueItem.DoesNotExist:
pass
if dataset:
perm_codename = dataset_type_to_manage_unpublished_permission_codename(
dataset.type)
if not dataset.published and not self.request.user.has_perm(
perm_codename):
dataset = None
if not dataset:
raise Http404('No dataset matches the given query.')
return dataset
def test_grants_manage_unpublished_visualisations_permission(self):
user = factories.UserFactory.create(
username='******',
is_staff=False,
is_superuser=False,
)
visualisation = factories.VisualisationCatalogueItemFactory.create(
published=False, visualisation_template__gitlab_project_id=1)
perm_codename = dataset_type_to_manage_unpublished_permission_codename(
DataSetType.VISUALISATION.value)
assert user.has_perm(perm_codename) is False
with requests_mock.Mocker() as rmock:
rmock.get(
f'http://127.0.0.1:8007/api/v4/users?extern_uid={user.profile.sso_id}&provider=oauth2_generic',
json=[{
"id": 1
}],
)
rmock.get(
'http://127.0.0.1:8007/api/v4/projects/1/members/all?user_ids=1',
json=[{
"id": 1,
"access_level": 50
}],
)
has_access = gitlab_has_developer_access(
user, visualisation.visualisation_template.gitlab_project_id)
# Permissions are cached on the instance so we need to re-fetch it entirely - refresh_from_db insufficient.
# https://docs.djangoproject.com/en/3.0/topics/auth/default/#permission-caching
user = get_object_or_404(get_user_model(), pk=user.id)
assert has_access is True
assert user.has_perm(perm_codename) is True
def filter_visualisations(query, access, source, user=None):
search = SearchVector('name', 'short_description', config='english')
search_query = SearchQuery(query, config='english')
if user and user.has_perm(
dataset_type_to_manage_unpublished_permission_codename(
DataSetType.VISUALISATION.value)):
published_filter = Q()
else:
published_filter = Q(published=True)
visualisations = VisualisationCatalogueItem.objects.filter(
published_filter).annotate(search=search,
search_rank=SearchRank(
search, search_query))
if query:
visualisations = visualisations.filter(search=query)
if user and access:
access_filter = (Q(user_access_type='REQUIRES_AUTHENTICATION')
&
(Q(visualisationuserpermission__user=user)
| Q(visualisationuserpermission__isnull=True))) | Q(
user_access_type='REQUIRES_AUTHORIZATION',
visualisationuserpermission__user=user,
)
visualisations = visualisations.filter(access_filter)
if source:
visualisations = visualisations.filter(
visualisation_template__datasetapplicationtemplatepermission__dataset__source_tags__in
=source)
return visualisations
def has_unpublished_dataset_access(user):
access = user.is_superuser
for dataset_type in DataSetType:
access = access or user.has_perm(
dataset_type_to_manage_unpublished_permission_codename(
dataset_type.value))
return access
def _ensure_user_has_manage_unpublish_perm(user):
# Update the django permission controlling whether the user can preview unpublished visualisation catalogue pages.
perm_codename = dataset_type_to_manage_unpublished_permission_codename(
DataSetType.VISUALISATION.value
)
app_label, codename = perm_codename.split('.')
perm = Permission.objects.get(content_type__app_label=app_label, codename=codename)
if not user.has_perm(perm_codename):
with transaction.atomic():
user.user_permissions.add(perm)
user.save()
LogEntry.objects.log_action(
user_id=user.pk,
content_type_id=ContentType.objects.get_for_model(user).pk,
object_id=user.pk,
object_repr=force_str(user),
action_flag=ADDITION,
change_message="Added 'manage unpublished visualisations' permission",
)
def get_datasets_data_for_user_matching_query(datasets: QuerySet,
query,
use=None,
user=None,
id_field='id'):
"""
Filters the dataset queryset for:
1) visibility (whether the user can know if the dataset exists)
2) matches the search terms
Annotates the dataset queryset with:
1) `has_access`, if the user can use the dataset's data.
"""
is_reference_query = datasets.model is ReferenceDataset
# Filter out datasets that the user is not allowed to even know about.
visibility_filter = Q(published=True)
if user:
if is_reference_query:
reference_type = DataSetType.REFERENCE.value
reference_perm = dataset_type_to_manage_unpublished_permission_codename(
reference_type)
if user.has_perm(reference_perm):
visibility_filter |= Q(published=False)
if datasets.model is DataSet:
master_type, datacut_type = (
DataSetType.MASTER.value,
DataSetType.DATACUT.value,
)
master_perm = dataset_type_to_manage_unpublished_permission_codename(
master_type)
datacut_perm = dataset_type_to_manage_unpublished_permission_codename(
datacut_type)
if user.has_perm(master_perm):
visibility_filter |= Q(published=False, type=master_type)
if user.has_perm(datacut_perm):
visibility_filter |= Q(published=False, type=datacut_type)
datasets = datasets.filter(visibility_filter)
# Filter out datasets that don't match the search terms
search = (SearchVector('name', weight='A', config='english') +
SearchVector('short_description', weight='B', config='english') +
SearchVector(StringAgg('tags__name', delimiter='\n'),
weight='B',
config='english'))
search_query = SearchQuery(query, config='english')
datasets = datasets.annotate(search=search,
search_rank=SearchRank(search, search_query))
if query:
datasets = datasets.filter(search=search_query)
# Mark up whether the user can access the data in the dataset.
access_filter = Q()
if user and datasets.model is not ReferenceDataset:
access_filter &= (Q(user_access_type='REQUIRES_AUTHENTICATION')
& (Q(datasetuserpermission__user=user)
| Q(datasetuserpermission__isnull=True))) | Q(
user_access_type='REQUIRES_AUTHORIZATION',
datasetuserpermission__user=user)
datasets = datasets.annotate(_has_access=Case(
When(access_filter, then=True),
default=False,
output_field=BooleanField(),
) if access_filter else Value(True, BooleanField()), )
# Pull in the source tag IDs for the dataset
datasets = datasets.annotate(source_tag_ids=ArrayAgg(
'tags', filter=Q(tags__type=TagType.SOURCE.value), distinct=True))
datasets = datasets.annotate(source_tag_names=ArrayAgg(
'tags__name', filter=Q(
tags__type=TagType.SOURCE.value), distinct=True))
# Pull in the topic tag IDs for the dataset
datasets = datasets.annotate(topic_tag_ids=ArrayAgg(
'tags', filter=Q(tags__type=TagType.TOPIC.value), distinct=True))
datasets = datasets.annotate(topic_tag_names=ArrayAgg(
'tags__name', filter=Q(tags__type=TagType.TOPIC.value), distinct=True))
# Define a `purpose` column denoting the dataset type.
if is_reference_query:
datasets = datasets.annotate(
purpose=Value(DataSetType.REFERENCE.value, IntegerField()))
else:
datasets = datasets.annotate(purpose=F('type'))
# We are joining on the user permissions table to determine `_has_access`` to the dataset, so we need to
# group them and remove duplicates. We aggregate all the `_has_access` fields together and return true if any
# of the records say that access is available.
datasets = datasets.values(
id_field,
'name',
'slug',
'short_description',
'search_rank',
'source_tag_names',
'source_tag_ids',
'topic_tag_names',
'topic_tag_ids',
'purpose',
'published',
'published_at',
).annotate(has_access=BoolOr('_has_access'))
return datasets.values(
id_field,
'name',
'slug',
'short_description',
'search_rank',
'source_tag_names',
'source_tag_ids',
'topic_tag_names',
'topic_tag_ids',
'purpose',
'published',
'published_at',
'has_access',
)
def get_visualisations_data_for_user_matching_query(visualisations: QuerySet,
query,
user=None):
"""
Filters the visualisation queryset for:
1) visibility (whether the user can know if the visualisation exists)
2) matches the search terms
Annotates the visualisation queryset with:
1) `has_access`, if the user can use the visualisation.
"""
# Filter out visualisations that the user is not allowed to even know about.
if not (user and user.has_perm(
dataset_type_to_manage_unpublished_permission_codename(
DataSetType.VISUALISATION.value))):
visualisations = visualisations.filter(published=True)
# Filter out visualisations that don't match the search terms
search = SearchVector('name', weight='A', config='english') + SearchVector(
'short_description', weight='B', config='english')
search_query = SearchQuery(query, config='english')
visualisations = visualisations.annotate(search=search,
search_rank=SearchRank(
search, search_query))
if query:
visualisations = visualisations.filter(search=search_query)
# Mark up whether the user can access the visualisation.
if user:
access_filter = (Q(user_access_type='REQUIRES_AUTHENTICATION')
&
(Q(visualisationuserpermission__user=user)
| Q(visualisationuserpermission__isnull=True))) | Q(
user_access_type='REQUIRES_AUTHORIZATION',
visualisationuserpermission__user=user,
)
else:
access_filter = Q()
visualisations = visualisations.annotate(_has_access=Case(
When(access_filter, then=True),
default=False,
output_field=BooleanField(),
) if access_filter else Value(True, BooleanField()), )
# Pull in the source tag IDs for the dataset
visualisations = visualisations.annotate(source_tag_ids=ArrayAgg(
'tags', filter=Q(tags__type=TagType.SOURCE.value), distinct=True))
visualisations = visualisations.annotate(source_tag_names=ArrayAgg(
'tags__name', filter=Q(
tags__type=TagType.SOURCE.value), distinct=True))
# Pull in the topic tag IDs for the dataset
visualisations = visualisations.annotate(topic_tag_ids=ArrayAgg(
'tags', filter=Q(tags__type=TagType.TOPIC.value), distinct=True))
visualisations = visualisations.annotate(topic_tag_names=ArrayAgg(
'tags__name', filter=Q(tags__type=TagType.TOPIC.value), distinct=True))
# Define a `purpose` column denoting the dataset type
visualisations = visualisations.annotate(
purpose=Value(DataSetType.VISUALISATION.value, IntegerField()))
# We are joining on the user permissions table to determine `_has_access`` to the visualisation, so we need to
# group them and remove duplicates. We aggregate all the `_has_access` fields together and return true if any
# of the records say that access is available.
visualisations = visualisations.values(
'id',
'name',
'slug',
'short_description',
'search_rank',
'source_tag_names',
'source_tag_ids',
'topic_tag_names',
'topic_tag_ids',
'purpose',
'published',
'published_at',
).annotate(has_access=BoolOr('_has_access'))
return visualisations.values(
'id',
'name',
'slug',
'short_description',
'search_rank',
'source_tag_names',
'source_tag_ids',
'topic_tag_names',
'topic_tag_ids',
'purpose',
'published',
'published_at',
'has_access',
)
def filter_datasets(
datasets: Union[ReferenceDataset, DataSet],
query,
access,
source,
use=None,
user=None,
):
search = (SearchVector('name', weight='A', config='english') +
SearchVector('short_description', weight='B', config='english') +
SearchVector('source_tags__name', weight='B', config='english'))
search_query = SearchQuery(query, config='english')
dataset_filter = Q(published=True)
if user:
if datasets.model is ReferenceDataset:
reference_type = DataSetType.REFERENCE.value
reference_perm = dataset_type_to_manage_unpublished_permission_codename(
reference_type)
if user.has_perm(reference_perm):
dataset_filter |= Q(published=False)
if datasets.model is DataSet:
master_type, datacut_type = (
DataSetType.MASTER.value,
DataSetType.DATACUT.value,
)
master_perm = dataset_type_to_manage_unpublished_permission_codename(
master_type)
datacut_perm = dataset_type_to_manage_unpublished_permission_codename(
datacut_type)
if user.has_perm(master_perm) and (not use
or str(master_type) in use):
dataset_filter |= Q(published=False, type=master_type)
if user.has_perm(datacut_perm) and (not use
or str(datacut_type) in use):
dataset_filter |= Q(published=False, type=datacut_type)
datasets = datasets.filter(dataset_filter).annotate(search=search,
search_rank=SearchRank(
search,
search_query))
if user and access and datasets.model is not ReferenceDataset:
access_filter = (Q(user_access_type='REQUIRES_AUTHENTICATION')
& (Q(datasetuserpermission__user=user)
| Q(datasetuserpermission__isnull=True))) | Q(
user_access_type='REQUIRES_AUTHORIZATION',
datasetuserpermission__user=user)
datasets = datasets.filter(access_filter)
if query:
datasets = datasets.filter(search=search_query)
if source:
datasets = datasets.filter(source_tags__in=source)
if use:
datasets = datasets.filter(type__in=use)
return datasets
