def filter_queryset(self, request, queryset, view): # We want to defer this import until runtime, rather than import-time. # See https://github.com/encode/django-rest-framework/issues/4608 # (Also see #1624 for why we need to make this import explicitly) from guardian.shortcuts import get_objects_for_user from geonode.security.utils import get_visible_resources user = request.user # perm_format = '%(app_label)s.view_%(model_name)s' # permission = self.perm_format % { # 'app_label': queryset.model._meta.app_label, # 'model_name': queryset.model._meta.model_name, # } resources = get_objects_for_user( user, 'base.view_resourcebase', **self.shortcut_kwargs).filter(polymorphic_ctype__model='dataset') obj_with_perms = get_visible_resources( resources, user, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) return queryset.filter(id__in=obj_with_perms.values('id'))
def filter_queryset(self, request, queryset, view): # We want to defer this import until runtime, rather than import-time. # See https://github.com/encode/django-rest-framework/issues/4608 # (Also see #1624 for why we need to make this import explicitly) from guardian.shortcuts import get_objects_for_user from geonode.security.utils import get_visible_resources user = request.user resources = get_objects_for_user( user, 'base.view_resourcebase', **self.shortcut_kwargs ) _allowed_ids = get_visible_resources( resources, user, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES ).values_list('id', flat=True) obj_with_perms = [_app.id for _app in GeoApp.objects.filter(id__in=_allowed_ids)] return queryset.filter(id__in=obj_with_perms)
def filter_group(self, queryset, request): filter_set = get_visible_resources( queryset, request.user if request else None, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) return filter_set
def _get_resource_counts(request, resourcebase_filter_kwargs): """Return a dict with counts of resources of various types The ``resourcebase_filter_kwargs`` argument should be a dict with a suitable queryset filter that can be applied to select only the relevant ``ResourceBase`` objects to use when retrieving counts. For example:: _get_resource_counts( request, { 'group__slug': 'my-group', } ) The above function call would result in only counting ``ResourceBase`` objects that belong to the group that has ``my-group`` as slug """ resources = get_visible_resources( ResourceBase.objects.filter(**resourcebase_filter_kwargs), request.user, request=request, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) values = resources.values( 'polymorphic_ctype__model', 'is_approved', 'is_published', ) qs = values.annotate(counts=Count('polymorphic_ctype__model')) types = [ 'layer', 'document', 'map', 'all' ] counts = {} for type_ in types: counts[type_] = { 'total': 0, 'visible': 0, 'published': 0, 'approved': 0, } for record in qs: resource_type = record['polymorphic_ctype__model'] is_visible = all((record['is_approved'], record['is_published'])) counts['all']['total'] += record['counts'] counts['all']['visible'] += record['counts'] if is_visible else 0 counts['all']['published'] += record['counts'] if record['is_published'] else 0 counts['all']['approved'] += record['counts'] if record['is_approved'] else 0 section = counts.get(resource_type) if section is not None: section['total'] += record['counts'] section['visible'] += record['counts'] if is_visible else 0 section['published'] += record['counts'] if record['is_published'] else 0 section['approved'] += record['counts'] if record['is_approved'] else 0 return counts
def get_resources_counts(self, options): if settings.SKIP_PERMS_FILTER: resources = ResourceBase.objects.all() else: resources = get_objects_for_user(options['user'], 'base.view_resourcebase') resources = get_visible_resources( resources, options['user'], admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) if resources and resources.count() > 0: if options['title_filter']: resources = resources.filter( title__icontains=options['title_filter']) if options['type_filter']: _type_filter = options['type_filter'] if not isinstance(_type_filter, str): _type_filter = _type_filter.__name__.lower() resources = resources.filter( polymorphic_ctype__model=_type_filter) counts = list( resources.values(options['count_type']).annotate( count=Count(options['count_type']))) return dict([(c[options['count_type']], c['count']) for c in counts])
def get_resources_counts(self, options): if settings.SKIP_PERMS_FILTER: resources = ResourceBase.objects.all() else: resources = get_objects_for_user( options['user'], 'base.view_resourcebase' ) resources = get_visible_resources( resources, options['user'], admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) if resources and resources.count() > 0: if options['title_filter']: resources = resources.filter(title__icontains=options['title_filter']) if options['type_filter']: resources = resources.instance_of(options['type_filter']) counts = list(resources.values(options['count_type']).annotate(count=Count(options['count_type']))) return dict([(c[options['count_type']], c['count']) for c in counts])
def apply_filters(self, request, applicable_filters): types = applicable_filters.pop('type', None) extent = applicable_filters.pop('extent', None) keywords = applicable_filters.pop('keywords__slug__in', None) metadata_only = applicable_filters.pop('metadata_only', False) filtering_method = applicable_filters.pop('f_method', 'and') if filtering_method == 'or': filters = Q() for f in applicable_filters.items(): filters |= Q(f) semi_filtered = self.get_object_list(request).filter(filters) else: semi_filtered = super().apply_filters( request, applicable_filters) filtered = None if types: for the_type in types: if the_type in LAYER_TYPES: super_type = the_type if 'vector_time' == the_type: super_type = 'vector' if filtered: if 'time' in the_type: filtered = filtered | semi_filtered.filter( Layer___subtype=super_type).exclude(Layer___has_time=False) else: filtered = filtered | semi_filtered.filter( Layer___subtype=super_type) else: if 'time' in the_type: filtered = semi_filtered.filter( Layer___subtype=super_type).exclude(Layer___has_time=False) else: filtered = semi_filtered.filter( Layer___subtype=super_type) else: _type_filter = FILTER_TYPES[the_type].__name__.lower() if filtered: filtered = filtered | semi_filtered.filter(polymorphic_ctype__model=_type_filter) else: filtered = semi_filtered.filter(polymorphic_ctype__model=_type_filter) else: filtered = semi_filtered if extent: filtered = filter_bbox(filtered, extent) if keywords: filtered = self.filter_h_keywords(filtered, keywords) # return filtered return get_visible_resources( filtered, request.user if request else None, metadata_only=metadata_only, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES)
def filter_published(self, queryset, request): filter_set = get_visible_resources( queryset, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING) return filter_set
def get_resources_counts(self, options): if settings.SKIP_PERMS_FILTER: resources = ResourceBase.objects.all() else: resources = get_objects_for_user(options['user'], 'base.view_resourcebase') resources = get_visible_resources( resources, options['user'], admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) subtypes = [] if resources and resources.count() > 0: if options['title_filter']: resources = resources.filter( title__icontains=options['title_filter']) if options['type_filter']: _type_filter = options['type_filter'] for label, app in apps.app_configs.items(): if hasattr(app, 'type') and app.type == 'GEONODE_APP': if hasattr(app, 'default_model'): _model = apps.get_model(label, app.default_model) if issubclass(_model, _type_filter): subtypes.append( resources.filter( polymorphic_ctype__model=_model. __name__.lower())) if not isinstance(_type_filter, str): _type_filter = _type_filter.__name__.lower() resources = resources.filter( polymorphic_ctype__model=_type_filter) counts = list() if subtypes: for subtype in subtypes: counts.extend( list( subtype.values(options['count_type']).annotate( count=Count(options['count_type'])))) else: counts = list( resources.values(options['count_type']).annotate( count=Count(options['count_type']))) _counts = {} for c in counts: if c and c['count'] and options['count_type']: if not _counts.get(c[options['count_type']], None): _counts.update({c[options['count_type']]: c['count']}) else: _counts[c[options['count_type']]] += c['count'] return _counts
def resources(self, request, pk=None): user = self.get_object() permitted = get_objects_for_user(user, 'base.view_resourcebase') qs = ResourceBase.objects.all().filter(id__in=permitted).order_by('title') resources = get_visible_resources( qs, user, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) return Response(ResourceBaseSerializer(embed=True, many=True).to_representation(resources))
def choices_for_request(self): request = self.request self.choices = self.choices.filter(level__in=[2, 3]) self.choices = get_visible_resources( self.choices, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) return super(CampUnionBaseAutocomplete, self).choices_for_request()
def choices_for_request(self): request = self.request permitted = get_objects_for_user(request.user, 'base.view_resourcebase') self.choices = self.choices.filter(id__in=permitted) self.choices = get_visible_resources( self.choices, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) return super(ResourceBaseAutocomplete, self).choices_for_request()
def get_queryset(self): request = self.request permitted = get_objects_for_user(request.user, 'base.view_resourcebase') qs = Document.objects.all().filter(id__in=permitted) if self.q: qs = qs.filter(title__icontains=self.q) return get_visible_resources( qs, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES)
def choices_for_request(self): request = self.request permitted = get_objects_for_user( request.user, 'base.view_resourcebase') self.choices = self.choices.filter(id__in=permitted) self.choices = get_visible_resources( self.choices, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) return super(LayerAutocomplete, self).choices_for_request()
def dehydrate_layers_count(self, bundle): request = bundle.request obj_with_perms = get_objects_for_user(request.user, 'base.view_resourcebase').instance_of(Layer) filter_set = bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')) if not settings.SKIP_PERMS_FILTER: filter_set = get_visible_resources( filter_set, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) return filter_set.distinct().count()
def dehydrate_layers_count(self, bundle): request = bundle.request obj_with_perms = get_objects_for_user(request.user, 'base.view_resourcebase').filter(polymorphic_ctype__model='layer') filter_set = bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')) if not settings.SKIP_PERMS_FILTER: filter_set = get_visible_resources( filter_set, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) return filter_set.distinct().count()
def get_dataset_count_by_services(service_id, user): try: service = Service.objects.get(id=service_id) harvested_resources_ids = [] if service.harvester: _h = service.harvester harvested_resources_ids = list( _h.harvestable_resources.filter( should_be_harvested=True, geonode_resource__isnull=False).values_list( "geonode_resource__id", flat=True)) return get_visible_resources(queryset=ResourceBase.objects.filter( id__in=harvested_resources_ids), user=user).count() except Exception: return 0
def facets(context): request = context['request'] title_filter = request.GET.get('title__icontains', '') extent_filter = request.GET.get('extent', None) keywords_filter = request.GET.getlist('keywords__slug__in', None) category_filter = request.GET.getlist('category__identifier__in', None) regions_filter = request.GET.getlist('regions__name__in', None) owner_filter = request.GET.getlist('owner__username__in', None) date_gte_filter = request.GET.get('date__gte', None) date_lte_filter = request.GET.get('date__lte', None) date_range_filter = request.GET.get('date__range', None) facet_type = context['facet_type'] if 'facet_type' in context else 'all' if not settings.SKIP_PERMS_FILTER: authorized = [] try: authorized = get_objects_for_user( request.user, 'base.view_resourcebase').values('id') except Exception: pass if facet_type == 'documents': documents = Document.objects.filter(title__icontains=title_filter) if category_filter: documents = documents.filter(category__identifier__in=category_filter) if regions_filter: documents = documents.filter(regions__name__in=regions_filter) if owner_filter: documents = documents.filter(owner__username__in=owner_filter) if date_gte_filter: documents = documents.filter(date__gte=date_gte_filter) if date_lte_filter: documents = documents.filter(date__lte=date_lte_filter) if date_range_filter: documents = documents.filter(date__range=date_range_filter.split(',')) documents = get_visible_resources( documents, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) if keywords_filter: treeqs = HierarchicalKeyword.objects.none() for keyword in keywords_filter: try: kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) for kw in kws: treeqs = treeqs | HierarchicalKeyword.get_tree(kw) except Exception: # Ignore keywords not actually used? pass documents = documents.filter(Q(keywords__in=treeqs)) if not settings.SKIP_PERMS_FILTER: documents = documents.filter(id__in=authorized) counts = documents.values('doc_type').annotate(count=Count('doc_type')) facets = dict([(count['doc_type'], count['count']) for count in counts]) return facets else: layers = Layer.objects.filter(title__icontains=title_filter) if category_filter: layers = layers.filter(category__identifier__in=category_filter) if regions_filter: layers = layers.filter(regions__name__in=regions_filter) if owner_filter: layers = layers.filter(owner__username__in=owner_filter) if date_gte_filter: layers = layers.filter(date__gte=date_gte_filter) if date_lte_filter: layers = layers.filter(date__lte=date_lte_filter) if date_range_filter: layers = layers.filter(date__range=date_range_filter.split(',')) layers = get_visible_resources( layers, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) if extent_filter: from geonode.utils import bbox_to_projection bbox = extent_filter.split(',') bbox = list(map(str, bbox)) intersects = (Q(bbox_x0__gt=bbox[0]) & Q(bbox_x1__lt=bbox[2]) & Q(bbox_y0__gt=bbox[1]) & Q(bbox_y1__lt=bbox[3])) for proj in Layer.objects.order_by('srid').values('srid').distinct(): if proj['srid'] != 'EPSG:4326': proj_bbox = bbox_to_projection(bbox + ['4326', ], target_srid=int(proj['srid'][5:])) if proj_bbox[-1] != 4326: intersects = intersects | (Q(bbox_x0__gt=proj_bbox[0]) & Q(bbox_x1__lt=proj_bbox[2]) & Q( bbox_y0__gt=proj_bbox[1]) & Q(bbox_y1__lt=proj_bbox[3])) layers = layers.filter(intersects) if keywords_filter: treeqs = HierarchicalKeyword.objects.none() for keyword in keywords_filter: try: kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) for kw in kws: treeqs = treeqs | HierarchicalKeyword.get_tree(kw) except Exception: # Ignore keywords not actually used? pass layers = layers.filter(Q(keywords__in=treeqs)) if not settings.SKIP_PERMS_FILTER: layers = layers.filter(id__in=authorized) counts = layers.values('storeType').annotate(count=Count('storeType')) counts_array = [] try: for count in counts: counts_array.append((count['storeType'], count['count'])) except Exception: pass count_dict = dict(counts_array) vector_time_series = layers.exclude(has_time=False).filter(storeType='dataStore'). \ values('storeType').annotate(count=Count('storeType')) if vector_time_series: count_dict['vectorTimeSeries'] = vector_time_series[0]['count'] facets = { 'raster': count_dict.get('coverageStore', 0), 'vector': count_dict.get('dataStore', 0), 'vector_time': count_dict.get('vectorTimeSeries', 0), 'remote': count_dict.get('remoteStore', 0), 'wms': count_dict.get('wmsStore', 0), } # Break early if only_layers is set. if facet_type == 'layers': return facets maps = Map.objects.filter(title__icontains=title_filter) documents = Document.objects.filter(title__icontains=title_filter) if category_filter: maps = maps.filter(category__identifier__in=category_filter) documents = documents.filter(category__identifier__in=category_filter) if regions_filter: maps = maps.filter(regions__name__in=regions_filter) documents = documents.filter(regions__name__in=regions_filter) if owner_filter: maps = maps.filter(owner__username__in=owner_filter) documents = documents.filter(owner__username__in=owner_filter) if date_gte_filter: maps = maps.filter(date__gte=date_gte_filter) documents = documents.filter(date__gte=date_gte_filter) if date_lte_filter: maps = maps.filter(date__lte=date_lte_filter) documents = documents.filter(date__lte=date_lte_filter) if date_range_filter: maps = maps.filter(date__range=date_range_filter.split(',')) documents = documents.filter(date__range=date_range_filter.split(',')) maps = get_visible_resources( maps, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) documents = get_visible_resources( documents, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) if extent_filter: bbox = extent_filter.split( ',') # TODO: Why is this different when done through haystack? bbox = map(str, bbox) # 2.6 compat - float to decimal conversion intersects = ~(Q(bbox_x0__gt=bbox[2]) | Q(bbox_x1__lt=bbox[0]) | Q(bbox_y0__gt=bbox[3]) | Q(bbox_y1__lt=bbox[1])) maps = maps.filter(intersects) documents = documents.filter(intersects) if keywords_filter: treeqs = HierarchicalKeyword.objects.none() for keyword in keywords_filter: try: kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) for kw in kws: treeqs = treeqs | HierarchicalKeyword.get_tree(kw) except Exception: # Ignore keywords not actually used? pass maps = maps.filter(Q(keywords__in=treeqs)) documents = documents.filter(Q(keywords__in=treeqs)) if not settings.SKIP_PERMS_FILTER: maps = maps.filter(id__in=authorized) documents = documents.filter(id__in=authorized) facets['map'] = maps.count() facets['document'] = documents.count() if facet_type == 'home': facets['user'] = get_user_model().objects.exclude( username='******').count() facets['group'] = GroupProfile.objects.exclude( access="private").count() facets['layer'] = facets['raster'] + facets['vector'] + facets['remote'] + facets['wms'] return facets
def get_search(self, request, **kwargs): self.method_check(request, allowed=['get']) self.is_authenticated(request) self.throttle_check(request) # Get the list of objects that matches the filter sqs = self.build_haystack_filters(request.GET) if not settings.SKIP_PERMS_FILTER: filter_set = get_objects_for_user(request.user, 'base.view_resourcebase') filter_set = get_visible_resources( filter_set, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) filter_set_ids = filter_set.values_list('id') # Do the query using the filterset and the query term. Facet the # results if len(filter_set) > 0: sqs = sqs.filter(id__in=filter_set_ids).facet('type').facet( 'subtype').facet('owner').facet('keywords').facet( 'regions').facet('category') else: sqs = None else: sqs = sqs.facet('type').facet('subtype').facet('owner').facet( 'keywords').facet('regions').facet('category') if sqs: # Build the Facet dict facets = {} for facet in sqs.facet_counts()['fields']: facets[facet] = {} for item in sqs.facet_counts()['fields'][facet]: facets[facet][item[0]] = item[1] # Paginate the results paginator = Paginator(sqs, request.GET.get('limit')) try: page = paginator.page( int(request.GET.get('offset') or 0) / int(request.GET.get('limit') or 0 + 1)) except InvalidPage: raise Http404("Sorry, no results on that page.") if page.has_previous(): previous_page = page.previous_page_number() else: previous_page = 1 if page.has_next(): next_page = page.next_page_number() else: next_page = 1 total_count = sqs.count() objects = page.object_list else: next_page = 0 previous_page = 0 total_count = 0 facets = {} objects = [] object_list = { "meta": { "limit": settings.CLIENT_RESULTS_LIMIT, "next": next_page, "offset": int(getattr(request.GET, 'offset', 0)), "previous": previous_page, "total_count": total_count, "facets": facets, }, "objects": [self.get_haystack_api_fields(x) for x in objects], } self.log_throttled_access(request) return self.create_response(request, object_list)
def get_dataset_count_by_services(service_id, user): return get_visible_resources( queryset=Dataset.objects.filter(remote_service=service_id), user=user).count()
def get_search(self, request, **kwargs): self.method_check(request, allowed=['get']) self.is_authenticated(request) self.throttle_check(request) # Get the list of objects that matches the filter sqs = self.build_haystack_filters(request.GET) if not settings.SKIP_PERMS_FILTER: filter_set = get_objects_for_user( request.user, 'base.view_resourcebase') filter_set = get_visible_resources( filter_set, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) filter_set_ids = filter_set.values_list('id') # Do the query using the filterset and the query term. Facet the # results if len(filter_set) > 0: sqs = sqs.filter(id__in=filter_set_ids).facet('type').facet('subtype').facet( 'owner') .facet('keywords').facet('regions').facet('category') else: sqs = None else: sqs = sqs.facet('type').facet('subtype').facet( 'owner').facet('keywords').facet('regions').facet('category') if sqs: # Build the Facet dict facets = {} for facet in sqs.facet_counts()['fields']: facets[facet] = {} for item in sqs.facet_counts()['fields'][facet]: facets[facet][item[0]] = item[1] # Paginate the results paginator = Paginator(sqs, request.GET.get('limit')) try: page = paginator.page( int(request.GET.get('offset') or 0) / int(request.GET.get('limit'), 0) + 1) except InvalidPage: raise Http404("Sorry, no results on that page.") if page.has_previous(): previous_page = page.previous_page_number() else: previous_page = 1 if page.has_next(): next_page = page.next_page_number() else: next_page = 1 total_count = sqs.count() objects = page.object_list else: next_page = 0 previous_page = 0 total_count = 0 facets = {} objects = [] object_list = { "meta": { "limit": settings.CLIENT_RESULTS_LIMIT, "next": next_page, "offset": int(getattr(request.GET, 'offset', 0)), "previous": previous_page, "total_count": total_count, "facets": facets, }, "objects": map(lambda x: self.get_haystack_api_fields(x), objects), } self.log_throttled_access(request) return self.create_response(request, object_list)
def dataresources(context): request = context['request'] title_filter = request.GET.get('title__icontains', '') extent_filter = request.GET.get('extent', None) keywords_filter = request.GET.getlist('keywords__slug__in', None) category_filter = request.GET.getlist('category__identifier__in', None) regions_filter = request.GET.getlist('regions__name__in', None) owner_filter = request.GET.getlist('owner__username__in', None) date_gte_filter = request.GET.get('date__gte', None) date_lte_filter = request.GET.get('date__lte', None) date_range_filter = request.GET.get('date__range', None) facet_type = context['facet_type'] if 'facet_type' in context else 'all' if facet_type == 'documents': documents = context.get('basemodel', Document).objects.filter(title__icontains=title_filter) if category_filter: documents = documents.filter(category__identifier__in=category_filter) if regions_filter: documents = documents.filter(regions__name__in=regions_filter) if owner_filter: documents = documents.filter(owner__username__in=owner_filter) if date_gte_filter: documents = documents.filter(date__gte=date_gte_filter) if date_lte_filter: documents = documents.filter(date__lte=date_lte_filter) if date_range_filter: documents = documents.filter(date__range=date_range_filter.split(',')) if 'datasource__in' in request.GET: documents = documents.filter(datasource__in=request.GET.get('datasource__in')) documents = get_visible_resources( documents, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) if keywords_filter: treeqs = HierarchicalKeyword.objects.none() for keyword in keywords_filter: try: kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) for kw in kws: treeqs = treeqs | HierarchicalKeyword.get_tree(kw) except BaseException: # Ignore keywords not actually used? pass documents = documents.filter(Q(keywords__in=treeqs)) # if not settings.SKIP_PERMS_FILTER: # documents = documents.filter(id__in=authorized) # return facets counts = documents.values('datasource').annotate(datasource_count=Count('datasource')) facets = dict([(count['datasource'], {'title':count['datasource'],'count':count['datasource_count'], 'facet_type':'documents'}) for count in counts if count['datasource'] ]) return facets else: layers = Layer.objects.filter(title__icontains=title_filter) if category_filter: layers = layers.filter(category__identifier__in=category_filter) if regions_filter: layers = layers.filter(regions__name__in=regions_filter) if owner_filter: layers = layers.filter(owner__username__in=owner_filter) if date_gte_filter: layers = layers.filter(date__gte=date_gte_filter) if date_lte_filter: layers = layers.filter(date__lte=date_lte_filter) if date_range_filter: layers = layers.filter(date__range=date_range_filter.split(',')) if 'store__in' in request.GET: layers = layers.filter(store__in=request.GET.get('store__in')) layers = get_visible_resources( layers, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) if extent_filter: bbox = extent_filter.split( ',') # TODO: Why is this different when done through haystack? bbox = map(str, bbox) # 2.6 compat - float to decimal conversion intersects = ~(Q(bbox_x0__gt=bbox[2]) | Q(bbox_x1__lt=bbox[0]) | Q(bbox_y0__gt=bbox[3]) | Q(bbox_y1__lt=bbox[1])) layers = layers.filter(intersects) if keywords_filter: treeqs = HierarchicalKeyword.objects.none() for keyword in keywords_filter: try: kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) for kw in kws: treeqs = treeqs | HierarchicalKeyword.get_tree(kw) except BaseException: # Ignore keywords not actually used? pass layers = layers.filter(Q(keywords__in=treeqs)) # if not settings.SKIP_PERMS_FILTER: # layers = layers.filter(id__in=authorized) counts = layers.values('storeType').annotate(count=Count('storeType')) facets = {} for count in counts: if count['storeType'] == 'remoteStore': count_service = layers.values('remote_service').filter(storeType__exact='remoteStore').annotate(store_count=Count('store')) for cs in count_service: remoteId = cs['remote_service'] # print(remoteId) getremoteurl = Service.objects.values('base_url','name','title').filter(id=remoteId) remoteurlcount = dict([(gm['name'], {'title':gm['title'], 'count':cs['store_count'], 'facet_type':'layers'}) for gm in getremoteurl]) facets.update(remoteurlcount) if facet_type == 'layers': return facets documents = Document.objects.filter(title__icontains=title_filter) if category_filter: documents = documents.filter(category__identifier__in=category_filter) if regions_filter: documents = documents.filter(regions__name__in=regions_filter) if owner_filter: documents = documents.filter(owner__username__in=owner_filter) if date_gte_filter: documents = documents.filter(date__gte=date_gte_filter) if date_lte_filter: documents = documents.filter(date__lte=date_lte_filter) if date_range_filter: documents = documents.filter(date__range=date_range_filter.split(',')) if 'datasource__in' in request.GET: documents = documents.filter(datasource__in=request.GET.get('datasource__in')) documents = get_visible_resources( documents, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) if extent_filter: bbox = extent_filter.split( ',') # TODO: Why is this different when done through haystack? bbox = map(str, bbox) # 2.6 compat - float to decimal conversion intersects = ~(Q(bbox_x0__gt=bbox[2]) | Q(bbox_x1__lt=bbox[0]) | Q(bbox_y0__gt=bbox[3]) | Q(bbox_y1__lt=bbox[1])) documents = documents.filter(intersects) if keywords_filter: treeqs = HierarchicalKeyword.objects.none() for keyword in keywords_filter: try: kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) for kw in kws: treeqs = treeqs | HierarchicalKeyword.get_tree(kw) except BaseException: # Ignore keywords not actually used? pass documents = documents.filter(Q(keywords__in=treeqs)) # if not settings.SKIP_PERMS_FILTER: # maps = maps.filter(id__in=authorized) # documents = documents.filter(id__in=authorized) documentcounts = documents.values('datasource').annotate(datasource_count=Count('datasource')) facets.update(dict([(count['datasource'], {'title':count['datasource'],'count':count['datasource_count'], 'facet_type':'documents'}) for count in documentcounts if count['datasource'] ])) return facets
def service_detail(request, service_id): """This view shows the details of a service""" services = Service.objects.filter(resourcebase_ptr_id=service_id) if not services.exists(): messages.add_message( request, messages.ERROR, _("You dont have enougth rigths to see the resource detail")) return redirect(reverse("services")) service = services.first() permissions_json = _perms_info_json(service) perms_list = list(service.get_self_resource().get_user_perms( request.user).union(service.get_user_perms(request.user))) already_imported_datasets = get_visible_resources( queryset=Dataset.objects.filter(remote_service=service), user=request.user) resources_being_harvested = HarvestJob.objects.filter(service=service) service_list = service.service_set.all() all_resources = (list(resources_being_harvested) + list(already_imported_datasets) + list(service_list)) paginator = Paginator(all_resources, getattr(settings, "CLIENT_RESULTS_LIMIT", 25), orphans=3) page = request.GET.get("page") try: resources = paginator.page(page) except PageNotAnInteger: resources = paginator.page(1) except EmptyPage: resources = paginator.page(paginator.num_pages) # pop the handler out of the session in order to free resources # - we had stored the service handler on the session in order to # speed up the register/harvest resources flow. However, for services # with many resources, keeping the handler in the session leads to degraded # performance try: request.session.pop(service.base_url) except KeyError: pass return render(request, template_name="services/service_detail.html", context={ "service": service, "layers": already_imported_datasets, "resource_jobs": (r for r in resources if isinstance(r, HarvestJob)), "permissions_json": permissions_json, "permissions_list": perms_list, "can_add_resorces": request.user.has_perm('base.add_resourcebase'), "resources": resources, "total_resources": len(already_imported_datasets), })
def facets(context): request = context['request'] title_filter = request.GET.get('title__icontains', '') extent_filter = request.GET.get('extent', None) keywords_filter = request.GET.getlist('keywords__slug__in', None) category_filter = request.GET.getlist('category__identifier__in', None) regions_filter = request.GET.getlist('regions__name__in', None) owner_filter = request.GET.getlist('owner__username__in', None) date_gte_filter = request.GET.get('date__gte', None) date_lte_filter = request.GET.get('date__lte', None) date_range_filter = request.GET.get('date__range', None) facet_type = context.get('facet_type', 'all') if not settings.SKIP_PERMS_FILTER: authorized = [] try: authorized = get_objects_for_user( request.user, 'base.view_resourcebase').values('id') except Exception: pass if facet_type == 'geoapps': facets = {} from django.apps import apps for label, app in apps.app_configs.items(): if hasattr(app, 'type') and app.type == 'GEONODE_APP': if hasattr(app, 'default_model'): geoapps = get_visible_resources( apps.get_model(label, app.default_model).objects.all(), request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) if category_filter: geoapps = geoapps.filter(category__identifier__in=category_filter) if regions_filter: geoapps = geoapps.filter(regions__name__in=regions_filter) if owner_filter: geoapps = geoapps.filter(owner__username__in=owner_filter) if date_gte_filter: geoapps = geoapps.filter(date__gte=date_gte_filter) if date_lte_filter: geoapps = geoapps.filter(date__lte=date_lte_filter) if date_range_filter: geoapps = geoapps.filter(date__range=date_range_filter.split(',')) if extent_filter: geoapps = filter_bbox(geoapps, extent_filter) if keywords_filter: treeqs = HierarchicalKeyword.objects.none() for keyword in keywords_filter: try: kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) for kw in kws: treeqs = treeqs | HierarchicalKeyword.get_tree(kw) except Exception: # Ignore keywords not actually used? pass geoapps = geoapps.filter(Q(keywords__in=treeqs)) if not settings.SKIP_PERMS_FILTER: geoapps = geoapps.filter(id__in=authorized) facets[app.default_model] = geoapps.count() return facets elif facet_type == 'documents': documents = Document.objects.filter(title__icontains=title_filter) if category_filter: documents = documents.filter(category__identifier__in=category_filter) if regions_filter: documents = documents.filter(regions__name__in=regions_filter) if owner_filter: documents = documents.filter(owner__username__in=owner_filter) if date_gte_filter: documents = documents.filter(date__gte=date_gte_filter) if date_lte_filter: documents = documents.filter(date__lte=date_lte_filter) if date_range_filter: documents = documents.filter(date__range=date_range_filter.split(',')) documents = get_visible_resources( documents, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) if keywords_filter: treeqs = HierarchicalKeyword.objects.none() for keyword in keywords_filter: try: kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) for kw in kws: treeqs = treeqs | HierarchicalKeyword.get_tree(kw) except Exception: # Ignore keywords not actually used? pass documents = documents.filter(Q(keywords__in=treeqs)) if not settings.SKIP_PERMS_FILTER: documents = documents.filter(id__in=authorized) counts = documents.values('doc_type').annotate(count=Count('doc_type')) facets = {count['doc_type']: count['count'] for count in counts} return facets else: layers = Layer.objects.filter(title__icontains=title_filter) if category_filter: layers = layers.filter(category__identifier__in=category_filter) if regions_filter: layers = layers.filter(regions__name__in=regions_filter) if owner_filter: layers = layers.filter(owner__username__in=owner_filter) if date_gte_filter: layers = layers.filter(date__gte=date_gte_filter) if date_lte_filter: layers = layers.filter(date__lte=date_lte_filter) if date_range_filter: layers = layers.filter(date__range=date_range_filter.split(',')) layers = get_visible_resources( layers, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) if extent_filter: layers = filter_bbox(layers, extent_filter) if keywords_filter: treeqs = HierarchicalKeyword.objects.none() for keyword in keywords_filter: try: kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) for kw in kws: treeqs = treeqs | HierarchicalKeyword.get_tree(kw) except Exception: # Ignore keywords not actually used? pass layers = layers.filter(Q(keywords__in=treeqs)) if not settings.SKIP_PERMS_FILTER: layers = layers.filter(id__in=authorized) counts = layers.values('storeType').annotate(count=Count('storeType')) counts_array = [] try: for count in counts: counts_array.append((count['storeType'], count['count'])) except Exception: pass count_dict = dict(counts_array) vector_time_series = layers.exclude(has_time=False).filter(storeType='dataStore'). \ values('storeType').annotate(count=Count('storeType')) if vector_time_series: count_dict['vectorTimeSeries'] = vector_time_series[0]['count'] facets = { 'raster': count_dict.get('coverageStore', 0), 'vector': count_dict.get('dataStore', 0), 'vector_time': count_dict.get('vectorTimeSeries', 0), 'remote': count_dict.get('remoteStore', 0), 'wms': count_dict.get('wmsStore', 0), } # Break early if only_layers is set. if facet_type == 'layers': return facets maps = Map.objects.filter(title__icontains=title_filter) documents = Document.objects.filter(title__icontains=title_filter) if category_filter: maps = maps.filter(category__identifier__in=category_filter) documents = documents.filter(category__identifier__in=category_filter) if regions_filter: maps = maps.filter(regions__name__in=regions_filter) documents = documents.filter(regions__name__in=regions_filter) if owner_filter: maps = maps.filter(owner__username__in=owner_filter) documents = documents.filter(owner__username__in=owner_filter) if date_gte_filter: maps = maps.filter(date__gte=date_gte_filter) documents = documents.filter(date__gte=date_gte_filter) if date_lte_filter: maps = maps.filter(date__lte=date_lte_filter) documents = documents.filter(date__lte=date_lte_filter) if date_range_filter: maps = maps.filter(date__range=date_range_filter.split(',')) documents = documents.filter(date__range=date_range_filter.split(',')) maps = get_visible_resources( maps, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) documents = get_visible_resources( documents, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) if extent_filter: documents = filter_bbox(documents, extent_filter) if keywords_filter: treeqs = HierarchicalKeyword.objects.none() for keyword in keywords_filter: try: kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) for kw in kws: treeqs = treeqs | HierarchicalKeyword.get_tree(kw) except Exception: # Ignore keywords not actually used? pass maps = maps.filter(Q(keywords__in=treeqs)) documents = documents.filter(Q(keywords__in=treeqs)) if not settings.SKIP_PERMS_FILTER: maps = maps.filter(id__in=authorized) documents = documents.filter(id__in=authorized) facets['map'] = maps.count() facets['document'] = documents.count() if facet_type == 'home': facets['user'] = get_user_model().objects.exclude( username='******').count() facets['group'] = GroupProfile.objects.exclude( access="private").count() facets['layer'] = facets['raster'] + facets['vector'] + facets['remote'] + facets['wms'] return facets
def facets(context): request = context['request'] title_filter = request.GET.get('title__icontains', '') extent_filter = request.GET.get('extent', None) keywords_filter = request.GET.getlist('keywords__slug__in', None) category_filter = request.GET.getlist('category__identifier__in', None) regions_filter = request.GET.getlist('regions__name__in', None) owner_filter = request.GET.getlist('owner__username__in', None) date_gte_filter = request.GET.get('date__gte', None) date_lte_filter = request.GET.get('date__lte', None) date_range_filter = request.GET.get('date__range', None) facet_type = context['facet_type'] if 'facet_type' in context else 'all' if not settings.SKIP_PERMS_FILTER: authorized = [] try: authorized = get_objects_for_user( request.user, 'base.view_resourcebase').values('id') except BaseException: pass if facet_type == 'documents': documents = Document.objects.filter(title__icontains=title_filter) if category_filter: documents = documents.filter(category__identifier__in=category_filter) if regions_filter: documents = documents.filter(regions__name__in=regions_filter) if owner_filter: documents = documents.filter(owner__username__in=owner_filter) if date_gte_filter: documents = documents.filter(date__gte=date_gte_filter) if date_lte_filter: documents = documents.filter(date__lte=date_lte_filter) if date_range_filter: documents = documents.filter(date__range=date_range_filter.split(',')) documents = get_visible_resources( documents, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) if keywords_filter: treeqs = HierarchicalKeyword.objects.none() for keyword in keywords_filter: try: kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) for kw in kws: treeqs = treeqs | HierarchicalKeyword.get_tree(kw) except BaseException: # Ignore keywords not actually used? pass documents = documents.filter(Q(keywords__in=treeqs)) if not settings.SKIP_PERMS_FILTER: documents = documents.filter(id__in=authorized) counts = documents.values('doc_type').annotate(count=Count('doc_type')) facets = dict([(count['doc_type'], count['count']) for count in counts]) return facets else: layers = Layer.objects.filter(title__icontains=title_filter) if category_filter: layers = layers.filter(category__identifier__in=category_filter) if regions_filter: layers = layers.filter(regions__name__in=regions_filter) if owner_filter: layers = layers.filter(owner__username__in=owner_filter) if date_gte_filter: layers = layers.filter(date__gte=date_gte_filter) if date_lte_filter: layers = layers.filter(date__lte=date_lte_filter) if date_range_filter: layers = layers.filter(date__range=date_range_filter.split(',')) layers = get_visible_resources( layers, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) if extent_filter: bbox = extent_filter.split( ',') # TODO: Why is this different when done through haystack? bbox = map(str, bbox) # 2.6 compat - float to decimal conversion intersects = ~(Q(bbox_x0__gt=bbox[2]) | Q(bbox_x1__lt=bbox[0]) | Q(bbox_y0__gt=bbox[3]) | Q(bbox_y1__lt=bbox[1])) layers = layers.filter(intersects) if keywords_filter: treeqs = HierarchicalKeyword.objects.none() for keyword in keywords_filter: try: kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) for kw in kws: treeqs = treeqs | HierarchicalKeyword.get_tree(kw) except BaseException: # Ignore keywords not actually used? pass layers = layers.filter(Q(keywords__in=treeqs)) if not settings.SKIP_PERMS_FILTER: layers = layers.filter(id__in=authorized) counts = layers.values('storeType').annotate(count=Count('storeType')) counts_array = [] try: for count in counts: counts_array.append((count['storeType'], count['count'])) except BaseException: pass count_dict = dict(counts_array) vector_time_series = layers.exclude(has_time=False).filter(storeType='dataStore'). \ values('storeType').annotate(count=Count('storeType')) if vector_time_series: count_dict['vectorTimeSeries'] = vector_time_series[0]['count'] facets = { 'raster': count_dict.get('coverageStore', 0), 'vector': count_dict.get('dataStore', 0), 'vector_time': count_dict.get('vectorTimeSeries', 0), 'remote': count_dict.get('remoteStore', 0), 'wms': count_dict.get('wmsStore', 0), } # Break early if only_layers is set. if facet_type == 'layers': return facets maps = Map.objects.filter(title__icontains=title_filter) documents = Document.objects.filter(title__icontains=title_filter) if category_filter: maps = maps.filter(category__identifier__in=category_filter) documents = documents.filter(category__identifier__in=category_filter) if regions_filter: maps = maps.filter(regions__name__in=regions_filter) documents = documents.filter(regions__name__in=regions_filter) if owner_filter: maps = maps.filter(owner__username__in=owner_filter) documents = documents.filter(owner__username__in=owner_filter) if date_gte_filter: maps = maps.filter(date__gte=date_gte_filter) documents = documents.filter(date__gte=date_gte_filter) if date_lte_filter: maps = maps.filter(date__lte=date_lte_filter) documents = documents.filter(date__lte=date_lte_filter) if date_range_filter: maps = maps.filter(date__range=date_range_filter.split(',')) documents = documents.filter(date__range=date_range_filter.split(',')) maps = get_visible_resources( maps, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) documents = get_visible_resources( documents, request.user if request else None, admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, unpublished_not_visible=settings.RESOURCE_PUBLISHING, private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES) if extent_filter: bbox = extent_filter.split( ',') # TODO: Why is this different when done through haystack? bbox = map(str, bbox) # 2.6 compat - float to decimal conversion intersects = ~(Q(bbox_x0__gt=bbox[2]) | Q(bbox_x1__lt=bbox[0]) | Q(bbox_y0__gt=bbox[3]) | Q(bbox_y1__lt=bbox[1])) maps = maps.filter(intersects) documents = documents.filter(intersects) if keywords_filter: treeqs = HierarchicalKeyword.objects.none() for keyword in keywords_filter: try: kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) for kw in kws: treeqs = treeqs | HierarchicalKeyword.get_tree(kw) except BaseException: # Ignore keywords not actually used? pass maps = maps.filter(Q(keywords__in=treeqs)) documents = documents.filter(Q(keywords__in=treeqs)) if not settings.SKIP_PERMS_FILTER: maps = maps.filter(id__in=authorized) documents = documents.filter(id__in=authorized) facets['map'] = maps.count() facets['document'] = documents.count() if facet_type == 'home': facets['user'] = get_user_model().objects.exclude( username='******').count() facets['group'] = GroupProfile.objects.exclude( access="private").count() facets['layer'] = facets['raster'] + \ facets['vector'] + facets['remote'] + facets['wms'] # + facets['vector_time'] return facets