def get_resource_uri(self, bundle_or_obj):
"""
Build a canonical uri for a datum.
If the resource doesn't have an external_id it is
considered "unaddressable" and this will return None.
"""
dr = DatasetResource()
kwargs = {
'api_name': self._meta.api_name,
'dataset_resource_name': dr._meta.resource_name,
'resource_name': self._meta.resource_name,
}
if isinstance(bundle_or_obj, Bundle):
kwargs['dataset_slug'] = bundle_or_obj.obj.dataset_slug
kwargs['external_id'] = bundle_or_obj.obj.external_id
else:
kwargs['dataset_slug'] = bundle_or_obj.dataset_slug
kwargs['external_id'] = bundle_or_obj.external_id
if not kwargs['external_id']:
return None
return dr._build_reverse_url('api_dataset_data_detail', kwargs=kwargs)
def get_resource_uri(self, bundle_or_obj):
"""
Build a canonical uri for a datum.
If the resource doesn't have an external_id it is
considered "unaddressable" and this will return None.
"""
dr = DatasetResource()
kwargs = {
"api_name": self._meta.api_name,
"dataset_resource_name": dr._meta.resource_name,
"resource_name": self._meta.resource_name,
}
if isinstance(bundle_or_obj, Bundle):
kwargs["dataset_slug"] = bundle_or_obj.obj.dataset_slug
kwargs["external_id"] = bundle_or_obj.obj.external_id
else:
kwargs["dataset_slug"] = bundle_or_obj.dataset_slug
kwargs["external_id"] = bundle_or_obj.external_id
if not kwargs["external_id"]:
return None
return dr._build_reverse_url("api_dataset_data_detail", kwargs=kwargs)
def search_dataset_data(self, request, **kwargs):
"""
Perform a full-text search on only one dataset.
See ``get_list``.
"""
dataset = Dataset.objects.get(slug=kwargs['dataset_slug'])
query = request.GET.get('q', '')
since = request.GET.get('since', None)
limit = int(
request.GET.get('limit', settings.PANDA_DEFAULT_SEARCH_ROWS))
offset = int(request.GET.get('offset', 0))
sort = request.GET.get('sort', '_docid_ asc')
if query:
solr_query = 'dataset_slug:%s AND (%s)' % (dataset.slug, query)
else:
solr_query = 'dataset_slug:%s' % dataset.slug
if since:
solr_query += ' AND last_modified:[' + since + 'Z TO *]'
response = solr.query(settings.SOLR_DATA_CORE,
solr_query,
offset=offset,
sort=sort,
limit=limit)
dataset_resource = DatasetResource()
dataset_bundle = dataset_resource.build_bundle(obj=dataset,
request=request)
dataset_bundle = dataset_resource.full_dehydrate(dataset_bundle)
dataset_bundle = dataset_resource.simplify_bundle(dataset_bundle)
results = [SolrObject(d) for d in response['response']['docs']]
page = PandaPaginator(request.GET,
results,
resource_uri=request.path_info,
count=response['response']['numFound']).page()
dataset_bundle.data.update(page)
dataset_bundle.data['objects'] = []
for obj in results:
bundle = self.build_bundle(obj=obj, request=request)
bundle = self.full_dehydrate(bundle)
dataset_bundle.data['objects'].append(bundle.data)
# Because users may have authenticated via headers the request.user may
# not be a full User instance. To be sure, we fetch one.
user = UserProxy.objects.get(id=request.user.id)
SearchLog.objects.create(user=user, dataset=dataset, query=query)
return dataset_bundle
def search_dataset_data(self, request, **kwargs):
"""
Perform a full-text search on only one dataset.
See ``get_list``.
"""
dataset = Dataset.objects.get(slug=kwargs['dataset_slug'])
query = request.GET.get('q', '')
since = request.GET.get('since', None)
limit = int(request.GET.get('limit', settings.PANDA_DEFAULT_SEARCH_ROWS))
offset = int(request.GET.get('offset', 0))
if query:
solr_query = 'dataset_slug:%s AND (%s)' % (dataset.slug, query)
else:
solr_query = 'dataset_slug:%s' % dataset.slug
if since:
solr_query += ' AND last_modified:[' + since + 'Z TO *]'
response = solr.query(
settings.SOLR_DATA_CORE,
solr_query,
offset=offset,
limit=limit
)
dataset_resource = DatasetResource()
dataset_bundle = dataset_resource.build_bundle(obj=dataset, request=request)
dataset_bundle = dataset_resource.full_dehydrate(dataset_bundle)
dataset_bundle = dataset_resource.simplify_bundle(dataset_bundle)
results = [SolrObject(d) for d in response['response']['docs']]
page = PandaPaginator(
request.GET,
results,
resource_uri=request.path_info,
count=response['response']['numFound']
).page()
dataset_bundle.data.update(page)
dataset_bundle.data['objects'] = []
for obj in results:
bundle = self.build_bundle(obj=obj, request=request)
bundle = self.full_dehydrate(bundle)
dataset_bundle.data['objects'].append(bundle.data)
SearchLog.objects.create(user=request.user, dataset=dataset, query=query)
return dataset_bundle
def search_dataset_data(self, request, **kwargs):
"""
Perform a full-text search on only one dataset.
See ``get_list``.
"""
dataset = Dataset.objects.get(slug=kwargs["dataset_slug"])
query = request.GET.get("q", "")
since = request.GET.get("since", None)
limit = int(request.GET.get("limit", settings.PANDA_DEFAULT_SEARCH_ROWS))
offset = int(request.GET.get("offset", 0))
sort = request.GET.get("sort", "_docid_ asc")
if query:
solr_query = "dataset_slug:%s AND (%s)" % (dataset.slug, query)
else:
solr_query = "dataset_slug:%s" % dataset.slug
if since:
solr_query += " AND last_modified:[" + since + "Z TO *]"
response = solr.query(settings.SOLR_DATA_CORE, solr_query, offset=offset, sort=sort, limit=limit)
dataset_resource = DatasetResource()
dataset_bundle = dataset_resource.build_bundle(obj=dataset, request=request)
dataset_bundle = dataset_resource.full_dehydrate(dataset_bundle)
dataset_bundle = dataset_resource.simplify_bundle(dataset_bundle)
results = [SolrObject(d) for d in response["response"]["docs"]]
page = PandaPaginator(
request.GET, results, resource_uri=request.path_info, count=response["response"]["numFound"]
).page()
dataset_bundle.data.update(page)
dataset_bundle.data["objects"] = []
for obj in results:
bundle = self.build_bundle(obj=obj, request=request)
bundle = self.full_dehydrate(bundle)
dataset_bundle.data["objects"].append(bundle.data)
# Because users may have authenticated via headers the request.user may
# not be a full User instance. To be sure, we fetch one.
user = UserProxy.objects.get(id=request.user.id)
SearchLog.objects.create(user=user, dataset=dataset, query=query)
return dataset_bundle
def dehydrate(self, bundle):
"""
Trim the dataset_slug field and add a proper relationship.
"""
dataset = Dataset.objects.get(slug=bundle.data['dataset_slug'])
del bundle.data['dataset_slug']
bundle.data['dataset'] = DatasetResource().get_resource_uri(dataset)
return bundle
def search_dataset_data(self, request, **kwargs):
"""
Perform a full-text search on only one dataset.
See ``get_list``.
"""
dataset = Dataset.objects.get(slug=kwargs["dataset_slug"])
query = request.GET.get("q", None)
limit = int(request.GET.get("limit", settings.PANDA_DEFAULT_SEARCH_ROWS))
offset = int(request.GET.get("offset", 0))
if query:
solr_query = "dataset_slug:%s AND %s" % (dataset.slug, query)
else:
solr_query = "dataset_slug:%s" % dataset.slug
response = solr.query(settings.SOLR_DATA_CORE, solr_query, offset=offset, limit=limit)
dataset_resource = DatasetResource()
dataset_bundle = dataset_resource.build_bundle(obj=dataset, request=request)
dataset_bundle = dataset_resource.full_dehydrate(dataset_bundle)
dataset_bundle = dataset_resource.simplify_bundle(dataset_bundle)
results = [SolrObject(d) for d in response["response"]["docs"]]
page = PandaPaginator(
request.GET, results, resource_uri=request.path_info, count=response["response"]["numFound"]
).page()
dataset_bundle.data.update(page)
dataset_bundle.data["objects"] = []
for obj in results:
bundle = self.build_bundle(obj=obj, request=request)
bundle = self.full_dehydrate(bundle)
dataset_bundle.data["objects"].append(bundle.data)
return dataset_bundle
def search_all_data(self, request, **kwargs):
"""
List endpoint using Solr. Provides full-text search via the "q" parameter."
"""
self.method_check(request, allowed=["get"])
self.is_authenticated(request)
self.throttle_check(request)
query = request.GET.get("q", "")
category = request.GET.get("category", "")
since = request.GET.get("since", None)
limit = int(request.GET.get("limit", settings.PANDA_DEFAULT_SEARCH_GROUPS))
offset = int(request.GET.get("offset", 0))
group_limit = int(request.GET.get("group_limit", settings.PANDA_DEFAULT_SEARCH_ROWS_PER_GROUP))
group_offset = int(request.GET.get("group_offset", 0))
export = bool(request.GET.get("export", False))
if category:
if category != "uncategorized":
category = Category.objects.get(slug=category)
dataset_slugs = category.datasets.values_list("slug", flat=True)
else:
dataset_slugs = Dataset.objects.filter(categories=None).values_list("slug", flat=True)
query += " dataset_slug:(%s)" % " ".join(dataset_slugs)
if since:
query = "last_modified:[" + since + "Z TO *] AND (%s)" % query
# Because users may have authenticated via headers the request.user may
# not be a full User instance. To be sure, we fetch one.
user = UserProxy.objects.get(id=request.user.id)
if export:
task_type = ExportSearchTask
task = TaskStatus.objects.create(
task_name=task_type.name, task_description='Export search results for "%s".' % query, creator=user
)
task_type.apply_async(args=[query, task.id], kwargs={}, task_id=task.id)
else:
response = solr.query_grouped(
settings.SOLR_DATA_CORE,
query,
"dataset_slug",
offset=offset,
limit=limit,
group_limit=group_limit,
group_offset=group_offset,
)
groups = response["grouped"]["dataset_slug"]["groups"]
page = PandaPaginator(
request.GET,
groups,
resource_uri=request.path_info,
count=response["grouped"]["dataset_slug"]["ngroups"],
).page()
datasets = []
for group in groups:
dataset_slug = group["groupValue"]
results = group["doclist"]
try:
dataset = Dataset.objects.get(slug=dataset_slug)
# In the event that stale data exists in Solr, skip this dataset,
# request the invalid data be purged and return the other results.
# Pagination may be wrong, but this is the most functional solution. (#793)
except Dataset.DoesNotExist:
PurgeDataTask.apply_async(args=[dataset_slug])
solr.delete(settings.SOLR_DATASETS_CORE, "slug:%s" % dataset_slug)
page["meta"]["total_count"] -= 1
continue
dataset_resource = DatasetResource()
dataset_bundle = dataset_resource.build_bundle(obj=dataset, request=request)
dataset_bundle = dataset_resource.full_dehydrate(dataset_bundle)
dataset_bundle = dataset_resource.simplify_bundle(dataset_bundle)
objects = [SolrObject(obj) for obj in results["docs"]]
dataset_search_url = reverse(
"api_dataset_data_list",
kwargs={
"api_name": self._meta.api_name,
"dataset_resource_name": "dataset",
"resource_name": "data",
"dataset_slug": dataset.slug,
},
)
data_page = PandaPaginator(
{"limit": str(group_limit), "offset": str(group_offset), "q": query},
objects,
resource_uri=dataset_search_url,
count=results["numFound"],
).page()
dataset_bundle.data.update(data_page)
dataset_bundle.data["objects"] = []
for obj in objects:
data_bundle = self.build_bundle(obj=obj, request=request)
data_bundle = self.full_dehydrate(data_bundle)
dataset_bundle.data["objects"].append(data_bundle)
datasets.append(dataset_bundle.data)
page["objects"] = datasets
# Log query
SearchLog.objects.create(user=user, dataset=None, query=query)
self.log_throttled_access(request)
if export:
return self.create_response(request, "Export queued.")
else:
return self.create_response(request, page)
def search_all_data(self, request, **kwargs):
"""
List endpoint using Solr. Provides full-text search via the "q" parameter."
"""
self.method_check(request, allowed=['get'])
self.is_authenticated(request)
self.throttle_check(request)
query = request.GET.get('q', '')
limit = int(request.GET.get('limit', settings.PANDA_DEFAULT_SEARCH_GROUPS))
offset = int(request.GET.get('offset', 0))
group_limit = int(request.GET.get('group_limit', settings.PANDA_DEFAULT_SEARCH_ROWS_PER_GROUP))
group_offset = int(request.GET.get('group_offset', 0))
response = solr.query_grouped(
settings.SOLR_DATA_CORE,
query,
'dataset_slug',
offset=offset,
limit=limit,
group_limit=group_limit,
group_offset=group_offset
)
groups = response['grouped']['dataset_slug']['groups']
page = PandaPaginator(
request.GET,
groups,
resource_uri=request.path_info,
count=response['grouped']['dataset_slug']['ngroups']
).page()
datasets = []
for group in groups:
dataset_slug = group['groupValue']
results = group['doclist']
dataset_resource = DatasetResource()
dataset = Dataset.objects.get(slug=dataset_slug)
dataset_bundle = dataset_resource.build_bundle(obj=dataset, request=request)
dataset_bundle = dataset_resource.full_dehydrate(dataset_bundle)
dataset_bundle = dataset_resource.simplify_bundle(dataset_bundle)
objects = [SolrObject(obj) for obj in results['docs']]
dataset_search_url = reverse('api_dataset_data_list', kwargs={ 'api_name': self._meta.api_name, 'dataset_resource_name': 'dataset', 'resource_name': 'data', 'dataset_slug': dataset.slug })
data_page = PandaPaginator(
{ 'limit': str(group_limit), 'offset': str(group_offset), 'q': query },
objects,
resource_uri=dataset_search_url,
count=results['numFound']
).page()
dataset_bundle.data.update(data_page)
dataset_bundle.data['objects'] = []
for obj in objects:
data_bundle = self.build_bundle(obj=obj, request=request)
data_bundle = self.full_dehydrate(data_bundle)
dataset_bundle.data['objects'].append(data_bundle)
datasets.append(dataset_bundle.data)
page['objects'] = datasets
self.log_throttled_access(request)
return self.create_response(request, page)
def search_all_data(self, request, **kwargs):
"""
List endpoint using Solr. Provides full-text search via the "q" parameter."
"""
self.method_check(request, allowed=['get'])
self.is_authenticated(request)
self.throttle_check(request)
query = request.GET.get('q', '')
category = request.GET.get('category', '')
since = request.GET.get('since', None)
limit = int(
request.GET.get('limit', settings.PANDA_DEFAULT_SEARCH_GROUPS))
offset = int(request.GET.get('offset', 0))
group_limit = int(
request.GET.get('group_limit',
settings.PANDA_DEFAULT_SEARCH_ROWS_PER_GROUP))
group_offset = int(request.GET.get('group_offset', 0))
export = bool(request.GET.get('export', False))
if category:
if category != 'uncategorized':
category = Category.objects.get(slug=category)
dataset_slugs = category.datasets.values_list('slug',
flat=True)
else:
dataset_slugs = Dataset.objects.filter(
categories=None).values_list('slug', flat=True)
query += ' dataset_slug:(%s)' % ' '.join(dataset_slugs)
if since:
query = 'last_modified:[' + since + 'Z TO *] AND (%s)' % query
# Because users may have authenticated via headers the request.user may
# not be a full User instance. To be sure, we fetch one.
user = UserProxy.objects.get(id=request.user.id)
if export:
task_type = ExportSearchTask
task = TaskStatus.objects.create(
task_name=task_type.name,
task_description='Export search results for "%s".' % query,
creator=user)
task_type.apply_async(args=[query, task.id],
kwargs={},
task_id=task.id)
else:
response = solr.query_grouped(settings.SOLR_DATA_CORE,
query,
'dataset_slug',
offset=offset,
limit=limit,
group_limit=group_limit,
group_offset=group_offset)
groups = response['grouped']['dataset_slug']['groups']
page = PandaPaginator(
request.GET,
groups,
resource_uri=request.path_info,
count=response['grouped']['dataset_slug']['ngroups']).page()
datasets = []
for group in groups:
dataset_slug = group['groupValue']
results = group['doclist']
try:
dataset = Dataset.objects.get(slug=dataset_slug)
# In the event that stale data exists in Solr, skip this dataset,
# request the invalid data be purged and return the other results.
# Pagination may be wrong, but this is the most functional solution. (#793)
except Dataset.DoesNotExist:
PurgeDataTask.apply_async(args=[dataset_slug])
solr.delete(settings.SOLR_DATASETS_CORE,
'slug:%s' % dataset_slug)
page['meta']['total_count'] -= 1
continue
dataset_resource = DatasetResource()
dataset_bundle = dataset_resource.build_bundle(obj=dataset,
request=request)
dataset_bundle = dataset_resource.full_dehydrate(
dataset_bundle)
dataset_bundle = dataset_resource.simplify_bundle(
dataset_bundle)
objects = [SolrObject(obj) for obj in results['docs']]
dataset_search_url = reverse('api_dataset_data_list',
kwargs={
'api_name':
self._meta.api_name,
'dataset_resource_name':
'dataset',
'resource_name': 'data',
'dataset_slug': dataset.slug
})
data_page = PandaPaginator(
{
'limit': str(group_limit),
'offset': str(group_offset),
'q': query
},
objects,
resource_uri=dataset_search_url,
count=results['numFound']).page()
dataset_bundle.data.update(data_page)
dataset_bundle.data['objects'] = []
for obj in objects:
data_bundle = self.build_bundle(obj=obj, request=request)
data_bundle = self.full_dehydrate(data_bundle)
dataset_bundle.data['objects'].append(data_bundle)
datasets.append(dataset_bundle.data)
page['objects'] = datasets
# Log query
SearchLog.objects.create(user=user, dataset=None, query=query)
self.log_throttled_access(request)
if export:
return self.create_response(request, 'Export queued.')
else:
return self.create_response(request, page)
def search_all_data(self, request, **kwargs):
"""
List endpoint using Solr. Provides full-text search via the "q" parameter."
"""
self.method_check(request, allowed=["get"])
self.is_authenticated(request)
self.throttle_check(request)
query = request.GET.get("q", "")
limit = int(request.GET.get("limit", settings.PANDA_DEFAULT_SEARCH_GROUPS))
offset = int(request.GET.get("offset", 0))
group_limit = int(request.GET.get("group_limit", settings.PANDA_DEFAULT_SEARCH_ROWS_PER_GROUP))
group_offset = int(request.GET.get("group_offset", 0))
response = solr.query_grouped(
settings.SOLR_DATA_CORE,
query,
"dataset_slug",
offset=offset,
limit=limit,
group_limit=group_limit,
group_offset=group_offset,
)
groups = response["grouped"]["dataset_slug"]["groups"]
page = PandaPaginator(
request.GET, groups, resource_uri=request.path_info, count=response["grouped"]["dataset_slug"]["ngroups"]
).page()
datasets = []
for group in groups:
dataset_slug = group["groupValue"]
results = group["doclist"]
dataset_resource = DatasetResource()
dataset = Dataset.objects.get(slug=dataset_slug)
dataset_bundle = dataset_resource.build_bundle(obj=dataset, request=request)
dataset_bundle = dataset_resource.full_dehydrate(dataset_bundle)
dataset_bundle = dataset_resource.simplify_bundle(dataset_bundle)
objects = [SolrObject(obj) for obj in results["docs"]]
dataset_search_url = reverse(
"api_dataset_data_list",
kwargs={
"api_name": self._meta.api_name,
"dataset_resource_name": "dataset",
"resource_name": "data",
"dataset_slug": dataset.slug,
},
)
data_page = PandaPaginator(
{"limit": str(group_limit), "offset": str(group_offset), "q": query},
objects,
resource_uri=dataset_search_url,
count=results["numFound"],
).page()
dataset_bundle.data.update(data_page)
dataset_bundle.data["objects"] = []
for obj in objects:
data_bundle = self.build_bundle(obj=obj, request=request)
data_bundle = self.full_dehydrate(data_bundle)
dataset_bundle.data["objects"].append(data_bundle)
datasets.append(dataset_bundle.data)
page["objects"] = datasets
self.log_throttled_access(request)
return self.create_response(request, page)
def search_all_data(self, request, **kwargs):
"""
List endpoint using Solr. Provides full-text search via the "q" parameter."
"""
self.method_check(request, allowed=['get'])
self.is_authenticated(request)
self.throttle_check(request)
query = request.GET.get('q', '')
category = request.GET.get('category', '')
since = request.GET.get('since', None)
limit = int(request.GET.get('limit', settings.PANDA_DEFAULT_SEARCH_GROUPS))
offset = int(request.GET.get('offset', 0))
group_limit = int(request.GET.get('group_limit', settings.PANDA_DEFAULT_SEARCH_ROWS_PER_GROUP))
group_offset = int(request.GET.get('group_offset', 0))
if category:
if category != 'uncategorized':
category = Category.objects.get(slug=category)
dataset_slugs = category.datasets.values_list('slug', flat=True)
else:
dataset_slugs = Dataset.objects.filter(categories=None).values_list('slug', flat=True)
query += ' dataset_slug:(%s)' % ' '.join(dataset_slugs)
if since:
query = 'last_modified:[' + since + 'Z TO *] AND (%s)' % query
response = solr.query_grouped(
settings.SOLR_DATA_CORE,
query,
'dataset_slug',
offset=offset,
limit=limit,
group_limit=group_limit,
group_offset=group_offset
)
groups = response['grouped']['dataset_slug']['groups']
page = PandaPaginator(
request.GET,
groups,
resource_uri=request.path_info,
count=response['grouped']['dataset_slug']['ngroups']
).page()
datasets = []
for group in groups:
dataset_slug = group['groupValue']
results = group['doclist']
dataset_resource = DatasetResource()
dataset = Dataset.objects.get(slug=dataset_slug)
dataset_bundle = dataset_resource.build_bundle(obj=dataset, request=request)
dataset_bundle = dataset_resource.full_dehydrate(dataset_bundle)
dataset_bundle = dataset_resource.simplify_bundle(dataset_bundle)
objects = [SolrObject(obj) for obj in results['docs']]
dataset_search_url = reverse('api_dataset_data_list', kwargs={ 'api_name': self._meta.api_name, 'dataset_resource_name': 'dataset', 'resource_name': 'data', 'dataset_slug': dataset.slug })
data_page = PandaPaginator(
{ 'limit': str(group_limit), 'offset': str(group_offset), 'q': query },
objects,
resource_uri=dataset_search_url,
count=results['numFound']
).page()
dataset_bundle.data.update(data_page)
dataset_bundle.data['objects'] = []
for obj in objects:
data_bundle = self.build_bundle(obj=obj, request=request)
data_bundle = self.full_dehydrate(data_bundle)
dataset_bundle.data['objects'].append(data_bundle)
datasets.append(dataset_bundle.data)
page['objects'] = datasets
# Log query
SearchLog.objects.create(user=request.user, dataset=None, query=query)
self.log_throttled_access(request)
return self.create_response(request, page)
def search_dataset_data(self, request, **kwargs):
"""
Perform a full-text search on only one dataset.
See ``get_list``.
"""
dataset = Dataset.objects.get(slug=kwargs['dataset_slug'])
try:
query = '(%s)' % request.GET['q']
except KeyError:
query = ''
since = request.GET.get('since', None)
limit = int(request.GET.get('limit', settings.PANDA_DEFAULT_SEARCH_ROWS))
offset = int(request.GET.get('offset', 0))
sort = request.GET.get('sort', '_docid_ asc')
solr_query_bits = [query]
solr_query_bits.append('dataset_slug:%s' % dataset.slug)
if since:
solr_query_bits.append('last_modified:[' + since + 'Z TO *]')
response = solr.query(
settings.SOLR_DATA_CORE,
' AND '.join(solr_query_bits),
offset=offset,
sort=sort,
limit=limit
)
dataset_resource = DatasetResource()
dataset_bundle = dataset_resource.build_bundle(obj=dataset, request=request)
dataset_bundle = dataset_resource.full_dehydrate(dataset_bundle)
dataset_bundle = dataset_resource.simplify_bundle(dataset_bundle)
results = [SolrObject(d) for d in response['response']['docs']]
page = PandaPaginator(
request.GET,
results,
resource_uri=request.path_info,
count=response['response']['numFound']
).page()
dataset_bundle.data.update(page)
dataset_bundle.data['objects'] = []
for obj in results:
bundle = self.build_bundle(obj=obj, request=request)
bundle = self.full_dehydrate(bundle)
dataset_bundle.data['objects'].append(bundle.data)
# Because users may have authenticated via headers the request.user may
# not be a full User instance. To be sure, we fetch one.
user = UserProxy.objects.get(id=request.user.id)
SearchLog.objects.create(user=user, dataset=dataset, query=query)
return dataset_bundle
def search_all_data(self, request, **kwargs):
"""
List endpoint using Solr. Provides full-text search via the "q" parameter."
"""
self.method_check(request, allowed=['get'])
self.is_authenticated(request)
self.throttle_check(request)
try:
query = '(%s)' % request.GET['q']
except KeyError:
query = ''
category = request.GET.get('category', '')
since = request.GET.get('since', None)
limit = int(request.GET.get('limit', settings.PANDA_DEFAULT_SEARCH_GROUPS))
offset = int(request.GET.get('offset', 0))
group_limit = int(request.GET.get('group_limit', settings.PANDA_DEFAULT_SEARCH_ROWS_PER_GROUP))
group_offset = int(request.GET.get('group_offset', 0))
export = bool(request.GET.get('export', False))
solr_query_bits = [query]
if category:
if category != 'uncategorized':
category = Category.objects.get(slug=category)
dataset_slugs = category.datasets.values_list('slug', flat=True)
else:
dataset_slugs = Dataset.objects.filter(categories=None).values_list('slug', flat=True)
solr_query_bits.append('dataset_slug:(%s)' % ' '.join(dataset_slugs))
if since:
solr_query_bits.append('last_modified:[' + since + 'Z TO *]')
# Because users may have authenticated via headers the request.user may
# not be a full User instance. To be sure, we fetch one.
user = UserProxy.objects.get(id=request.user.id)
if export:
task_type = ExportSearchTask
task = TaskStatus.objects.create(
task_name=task_type.name,
task_description=_('Export search results for "%s".') % query,
creator=user
)
task_type.apply_async(
args=[query, task.id],
kwargs={},
task_id=task.id
)
else:
response = solr.query_grouped(
settings.SOLR_DATA_CORE,
' AND '.join(solr_query_bits),
'dataset_slug',
offset=offset,
limit=limit,
group_limit=group_limit,
group_offset=group_offset
)
groups = response['grouped']['dataset_slug']['groups']
page = PandaPaginator(
request.GET,
groups,
resource_uri=request.path_info,
count=response['grouped']['dataset_slug']['ngroups']
).page()
datasets = []
for group in groups:
dataset_slug = group['groupValue']
results = group['doclist']
try:
dataset = Dataset.objects.get(slug=dataset_slug)
# In the event that stale data exists in Solr, skip this dataset,
# request the invalid data be purged and return the other results.
# Pagination may be wrong, but this is the most functional solution. (#793)
except Dataset.DoesNotExist:
PurgeDataTask.apply_async(args=[dataset_slug])
solr.delete(settings.SOLR_DATASETS_CORE, 'slug:%s' % dataset_slug)
page['meta']['total_count'] -= 1
continue
dataset_resource = DatasetResource()
dataset_bundle = dataset_resource.build_bundle(obj=dataset, request=request)
dataset_bundle = dataset_resource.full_dehydrate(dataset_bundle)
dataset_bundle = dataset_resource.simplify_bundle(dataset_bundle)
objects = [SolrObject(obj) for obj in results['docs']]
dataset_search_url = reverse('api_dataset_data_list', kwargs={ 'api_name': self._meta.api_name, 'dataset_resource_name': 'dataset', 'resource_name': 'data', 'dataset_slug': dataset.slug })
data_page = PandaPaginator(
{ 'limit': str(group_limit), 'offset': str(group_offset), 'q': query },
objects,
resource_uri=dataset_search_url,
count=results['numFound']
).page()
dataset_bundle.data.update(data_page)
dataset_bundle.data['objects'] = []
for obj in objects:
data_bundle = self.build_bundle(obj=obj, request=request)
data_bundle = self.full_dehydrate(data_bundle)
dataset_bundle.data['objects'].append(data_bundle)
datasets.append(dataset_bundle.data)
page['objects'] = datasets
# Log query
SearchLog.objects.create(user=user, dataset=None, query=query)
self.log_throttled_access(request)
if export:
return self.create_response(request, _('Export queued.'))
else:
return self.create_response(request, page)
def dehydrate(self, bundle):
"""
Always remove the password form the serialized bundle.
"""
del bundle.data['password']
user = bundle.obj
if 'notifications' in bundle.request.GET and bundle.request.GET[
'notifications'].lower() == 'true':
from panda.api.notifications import NotificationResource
resource = NotificationResource()
notifications = user.notifications.all(
)[:settings.PANDA_NOTIFICATIONS_TO_SHOW]
bundles = [resource.build_bundle(obj=n) for n in notifications]
notifications = [resource.full_dehydrate(b) for b in bundles]
bundle.data['notifications'] = notifications
if 'exports' in bundle.request.GET and bundle.request.GET[
'exports'].lower() == 'true':
from panda.api.exports import ExportResource
resource = ExportResource()
exports = Export.objects.filter(creator=user)
bundles = [resource.build_bundle(obj=e) for e in exports]
exports = [resource.full_dehydrate(b) for b in bundles]
bundle.data['exports'] = exports
if 'datasets' in bundle.request.GET and bundle.request.GET[
'datasets'].lower() == 'true':
from panda.api.datasets import DatasetResource
resource = DatasetResource()
datasets = user.datasets.all()
bundles = [resource.build_bundle(obj=d) for d in datasets]
datasets = [
resource.simplify_bundle(resource.full_dehydrate(b))
for b in bundles
]
bundle.data['datasets'] = datasets
if 'search_subscriptions' in bundle.request.GET and bundle.request.GET[
'search_subscriptions'].lower() == 'true':
from panda.api.search_subscriptions import SearchSubscriptionResource
resource = SearchSubscriptionResource()
subscriptions = user.search_subscriptions.all()
bundles = [resource.build_bundle(obj=s) for s in subscriptions]
datasets = [resource.full_dehydrate(b) for b in bundles]
bundle.data['subscriptions'] = datasets
return bundle
