def grants_json_generator(query):
yield '''{
"license": "See dataset/license within each grant. This file also contains OS data © Crown copyright and database right 2016, Royal Mail data © Royal Mail copyright and Database right 2016, National Statistics data © Crown copyright and database right 2016, see http://grantnav.org/datasets/ for more information.",
"grants": [\n'''
es = get_es()
for num, result in enumerate(scan(es, query, index=settings.ES_INDEX)):
result["_source"]["dataset"] = provenance.by_identifier.get(provenance.identifier_from_filename(result['_source']['filename']), {})
if num == 0:
yield json.dumps(result["_source"]) + "\n"
else:
yield ", " + json.dumps(result["_source"]) + "\n"
yield ']}'
def grants_csv_generator(query):
yield csv_layout.grant_csv_titles
es = get_es()
for result in scan(es, query, index=settings.ES_INDEX):
result_with_provenance = {
"result": result["_source"],
"dataset": provenance.by_identifier.get(provenance.identifier_from_filename(result['_source']['filename']), {})
}
line = []
for path in csv_layout.grant_csv_paths:
line.append(get_data_from_path(path, result_with_provenance))
yield line
def grants_json_generator(query):
yield '''{
"license": "See dataset/license within each grant. This file also contains OS data © Crown copyright and database right 2016, Royal Mail data © Royal Mail copyright and Database right 2016, National Statistics data © Crown copyright and database right 2016, see http://grantnav.org/datasets/ for more information.",
"grants": [\n'''
es = get_es()
for num, result in enumerate(scan(es, query, index=settings.ES_INDEX)):
result["_source"]["dataset"] = provenance.by_identifier.get(provenance.identifier_from_filename(result['_source']['filename']), {})
if num == 0:
yield json.dumps(result["_source"]) + "\n"
else:
yield ", " + json.dumps(result["_source"]) + "\n"
yield ']}'
def grants_csv_generator(query):
yield csv_layout.grant_csv_titles
es = get_es()
for result in scan(es, query, index=settings.ES_INDEX):
result_with_provenance = {
"result": result["_source"],
"dataset": provenance.by_identifier.get(provenance.identifier_from_filename(result['_source']['filename']), {})
}
line = []
for path in csv_layout.grant_csv_paths:
line.append(get_data_from_path(path, result_with_provenance))
yield line
def funder(request, funder_id):
[result_format, results_size] = get_request_type_and_size(request)
if result_format != "html":
funder_id = re.match('(.*)\.\w*$', funder_id).group(1)
query = {"query": {"bool": {"filter":
[{"term": {"fundingOrganization.id": funder_id}}]}},
"aggs": {
"recipient_orgs": {"cardinality": {"field": "recipientOrganization.id", "precision_threshold": 40000}},
"filenames": {"terms": {"field": "filename", "size": 10}},
"currency_stats": {"terms": {"field": "currency"}, "aggs": {"amount_stats": {"stats": {"field": "amountAwarded"}}}},
"min_date": {"min": {"field": "awardDate"}},
"max_date": {"max": {"field": "awardDate"}},
"recipients": {"terms": {"field": "recipientOrganization.id_and_name", "size": 10},
"aggs": {"recipient_stats": {"stats": {"field": "amountAwarded"}}}}
}
}
if result_format == "csv":
return grants_csv_paged(query)
elif result_format == "json":
return grants_json_paged(query)
results = get_results(query, results_size)
if results['hits']['total'] == 0:
raise Http404
context = {}
context['results'] = results
funder = results['hits']['hits'][0]["_source"]["fundingOrganization"][0].copy()
# funder name for this case to come from same place as Filters.
funder['name'] = json.loads(funder['id_and_name'])[0]
context['funder'] = funder
try:
context['publisher'] = provenance.by_identifier[provenance.identifier_from_filename(results['aggregations']['filenames']['buckets'][0]['key'])]['publisher']
except KeyError:
pass
return render(request, "funder.html", context=context)
def funder(request, funder_id):
[result_format, results_size] = get_request_type_and_size(request)
if result_format != "html":
funder_id = re.match('(.*)\.\w*$', funder_id).group(1)
query = {"query": {"bool": {"filter":
[{"term": {"fundingOrganization.id": funder_id}}]}},
"aggs": {
"recipient_orgs": {"cardinality": {"field": "recipientOrganization.id", "precision_threshold": 40000}},
"filenames": {"terms": {"field": "filename", "size": 10}},
"total_amount": {"sum": {"field": "amountAwarded"}},
"avg_amount": {"avg": {"field": "amountAwarded"}},
"min_amount": {"min": {"field": "amountAwarded"}},
"max_amount": {"max": {"field": "amountAwarded"}},
"min_date": {"min": {"field": "awardDate"}},
"max_date": {"max": {"field": "awardDate"}},
"currencies": {"terms": {"field": "currency", "size": 0}},
"recipients": {"terms": {"field": "recipientOrganization.id_and_name", "size": 10},
"aggs": {"recipient_stats": {"stats": {"field": "amountAwarded"}}}}
}
}
if result_format == "csv":
return grants_csv_paged(query)
elif result_format == "json":
return grants_json_paged(query)
results = get_results(query, results_size)
if results['hits']['total'] == 0:
raise Http404
context = {}
context['results'] = results
context['funder'] = results['hits']['hits'][0]["_source"]["fundingOrganization"][0]
try:
context['publisher'] = provenance.by_identifier[provenance.identifier_from_filename(results['aggregations']['filenames']['buckets'][0]['key'])]['publisher']
except KeyError:
pass
return render(request, "funder.html", context=context)
def get_dataset(grant):
try:
return provenance.by_identifier[provenance.identifier_from_filename(
grant['source']['filename'])]
except KeyError:
return None
def get_dataset(grant):
try:
return provenance.by_identifier[provenance.identifier_from_filename(grant["source"]["filename"])]
except KeyError:
return None
