def grants_json_generator(query): yield '''{ "license": "See dataset/license within each grant. This file also contains OS data © Crown copyright and database right 2016, Royal Mail data © Royal Mail copyright and Database right 2016, National Statistics data © Crown copyright and database right 2016, see http://grantnav.org/datasets/ for more information.", "grants": [\n''' es = get_es() for num, result in enumerate(scan(es, query, index=settings.ES_INDEX)): result["_source"]["dataset"] = provenance.by_identifier.get(provenance.identifier_from_filename(result['_source']['filename']), {}) if num == 0: yield json.dumps(result["_source"]) + "\n" else: yield ", " + json.dumps(result["_source"]) + "\n" yield ']}'
def grants_csv_generator(query): yield csv_layout.grant_csv_titles es = get_es() for result in scan(es, query, index=settings.ES_INDEX): result_with_provenance = { "result": result["_source"], "dataset": provenance.by_identifier.get(provenance.identifier_from_filename(result['_source']['filename']), {}) } line = [] for path in csv_layout.grant_csv_paths: line.append(get_data_from_path(path, result_with_provenance)) yield line
def funder(request, funder_id): [result_format, results_size] = get_request_type_and_size(request) if result_format != "html": funder_id = re.match('(.*)\.\w*$', funder_id).group(1) query = {"query": {"bool": {"filter": [{"term": {"fundingOrganization.id": funder_id}}]}}, "aggs": { "recipient_orgs": {"cardinality": {"field": "recipientOrganization.id", "precision_threshold": 40000}}, "filenames": {"terms": {"field": "filename", "size": 10}}, "currency_stats": {"terms": {"field": "currency"}, "aggs": {"amount_stats": {"stats": {"field": "amountAwarded"}}}}, "min_date": {"min": {"field": "awardDate"}}, "max_date": {"max": {"field": "awardDate"}}, "recipients": {"terms": {"field": "recipientOrganization.id_and_name", "size": 10}, "aggs": {"recipient_stats": {"stats": {"field": "amountAwarded"}}}} } } if result_format == "csv": return grants_csv_paged(query) elif result_format == "json": return grants_json_paged(query) results = get_results(query, results_size) if results['hits']['total'] == 0: raise Http404 context = {} context['results'] = results funder = results['hits']['hits'][0]["_source"]["fundingOrganization"][0].copy() # funder name for this case to come from same place as Filters. funder['name'] = json.loads(funder['id_and_name'])[0] context['funder'] = funder try: context['publisher'] = provenance.by_identifier[provenance.identifier_from_filename(results['aggregations']['filenames']['buckets'][0]['key'])]['publisher'] except KeyError: pass return render(request, "funder.html", context=context)
def funder(request, funder_id): [result_format, results_size] = get_request_type_and_size(request) if result_format != "html": funder_id = re.match('(.*)\.\w*$', funder_id).group(1) query = {"query": {"bool": {"filter": [{"term": {"fundingOrganization.id": funder_id}}]}}, "aggs": { "recipient_orgs": {"cardinality": {"field": "recipientOrganization.id", "precision_threshold": 40000}}, "filenames": {"terms": {"field": "filename", "size": 10}}, "total_amount": {"sum": {"field": "amountAwarded"}}, "avg_amount": {"avg": {"field": "amountAwarded"}}, "min_amount": {"min": {"field": "amountAwarded"}}, "max_amount": {"max": {"field": "amountAwarded"}}, "min_date": {"min": {"field": "awardDate"}}, "max_date": {"max": {"field": "awardDate"}}, "currencies": {"terms": {"field": "currency", "size": 0}}, "recipients": {"terms": {"field": "recipientOrganization.id_and_name", "size": 10}, "aggs": {"recipient_stats": {"stats": {"field": "amountAwarded"}}}} } } if result_format == "csv": return grants_csv_paged(query) elif result_format == "json": return grants_json_paged(query) results = get_results(query, results_size) if results['hits']['total'] == 0: raise Http404 context = {} context['results'] = results context['funder'] = results['hits']['hits'][0]["_source"]["fundingOrganization"][0] try: context['publisher'] = provenance.by_identifier[provenance.identifier_from_filename(results['aggregations']['filenames']['buckets'][0]['key'])]['publisher'] except KeyError: pass return render(request, "funder.html", context=context)
def get_dataset(grant): try: return provenance.by_identifier[provenance.identifier_from_filename( grant['source']['filename'])] except KeyError: return None
def get_dataset(grant): try: return provenance.by_identifier[provenance.identifier_from_filename(grant["source"]["filename"])] except KeyError: return None