def build_elasticsearch_result(self,
info_buckets: List[dict]) -> List[dict]:
results = []
cfda_prefetch_pks = [
json_str_to_dict(bucket.get("key")).get("code")
for bucket in info_buckets
]
prefetched_cfdas = Cfda.objects.filter(
program_number__in=cfda_prefetch_pks)
for bucket in info_buckets:
info = json_str_to_dict(bucket.get("key"))
toAdd = {
"id": int(info.get("id")) if info.get("id") else None,
"code": info.get("code") or None,
"description": info.get("description") or None,
"award_count": int(bucket.get("doc_count", 0)),
"resource_link": info.get("url") or None,
**{
column: get_summed_value_as_float(
bucket, self.sum_column_mapping[column])
for column in self.sum_column_mapping
},
}
cfda = [
cfda for cfda in prefetched_cfdas
if cfda.program_number == info.get("code")
]
if cfda:
toAdd.update({
"cfda_federal_agency":
cfda[0].federal_agency,
"cfda_objectives":
cfda[0].objectives,
"cfda_website":
cfda[0].website_address,
"applicant_eligibility":
cfda[0].applicant_eligibility,
"beneficiary_eligibility":
cfda[0].beneficiary_eligibility,
})
else:
toAdd.update({
"cfda_federal_agency": None,
"cfda_objectives": None,
"cfda_website": None,
"applicant_eligibility": None,
"beneficiary_eligibility": None,
})
results.append(toAdd)
return results
def build_elasticsearch_result(self, response: dict) -> List[dict]:
results = []
location_info_buckets = response.get("group_by_agg_key",
{}).get("buckets", [])
for bucket in location_info_buckets:
location_info = json_str_to_dict(bucket.get("key"))
if self.location_type == LocationType.CONGRESSIONAL_DISTRICT:
if location_info.get("congressional_code") == "90":
congressional_code = "MULTIPLE DISTRICTS"
else:
congressional_code = location_info.get(
"congressional_code")
name = f"{location_info.get('state_code')}-{congressional_code}"
else:
name = location_info.get(f"{self.location_type.value}_name")
if self.location_type == LocationType.STATE_TERRITORY:
name = name.title()
else:
name = name.upper()
results.append({
"amount":
int(bucket.get("sum_field", {"value": 0})["value"]) /
Decimal("100"),
"code":
location_info.get(f"{self.location_type.value}_code"),
"id":
None,
"name":
name,
})
return results
def build_elasticsearch_result(self, response: dict) -> List[dict]:
results = []
info_buckets = response.get("group_by_agg_key", {}).get("buckets", [])
for bucket in info_buckets:
info = json_str_to_dict(bucket.get("key"))
# Build a list of hash IDs to handle multiple levels
recipient_hash = info.get("hash")
recipient_levels = sorted(
list(re.sub("[{},]", "", info.get("levels", ""))))
if recipient_hash and recipient_levels:
recipient_hash_list = [
f"{recipient_hash}-{level}" for level in recipient_levels
]
else:
recipient_hash_list = None
results.append({
"id": recipient_hash_list,
"code": info["unique_id"] or "DUNS Number not provided",
"description": info["name"] or None,
"award_count": int(bucket.get("doc_count", 0)),
**{
column: int(
bucket.get(self.sum_column_mapping[column], {
"value": 0
})["value"]) / Decimal("100")
for column in self.sum_column_mapping
},
})
return results
def build_elasticsearch_result(self, info_buckets: List[dict]) -> List[dict]:
results = []
for bucket in info_buckets:
info = json_str_to_dict(bucket.get("key"))
# Build a list of hash IDs to handle multiple levels
recipient_hash = info.get("hash")
recipient_levels = sorted(list(re.sub("[{},]", "", info.get("levels", ""))))
if recipient_hash and recipient_levels:
recipient_hash_list = [f"{recipient_hash}-{level}" for level in recipient_levels]
else:
recipient_hash_list = None
results.append(
{
"id": recipient_hash_list,
"code": info["unique_id"] or "DUNS Number not provided",
"description": info["name"] or None,
"award_count": int(bucket.get("doc_count", 0)),
**{
column: get_summed_value_as_float(bucket, self.sum_column_mapping[column])
for column in self.sum_column_mapping
},
}
)
return results
def build_elasticsearch_result(self, response: dict) -> Dict[str, dict]:
results = {}
geo_info_buckets = response.get("group_by_agg_key", {}).get("buckets", [])
for bucket in geo_info_buckets:
geo_info = json_str_to_dict(bucket.get("key"))
if self.geo_layer == GeoLayer.STATE:
display_name = geo_info.get("state_name").title()
shape_code = geo_info.get("state_code").upper()
elif self.geo_layer == GeoLayer.COUNTY:
display_name = geo_info["county_name"].title()
shape_code = f"{geo_info['state_fips']}{geo_info['county_code']}"
else:
display_name = f"{geo_info['state_code']}-{geo_info['congressional_code']}".upper()
shape_code = f"{geo_info['state_fips']}{geo_info['congressional_code']}"
per_capita = None
amount = int(bucket.get("sum_field", {"value": 0})["value"]) / Decimal("100")
population = int(geo_info["population"]) if geo_info["population"] else None
if population:
per_capita = (Decimal(amount) / Decimal(population)).quantize(Decimal(".01"))
results[shape_code] = {
"amount": amount,
"display_name": display_name or None,
"shape_code": shape_code or None,
"population": population,
"per_capita": per_capita,
"award_count": int(bucket.get("doc_count", 0)),
}
return results
def _build_json_result(self, bucket: dict):
info = json_str_to_dict(bucket.get("key"))
return {
"id": int(info["id"]),
"code": info["code"],
"description": info["name"],
# the count of distinct awards contributing to the totals
"award_count": int(bucket.get("doc_count", 0)),
**{
column: get_summed_value_as_float(bucket, self.sum_column_mapping[column])
for column in self.sum_column_mapping
},
}
def build_elasticsearch_result(self, response: dict) -> List[dict]:
results = []
agency_info_buckets = response.get("group_by_agg_key", {}).get("buckets", [])
for bucket in agency_info_buckets:
agency_info = json_str_to_dict(bucket.get("key"))
results.append(
{
"amount": int(bucket.get("sum_field", {"value": 0})["value"]) / Decimal("100"),
"name": agency_info.get("name"),
"code": agency_info.get("abbreviation") or None,
"id": int(agency_info.get("id")) if len(agency_info.get("id")) > 0 else None,
}
)
return results
def build_elasticsearch_result(self, response: dict) -> List[dict]:
results = []
location_info_buckets = response.get("group_by_agg_key", {}).get("buckets", [])
for bucket in location_info_buckets:
recipient_info = json_str_to_dict(bucket.get("key"))
results.append(
{
"amount": int(bucket.get("sum_field", {"value": 0})["value"]) / Decimal("100"),
"recipient_id": recipient_info["hash_with_level"] or None,
"name": recipient_info["name"] or None,
"code": recipient_info["unique_id"] or "DUNS Number not provided",
}
)
return results
def build_elasticsearch_result(self, response: dict) -> List[dict]:
results = []
info_buckets = response.get("group_by_agg_key", {}).get("buckets", [])
for bucket in info_buckets:
info = json_str_to_dict(bucket.get("key"))
results.append(
{
"id": int(info.get("id")) if info.get("id") else None,
"code": info.get("code") or None,
"description": info.get("description") or None,
"award_count": int(bucket.get("doc_count", 0)),
"resource_link": info.get("url") or None,
**{
column: int(bucket.get(self.sum_column_mapping[column], {"value": 0})["value"]) / Decimal("100")
for column in self.sum_column_mapping
},
}
)
return results
def build_elasticsearch_result(self, response: dict) -> List[dict]:
results = []
account_info_buckets = response.get("group_by_agg_key",
{}).get("buckets", [])
for bucket in account_info_buckets:
account_info = json_str_to_dict(bucket.get("key"))
results.append({
"amount":
int(bucket.get("sum_field", {"value": 0})["value"]) /
Decimal("100"),
"id":
int(account_info.get("id"))
if account_info.get("id") else None,
"code":
account_info.get("federal_account_code") or None,
"name":
account_info.get("account_title"),
})
return results
def build_elasticsearch_result(self, response: dict) -> List[dict]:
results = []
industry_code_info_buckets = response.get("group_by_agg_key",
{}).get("buckets", [])
for bucket in industry_code_info_buckets:
industry_code_info = json_str_to_dict(bucket.get("key"))
results.append({
"amount":
int(bucket.get("sum_field", {"value": 0})["value"]) /
Decimal("100"),
"code":
industry_code_info.get("code"),
"id":
int(industry_code_info.get("id"))
if industry_code_info.get("id") else None,
"name":
industry_code_info.get("description") or None,
})
return results
def build_elasticsearch_result(self, response: dict) -> List[dict]:
results = []
cfda_code_list = [
] # Separate lookup is done for CFDA to join to references_cfda during ES indexing
industry_code_info_buckets = response.get("group_by_agg_key",
{}).get("buckets", [])
for bucket in industry_code_info_buckets:
if self.industry_code_type == IndustryCodeType.CFDA:
industry_code_info = {"code": bucket.get("key")}
cfda_code_list.append(industry_code_info["code"])
else:
industry_code_info = json_str_to_dict(bucket.get("key"))
results.append({
"amount":
int(bucket.get("sum_field", {"value": 0})["value"]) /
Decimal("100"),
"code":
industry_code_info.get("code"),
"id":
int(industry_code_info.get("id"))
if industry_code_info.get("id") else None,
"name":
industry_code_info.get("description") or None,
})
if cfda_code_list:
cfda_matches = {
cfda["program_number"]: cfda
for cfda in Cfda.objects.filter(
program_number__in=cfda_code_list).values(
"id", "program_number", "program_title")
}
for val in results:
val["id"] = cfda_matches.get(val["code"], {}).get("id")
val["name"] = cfda_matches.get(val["code"],
{}).get("program_title")
return results
def obtain_recipient_totals(recipient_id, children=False, year="latest"):
""" Extract the total amount and transaction count for the recipient_hash given the timeframe
Args:
recipient_id: string of hash(duns, name)-[recipient-level]
children: whether or not to group by children
year: the year the totals/counts are based on
Returns:
list of dictionaries representing hashes and their totals/counts
"""
filters = reshape_filters(recipient_id=recipient_id, year=year)
filter_query = QueryWithFilters.generate_transactions_elasticsearch_query(
filters)
search = TransactionSearch().filter(filter_query)
if children:
group_by_field = "recipient_agg_key"
elif recipient_id[-2:] == "-P":
group_by_field = "parent_recipient_hash"
else:
group_by_field = "recipient_hash"
bucket_count = get_number_of_unique_terms_for_transactions(
filter_query, f"{group_by_field}.hash")
if bucket_count == 0:
return []
# Not setting the shard_size since the number of child recipients under a
# parent recipient will not exceed 10k
group_by_recipient = A("terms", field=group_by_field, size=bucket_count)
sum_obligation = get_scaled_sum_aggregations(
"generated_pragmatic_obligation")["sum_field"]
filter_loans = A("filter", terms={"type": list(loan_type_mapping.keys())})
sum_face_value_loan = get_scaled_sum_aggregations(
"face_value_loan_guarantee")["sum_field"]
search.aggs.bucket("group_by_recipient", group_by_recipient)
search.aggs["group_by_recipient"].metric("sum_obligation", sum_obligation)
search.aggs["group_by_recipient"].bucket("filter_loans", filter_loans)
search.aggs["group_by_recipient"]["filter_loans"].metric(
"sum_face_value_loan", sum_face_value_loan)
response = search.handle_execute()
response_as_dict = response.aggs.to_dict()
recipient_info_buckets = response_as_dict.get("group_by_recipient",
{}).get("buckets", [])
result_list = []
for bucket in recipient_info_buckets:
result = {}
if children:
recipient_info = json_str_to_dict(bucket.get("key"))
hash_with_level = recipient_info.get("hash_with_level") or None
result = {
"recipient_hash":
hash_with_level[:-2] if hash_with_level else None,
"recipient_unique_id": recipient_info.get("unique_id") or None,
"recipient_name": recipient_info.get("name") or None,
}
loan_info = bucket.get("filter_loans", {})
result.update({
"total_obligation_amount":
int(bucket.get("sum_obligation", {"value": 0})["value"]) /
Decimal("100"),
"total_obligation_count":
bucket.get("doc_count", 0),
"total_face_value_loan_amount":
int(loan_info.get("sum_face_value_loan", {"value": 0})["value"]) /
Decimal("100"),
"total_face_value_loan_count":
loan_info.get("doc_count", 0),
})
result_list.append(result)
return result_list
