def build_elasticsearch_result(self, info_buckets: List[dict]) -> List[dict]: results = [] cfda_prefetch_pks = [ json_str_to_dict(bucket.get("key")).get("code") for bucket in info_buckets ] prefetched_cfdas = Cfda.objects.filter( program_number__in=cfda_prefetch_pks) for bucket in info_buckets: info = json_str_to_dict(bucket.get("key")) toAdd = { "id": int(info.get("id")) if info.get("id") else None, "code": info.get("code") or None, "description": info.get("description") or None, "award_count": int(bucket.get("doc_count", 0)), "resource_link": info.get("url") or None, **{ column: get_summed_value_as_float( bucket, self.sum_column_mapping[column]) for column in self.sum_column_mapping }, } cfda = [ cfda for cfda in prefetched_cfdas if cfda.program_number == info.get("code") ] if cfda: toAdd.update({ "cfda_federal_agency": cfda[0].federal_agency, "cfda_objectives": cfda[0].objectives, "cfda_website": cfda[0].website_address, "applicant_eligibility": cfda[0].applicant_eligibility, "beneficiary_eligibility": cfda[0].beneficiary_eligibility, }) else: toAdd.update({ "cfda_federal_agency": None, "cfda_objectives": None, "cfda_website": None, "applicant_eligibility": None, "beneficiary_eligibility": None, }) results.append(toAdd) return results
def build_elasticsearch_result(self, response: dict) -> List[dict]: results = [] location_info_buckets = response.get("group_by_agg_key", {}).get("buckets", []) for bucket in location_info_buckets: location_info = json_str_to_dict(bucket.get("key")) if self.location_type == LocationType.CONGRESSIONAL_DISTRICT: if location_info.get("congressional_code") == "90": congressional_code = "MULTIPLE DISTRICTS" else: congressional_code = location_info.get( "congressional_code") name = f"{location_info.get('state_code')}-{congressional_code}" else: name = location_info.get(f"{self.location_type.value}_name") if self.location_type == LocationType.STATE_TERRITORY: name = name.title() else: name = name.upper() results.append({ "amount": int(bucket.get("sum_field", {"value": 0})["value"]) / Decimal("100"), "code": location_info.get(f"{self.location_type.value}_code"), "id": None, "name": name, }) return results
def build_elasticsearch_result(self, response: dict) -> List[dict]: results = [] info_buckets = response.get("group_by_agg_key", {}).get("buckets", []) for bucket in info_buckets: info = json_str_to_dict(bucket.get("key")) # Build a list of hash IDs to handle multiple levels recipient_hash = info.get("hash") recipient_levels = sorted( list(re.sub("[{},]", "", info.get("levels", "")))) if recipient_hash and recipient_levels: recipient_hash_list = [ f"{recipient_hash}-{level}" for level in recipient_levels ] else: recipient_hash_list = None results.append({ "id": recipient_hash_list, "code": info["unique_id"] or "DUNS Number not provided", "description": info["name"] or None, "award_count": int(bucket.get("doc_count", 0)), **{ column: int( bucket.get(self.sum_column_mapping[column], { "value": 0 })["value"]) / Decimal("100") for column in self.sum_column_mapping }, }) return results
def build_elasticsearch_result(self, info_buckets: List[dict]) -> List[dict]: results = [] for bucket in info_buckets: info = json_str_to_dict(bucket.get("key")) # Build a list of hash IDs to handle multiple levels recipient_hash = info.get("hash") recipient_levels = sorted(list(re.sub("[{},]", "", info.get("levels", "")))) if recipient_hash and recipient_levels: recipient_hash_list = [f"{recipient_hash}-{level}" for level in recipient_levels] else: recipient_hash_list = None results.append( { "id": recipient_hash_list, "code": info["unique_id"] or "DUNS Number not provided", "description": info["name"] or None, "award_count": int(bucket.get("doc_count", 0)), **{ column: get_summed_value_as_float(bucket, self.sum_column_mapping[column]) for column in self.sum_column_mapping }, } ) return results
def build_elasticsearch_result(self, response: dict) -> Dict[str, dict]: results = {} geo_info_buckets = response.get("group_by_agg_key", {}).get("buckets", []) for bucket in geo_info_buckets: geo_info = json_str_to_dict(bucket.get("key")) if self.geo_layer == GeoLayer.STATE: display_name = geo_info.get("state_name").title() shape_code = geo_info.get("state_code").upper() elif self.geo_layer == GeoLayer.COUNTY: display_name = geo_info["county_name"].title() shape_code = f"{geo_info['state_fips']}{geo_info['county_code']}" else: display_name = f"{geo_info['state_code']}-{geo_info['congressional_code']}".upper() shape_code = f"{geo_info['state_fips']}{geo_info['congressional_code']}" per_capita = None amount = int(bucket.get("sum_field", {"value": 0})["value"]) / Decimal("100") population = int(geo_info["population"]) if geo_info["population"] else None if population: per_capita = (Decimal(amount) / Decimal(population)).quantize(Decimal(".01")) results[shape_code] = { "amount": amount, "display_name": display_name or None, "shape_code": shape_code or None, "population": population, "per_capita": per_capita, "award_count": int(bucket.get("doc_count", 0)), } return results
def _build_json_result(self, bucket: dict): info = json_str_to_dict(bucket.get("key")) return { "id": int(info["id"]), "code": info["code"], "description": info["name"], # the count of distinct awards contributing to the totals "award_count": int(bucket.get("doc_count", 0)), **{ column: get_summed_value_as_float(bucket, self.sum_column_mapping[column]) for column in self.sum_column_mapping }, }
def build_elasticsearch_result(self, response: dict) -> List[dict]: results = [] agency_info_buckets = response.get("group_by_agg_key", {}).get("buckets", []) for bucket in agency_info_buckets: agency_info = json_str_to_dict(bucket.get("key")) results.append( { "amount": int(bucket.get("sum_field", {"value": 0})["value"]) / Decimal("100"), "name": agency_info.get("name"), "code": agency_info.get("abbreviation") or None, "id": int(agency_info.get("id")) if len(agency_info.get("id")) > 0 else None, } ) return results
def build_elasticsearch_result(self, response: dict) -> List[dict]: results = [] location_info_buckets = response.get("group_by_agg_key", {}).get("buckets", []) for bucket in location_info_buckets: recipient_info = json_str_to_dict(bucket.get("key")) results.append( { "amount": int(bucket.get("sum_field", {"value": 0})["value"]) / Decimal("100"), "recipient_id": recipient_info["hash_with_level"] or None, "name": recipient_info["name"] or None, "code": recipient_info["unique_id"] or "DUNS Number not provided", } ) return results
def build_elasticsearch_result(self, response: dict) -> List[dict]: results = [] info_buckets = response.get("group_by_agg_key", {}).get("buckets", []) for bucket in info_buckets: info = json_str_to_dict(bucket.get("key")) results.append( { "id": int(info.get("id")) if info.get("id") else None, "code": info.get("code") or None, "description": info.get("description") or None, "award_count": int(bucket.get("doc_count", 0)), "resource_link": info.get("url") or None, **{ column: int(bucket.get(self.sum_column_mapping[column], {"value": 0})["value"]) / Decimal("100") for column in self.sum_column_mapping }, } ) return results
def build_elasticsearch_result(self, response: dict) -> List[dict]: results = [] account_info_buckets = response.get("group_by_agg_key", {}).get("buckets", []) for bucket in account_info_buckets: account_info = json_str_to_dict(bucket.get("key")) results.append({ "amount": int(bucket.get("sum_field", {"value": 0})["value"]) / Decimal("100"), "id": int(account_info.get("id")) if account_info.get("id") else None, "code": account_info.get("federal_account_code") or None, "name": account_info.get("account_title"), }) return results
def build_elasticsearch_result(self, response: dict) -> List[dict]: results = [] industry_code_info_buckets = response.get("group_by_agg_key", {}).get("buckets", []) for bucket in industry_code_info_buckets: industry_code_info = json_str_to_dict(bucket.get("key")) results.append({ "amount": int(bucket.get("sum_field", {"value": 0})["value"]) / Decimal("100"), "code": industry_code_info.get("code"), "id": int(industry_code_info.get("id")) if industry_code_info.get("id") else None, "name": industry_code_info.get("description") or None, }) return results
def build_elasticsearch_result(self, response: dict) -> List[dict]: results = [] cfda_code_list = [ ] # Separate lookup is done for CFDA to join to references_cfda during ES indexing industry_code_info_buckets = response.get("group_by_agg_key", {}).get("buckets", []) for bucket in industry_code_info_buckets: if self.industry_code_type == IndustryCodeType.CFDA: industry_code_info = {"code": bucket.get("key")} cfda_code_list.append(industry_code_info["code"]) else: industry_code_info = json_str_to_dict(bucket.get("key")) results.append({ "amount": int(bucket.get("sum_field", {"value": 0})["value"]) / Decimal("100"), "code": industry_code_info.get("code"), "id": int(industry_code_info.get("id")) if industry_code_info.get("id") else None, "name": industry_code_info.get("description") or None, }) if cfda_code_list: cfda_matches = { cfda["program_number"]: cfda for cfda in Cfda.objects.filter( program_number__in=cfda_code_list).values( "id", "program_number", "program_title") } for val in results: val["id"] = cfda_matches.get(val["code"], {}).get("id") val["name"] = cfda_matches.get(val["code"], {}).get("program_title") return results
def obtain_recipient_totals(recipient_id, children=False, year="latest"): """ Extract the total amount and transaction count for the recipient_hash given the timeframe Args: recipient_id: string of hash(duns, name)-[recipient-level] children: whether or not to group by children year: the year the totals/counts are based on Returns: list of dictionaries representing hashes and their totals/counts """ filters = reshape_filters(recipient_id=recipient_id, year=year) filter_query = QueryWithFilters.generate_transactions_elasticsearch_query( filters) search = TransactionSearch().filter(filter_query) if children: group_by_field = "recipient_agg_key" elif recipient_id[-2:] == "-P": group_by_field = "parent_recipient_hash" else: group_by_field = "recipient_hash" bucket_count = get_number_of_unique_terms_for_transactions( filter_query, f"{group_by_field}.hash") if bucket_count == 0: return [] # Not setting the shard_size since the number of child recipients under a # parent recipient will not exceed 10k group_by_recipient = A("terms", field=group_by_field, size=bucket_count) sum_obligation = get_scaled_sum_aggregations( "generated_pragmatic_obligation")["sum_field"] filter_loans = A("filter", terms={"type": list(loan_type_mapping.keys())}) sum_face_value_loan = get_scaled_sum_aggregations( "face_value_loan_guarantee")["sum_field"] search.aggs.bucket("group_by_recipient", group_by_recipient) search.aggs["group_by_recipient"].metric("sum_obligation", sum_obligation) search.aggs["group_by_recipient"].bucket("filter_loans", filter_loans) search.aggs["group_by_recipient"]["filter_loans"].metric( "sum_face_value_loan", sum_face_value_loan) response = search.handle_execute() response_as_dict = response.aggs.to_dict() recipient_info_buckets = response_as_dict.get("group_by_recipient", {}).get("buckets", []) result_list = [] for bucket in recipient_info_buckets: result = {} if children: recipient_info = json_str_to_dict(bucket.get("key")) hash_with_level = recipient_info.get("hash_with_level") or None result = { "recipient_hash": hash_with_level[:-2] if hash_with_level else None, "recipient_unique_id": recipient_info.get("unique_id") or None, "recipient_name": recipient_info.get("name") or None, } loan_info = bucket.get("filter_loans", {}) result.update({ "total_obligation_amount": int(bucket.get("sum_obligation", {"value": 0})["value"]) / Decimal("100"), "total_obligation_count": bucket.get("doc_count", 0), "total_face_value_loan_amount": int(loan_info.get("sum_face_value_loan", {"value": 0})["value"]) / Decimal("100"), "total_face_value_loan_count": loan_info.get("doc_count", 0), }) result_list.append(result) return result_list