def filters(self):
all_def_codes = sorted(
DisasterEmergencyFundCode.objects.values_list("code", flat=True))
object_keys_lookup = {
"def_codes": {
"key": "filter|def_codes",
"name": "def_codes",
"type": "array",
"array_type": "enum",
"enum_values": all_def_codes,
"allow_nulls": False,
"optional": False,
},
"query": {
"key": "filter|query",
"name": "query",
"type": "text",
"text_type": "search",
"allow_nulls": True,
"optional": True,
},
"award_type_codes": {
"key": "filter|award_type_codes",
"name": "award_type_codes",
"type": "array",
"array_type": "enum",
"enum_values": sorted(award_type_mapping.keys()),
"allow_nulls": True,
"optional": True,
},
"_loan_award_type_codes": {
"key": "filter|award_type_codes",
"name": "award_type_codes",
"type": "array",
"array_type": "enum",
"enum_values": sorted(loan_type_mapping.keys()),
"allow_nulls": True,
"optional": True,
"default": list(loan_type_mapping.keys()),
},
"_assistance_award_type_codes": {
"key": "filter|award_type_codes",
"name": "award_type_codes",
"type": "array",
"array_type": "enum",
"enum_values": sorted(assistance_type_mapping.keys()),
"allow_nulls": True,
"optional": True,
"default": list(assistance_type_mapping.keys()),
},
}
model = [object_keys_lookup[key] for key in self.required_filters]
json_request = TinyShield(model).block(self.request.data)
return json_request["filter"]
def _test_correct_response_of_loans(client):
resp = post(
client,
award_type_codes=list(loan_type_mapping.keys()),
def_codes=["L", "M"],
geo_layer="county",
geo_layer_filters=["45001", "45005"],
spending_type="obligation",
)
expected_response = {
"geo_layer":
"county",
"scope":
"recipient_location",
"spending_type":
"obligation",
"results": [{
"amount": 220.0,
"display_name": "Charleston",
"per_capita": 220.0,
"population": 1,
"shape_code": "45001",
"award_count": 2,
}],
}
assert resp.status_code == status.HTTP_200_OK, "Failed to return 200 Response"
resp_json = resp.json()
resp_json["results"].sort(key=_get_shape_code_for_sort)
assert resp_json == expected_response
def _file_d_aggregation(self):
aggregations = {
"award_count": Count("award_id"),
"obligation": Coalesce(Sum("obligation"), 0),
"outlay": Coalesce(Sum("outlay"), 0),
}
if set(self.award_type_codes) <= set(loan_type_mapping.keys()):
aggregations["face_value_of_loan"] = Coalesce(
Sum("total_loan_value"), 0)
return (CovidFinancialAccountMatview.objects.annotate(
cast_def_codes=Cast("def_codes", ArrayField(TextField()))).filter(
type__in=self.award_type_codes,
cast_def_codes__overlap=self.def_codes).values(
"award_id").aggregate(**aggregations))
def validate_disaster_recipient_request(request_data):
_validate_required_parameters(request_data, ["filters"])
model = [
{
"key":
"filters|def_codes",
"name":
"def_codes",
"type":
"array",
"array_type":
"enum",
"enum_values":
sorted(
DisasterEmergencyFundCode.objects.values_list("code",
flat=True)),
"allow_nulls":
False,
"optional":
False,
},
{
"key": "filters|query",
"name": "query",
"type": "text",
"text_type": "search",
"allow_nulls": False,
"optional": True,
},
{
"key": "filters|award_type_codes",
"name": "award_type_codes",
"type": "array",
"array_type": "enum",
"enum_values": sorted(award_type_mapping.keys()),
"allow_nulls": False,
"optional": True,
},
]
filters = TinyShield(model).block(request_data)["filters"]
# Determine what to use in the filename based on "award_type_codes" filter;
# Also add "face_value_of_loans" column if only loan types
award_category = "All-Awards"
award_type_codes = set(
filters.get("award_type_codes", award_type_mapping.keys()))
columns = [
"recipient", "award_obligations", "award_outlays", "number_of_awards"
]
if award_type_codes <= set(contract_type_mapping.keys()):
award_category = "Contracts"
elif award_type_codes <= set(idv_type_mapping.keys()):
award_category = "Contract-IDVs"
elif award_type_codes <= set(grant_type_mapping.keys()):
award_category = "Grants"
elif award_type_codes <= set(loan_type_mapping.keys()):
award_category = "Loans"
columns.insert(3, "face_value_of_loans")
elif award_type_codes <= set(direct_payment_type_mapping.keys()):
award_category = "Direct-Payments"
elif award_type_codes <= set(other_type_mapping.keys()):
award_category = "Other-Financial-Assistance"
# Need to specify the field to use "query" filter on if present
query_text = filters.pop("query", None)
if query_text:
filters["query"] = {"text": query_text, "fields": ["recipient_name"]}
json_request = {
"award_category": award_category,
"columns": tuple(columns),
"download_types": ["disaster_recipient"],
"file_format": str(request_data.get("file_format", "csv")).lower(),
"filters": filters,
}
_validate_file_format(json_request)
return json_request
},
"direct_payments": {
"model": DirectPaymentAwardSearchMatview,
"types": set(direct_payment_type_mapping.keys())
},
"grants": {
"model": GrantAwardSearchMatview,
"types": set(grant_type_mapping.keys())
},
"idvs": {
"model": IDVAwardSearchMatview,
"types": set(idv_type_mapping.keys())
},
"loans": {
"model": LoanAwardSearchMatview,
"types": set(loan_type_mapping.keys())
},
"other": {
"model": OtherAwardSearchMatview,
"types": set(other_type_mapping.keys())
},
}
class AwardGroupsException(Exception):
"""Custom Exception for a specific event"""
class FiscalMonth(Func):
function = "EXTRACT"
template = "%(function)s(MONTH from (%(expressions)s) + INTERVAL '3 months')"
def obtain_recipient_totals(recipient_id, children=False, year="latest"):
"""Extract the total amount and transaction count for the recipient_hash given the time frame
Args:
recipient_id: string of hash(duns, name)-[recipient-level]
children: whether or not to group by children
year: the year the totals/counts are based on
Returns:
list of dictionaries representing hashes and their totals/counts
"""
filters = reshape_filters(recipient_id=recipient_id, year=year)
filter_query = QueryWithFilters.generate_transactions_elasticsearch_query(
filters)
search = TransactionSearch().filter(filter_query)
if children:
group_by_field = "recipient_agg_key"
elif recipient_id[-2:] == "-P":
group_by_field = "parent_recipient_hash"
else:
group_by_field = "recipient_hash"
bucket_count = get_number_of_unique_terms_for_transactions(
filter_query, f"{group_by_field}.hash")
if bucket_count == 0:
return []
# Not setting the shard_size since the number of child recipients under a
# parent recipient will not exceed 10k
group_by_recipient = A("terms", field=group_by_field, size=bucket_count)
sum_obligation = get_scaled_sum_aggregations(
"generated_pragmatic_obligation")["sum_field"]
filter_loans = A("filter", terms={"type": list(loan_type_mapping.keys())})
sum_face_value_loan = get_scaled_sum_aggregations(
"face_value_loan_guarantee")["sum_field"]
search.aggs.bucket("group_by_recipient", group_by_recipient)
search.aggs["group_by_recipient"].metric("sum_obligation", sum_obligation)
search.aggs["group_by_recipient"].bucket("filter_loans", filter_loans)
search.aggs["group_by_recipient"]["filter_loans"].metric(
"sum_face_value_loan", sum_face_value_loan)
response = search.handle_execute()
response_as_dict = response.aggs.to_dict()
recipient_info_buckets = response_as_dict.get("group_by_recipient",
{}).get("buckets", [])
result_list = []
for bucket in recipient_info_buckets:
result = {}
if children:
recipient_info = json.loads(bucket.get("key"))
hash_with_level = recipient_info.get("hash_with_level") or None
result = {
"recipient_hash":
hash_with_level[:-2] if hash_with_level else None,
"recipient_unique_id": recipient_info.get("unique_id"),
"recipient_name": recipient_info.get("name"),
}
loan_info = bucket.get("filter_loans", {})
result.update({
"total_obligation_amount":
int(bucket.get("sum_obligation", {"value": 0})["value"]) /
Decimal("100"),
"total_obligation_count":
bucket.get("doc_count", 0),
"total_face_value_loan_amount":
int(loan_info.get("sum_face_value_loan", {"value": 0})["value"]) /
Decimal("100"),
"total_face_value_loan_count":
loan_info.get("doc_count", 0),
})
result_list.append(result)
return result_list
def __init__(self, request_data: dict):
super().__init__(request_data)
self.tinyshield_models.extend([
{
"key":
"filters|def_codes",
"name":
"def_codes",
"type":
"array",
"array_type":
"enum",
"enum_values":
sorted(
DisasterEmergencyFundCode.objects.values_list("code",
flat=True)),
"allow_nulls":
False,
"optional":
False,
},
{
"key": "filters|query",
"name": "query",
"type": "text",
"text_type": "search",
"allow_nulls": False,
"optional": True,
},
{
"key": "filters|award_type_codes",
"name": "award_type_codes",
"type": "array",
"array_type": "enum",
"enum_values": sorted(award_type_mapping.keys()),
"allow_nulls": False,
"optional": True,
},
])
self._json_request["filters"] = request_data.get("filters")
self._json_request = self.get_validated_request()
self._json_request["download_types"] = [self.name]
# Determine what to use in the filename based on "award_type_codes" filter;
# Also add "face_value_of_loans" column if only loan types
award_category = "All-Awards"
award_type_codes = set(self._json_request["filters"].get(
"award_type_codes", award_type_mapping.keys()))
columns = [
"recipient", "award_obligations", "award_outlays",
"number_of_awards"
]
if award_type_codes <= set(contract_type_mapping.keys()):
award_category = "Contracts"
elif award_type_codes <= set(idv_type_mapping.keys()):
award_category = "Contract-IDVs"
elif award_type_codes <= set(grant_type_mapping.keys()):
award_category = "Grants"
elif award_type_codes <= set(loan_type_mapping.keys()):
award_category = "Loans"
columns.insert(3, "face_value_of_loans")
elif award_type_codes <= set(direct_payment_type_mapping.keys()):
award_category = "Direct-Payments"
elif award_type_codes <= set(other_type_mapping.keys()):
award_category = "Other-Financial-Assistance"
self._json_request["award_category"] = award_category
self._json_request["columns"] = self._json_request.get(
"columns") or tuple(columns)
# Need to specify the field to use "query" filter on if present
query_text = self._json_request["filters"].pop("query", None)
if query_text:
self._json_request["filters"]["query"] = {
"text": query_text,
"fields": ["recipient_name"]
}
idv_type_mapping,
loan_type_mapping,
other_type_mapping,
procurement_type_mapping,
)
TYPES_TO_QUOTE_IN_SQL = (str, date)
CATEGORY_TO_MODEL = {
"contracts": {"model": ContractAwardSearchMatview, "types": set(contract_type_mapping.keys())},
"direct_payments": {"model": DirectPaymentAwardSearchMatview, "types": set(direct_payment_type_mapping.keys())},
"grants": {"model": GrantAwardSearchMatview, "types": set(grant_type_mapping.keys())},
"idvs": {"model": IDVAwardSearchMatview, "types": set(idv_type_mapping.keys())},
"loans": {"model": LoanAwardSearchMatview, "types": set(loan_type_mapping.keys())},
"other": {"model": OtherAwardSearchMatview, "types": set(other_type_mapping.keys())},
}
class AwardGroupsException(Exception):
"""Custom Exception for a specific event"""
class FiscalMonth(Func):
function = "EXTRACT"
template = "%(function)s(MONTH from (%(expressions)s) + INTERVAL '3 months')"
output_field = IntegerField()
class FiscalQuarter(Func):