def post(self, request: Request) -> Response:
self.original_filters = request.data.get("filters")
json_request = self.validate_request_data(request.data)
self.group = GROUPING_LOOKUP[json_request["group"]]
self.subawards = json_request["subawards"]
self.filters = json_request["filters"]
# time_period is optional so we're setting a default window from API_SEARCH_MIN_DATE to end of the current FY.
# Otherwise, users will see blank results for years
current_fy = generate_fiscal_year(datetime.now(timezone.utc))
if self.group == "fiscal_year":
end_date = "{}-09-30".format(current_fy)
else:
current_fiscal_month = generate_fiscal_month(datetime.now(timezone.utc))
days_in_month = monthrange(current_fy, current_fiscal_month)[1]
end_date = f"{current_fy}-{current_fiscal_month}-{days_in_month}"
default_time_period = {"start_date": settings.API_SEARCH_MIN_DATE, "end_date": end_date}
time_periods = self.filters.get("time_period", [default_time_period])
if self.subawards:
db_results, values = self.database_data_layer()
results = bolster_missing_time_periods(
filter_time_periods=time_periods,
queryset=db_results,
date_range_type=values[-1],
columns={"aggregated_amount": "aggregated_amount"},
)
else:
results = self.query_elasticsearch(time_periods)
return Response(
OrderedDict(
[
("group", self.group),
("results", results),
(
"messages",
get_generic_filters_message(
self.original_filters.keys(), [elem["name"] for elem in AWARD_FILTER]
),
),
]
)
)
def post(self, request):
"""Returns boolean of whether a download request is greater than the max limit. """
models = [{
"name": "subawards",
"key": "subawards",
"type": "boolean",
"default": False
}]
models.extend(copy.deepcopy(AWARD_FILTER))
self.original_filters = request.data.get("filters")
json_request = TinyShield(models).block(request.data)
# If no filters in request return empty object to return all transactions
filters = json_request.get("filters", {})
if json_request["subawards"]:
total_count = subaward_filter(filters).count()
else:
queryset, model = download_transaction_count(filters)
if model in ["UniversalTransactionView"]:
total_count = queryset.count()
else:
# "summary" materialized views are pre-aggregated and contain a counts col
total_count = queryset.aggregate(
total_count=Sum("counts"))["total_count"]
if total_count is None:
total_count = 0
result = {
"calculated_transaction_count":
total_count,
"maximum_transaction_limit":
settings.MAX_DOWNLOAD_LIMIT,
"transaction_rows_gt_limit":
total_count > settings.MAX_DOWNLOAD_LIMIT,
"messages": [
get_generic_filters_message(
self.original_filters.keys(),
[elem["name"] for elem in AWARD_FILTER])
],
}
return Response(result)
def post(self, request):
"""Returns boolean of whether a download request is greater than the max limit. """
models = [{
"name": "subawards",
"key": "subawards",
"type": "boolean",
"default": False
}]
models.extend(copy.deepcopy(AWARD_FILTER))
self.original_filters = request.data.get("filters")
json_request = TinyShield(models).block(request.data)
# If no filters in request return empty object to return all transactions
filters = json_request.get("filters", {})
if json_request["subawards"]:
total_count = subaward_filter(filters).count()
else:
filter_query = QueryWithFilters.generate_transactions_elasticsearch_query(
filters)
search = TransactionSearch().filter(filter_query)
total_count = search.handle_count()
if total_count is None:
total_count = 0
result = {
"calculated_transaction_count":
total_count,
"maximum_transaction_limit":
settings.MAX_DOWNLOAD_LIMIT,
"transaction_rows_gt_limit":
total_count > settings.MAX_DOWNLOAD_LIMIT,
"messages": [
get_generic_filters_message(
self.original_filters.keys(),
[elem["name"] for elem in AWARD_FILTER])
],
}
return Response(result)
def post(self, request: Request) -> Response:
models = [
{
"name": "subawards",
"key": "subawards",
"type": "boolean",
"default": False
},
{
"name": "scope",
"key": "scope",
"type": "enum",
"optional": False,
"enum_values": ["place_of_performance", "recipient_location"],
},
{
"name": "geo_layer",
"key": "geo_layer",
"type": "enum",
"optional": False,
"enum_values": ["state", "county", "district"],
},
{
"name": "geo_layer_filters",
"key": "geo_layer_filters",
"type": "array",
"array_type": "text",
"text_type": "search",
},
]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
original_filters = request.data.get("filters")
json_request = TinyShield(models).block(request.data)
agg_key_dict = {
"county": "county_agg_key",
"district": "congressional_agg_key",
"state": "state_agg_key",
}
location_dict = {
"county": "county_code",
"district": "congressional_code",
"state": "state_code"
}
model_dict = {
"place_of_performance": "pop",
"recipient_location": "recipient_location",
# 'subawards_place_of_performance': 'pop',
# 'subawards_recipient_location': 'recipient_location'
}
self.scope_field_name = model_dict[json_request["scope"]]
self.agg_key = f"{self.scope_field_name}_{agg_key_dict[json_request['geo_layer']]}"
self.filters = json_request.get("filters")
self.geo_layer = GeoLayer(json_request["geo_layer"])
self.geo_layer_filters = json_request.get("geo_layer_filters")
self.loc_field_name = location_dict[self.geo_layer.value]
self.loc_lookup = f"{self.scope_field_name}_{self.loc_field_name}"
self.subawards = json_request["subawards"]
if self.subawards:
# We do not use matviews for Subaward filtering, just the Subaward download filters
self.model_name = SubawardView
self.queryset = subaward_filter(self.filters)
self.obligation_column = "amount"
result = self.query_django()
elif is_experimental_elasticsearch_api(request):
if self.scope_field_name == "pop":
scope_filter_name = "place_of_performance_scope"
else:
scope_filter_name = "recipient_scope"
# Only search for values within USA, but don't overwrite a user's search
if scope_filter_name not in self.filters:
self.filters[scope_filter_name] = "domestic"
self.obligation_column = "generated_pragmatic_obligation"
filter_query = QueryWithFilters.generate_transactions_elasticsearch_query(
self.filters)
result = self.query_elasticsearch(filter_query)
else:
self.queryset, self.model_name = spending_by_geography(
self.filters)
self.obligation_column = "generated_pragmatic_obligation"
result = self.query_django()
return Response({
"scope":
json_request["scope"],
"geo_layer":
self.geo_layer.value,
"results":
result,
"messages":
get_generic_filters_message(
original_filters.keys(),
[elem["name"] for elem in AWARD_FILTER]),
})
def construct_es_response(self, response) -> dict:
results = []
for res in response:
hit = res.to_dict()
row = {
k: hit[v]
for k, v in self.constants["internal_id_fields"].items()
}
# Parsing API response values from ES query result JSON
# We parse the `hit` (result from elasticsearch) to get the award type, use the type to determine
# which lookup dict to use, and then use that lookup to retrieve the correct value requested from `fields`
for field in self.fields:
row[field] = hit.get(
self.constants["elasticsearch_type_code_to_field_map"][hit[
self.constants["award_semaphore"]]].get(field))
row["internal_id"] = int(row["internal_id"])
if row.get("Loan Value"):
row["Loan Value"] = float(row["Loan Value"])
if row.get("Subsidy Cost"):
row["Subsidy Cost"] = float(row["Subsidy Cost"])
if row.get("Award Amount"):
row["Award Amount"] = float(row["Award Amount"])
row["generated_internal_id"] = hit["generated_unique_award_id"]
row["recipient_id"] = hit.get("recipient_unique_id")
row["parent_recipient_unique_id"] = hit.get(
"parent_recipient_unique_id")
if "Award ID" in self.fields:
row["Award ID"] = hit["display_award_id"]
row = self.append_recipient_hash_level(row)
row.pop("parent_recipient_unique_id")
results.append(row)
last_record_unique_id = None
last_record_sort_value = None
offset = 1
if self.last_record_unique_id is not None:
has_next = len(results) > self.pagination["limit"]
offset = 2
else:
has_next = (
response.hits.total.value -
(self.pagination["page"] - 1) * self.pagination["limit"] >
self.pagination["limit"])
if len(response) > 0 and has_next:
last_record_unique_id = response[len(response) -
offset].meta.sort[1]
last_record_sort_value = response[len(response) -
offset].meta.sort[0]
return {
"limit":
self.pagination["limit"],
"results":
results[:self.pagination["limit"]],
"page_metadata": {
"page": self.pagination["page"],
"hasNext": has_next,
"last_record_unique_id": last_record_unique_id,
"last_record_sort_value": str(last_record_sort_value),
},
"messages": [
get_generic_filters_message(
self.original_filters.keys(),
[elem["name"] for elem in AWARD_FILTER_NO_RECIPIENT_ID])
],
}
def post(self, request):
models = [
{
"name": "subawards",
"key": "subawards",
"type": "boolean",
"default": False
},
{
"name": "object_class",
"key": "filter|object_class",
"type": "array",
"array_type": "text",
"text_type": "search",
},
{
"name": "program_activity",
"key": "filter|program_activity",
"type": "array",
"array_type": "integer",
"array_max": maxsize,
},
]
models.extend(copy.deepcopy(AWARD_FILTER_NO_RECIPIENT_ID))
models.extend(copy.deepcopy(PAGINATION))
self.original_filters = request.data.get("filters")
json_request = TinyShield(models).block(request.data)
subawards = json_request["subawards"]
filters = add_date_range_comparison_types(json_request.get(
"filters", None),
subawards,
gte_date_type="action_date",
lte_date_type="date_signed")
elasticsearch = is_experimental_elasticsearch_api(request)
if not elasticsearch:
mirror_request_to_elasticsearch(request)
if elasticsearch and not subawards:
logger.info(
"Using experimental Elasticsearch functionality for 'spending_by_award_count'"
)
results = self.query_elasticsearch(filters)
return Response({
"results":
results,
"messages":
get_generic_filters_message(
self.original_filters.keys(),
[elem["name"] for elem in AWARD_FILTER_NO_RECIPIENT_ID]),
})
if filters is None:
raise InvalidParameterException(
"Missing required request parameters: 'filters'")
empty_results = {
"contracts": 0,
"idvs": 0,
"grants": 0,
"direct_payments": 0,
"loans": 0,
"other": 0
}
if subawards:
empty_results = {"subcontracts": 0, "subgrants": 0}
if "award_type_codes" in filters and "no intersection" in filters[
"award_type_codes"]:
# "Special case": there will never be results when the website provides this value
return Response({"results": empty_results})
if subawards:
results = self.handle_subawards(filters)
else:
results = self.handle_awards(filters, empty_results)
return Response({
"results":
results,
"messages":
get_generic_filters_message(
self.original_filters.keys(),
[elem["name"] for elem in AWARD_FILTER_NO_RECIPIENT_ID]),
})
def _get_messages(original_filters) -> List:
if original_filters:
return get_generic_filters_message(original_filters.keys(), [elem["name"] for elem in AWARD_FILTER])
else:
return get_generic_filters_message(set(), [elem["name"] for elem in AWARD_FILTER])
def post(self, request: Request) -> Response:
models = [
{
"key": "geo_layer",
"name": "geo_layer",
"type": "enum",
"enum_values": ["state", "county", "district"],
"text_type": "search",
"allow_nulls": False,
"optional": False,
},
{
"key": "geo_layer_filters",
"name": "geo_layer_filters",
"type": "array",
"array_type": "text",
"text_type": "search",
},
{
"key": "spending_type",
"name": "spending_type",
"type": "enum",
"enum_values": ["obligation", "outlay", "face_value_of_loan"],
"allow_nulls": False,
"optional": False,
},
]
# NOTE: filter object in request handled in base class: see self.filters
json_request = TinyShield(models).block(request.data)
agg_key_dict = {
"county": "county_agg_key",
"district": "congressional_agg_key",
"state": "state_agg_key",
}
location_dict = {"county": "county_code", "district": "congressional_code", "state": "state_code"}
self.agg_key = f"{self.scope_field_name}_{agg_key_dict[json_request['geo_layer']]}"
self.geo_layer = GeoLayer(json_request["geo_layer"])
self.geo_layer_filters = json_request.get("geo_layer_filters")
self.spending_type = json_request.get("spending_type")
self.loc_field_name = location_dict[self.geo_layer.value]
self.loc_lookup = f"{self.scope_field_name}_{self.loc_field_name}"
# Set which field will be the aggregation amount
if self.spending_type == "obligation":
self.metric_field = "total_covid_obligation"
elif self.spending_type == "outlay":
self.metric_field = "total_covid_outlay"
elif self.spending_type == "face_value_of_loan":
self.metric_field = "total_loan_value"
else:
raise UnprocessableEntityException(
f"Unrecognized value '{self.spending_type}' for field " f"'spending_type'"
)
filter_query = QueryWithFilters.generate_awards_elasticsearch_query(self.filters)
result = self.query_elasticsearch(filter_query)
return Response(
{
"geo_layer": self.geo_layer.value,
"spending_type": self.spending_type,
"results": result,
"messages": get_generic_filters_message(self.filters.keys(), self.required_filters),
}
)
