def get(self, request):
self.sortable_columns = [
"toptier_code",
"current_total_budget_authority_amount",
"missing_tas_accounts_count",
"tas_accounts_total",
"agency_name",
"obligation_difference",
"recent_publication_date",
"recent_publication_date_certified",
"tas_obligation_not_in_gtas_total",
"unlinked_contract_award_count",
"unlinked_assistance_award_count",
]
self.default_sort_column = "current_total_budget_authority_amount"
results = self.get_agency_overview()
page_metadata = get_pagination_metadata(len(results),
self.pagination.limit,
self.pagination.page)
results = results[self.pagination.lower_limit:self.pagination.
upper_limit]
return Response({
"page_metadata": page_metadata,
"results": results,
"messages": self.standard_response_messages
})
def perform_elasticsearch_search(self, loans=False) -> Response:
filters = {f"nested_{key}": val for key, val in self.filters.items() if key != "award_type_codes"}
if self.filters.get("award_type_codes") is not None:
filters["award_type_codes"] = self.filters["award_type_codes"]
# Need to update the value of "query" to have the fields to search on
query = filters.pop("nested_query", None)
if query:
filters["nested_query"] = {"text": query, "fields": self.query_fields}
# Ensure that only non-zero values are taken into consideration
filters["nested_nonzero_fields"] = list(self.nested_nonzero_fields.values())
self.filter_query = QueryWithFilters.generate_accounts_elasticsearch_query(filters)
# using a set value here as doing an extra ES query is detrimental to performance
# And the dimensions on which group-by aggregations are performed so far
# (agency, TAS, object_class) all have cardinality less than this number
# If the data increases to a point where there are more results than this, it should be changed
self.bucket_count = 1000
messages = []
if self.pagination.sort_key in ("id", "code"):
messages.append(
(
f"Notice! API Request to sort on '{self.pagination.sort_key}' field isn't fully implemented."
" Results were actually sorted using 'description' field."
)
)
response = self.query_elasticsearch(loans)
response["page_metadata"] = get_pagination_metadata(
len(response["results"]), self.pagination.limit, self.pagination.page
)
response["results"] = response["results"][self.pagination.lower_limit : self.pagination.upper_limit]
if messages:
response["messages"] = messages
return Response(response)
def get(self, request: Request, *args: Any, **kwargs: Any) -> Response:
request_data = self._parse_and_validate_request(request.query_params)
toptier_agency = ToptierAgencyPublishedDABSView.objects.filter(
toptier_code=self.toptier_code).first()
if not toptier_agency:
raise NotFound(
f"Agency with a toptier code of '{self.toptier_code}' does not exist"
)
request_data["toptier_agency"] = toptier_agency
self.validate_fiscal_period(request_data)
pagination = Pagination(
page=request_data["page"],
limit=request_data["limit"],
lower_limit=(request_data["page"] - 1) * request_data["limit"],
upper_limit=(request_data["page"] * request_data["limit"]),
sort_key=request_data.get("sort", "tas"),
sort_order=request_data["order"],
)
results = self.get_differences_queryset(request_data)
formatted_results = self.format_results(results, pagination)
page_metadata = get_pagination_metadata(len(results), pagination.limit,
pagination.page)
return Response({
"page_metadata":
page_metadata,
"results":
formatted_results[pagination.lower_limit:pagination.upper_limit],
"messages":
self.standard_response_messages,
})
def post(self, request: Request) -> Response:
request_data = self._parse_and_validate_request(request.data)
results, overall_count = self._business_logic(request_data)
page_metadata = get_pagination_metadata(overall_count,
request_data['limit'],
request_data['page'])
response = OrderedDict((('results', results[:request_data['limit']]),
('page_metadata', page_metadata)))
return Response(response)
def post(self, request):
results = list(
self.queryset.order_by(*self.pagination.robust_order_by_fields))
return Response({
"results":
results[self.pagination.lower_limit:self.pagination.upper_limit],
"page_metadata":
get_pagination_metadata(len(results), self.pagination.limit,
self.pagination.page),
})
def post(self, request: Request) -> Response:
request_data = self._parse_and_validate_request(request.data)
results, overall_count = self._business_logic(request_data)
page_metadata = get_pagination_metadata(overall_count,
request_data["limit"],
request_data["page"])
response = OrderedDict((("results", results[:request_data["limit"]]),
("page_metadata", page_metadata)))
return Response(response)
def construct_response(results: list, pagination: Pagination):
FederalAccounts = FedAcctResults()
for row in results:
FA = FedAccount(
id=row.pop("fa_id"), code=row.pop("fa_code"), award_count=0, description=row.pop("fa_description")
)
FederalAccounts[FA].include(TAS(**row))
return {
"results": FederalAccounts.finalize(pagination),
"page_metadata": get_pagination_metadata(len(FederalAccounts), pagination.limit, pagination.page),
}
def get_recipients(filters={}):
lower_limit = (filters["page"] - 1) * filters["limit"]
upper_limit = filters["page"] * filters["limit"]
qs_filter = Q()
if "keyword" in filters:
qs_filter |= Q(recipient_name__contains=filters["keyword"].upper())
qs_filter |= Q(recipient_unique_id__contains=filters["keyword"])
amount_column = "last_12_months"
if filters["award_type"] != "all":
amount_column = AWARD_TYPES[filters["award_type"]]["amount"]
qs_filter &= Q(award_types__overlap=[
AWARD_TYPES[filters["award_type"]]["filter"]
])
queryset = (RecipientProfile.objects.filter(qs_filter).values(
"recipient_level", "recipient_hash", "recipient_unique_id",
"recipient_name",
amount_column).exclude(recipient_name__in=SPECIAL_CASES))
api_to_db_mapper = {
"amount": amount_column,
"duns": "recipient_unique_id",
"name": "recipient_name"
}
if filters["order"] == "desc":
queryset = queryset.order_by(
F(api_to_db_mapper[filters["sort"]]).desc(nulls_last=True))
else:
queryset = queryset.order_by(
F(api_to_db_mapper[filters["sort"]]).asc(nulls_last=True))
count = queryset.count()
page_metadata = get_pagination_metadata(count, filters["limit"],
filters["page"])
results = [{
"id":
"{}-{}".format(row["recipient_hash"], row["recipient_level"]),
"duns":
row["recipient_unique_id"],
"name":
row["recipient_name"],
"recipient_level":
row["recipient_level"],
"amount":
row[amount_column],
} for row in queryset[lower_limit:upper_limit]]
return results, page_metadata
def construct_response(results: list, pagination: Pagination, strip_total_budgetary_resources=True):
object_classes = ObjectClassResults()
for row in results:
major_code = row.pop("major_code")
major_class = MajorClass(
id=major_code, code=major_code, award_count=0, description=row.pop("major_description")
)
object_classes[major_class].include(ObjectClass(**row))
return {
"results": object_classes.finalize(pagination, strip_total_budgetary_resources),
"page_metadata": get_pagination_metadata(len(object_classes), pagination.limit, pagination.page),
}
def get_recipients(filters={}, count=None):
lower_limit = (filters["page"] - 1) * filters["limit"]
upper_limit = filters["page"] * filters["limit"]
amount_column = "last_12_months"
if filters["award_type"] != "all":
amount_column = AWARD_TYPES[filters["award_type"]]["amount"]
qs_filter = build_duns_base_query(filters)
queryset = (RecipientProfile.objects.filter(qs_filter).values(
"recipient_level", "recipient_hash", "recipient_unique_id",
"recipient_name",
amount_column).exclude(recipient_name__in=SPECIAL_CASES))
api_to_db_mapper = {
"amount": amount_column,
"duns": "recipient_unique_id",
"name": "recipient_name"
}
# Nulls Last isn't enabled for the amount sort because it prevents queries sorted by amount columns DESC
# from using an index on those columns, even though they cannot contain nulls
nulls_last = filters["sort"] in ["name", "duns"]
if filters["order"] == "desc":
queryset = queryset.order_by(
F(api_to_db_mapper[filters["sort"]]).desc(nulls_last=nulls_last))
else:
queryset = queryset.order_by(
F(api_to_db_mapper[filters["sort"]]).asc(nulls_last=nulls_last))
if count is None:
count = get_recipient_count(filters=filters)
page_metadata = get_pagination_metadata(count, filters["limit"],
filters["page"])
results = [{
"id":
"{}-{}".format(row["recipient_hash"], row["recipient_level"]),
"duns":
row["recipient_unique_id"],
"name":
row["recipient_name"],
"recipient_level":
row["recipient_level"],
"amount":
row[amount_column],
} for row in queryset[lower_limit:upper_limit]]
return results, page_metadata
def post(self, request):
results = list(
self.queryset.order_by(*self.pagination.robust_order_by_fields))
return Response({
"totals":
self.accumulate_total_values(
results, ["award_count", "face_value_of_loan"]),
"results":
results[self.pagination.lower_limit:self.pagination.upper_limit],
"page_metadata":
get_pagination_metadata(len(results), self.pagination.limit,
self.pagination.page),
})
def get(self, request: Request, *args: Any, **kwargs: Any) -> Response:
results = list(self.get_program_activity_list())
page_metadata = get_pagination_metadata(len(results),
self.pagination.limit,
self.pagination.page)
results = results[self.pagination.lower_limit:self.pagination.
upper_limit]
return Response({
"toptier_code": self.toptier_code,
"fiscal_year": self.fiscal_year,
"page_metadata": page_metadata,
"results": results[:self.pagination.limit],
"messages": self.standard_response_messages,
})
def get(self, request: Request, *args: Any, **kwargs: Any) -> Response:
self.sortable_columns = ["name", "obligated_amount", "gross_outlay_amount"]
self.default_sort_column = "obligated_amount"
results = list(self.get_object_class_list())
page_metadata = get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page)
results = results[self.pagination.lower_limit : self.pagination.upper_limit]
return Response(
{
"toptier_code": self.toptier_code,
"fiscal_year": self.fiscal_year,
"page_metadata": page_metadata,
"results": results[: self.pagination.limit],
"messages": self.standard_response_messages,
}
)
def post(self, request):
if self.spending_type == "award":
results = self.award_queryset
else:
results = self.total_queryset
results = list(
results.order_by(*self.pagination.robust_order_by_fields))
return Response({
"results":
results[self.pagination.lower_limit:self.pagination.upper_limit],
"page_metadata":
get_pagination_metadata(len(results), self.pagination.limit,
self.pagination.page),
})
def get_recipients(filters={}):
lower_limit = (filters["page"] - 1) * filters["limit"]
upper_limit = filters["page"] * filters["limit"]
qs_filter = Q()
if "keyword" in filters:
qs_filter |= Q(recipient_name__contains=filters["keyword"].upper())
qs_filter |= Q(recipient_unique_id__contains=filters["keyword"])
amount_column = "last_12_months"
if filters["award_type"] != "all":
amount_column = AWARD_TYPES[filters["award_type"]]["amount"]
qs_filter &= Q(award_types__overlap=[AWARD_TYPES[filters["award_type"]]["filter"]])
queryset = (
RecipientProfile.objects.filter(qs_filter)
.values("recipient_level", "recipient_hash", "recipient_unique_id", "recipient_name", amount_column)
.exclude(recipient_name__in=SPECIAL_CASES)
)
api_to_db_mapper = {"amount": amount_column, "duns": "recipient_unique_id", "name": "recipient_name"}
if filters["order"] == "desc":
queryset = queryset.order_by(F(api_to_db_mapper[filters["sort"]]).desc(nulls_last=True))
else:
queryset = queryset.order_by(F(api_to_db_mapper[filters["sort"]]).asc(nulls_last=True))
count = queryset.count()
page_metadata = get_pagination_metadata(count, filters["limit"], filters["page"])
results = [
{
"id": "{}-{}".format(row["recipient_hash"], row["recipient_level"]),
"duns": row["recipient_unique_id"],
"name": row["recipient_name"],
"recipient_level": row["recipient_level"],
"amount": row[amount_column],
}
for row in queryset[lower_limit:upper_limit]
]
return results, page_metadata
def post(self, request: Request) -> Response:
# Need to update the value of "query" to have the fields to search on
query = self.filters.pop("query", None)
if query:
self.filters["query"] = {
"text": query,
"fields": self.query_fields
}
self.filter_query = QueryWithFilters.generate_awards_elasticsearch_query(
self.filters)
# Ensure that only non-zero values are taken into consideration
# TODO: Refactor to use new NonzeroFields filter in QueryWithFilters
non_zero_queries = []
for field in self.sum_column_mapping.values():
non_zero_queries.append(ES_Q("range", **{field: {"gt": 0}}))
non_zero_queries.append(ES_Q("range", **{field: {"lt": 0}}))
self.filter_query.must.append(
ES_Q("bool", should=non_zero_queries, minimum_should_match=1))
self.bucket_count = get_number_of_unique_terms_for_awards(
self.filter_query, f"{self.agg_key.replace('.keyword', '')}.hash")
messages = []
if self.pagination.sort_key in ("id", "code"):
messages.append((
f"Notice! API Request to sort on '{self.pagination.sort_key}' field isn't fully implemented."
" Results were actually sorted using 'description' field."))
if self.bucket_count > 10000 and self.agg_key == settings.ES_ROUTING_FIELD:
self.bucket_count = 10000
messages.append((
"Notice! API Request is capped at 10,000 results. Either download to view all results or"
" filter using the 'query' attribute."))
response = self.query_elasticsearch()
response["page_metadata"] = get_pagination_metadata(
self.bucket_count, self.pagination.limit, self.pagination.page)
if messages:
response["messages"] = messages
return Response(response)
def post(self, request):
if self.spending_type == "award":
results = self.award_queryset
extra_columns = ["award_count"]
else:
results = self.total_queryset
extra_columns = ["total_budgetary_resources"]
results = list(results.order_by(*self.pagination.robust_order_by_fields))
for item in results: # we're checking for items that do not have an agency profile page
if item.get("link") is not None:
if not item["link"]:
item["id"] = None # if they don't have a page (means they have no submission), we don't send the id
item.pop("link")
return Response(
{
"totals": self.accumulate_total_values(results, extra_columns),
"results": results[self.pagination.lower_limit : self.pagination.upper_limit],
"page_metadata": get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page),
}
)
def get(self, request, toptier_code):
model = [
{
"key": "fiscal_year",
"name": "fiscal_year",
"type": "integer",
"min": 2017,
"optional": False,
"default": None,
"allow_nulls": False,
},
{
"key": "fiscal_period",
"name": "fiscal_period",
"type": "integer",
"min": 2,
"max": 12,
"optional": False,
"default": None,
"allow_nulls": False,
},
]
validated = TinyShield(model).block(request.query_params)
self.sortable_columns = [
"amount",
"tas",
]
self.default_sort_column = "amount"
results = self.get_agency_discrepancies(
fiscal_year=validated["fiscal_year"],
fiscal_period=validated["fiscal_period"])
return Response({
"page_metadata":
get_pagination_metadata(len(results), self.pagination.limit,
self.pagination.page),
"results":
results[self.pagination.lower_limit:self.pagination.upper_limit],
"messages":
self.standard_response_messages,
})
def get(self, request):
if "publication_date" in self.pagination.sort_key:
self.validate_publication_sort(self.pagination.sort_key)
sort_key = deepcopy(self.pagination.sort_key)
# we get the index of the periods by subtracting 2, since we index from 0 and have no period 1
pub_sort = int(sort_key.split(",")[1]) - 2
self.pagination.sort_key = "publication_date"
results = sorted(
self.get_agency_data(),
key=lambda x: x["periods"][pub_sort]["submission_dates"]["publication_date"],
reverse=(self.pagination.sort_order == "desc"),
)
else:
results = sorted(
self.get_agency_data(),
key=lambda x: x[self.pagination.sort_key],
reverse=(self.pagination.sort_order == "desc"),
)
page_metadata = get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page)
results = results[self.pagination.lower_limit : self.pagination.upper_limit]
return Response(
{"page_metadata": page_metadata, "results": results, "messages": self.standard_response_messages}
)
def get(self, request, toptier_code):
self.sortable_columns = [
"current_total_budget_authority_amount",
"fiscal_year",
"missing_tas_accounts_count",
"tas_accounts_total",
"obligation_difference",
"percent_of_total_budgetary_resources",
"recent_publication_date",
"recent_publication_date_certified",
"tas_obligation_not_in_gtas_total",
"unlinked_contract_award_count",
"unlinked_assistance_award_count",
]
self.default_sort_column = "current_total_budget_authority_amount"
if self.pagination.sort_key in ("recent_publication_date"):
self.sort_value_when_null = datetime(2000,
1,
1,
tzinfo=timezone.utc)
elif self.pagination.sort_key in ("recent_publication_date_certified"):
self.sort_value_when_null = False
else:
self.sort_value_when_null = -sys.maxsize - 1 # greatest negative int value in Python
results = self.get_agency_overview()
page_metadata = get_pagination_metadata(len(results),
self.pagination.limit,
self.pagination.page)
results = results[self.pagination.lower_limit:self.pagination.
upper_limit]
return Response({
"page_metadata": page_metadata,
"results": results,
"messages": self.standard_response_messages
})
def get(self, request, toptier_code, fiscal_year, fiscal_period):
self.fiscal_year = int(fiscal_year)
self.sortable_columns = ["publication_date", "certification_date"]
self.default_sort_column = "publication_date"
self.validate_fiscal_period({"fiscal_period": int(fiscal_period)})
record = list(
SubmissionAttributes.objects.filter(
toptier_code=toptier_code,
reporting_fiscal_year=fiscal_year,
reporting_fiscal_period=fiscal_period,
submission_window_id__submission_reveal_date__lte=now(),
).values_list("history", flat=True)
)
if len(record) == 0:
raise NoDataFoundException("No Agency Account Submission History records match the provided parameters")
# Convoluted list comprehension and sort to
# A) construct the dict list
# B) add secondary sort key and handle nulls in `certification_date` for sorting
results = sorted(
[
{"publication_date": row["published_date"], "certification_date": row["certified_date"]}
for row in record[0]
],
key=lambda x: x["publication_date"]
if self.pagination.sort_key == "publication_date"
else (x["certification_date"] or "", x["publication_date"]),
reverse=self.pagination.sort_order == "desc",
)
page_metadata = get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page)
results = results[self.pagination.lower_limit : self.pagination.upper_limit]
return Response(
{"page_metadata": page_metadata, "results": results, "messages": self.standard_response_messages}
)
