def get(self, request): self.sortable_columns = [ "toptier_code", "current_total_budget_authority_amount", "missing_tas_accounts_count", "tas_accounts_total", "agency_name", "obligation_difference", "recent_publication_date", "recent_publication_date_certified", "tas_obligation_not_in_gtas_total", "unlinked_contract_award_count", "unlinked_assistance_award_count", ] self.default_sort_column = "current_total_budget_authority_amount" results = self.get_agency_overview() page_metadata = get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page) results = results[self.pagination.lower_limit:self.pagination. upper_limit] return Response({ "page_metadata": page_metadata, "results": results, "messages": self.standard_response_messages })
def perform_elasticsearch_search(self, loans=False) -> Response: filters = {f"nested_{key}": val for key, val in self.filters.items() if key != "award_type_codes"} if self.filters.get("award_type_codes") is not None: filters["award_type_codes"] = self.filters["award_type_codes"] # Need to update the value of "query" to have the fields to search on query = filters.pop("nested_query", None) if query: filters["nested_query"] = {"text": query, "fields": self.query_fields} # Ensure that only non-zero values are taken into consideration filters["nested_nonzero_fields"] = list(self.nested_nonzero_fields.values()) self.filter_query = QueryWithFilters.generate_accounts_elasticsearch_query(filters) # using a set value here as doing an extra ES query is detrimental to performance # And the dimensions on which group-by aggregations are performed so far # (agency, TAS, object_class) all have cardinality less than this number # If the data increases to a point where there are more results than this, it should be changed self.bucket_count = 1000 messages = [] if self.pagination.sort_key in ("id", "code"): messages.append( ( f"Notice! API Request to sort on '{self.pagination.sort_key}' field isn't fully implemented." " Results were actually sorted using 'description' field." ) ) response = self.query_elasticsearch(loans) response["page_metadata"] = get_pagination_metadata( len(response["results"]), self.pagination.limit, self.pagination.page ) response["results"] = response["results"][self.pagination.lower_limit : self.pagination.upper_limit] if messages: response["messages"] = messages return Response(response)
def get(self, request: Request, *args: Any, **kwargs: Any) -> Response: request_data = self._parse_and_validate_request(request.query_params) toptier_agency = ToptierAgencyPublishedDABSView.objects.filter( toptier_code=self.toptier_code).first() if not toptier_agency: raise NotFound( f"Agency with a toptier code of '{self.toptier_code}' does not exist" ) request_data["toptier_agency"] = toptier_agency self.validate_fiscal_period(request_data) pagination = Pagination( page=request_data["page"], limit=request_data["limit"], lower_limit=(request_data["page"] - 1) * request_data["limit"], upper_limit=(request_data["page"] * request_data["limit"]), sort_key=request_data.get("sort", "tas"), sort_order=request_data["order"], ) results = self.get_differences_queryset(request_data) formatted_results = self.format_results(results, pagination) page_metadata = get_pagination_metadata(len(results), pagination.limit, pagination.page) return Response({ "page_metadata": page_metadata, "results": formatted_results[pagination.lower_limit:pagination.upper_limit], "messages": self.standard_response_messages, })
def post(self, request: Request) -> Response: request_data = self._parse_and_validate_request(request.data) results, overall_count = self._business_logic(request_data) page_metadata = get_pagination_metadata(overall_count, request_data['limit'], request_data['page']) response = OrderedDict((('results', results[:request_data['limit']]), ('page_metadata', page_metadata))) return Response(response)
def post(self, request): results = list( self.queryset.order_by(*self.pagination.robust_order_by_fields)) return Response({ "results": results[self.pagination.lower_limit:self.pagination.upper_limit], "page_metadata": get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page), })
def post(self, request: Request) -> Response: request_data = self._parse_and_validate_request(request.data) results, overall_count = self._business_logic(request_data) page_metadata = get_pagination_metadata(overall_count, request_data["limit"], request_data["page"]) response = OrderedDict((("results", results[:request_data["limit"]]), ("page_metadata", page_metadata))) return Response(response)
def construct_response(results: list, pagination: Pagination): FederalAccounts = FedAcctResults() for row in results: FA = FedAccount( id=row.pop("fa_id"), code=row.pop("fa_code"), award_count=0, description=row.pop("fa_description") ) FederalAccounts[FA].include(TAS(**row)) return { "results": FederalAccounts.finalize(pagination), "page_metadata": get_pagination_metadata(len(FederalAccounts), pagination.limit, pagination.page), }
def get_recipients(filters={}): lower_limit = (filters["page"] - 1) * filters["limit"] upper_limit = filters["page"] * filters["limit"] qs_filter = Q() if "keyword" in filters: qs_filter |= Q(recipient_name__contains=filters["keyword"].upper()) qs_filter |= Q(recipient_unique_id__contains=filters["keyword"]) amount_column = "last_12_months" if filters["award_type"] != "all": amount_column = AWARD_TYPES[filters["award_type"]]["amount"] qs_filter &= Q(award_types__overlap=[ AWARD_TYPES[filters["award_type"]]["filter"] ]) queryset = (RecipientProfile.objects.filter(qs_filter).values( "recipient_level", "recipient_hash", "recipient_unique_id", "recipient_name", amount_column).exclude(recipient_name__in=SPECIAL_CASES)) api_to_db_mapper = { "amount": amount_column, "duns": "recipient_unique_id", "name": "recipient_name" } if filters["order"] == "desc": queryset = queryset.order_by( F(api_to_db_mapper[filters["sort"]]).desc(nulls_last=True)) else: queryset = queryset.order_by( F(api_to_db_mapper[filters["sort"]]).asc(nulls_last=True)) count = queryset.count() page_metadata = get_pagination_metadata(count, filters["limit"], filters["page"]) results = [{ "id": "{}-{}".format(row["recipient_hash"], row["recipient_level"]), "duns": row["recipient_unique_id"], "name": row["recipient_name"], "recipient_level": row["recipient_level"], "amount": row[amount_column], } for row in queryset[lower_limit:upper_limit]] return results, page_metadata
def construct_response(results: list, pagination: Pagination, strip_total_budgetary_resources=True): object_classes = ObjectClassResults() for row in results: major_code = row.pop("major_code") major_class = MajorClass( id=major_code, code=major_code, award_count=0, description=row.pop("major_description") ) object_classes[major_class].include(ObjectClass(**row)) return { "results": object_classes.finalize(pagination, strip_total_budgetary_resources), "page_metadata": get_pagination_metadata(len(object_classes), pagination.limit, pagination.page), }
def get_recipients(filters={}, count=None): lower_limit = (filters["page"] - 1) * filters["limit"] upper_limit = filters["page"] * filters["limit"] amount_column = "last_12_months" if filters["award_type"] != "all": amount_column = AWARD_TYPES[filters["award_type"]]["amount"] qs_filter = build_duns_base_query(filters) queryset = (RecipientProfile.objects.filter(qs_filter).values( "recipient_level", "recipient_hash", "recipient_unique_id", "recipient_name", amount_column).exclude(recipient_name__in=SPECIAL_CASES)) api_to_db_mapper = { "amount": amount_column, "duns": "recipient_unique_id", "name": "recipient_name" } # Nulls Last isn't enabled for the amount sort because it prevents queries sorted by amount columns DESC # from using an index on those columns, even though they cannot contain nulls nulls_last = filters["sort"] in ["name", "duns"] if filters["order"] == "desc": queryset = queryset.order_by( F(api_to_db_mapper[filters["sort"]]).desc(nulls_last=nulls_last)) else: queryset = queryset.order_by( F(api_to_db_mapper[filters["sort"]]).asc(nulls_last=nulls_last)) if count is None: count = get_recipient_count(filters=filters) page_metadata = get_pagination_metadata(count, filters["limit"], filters["page"]) results = [{ "id": "{}-{}".format(row["recipient_hash"], row["recipient_level"]), "duns": row["recipient_unique_id"], "name": row["recipient_name"], "recipient_level": row["recipient_level"], "amount": row[amount_column], } for row in queryset[lower_limit:upper_limit]] return results, page_metadata
def post(self, request): results = list( self.queryset.order_by(*self.pagination.robust_order_by_fields)) return Response({ "totals": self.accumulate_total_values( results, ["award_count", "face_value_of_loan"]), "results": results[self.pagination.lower_limit:self.pagination.upper_limit], "page_metadata": get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page), })
def get(self, request: Request, *args: Any, **kwargs: Any) -> Response: results = list(self.get_program_activity_list()) page_metadata = get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page) results = results[self.pagination.lower_limit:self.pagination. upper_limit] return Response({ "toptier_code": self.toptier_code, "fiscal_year": self.fiscal_year, "page_metadata": page_metadata, "results": results[:self.pagination.limit], "messages": self.standard_response_messages, })
def get(self, request: Request, *args: Any, **kwargs: Any) -> Response: self.sortable_columns = ["name", "obligated_amount", "gross_outlay_amount"] self.default_sort_column = "obligated_amount" results = list(self.get_object_class_list()) page_metadata = get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page) results = results[self.pagination.lower_limit : self.pagination.upper_limit] return Response( { "toptier_code": self.toptier_code, "fiscal_year": self.fiscal_year, "page_metadata": page_metadata, "results": results[: self.pagination.limit], "messages": self.standard_response_messages, } )
def post(self, request): if self.spending_type == "award": results = self.award_queryset else: results = self.total_queryset results = list( results.order_by(*self.pagination.robust_order_by_fields)) return Response({ "results": results[self.pagination.lower_limit:self.pagination.upper_limit], "page_metadata": get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page), })
def get_recipients(filters={}): lower_limit = (filters["page"] - 1) * filters["limit"] upper_limit = filters["page"] * filters["limit"] qs_filter = Q() if "keyword" in filters: qs_filter |= Q(recipient_name__contains=filters["keyword"].upper()) qs_filter |= Q(recipient_unique_id__contains=filters["keyword"]) amount_column = "last_12_months" if filters["award_type"] != "all": amount_column = AWARD_TYPES[filters["award_type"]]["amount"] qs_filter &= Q(award_types__overlap=[AWARD_TYPES[filters["award_type"]]["filter"]]) queryset = ( RecipientProfile.objects.filter(qs_filter) .values("recipient_level", "recipient_hash", "recipient_unique_id", "recipient_name", amount_column) .exclude(recipient_name__in=SPECIAL_CASES) ) api_to_db_mapper = {"amount": amount_column, "duns": "recipient_unique_id", "name": "recipient_name"} if filters["order"] == "desc": queryset = queryset.order_by(F(api_to_db_mapper[filters["sort"]]).desc(nulls_last=True)) else: queryset = queryset.order_by(F(api_to_db_mapper[filters["sort"]]).asc(nulls_last=True)) count = queryset.count() page_metadata = get_pagination_metadata(count, filters["limit"], filters["page"]) results = [ { "id": "{}-{}".format(row["recipient_hash"], row["recipient_level"]), "duns": row["recipient_unique_id"], "name": row["recipient_name"], "recipient_level": row["recipient_level"], "amount": row[amount_column], } for row in queryset[lower_limit:upper_limit] ] return results, page_metadata
def post(self, request: Request) -> Response: # Need to update the value of "query" to have the fields to search on query = self.filters.pop("query", None) if query: self.filters["query"] = { "text": query, "fields": self.query_fields } self.filter_query = QueryWithFilters.generate_awards_elasticsearch_query( self.filters) # Ensure that only non-zero values are taken into consideration # TODO: Refactor to use new NonzeroFields filter in QueryWithFilters non_zero_queries = [] for field in self.sum_column_mapping.values(): non_zero_queries.append(ES_Q("range", **{field: {"gt": 0}})) non_zero_queries.append(ES_Q("range", **{field: {"lt": 0}})) self.filter_query.must.append( ES_Q("bool", should=non_zero_queries, minimum_should_match=1)) self.bucket_count = get_number_of_unique_terms_for_awards( self.filter_query, f"{self.agg_key.replace('.keyword', '')}.hash") messages = [] if self.pagination.sort_key in ("id", "code"): messages.append(( f"Notice! API Request to sort on '{self.pagination.sort_key}' field isn't fully implemented." " Results were actually sorted using 'description' field.")) if self.bucket_count > 10000 and self.agg_key == settings.ES_ROUTING_FIELD: self.bucket_count = 10000 messages.append(( "Notice! API Request is capped at 10,000 results. Either download to view all results or" " filter using the 'query' attribute.")) response = self.query_elasticsearch() response["page_metadata"] = get_pagination_metadata( self.bucket_count, self.pagination.limit, self.pagination.page) if messages: response["messages"] = messages return Response(response)
def post(self, request): if self.spending_type == "award": results = self.award_queryset extra_columns = ["award_count"] else: results = self.total_queryset extra_columns = ["total_budgetary_resources"] results = list(results.order_by(*self.pagination.robust_order_by_fields)) for item in results: # we're checking for items that do not have an agency profile page if item.get("link") is not None: if not item["link"]: item["id"] = None # if they don't have a page (means they have no submission), we don't send the id item.pop("link") return Response( { "totals": self.accumulate_total_values(results, extra_columns), "results": results[self.pagination.lower_limit : self.pagination.upper_limit], "page_metadata": get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page), } )
def get(self, request, toptier_code): model = [ { "key": "fiscal_year", "name": "fiscal_year", "type": "integer", "min": 2017, "optional": False, "default": None, "allow_nulls": False, }, { "key": "fiscal_period", "name": "fiscal_period", "type": "integer", "min": 2, "max": 12, "optional": False, "default": None, "allow_nulls": False, }, ] validated = TinyShield(model).block(request.query_params) self.sortable_columns = [ "amount", "tas", ] self.default_sort_column = "amount" results = self.get_agency_discrepancies( fiscal_year=validated["fiscal_year"], fiscal_period=validated["fiscal_period"]) return Response({ "page_metadata": get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page), "results": results[self.pagination.lower_limit:self.pagination.upper_limit], "messages": self.standard_response_messages, })
def get(self, request): if "publication_date" in self.pagination.sort_key: self.validate_publication_sort(self.pagination.sort_key) sort_key = deepcopy(self.pagination.sort_key) # we get the index of the periods by subtracting 2, since we index from 0 and have no period 1 pub_sort = int(sort_key.split(",")[1]) - 2 self.pagination.sort_key = "publication_date" results = sorted( self.get_agency_data(), key=lambda x: x["periods"][pub_sort]["submission_dates"]["publication_date"], reverse=(self.pagination.sort_order == "desc"), ) else: results = sorted( self.get_agency_data(), key=lambda x: x[self.pagination.sort_key], reverse=(self.pagination.sort_order == "desc"), ) page_metadata = get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page) results = results[self.pagination.lower_limit : self.pagination.upper_limit] return Response( {"page_metadata": page_metadata, "results": results, "messages": self.standard_response_messages} )
def get(self, request, toptier_code): self.sortable_columns = [ "current_total_budget_authority_amount", "fiscal_year", "missing_tas_accounts_count", "tas_accounts_total", "obligation_difference", "percent_of_total_budgetary_resources", "recent_publication_date", "recent_publication_date_certified", "tas_obligation_not_in_gtas_total", "unlinked_contract_award_count", "unlinked_assistance_award_count", ] self.default_sort_column = "current_total_budget_authority_amount" if self.pagination.sort_key in ("recent_publication_date"): self.sort_value_when_null = datetime(2000, 1, 1, tzinfo=timezone.utc) elif self.pagination.sort_key in ("recent_publication_date_certified"): self.sort_value_when_null = False else: self.sort_value_when_null = -sys.maxsize - 1 # greatest negative int value in Python results = self.get_agency_overview() page_metadata = get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page) results = results[self.pagination.lower_limit:self.pagination. upper_limit] return Response({ "page_metadata": page_metadata, "results": results, "messages": self.standard_response_messages })
def get(self, request, toptier_code, fiscal_year, fiscal_period): self.fiscal_year = int(fiscal_year) self.sortable_columns = ["publication_date", "certification_date"] self.default_sort_column = "publication_date" self.validate_fiscal_period({"fiscal_period": int(fiscal_period)}) record = list( SubmissionAttributes.objects.filter( toptier_code=toptier_code, reporting_fiscal_year=fiscal_year, reporting_fiscal_period=fiscal_period, submission_window_id__submission_reveal_date__lte=now(), ).values_list("history", flat=True) ) if len(record) == 0: raise NoDataFoundException("No Agency Account Submission History records match the provided parameters") # Convoluted list comprehension and sort to # A) construct the dict list # B) add secondary sort key and handle nulls in `certification_date` for sorting results = sorted( [ {"publication_date": row["published_date"], "certification_date": row["certified_date"]} for row in record[0] ], key=lambda x: x["publication_date"] if self.pagination.sort_key == "publication_date" else (x["certification_date"] or "", x["publication_date"]), reverse=self.pagination.sort_order == "desc", ) page_metadata = get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page) results = results[self.pagination.lower_limit : self.pagination.upper_limit] return Response( {"page_metadata": page_metadata, "results": results, "messages": self.standard_response_messages} )