def post(self, request: Request) -> Response: self.original_filters = request.data.get("filters") json_request = self.validate_request_data(request.data) self.group = GROUPING_LOOKUP[json_request["group"]] self.subawards = json_request["subawards"] self.filters = json_request["filters"] # time_period is optional so we're setting a default window from API_SEARCH_MIN_DATE to end of the current FY. # Otherwise, users will see blank results for years current_fy = generate_fiscal_year(datetime.now(timezone.utc)) if self.group == "fiscal_year": end_date = "{}-09-30".format(current_fy) else: current_fiscal_month = generate_fiscal_month(datetime.now(timezone.utc)) days_in_month = monthrange(current_fy, current_fiscal_month)[1] end_date = f"{current_fy}-{current_fiscal_month}-{days_in_month}" default_time_period = {"start_date": settings.API_SEARCH_MIN_DATE, "end_date": end_date} time_periods = self.filters.get("time_period", [default_time_period]) if self.subawards: db_results, values = self.database_data_layer() results = bolster_missing_time_periods( filter_time_periods=time_periods, queryset=db_results, date_range_type=values[-1], columns={"aggregated_amount": "aggregated_amount"}, ) else: results = self.query_elasticsearch(time_periods) return Response( OrderedDict( [ ("group", self.group), ("results", results), ( "messages", get_generic_filters_message( self.original_filters.keys(), [elem["name"] for elem in AWARD_FILTER] ), ), ] ) )
def post(self, request): """Returns boolean of whether a download request is greater than the max limit. """ models = [{ "name": "subawards", "key": "subawards", "type": "boolean", "default": False }] models.extend(copy.deepcopy(AWARD_FILTER)) self.original_filters = request.data.get("filters") json_request = TinyShield(models).block(request.data) # If no filters in request return empty object to return all transactions filters = json_request.get("filters", {}) if json_request["subawards"]: total_count = subaward_filter(filters).count() else: queryset, model = download_transaction_count(filters) if model in ["UniversalTransactionView"]: total_count = queryset.count() else: # "summary" materialized views are pre-aggregated and contain a counts col total_count = queryset.aggregate( total_count=Sum("counts"))["total_count"] if total_count is None: total_count = 0 result = { "calculated_transaction_count": total_count, "maximum_transaction_limit": settings.MAX_DOWNLOAD_LIMIT, "transaction_rows_gt_limit": total_count > settings.MAX_DOWNLOAD_LIMIT, "messages": [ get_generic_filters_message( self.original_filters.keys(), [elem["name"] for elem in AWARD_FILTER]) ], } return Response(result)
def post(self, request): """Returns boolean of whether a download request is greater than the max limit. """ models = [{ "name": "subawards", "key": "subawards", "type": "boolean", "default": False }] models.extend(copy.deepcopy(AWARD_FILTER)) self.original_filters = request.data.get("filters") json_request = TinyShield(models).block(request.data) # If no filters in request return empty object to return all transactions filters = json_request.get("filters", {}) if json_request["subawards"]: total_count = subaward_filter(filters).count() else: filter_query = QueryWithFilters.generate_transactions_elasticsearch_query( filters) search = TransactionSearch().filter(filter_query) total_count = search.handle_count() if total_count is None: total_count = 0 result = { "calculated_transaction_count": total_count, "maximum_transaction_limit": settings.MAX_DOWNLOAD_LIMIT, "transaction_rows_gt_limit": total_count > settings.MAX_DOWNLOAD_LIMIT, "messages": [ get_generic_filters_message( self.original_filters.keys(), [elem["name"] for elem in AWARD_FILTER]) ], } return Response(result)
def post(self, request: Request) -> Response: models = [ { "name": "subawards", "key": "subawards", "type": "boolean", "default": False }, { "name": "scope", "key": "scope", "type": "enum", "optional": False, "enum_values": ["place_of_performance", "recipient_location"], }, { "name": "geo_layer", "key": "geo_layer", "type": "enum", "optional": False, "enum_values": ["state", "county", "district"], }, { "name": "geo_layer_filters", "key": "geo_layer_filters", "type": "array", "array_type": "text", "text_type": "search", }, ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) original_filters = request.data.get("filters") json_request = TinyShield(models).block(request.data) agg_key_dict = { "county": "county_agg_key", "district": "congressional_agg_key", "state": "state_agg_key", } location_dict = { "county": "county_code", "district": "congressional_code", "state": "state_code" } model_dict = { "place_of_performance": "pop", "recipient_location": "recipient_location", # 'subawards_place_of_performance': 'pop', # 'subawards_recipient_location': 'recipient_location' } self.scope_field_name = model_dict[json_request["scope"]] self.agg_key = f"{self.scope_field_name}_{agg_key_dict[json_request['geo_layer']]}" self.filters = json_request.get("filters") self.geo_layer = GeoLayer(json_request["geo_layer"]) self.geo_layer_filters = json_request.get("geo_layer_filters") self.loc_field_name = location_dict[self.geo_layer.value] self.loc_lookup = f"{self.scope_field_name}_{self.loc_field_name}" self.subawards = json_request["subawards"] if self.subawards: # We do not use matviews for Subaward filtering, just the Subaward download filters self.model_name = SubawardView self.queryset = subaward_filter(self.filters) self.obligation_column = "amount" result = self.query_django() elif is_experimental_elasticsearch_api(request): if self.scope_field_name == "pop": scope_filter_name = "place_of_performance_scope" else: scope_filter_name = "recipient_scope" # Only search for values within USA, but don't overwrite a user's search if scope_filter_name not in self.filters: self.filters[scope_filter_name] = "domestic" self.obligation_column = "generated_pragmatic_obligation" filter_query = QueryWithFilters.generate_transactions_elasticsearch_query( self.filters) result = self.query_elasticsearch(filter_query) else: self.queryset, self.model_name = spending_by_geography( self.filters) self.obligation_column = "generated_pragmatic_obligation" result = self.query_django() return Response({ "scope": json_request["scope"], "geo_layer": self.geo_layer.value, "results": result, "messages": get_generic_filters_message( original_filters.keys(), [elem["name"] for elem in AWARD_FILTER]), })
def construct_es_response(self, response) -> dict: results = [] for res in response: hit = res.to_dict() row = { k: hit[v] for k, v in self.constants["internal_id_fields"].items() } # Parsing API response values from ES query result JSON # We parse the `hit` (result from elasticsearch) to get the award type, use the type to determine # which lookup dict to use, and then use that lookup to retrieve the correct value requested from `fields` for field in self.fields: row[field] = hit.get( self.constants["elasticsearch_type_code_to_field_map"][hit[ self.constants["award_semaphore"]]].get(field)) row["internal_id"] = int(row["internal_id"]) if row.get("Loan Value"): row["Loan Value"] = float(row["Loan Value"]) if row.get("Subsidy Cost"): row["Subsidy Cost"] = float(row["Subsidy Cost"]) if row.get("Award Amount"): row["Award Amount"] = float(row["Award Amount"]) row["generated_internal_id"] = hit["generated_unique_award_id"] row["recipient_id"] = hit.get("recipient_unique_id") row["parent_recipient_unique_id"] = hit.get( "parent_recipient_unique_id") if "Award ID" in self.fields: row["Award ID"] = hit["display_award_id"] row = self.append_recipient_hash_level(row) row.pop("parent_recipient_unique_id") results.append(row) last_record_unique_id = None last_record_sort_value = None offset = 1 if self.last_record_unique_id is not None: has_next = len(results) > self.pagination["limit"] offset = 2 else: has_next = ( response.hits.total.value - (self.pagination["page"] - 1) * self.pagination["limit"] > self.pagination["limit"]) if len(response) > 0 and has_next: last_record_unique_id = response[len(response) - offset].meta.sort[1] last_record_sort_value = response[len(response) - offset].meta.sort[0] return { "limit": self.pagination["limit"], "results": results[:self.pagination["limit"]], "page_metadata": { "page": self.pagination["page"], "hasNext": has_next, "last_record_unique_id": last_record_unique_id, "last_record_sort_value": str(last_record_sort_value), }, "messages": [ get_generic_filters_message( self.original_filters.keys(), [elem["name"] for elem in AWARD_FILTER_NO_RECIPIENT_ID]) ], }
def post(self, request): models = [ { "name": "subawards", "key": "subawards", "type": "boolean", "default": False }, { "name": "object_class", "key": "filter|object_class", "type": "array", "array_type": "text", "text_type": "search", }, { "name": "program_activity", "key": "filter|program_activity", "type": "array", "array_type": "integer", "array_max": maxsize, }, ] models.extend(copy.deepcopy(AWARD_FILTER_NO_RECIPIENT_ID)) models.extend(copy.deepcopy(PAGINATION)) self.original_filters = request.data.get("filters") json_request = TinyShield(models).block(request.data) subawards = json_request["subawards"] filters = add_date_range_comparison_types(json_request.get( "filters", None), subawards, gte_date_type="action_date", lte_date_type="date_signed") elasticsearch = is_experimental_elasticsearch_api(request) if not elasticsearch: mirror_request_to_elasticsearch(request) if elasticsearch and not subawards: logger.info( "Using experimental Elasticsearch functionality for 'spending_by_award_count'" ) results = self.query_elasticsearch(filters) return Response({ "results": results, "messages": get_generic_filters_message( self.original_filters.keys(), [elem["name"] for elem in AWARD_FILTER_NO_RECIPIENT_ID]), }) if filters is None: raise InvalidParameterException( "Missing required request parameters: 'filters'") empty_results = { "contracts": 0, "idvs": 0, "grants": 0, "direct_payments": 0, "loans": 0, "other": 0 } if subawards: empty_results = {"subcontracts": 0, "subgrants": 0} if "award_type_codes" in filters and "no intersection" in filters[ "award_type_codes"]: # "Special case": there will never be results when the website provides this value return Response({"results": empty_results}) if subawards: results = self.handle_subawards(filters) else: results = self.handle_awards(filters, empty_results) return Response({ "results": results, "messages": get_generic_filters_message( self.original_filters.keys(), [elem["name"] for elem in AWARD_FILTER_NO_RECIPIENT_ID]), })
def _get_messages(original_filters) -> List: if original_filters: return get_generic_filters_message(original_filters.keys(), [elem["name"] for elem in AWARD_FILTER]) else: return get_generic_filters_message(set(), [elem["name"] for elem in AWARD_FILTER])
def post(self, request: Request) -> Response: models = [ { "key": "geo_layer", "name": "geo_layer", "type": "enum", "enum_values": ["state", "county", "district"], "text_type": "search", "allow_nulls": False, "optional": False, }, { "key": "geo_layer_filters", "name": "geo_layer_filters", "type": "array", "array_type": "text", "text_type": "search", }, { "key": "spending_type", "name": "spending_type", "type": "enum", "enum_values": ["obligation", "outlay", "face_value_of_loan"], "allow_nulls": False, "optional": False, }, ] # NOTE: filter object in request handled in base class: see self.filters json_request = TinyShield(models).block(request.data) agg_key_dict = { "county": "county_agg_key", "district": "congressional_agg_key", "state": "state_agg_key", } location_dict = {"county": "county_code", "district": "congressional_code", "state": "state_code"} self.agg_key = f"{self.scope_field_name}_{agg_key_dict[json_request['geo_layer']]}" self.geo_layer = GeoLayer(json_request["geo_layer"]) self.geo_layer_filters = json_request.get("geo_layer_filters") self.spending_type = json_request.get("spending_type") self.loc_field_name = location_dict[self.geo_layer.value] self.loc_lookup = f"{self.scope_field_name}_{self.loc_field_name}" # Set which field will be the aggregation amount if self.spending_type == "obligation": self.metric_field = "total_covid_obligation" elif self.spending_type == "outlay": self.metric_field = "total_covid_outlay" elif self.spending_type == "face_value_of_loan": self.metric_field = "total_loan_value" else: raise UnprocessableEntityException( f"Unrecognized value '{self.spending_type}' for field " f"'spending_type'" ) filter_query = QueryWithFilters.generate_awards_elasticsearch_query(self.filters) result = self.query_elasticsearch(filter_query) return Response( { "geo_layer": self.geo_layer.value, "spending_type": self.spending_type, "results": result, "messages": get_generic_filters_message(self.filters.keys(), self.required_filters), } )