def test_any_rule(): models = [{ 'name': 'value', 'key': 'value', 'type': 'any', 'models': [ {'type': 'integer'}, {'type': 'text', 'text_type': 'search'} ] }] # Test integer and random other key. ts = TinyShield(models).block({'value': 1, 'another_value': 2}) assert ts['value'] == 1 assert ts.get('another_value') is None # Test integer masquerading as a string. ts = TinyShield(models).block({'value': '1'}) assert ts['value'] == 1 # Test string. ts = TinyShield(models).block({'value': 'XYZ'}) assert ts['value'] == 'XYZ' # Test list (which should blow up). with pytest.raises(UnprocessableEntityException): TinyShield(models).block({'value': ['XYZ']}) # Test with optional 'value' missing. ts = TinyShield(models).block({'another_value': 2}) assert ts.get('value') is None assert ts.get('another_value') is None # Make 'value' required then run the same tests as above. models[0]['optional'] = False # Test integer and random other key. ts = TinyShield(models).block({'value': 1, 'another_value': 2}) assert ts['value'] == 1 assert ts.get('another_value') is None # Test integer masquerading as a string. ts = TinyShield(models).block({'value': '1'}) assert ts['value'] == 1 # Test string. ts = TinyShield(models).block({'value': 'XYZ'}) assert ts['value'] == 'XYZ' # Test list (which should blow up). with pytest.raises(UnprocessableEntityException): TinyShield(models).block({'value': ['XYZ']}) # Test with required 'value' missing. with pytest.raises(UnprocessableEntityException): TinyShield(models).block({'another_value': 2})
def get(self, request): model = [ { "key": "fiscal_year", "name": "fiscal_year", "type": "integer", "min": 2017, "optional": True, "default": None, "allow_nulls": True, }, { "key": "fiscal_period", "name": "fiscal_period", "type": "integer", "min": 2, "max": 12, "optional": True, "default": None, "allow_nulls": True, }, ] validated = TinyShield(model).block(request.query_params) fiscal_year = validated.get("fiscal_year", None) fiscal_period = validated.get("fiscal_period", None) gtas_queryset = GTASSF133Balances.objects.values( "fiscal_year", "fiscal_period") if fiscal_period: if not fiscal_year: raise InvalidParameterException( "fiscal_period was provided without fiscal_year.") else: gtas_queryset = gtas_queryset.filter( fiscal_year=fiscal_year, fiscal_period=fiscal_period) elif fiscal_year: gtas_queryset = gtas_queryset.filter(fiscal_year=fiscal_year) results = gtas_queryset.annotate(total_budgetary_resources=Sum( "total_budgetary_resources_cpe")).order_by("-fiscal_year", "-fiscal_period") return Response({ "results": list(results), "messages": [get_account_data_time_period_message()] if not fiscal_year or fiscal_year < 2017 else [], })
def validate_request_data(self, json_data): self.groupings = { "quarter": "quarter", "q": "quarter", "fiscal_year": "fiscal_year", "fy": "fiscal_year", "month": "month", "m": "month", } models = [ {"name": "subawards", "key": "subawards", "type": "boolean", "default": False}, { "name": "group", "key": "group", "type": "enum", "enum_values": list(self.groupings.keys()), "default": "fy", "optional": False, # allow to be optional in the future }, ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) validated_data = TinyShield(models).block(json_data) if validated_data.get("filters", None) is None: raise InvalidParameterException("Missing request parameters: filters") return validated_data
def validate_request_data(self, json_data): self.groupings = { "quarter": "quarter", "q": "quarter", "fiscal_year": "fiscal_year", "fy": "fiscal_year", "month": "month", "m": "month", } models = [ { "name": "subawards", "key": "subawards", "type": "boolean", "default": False }, { "name": "group", "key": "group", "type": "enum", "enum_values": list(self.groupings.keys()), "default": "fy", "optional": False, # allow to be optional in the future }, ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) validated_data = TinyShield(models).block(json_data) if validated_data.get("filters", None) is None: raise InvalidParameterException( "Missing request parameters: filters") return validated_data
def validate_request_data(json_data: dict) -> dict: models = [ { "name": "subawards", "key": "subawards", "type": "boolean", "default": False }, { "name": "group", "key": "group", "type": "enum", "enum_values": list(GROUPING_LOOKUP.keys()), "default": "fy", "optional": False, # allow to be optional in the future }, ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) validated_data = TinyShield(models).block(json_data) if validated_data.get("filters", None) is None: raise InvalidParameterException( "Missing request parameters: filters") return validated_data
def post(self, request): """Returns boolean of whether a download request is greater than the max limit. """ models = [ {'name': 'subawards', 'key': 'subawards', 'type': 'boolean', 'default': False}, ] models.extend(copy.deepcopy(AWARD_FILTER)) json_request = TinyShield(models).block(request.data) # If no filters in request return empty object to return all transactions filters = json_request.get('filters', {}) is_over_limit = False if json_request['subawards']: total_count = subaward_filter(filters).count() else: queryset, model = download_transaction_count(filters) if model in ['UniversalTransactionView']: total_count = queryset.count() else: # "summary" materialized views are pre-aggregated and contain a counts col total_count = queryset.aggregate(total_count=Sum('counts'))['total_count'] if total_count and total_count > settings.MAX_DOWNLOAD_LIMIT: is_over_limit = True result = { "transaction_rows_gt_limit": is_over_limit } return Response(result)
def post(self, request): models = [{'name': 'subawards', 'key': 'subawards', 'type': 'boolean', 'default': False}] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) json_request = TinyShield(models).block(request.data) filters = json_request.get("filters", None) subawards = json_request["subawards"] if filters is None: raise InvalidParameterException("Missing required request parameters: 'filters'") results = { "contracts": 0, "idvs": 0, "grants": 0, "direct_payments": 0, "loans": 0, "other": 0 } if not subawards else { "subcontracts": 0, "subgrants": 0 } if "award_type_codes" in filters and "no intersection" in filters["award_type_codes"]: # "Special case": there will never be results when the website provides this value return Response({"results": results}) if subawards: queryset = subaward_filter(filters) else: queryset, model = spending_by_award_count(filters) if subawards: queryset = queryset.values('award_type').annotate(category_count=Count('subaward_id')) elif model == 'SummaryAwardView': queryset = queryset.values('category').annotate(category_count=Sum('counts')) else: queryset = queryset.values('category').annotate(category_count=Count('category')) categories = { 'contract': 'contracts', 'idv': 'idvs', 'grant': 'grants', 'direct payment': 'direct_payments', 'loans': 'loans', 'other': 'other' } if not subawards else {'procurement': 'subcontracts', 'grant': 'subgrants'} category_name = 'category' if not subawards else 'award_type' # DB hit here for award in queryset: if award[category_name] is None: result_key = 'other' if not subawards else 'subcontracts' elif award[category_name] not in categories.keys(): result_key = 'other' else: result_key = categories[award[category_name]] results[result_key] += award['category_count'] return Response({"results": results})
def post(self, request): models = [ {"name": "subawards", "key": "subawards", "type": "boolean", "default": False}, { "name": "object_class", "key": "filter|object_class", "type": "array", "array_type": "text", "text_type": "search", }, { "name": "program_activity", "key": "filter|program_activity", "type": "array", "array_type": "integer", "array_max": maxsize, }, ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) json_request = TinyShield(models).block(request.data) subawards = json_request["subawards"] filters = add_date_range_comparison_types( json_request.get("filters", None), subawards, gte_date_type="action_date", lte_date_type="date_signed" ) elasticsearch = is_experimental_elasticsearch_api(request) if elasticsearch and not subawards: logger.info("Using experimental Elasticsearch functionality for 'spending_by_award_count'") results = self.query_elasticsearch(filters) return Response({"results": results, "messages": [get_time_period_message()]}) if filters is None: raise InvalidParameterException("Missing required request parameters: 'filters'") empty_results = {"contracts": 0, "idvs": 0, "grants": 0, "direct_payments": 0, "loans": 0, "other": 0} if subawards: empty_results = {"subcontracts": 0, "subgrants": 0} if "award_type_codes" in filters and "no intersection" in filters["award_type_codes"]: # "Special case": there will never be results when the website provides this value return Response({"results": empty_results}) if subawards: results = self.handle_subawards(filters) else: results = self.handle_awards(filters, empty_results) return Response({"results": results, "messages": [get_time_period_message()]})
def post(self, request): models = [ {"name": "subawards", "key": "subawards", "type": "boolean", "default": False}, { "name": "object_class", "key": "filter|object_class", "type": "array", "array_type": "text", "text_type": "search", }, { "name": "program_activity", "key": "filter|program_activity", "type": "array", "array_type": "integer", "array_max": maxsize, }, ] models.extend(copy.deepcopy(AWARD_FILTER_NO_RECIPIENT_ID)) models.extend(copy.deepcopy(PAGINATION)) self.original_filters = request.data.get("filters") json_request = TinyShield(models).block(request.data) subawards = json_request["subawards"] filters = add_date_range_comparison_types( json_request.get("filters", None), subawards, gte_date_type="action_date", lte_date_type="date_signed" ) if filters is None: raise InvalidParameterException("Missing required request parameters: 'filters'") if "award_type_codes" in filters and "no intersection" in filters["award_type_codes"]: # "Special case": there will never be results when the website provides this value empty_results = {"contracts": 0, "idvs": 0, "grants": 0, "direct_payments": 0, "loans": 0, "other": 0} if subawards: empty_results = {"subcontracts": 0, "subgrants": 0} results = empty_results elif subawards: results = self.handle_subawards(filters) else: results = self.query_elasticsearch_for_prime_awards(filters) return Response( { "results": results, "messages": get_generic_filters_message( self.original_filters.keys(), [elem["name"] for elem in AWARD_FILTER_NO_RECIPIENT_ID] ), } )
def post(self, request): """Returns boolean of whether a download request is greater than the max limit. """ models = [{ "name": "subawards", "key": "subawards", "type": "boolean", "default": False }] models.extend(copy.deepcopy(AWARD_FILTER)) self.original_filters = request.data.get("filters") json_request = TinyShield(models).block(request.data) # If no filters in request return empty object to return all transactions filters = json_request.get("filters", {}) if json_request["subawards"]: total_count = subaward_filter(filters).count() else: queryset, model = download_transaction_count(filters) if model in ["UniversalTransactionView"]: total_count = queryset.count() else: # "summary" materialized views are pre-aggregated and contain a counts col total_count = queryset.aggregate( total_count=Sum("counts"))["total_count"] if total_count is None: total_count = 0 result = { "calculated_transaction_count": total_count, "maximum_transaction_limit": settings.MAX_DOWNLOAD_LIMIT, "transaction_rows_gt_limit": total_count > settings.MAX_DOWNLOAD_LIMIT, "messages": [ get_generic_filters_message( self.original_filters.keys(), [elem["name"] for elem in AWARD_FILTER]) ], } return Response(result)
def post(self, request): models = [{ "name": "subawards", "key": "subawards", "type": "boolean", "default": False }] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) json_request = TinyShield(models).block(request.data) subawards = json_request["subawards"] filters = add_date_range_comparison_types(json_request.get( "filters", None), subawards, gte_date_type="action_date", lte_date_type="date_signed") if filters is None: raise InvalidParameterException( "Missing required request parameters: 'filters'") empty_results = { "contracts": 0, "idvs": 0, "grants": 0, "direct_payments": 0, "loans": 0, "other": 0 } if subawards: empty_results = {"subcontracts": 0, "subgrants": 0} if "award_type_codes" in filters and "no intersection" in filters[ "award_type_codes"]: # "Special case": there will never be results when the website provides this value return Response({"results": empty_results}) if subawards: results = self.handle_subawards(filters) else: results = self.handle_awards(filters, empty_results) return Response({"results": results})
def post(self, request): """Returns boolean of whether a download request is greater than the max limit. """ models = [{ "name": "subawards", "key": "subawards", "type": "boolean", "default": False }] models.extend(copy.deepcopy(AWARD_FILTER)) self.original_filters = request.data.get("filters") json_request = TinyShield(models).block(request.data) # If no filters in request return empty object to return all transactions filters = json_request.get("filters", {}) if json_request["subawards"]: total_count = subaward_filter(filters).count() else: filter_query = QueryWithFilters.generate_transactions_elasticsearch_query( filters) search = TransactionSearch().filter(filter_query) total_count = search.handle_count() if total_count is None: total_count = 0 result = { "calculated_transaction_count": total_count, "maximum_transaction_limit": settings.MAX_DOWNLOAD_LIMIT, "transaction_rows_gt_limit": total_count > settings.MAX_DOWNLOAD_LIMIT, "messages": [ get_generic_filters_message( self.original_filters.keys(), [elem["name"] for elem in AWARD_FILTER]) ], } return Response(result)
def test_any_rule(): models = [{ "name": "value", "key": "value", "type": "any", "models": [{ "type": "integer" }, { "type": "text", "text_type": "search" }], }] # Test integer and random other key. ts = TinyShield(models).block({"value": 1, "another_value": 2}) assert ts["value"] == 1 assert ts.get("another_value") is None # Test integer masquerading as a string. ts = TinyShield(models).block({"value": "1"}) assert ts["value"] == 1 # Test string. ts = TinyShield(models).block({"value": "XYZ"}) assert ts["value"] == "XYZ" # Test list (which should blow up). with pytest.raises(UnprocessableEntityException): TinyShield(models).block({"value": ["XYZ"]}) # Test with optional 'value' missing. ts = TinyShield(models).block({"another_value": 2}) assert ts.get("value") is None assert ts.get("another_value") is None # Make 'value' required then run the same tests as above. models[0]["optional"] = False # Test integer and random other key. ts = TinyShield(models).block({"value": 1, "another_value": 2}) assert ts["value"] == 1 assert ts.get("another_value") is None # Test integer masquerading as a string. ts = TinyShield(models).block({"value": "1"}) assert ts["value"] == 1 # Test string. ts = TinyShield(models).block({"value": "XYZ"}) assert ts["value"] == "XYZ" # Test list (which should blow up). with pytest.raises(UnprocessableEntityException): TinyShield(models).block({"value": ["XYZ"]}) # Test with required 'value' missing. with pytest.raises(UnprocessableEntityException): TinyShield(models).block({"another_value": 2})
def validate_disaster_recipient_request(request_data): _validate_required_parameters(request_data, ["filters"]) model = [ { "key": "filters|def_codes", "name": "def_codes", "type": "array", "array_type": "enum", "enum_values": sorted( DisasterEmergencyFundCode.objects.values_list("code", flat=True)), "allow_nulls": False, "optional": False, }, { "key": "filters|query", "name": "query", "type": "text", "text_type": "search", "allow_nulls": False, "optional": True, }, { "key": "filters|award_type_codes", "name": "award_type_codes", "type": "array", "array_type": "enum", "enum_values": sorted(award_type_mapping.keys()), "allow_nulls": False, "optional": True, }, ] filters = TinyShield(model).block(request_data)["filters"] # Determine what to use in the filename based on "award_type_codes" filter; # Also add "face_value_of_loans" column if only loan types award_category = "All-Awards" award_type_codes = set( filters.get("award_type_codes", award_type_mapping.keys())) columns = [ "recipient", "award_obligations", "award_outlays", "number_of_awards" ] if award_type_codes <= set(contract_type_mapping.keys()): award_category = "Contracts" elif award_type_codes <= set(idv_type_mapping.keys()): award_category = "Contract-IDVs" elif award_type_codes <= set(grant_type_mapping.keys()): award_category = "Grants" elif award_type_codes <= set(loan_type_mapping.keys()): award_category = "Loans" columns.insert(3, "face_value_of_loans") elif award_type_codes <= set(direct_payment_type_mapping.keys()): award_category = "Direct-Payments" elif award_type_codes <= set(other_type_mapping.keys()): award_category = "Other-Financial-Assistance" # Need to specify the field to use "query" filter on if present query_text = filters.pop("query", None) if query_text: filters["query"] = {"text": query_text, "fields": ["recipient_name"]} json_request = { "award_category": award_category, "columns": tuple(columns), "download_types": ["disaster_recipient"], "file_format": str(request_data.get("file_format", "csv")).lower(), "filters": filters, } _validate_file_format(json_request) return json_request
def post(self, request): models = [ {"name": "subawards", "key": "subawards", "type": "boolean", "default": False}, { "name": "scope", "key": "scope", "type": "enum", "optional": False, "enum_values": ["place_of_performance", "recipient_location"], }, { "name": "geo_layer", "key": "geo_layer", "type": "enum", "optional": False, "enum_values": ["state", "county", "district"], }, { "name": "geo_layer_filters", "key": "geo_layer_filters", "type": "array", "array_type": "text", "text_type": "search", }, ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) json_request = TinyShield(models).block(request.data) self.subawards = json_request["subawards"] self.scope = json_request["scope"] self.filters = json_request.get("filters", None) self.geo_layer = json_request["geo_layer"] self.geo_layer_filters = json_request.get("geo_layer_filters", None) fields_list = [] # fields to include in the aggregate query loc_dict = {"state": "state_code", "county": "county_code", "district": "congressional_code"} model_dict = { "place_of_performance": "pop", "recipient_location": "recipient_location", # 'subawards_place_of_performance': 'pop', # 'subawards_recipient_location': 'recipient_location' } # Build the query based on the scope fields and geo_layers # Fields not in the reference objects above then request is invalid scope_field_name = model_dict.get(self.scope) loc_field_name = loc_dict.get(self.geo_layer) loc_lookup = "{}_{}".format(scope_field_name, loc_field_name) if self.subawards: # We do not use matviews for Subaward filtering, just the Subaward download filters self.queryset = subaward_filter(self.filters) self.model_name = SubawardView else: self.queryset, self.model_name = spending_by_geography(self.filters) if self.geo_layer == "state": # State will have one field (state_code) containing letter A-Z column_isnull = "generated_pragmatic_obligation__isnull" if self.subawards: column_isnull = "amount__isnull" kwargs = {"{}_country_code".format(scope_field_name): "USA", column_isnull: False} # Only state scope will add its own state code # State codes are consistent in database i.e. AL, AK fields_list.append(loc_lookup) state_response = { "scope": self.scope, "geo_layer": self.geo_layer, "results": self.state_results(kwargs, fields_list, loc_lookup), "messages": [get_time_period_message()], } return Response(state_response) else: # County and district scope will need to select multiple fields # State code is needed for county/district aggregation state_lookup = "{}_{}".format(scope_field_name, loc_dict["state"]) fields_list.append(state_lookup) # Adding regex to county/district codes to remove entries with letters since can't be surfaced by map kwargs = {"{}__isnull".format("amount" if self.subawards else "generated_pragmatic_obligation"): False} if self.geo_layer == "county": # County name added to aggregation since consistent in db county_name_lookup = "{}_county_name".format(scope_field_name) fields_list.append(county_name_lookup) self.county_district_queryset(kwargs, fields_list, loc_lookup, state_lookup, scope_field_name) county_response = { "scope": self.scope, "geo_layer": self.geo_layer, "results": self.county_results(state_lookup, county_name_lookup), "messages": [get_time_period_message()], } return Response(county_response) else: self.county_district_queryset(kwargs, fields_list, loc_lookup, state_lookup, scope_field_name) district_response = { "scope": self.scope, "geo_layer": self.geo_layer, "results": self.district_results(state_lookup), "messages": [get_time_period_message()], } return Response(district_response)
def post(self, request: Request) -> Response: models = [ { "name": "subawards", "key": "subawards", "type": "boolean", "default": False }, { "name": "scope", "key": "scope", "type": "enum", "optional": False, "enum_values": ["place_of_performance", "recipient_location"], }, { "name": "geo_layer", "key": "geo_layer", "type": "enum", "optional": False, "enum_values": ["state", "county", "district"], }, { "name": "geo_layer_filters", "key": "geo_layer_filters", "type": "array", "array_type": "text", "text_type": "search", }, ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) original_filters = request.data.get("filters") json_request = TinyShield(models).block(request.data) agg_key_dict = { "county": "county_agg_key", "district": "congressional_agg_key", "state": "state_agg_key", } location_dict = { "county": "county_code", "district": "congressional_code", "state": "state_code" } model_dict = { "place_of_performance": "pop", "recipient_location": "recipient_location", # 'subawards_place_of_performance': 'pop', # 'subawards_recipient_location': 'recipient_location' } self.scope_field_name = model_dict[json_request["scope"]] self.agg_key = f"{self.scope_field_name}_{agg_key_dict[json_request['geo_layer']]}" self.filters = json_request.get("filters") self.geo_layer = GeoLayer(json_request["geo_layer"]) self.geo_layer_filters = json_request.get("geo_layer_filters") self.loc_field_name = location_dict[self.geo_layer.value] self.loc_lookup = f"{self.scope_field_name}_{self.loc_field_name}" self.subawards = json_request["subawards"] if self.subawards: # We do not use matviews for Subaward filtering, just the Subaward download filters self.model_name = SubawardView self.queryset = subaward_filter(self.filters) self.obligation_column = "amount" result = self.query_django() elif is_experimental_elasticsearch_api(request): if self.scope_field_name == "pop": scope_filter_name = "place_of_performance_scope" else: scope_filter_name = "recipient_scope" # Only search for values within USA, but don't overwrite a user's search if scope_filter_name not in self.filters: self.filters[scope_filter_name] = "domestic" self.obligation_column = "generated_pragmatic_obligation" filter_query = QueryWithFilters.generate_transactions_elasticsearch_query( self.filters) result = self.query_elasticsearch(filter_query) else: self.queryset, self.model_name = spending_by_geography( self.filters) self.obligation_column = "generated_pragmatic_obligation" result = self.query_django() return Response({ "scope": json_request["scope"], "geo_layer": self.geo_layer.value, "results": result, "messages": get_generic_filters_message( original_filters.keys(), [elem["name"] for elem in AWARD_FILTER]), })
def post(self, request): """Return all awards matching the provided filters and limits""" models = [{ 'name': 'fields', 'key': 'fields', 'type': 'array', 'array_type': 'text', 'text_type': 'search', 'min': 1 }, { 'name': 'subawards', 'key': 'subawards', 'type': 'boolean', 'default': False }] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) for m in models: if m['name'] in ('award_type_codes', 'fields'): m['optional'] = False json_request = TinyShield(models).block(request.data) fields = json_request["fields"] filters = json_request.get("filters", {}) subawards = json_request["subawards"] order = json_request["order"] limit = json_request["limit"] page = json_request["page"] if "no intersection" in filters["award_type_codes"]: # "Special case": there will never be results when the website provides this value return Response({ "limit": limit, "results": [], "page_metadata": { "page": page, "hasNext": False }, }) sort = json_request.get("sort", fields[0]) if sort not in fields: raise InvalidParameterException( "Sort value '{}' not found in requested fields: {}".format( sort, fields)) subawards_values = list(contract_subaward_mapping.keys()) + list( grant_subaward_mapping.keys()) awards_values = list(award_contracts_mapping.keys()) + list(loan_award_mapping.keys()) + \ list(non_loan_assistance_award_mapping.keys()) + list(award_idv_mapping.keys()) msg = "Sort value '{}' not found in {{}} mappings: {{}}".format(sort) if not subawards and sort not in awards_values: raise InvalidParameterException(msg.format("award", awards_values)) elif subawards and sort not in subawards_values: raise InvalidParameterException( msg.format("subaward", subawards_values)) # build sql query filters if subawards: queryset = subaward_filter(filters) values = {'subaward_number', 'piid', 'fain', 'award_type'} for field in fields: if contract_subaward_mapping.get(field): values.add(contract_subaward_mapping.get(field)) if grant_subaward_mapping.get(field): values.add(grant_subaward_mapping.get(field)) else: queryset = matview_search_filter(filters, UniversalAwardView).values() values = {'award_id', 'piid', 'fain', 'uri', 'type'} for field in fields: if award_contracts_mapping.get(field): values.add(award_contracts_mapping.get(field)) if loan_award_mapping.get(field): values.add(loan_award_mapping.get(field)) if non_loan_assistance_award_mapping.get(field): values.add(non_loan_assistance_award_mapping.get(field)) if award_idv_mapping.get(field): values.add(award_idv_mapping.get(field)) # Modify queryset to be ordered by requested "sort" in the request or default value(s) if sort: if subawards: if set(filters["award_type_codes"]) <= set( contract_type_mapping): # Subaward contracts sort_filters = [contract_subaward_mapping[sort]] elif set(filters["award_type_codes"]) <= set( grant_type_mapping): # Subaward grants sort_filters = [grant_subaward_mapping[sort]] else: msg = 'Award Type codes limited for Subawards. Only contracts {} or grants {} are available' msg = msg.format(list(contract_type_mapping.keys()), list(grant_type_mapping.keys())) raise UnprocessableEntityException(msg) else: if set(filters["award_type_codes"]) <= set( contract_type_mapping): # contracts sort_filters = [award_contracts_mapping[sort]] elif set(filters["award_type_codes"]) <= set( loan_type_mapping): # loans sort_filters = [loan_award_mapping[sort]] elif set(filters["award_type_codes"]) <= set( idv_type_mapping): # idvs sort_filters = [award_idv_mapping[sort]] else: # assistance data sort_filters = [non_loan_assistance_award_mapping[sort]] # Explictly set NULLS LAST in the ordering to encourage the usage of the indexes if sort == "Award ID" and subawards: if order == "desc": queryset = queryset.order_by( F('award__piid').desc(nulls_last=True), F('award__fain').desc(nulls_last=True)).values( *list(values)) else: queryset = queryset.order_by( F('award__piid').asc(nulls_last=True), F('award__fain').asc(nulls_last=True)).values( *list(values)) elif sort == "Award ID": if order == "desc": queryset = queryset.order_by( F('piid').desc(nulls_last=True), F('fain').desc(nulls_last=True), F('uri').desc(nulls_last=True)).values(*list(values)) else: queryset = queryset.order_by( F('piid').asc(nulls_last=True), F('fain').asc(nulls_last=True), F('uri').asc(nulls_last=True)).values(*list(values)) elif order == "desc": queryset = queryset.order_by( F(sort_filters[0]).desc(nulls_last=True)).values( *list(values)) else: queryset = queryset.order_by( F(sort_filters[0]).asc(nulls_last=True)).values( *list(values)) limited_queryset = queryset[(page - 1) * limit:page * limit + 1] # lower limit : upper limit has_next = len(limited_queryset) > limit results = [] for award in limited_queryset[:limit]: if subawards: row = {"internal_id": award["subaward_number"]} if award['award_type'] == 'procurement': for field in fields: row[field] = award.get( contract_subaward_mapping[field]) elif award['award_type'] == 'grant': for field in fields: row[field] = award.get(grant_subaward_mapping[field]) else: row = {"internal_id": award["award_id"]} if award['type'] in loan_type_mapping: # loans for field in fields: row[field] = award.get(loan_award_mapping.get(field)) elif award[ 'type'] in non_loan_assistance_type_mapping: # assistance data for field in fields: row[field] = award.get( non_loan_assistance_award_mapping.get(field)) elif award['type'] in idv_type_mapping: for field in fields: row[field] = award.get(award_idv_mapping.get(field)) elif (award['type'] is None and award['piid']) or award['type'] in contract_type_mapping: # IDV + contract for field in fields: row[field] = award.get( award_contracts_mapping.get(field)) if "Award ID" in fields: for id_type in ["piid", "fain", "uri"]: if award[id_type]: row["Award ID"] = award[id_type] break results.append(row) return Response({ "limit": limit, "results": results, "page_metadata": { "page": page, "hasNext": has_next } })
def post(self, request): models = [{ 'name': 'subawards', 'key': 'subawards', 'type': 'boolean', 'default': False }] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) json_request = TinyShield(models).block(request.data) filters = json_request.get("filters", None) subawards = json_request["subawards"] if filters is None: raise InvalidParameterException( "Missing required request parameters: 'filters'") results = { "contracts": 0, "idvs": 0, "grants": 0, "direct_payments": 0, "loans": 0, "other": 0 } if not subawards else { "subcontracts": 0, "subgrants": 0 } if "award_type_codes" in filters and "no intersection" in filters[ "award_type_codes"]: # "Special case": there will never be results when the website provides this value return Response({"results": results}) if subawards: queryset = subaward_filter(filters) else: queryset, model = spending_by_award_count(filters) if subawards: queryset = queryset.values('award_type').annotate( category_count=Count('subaward_id')) elif model == 'SummaryAwardView': queryset = queryset.values('category').annotate( category_count=Sum('counts')) else: queryset = queryset.values('category').annotate( category_count=Count('category')) categories = { 'contract': 'contracts', 'idv': 'idvs', 'grant': 'grants', 'direct payment': 'direct_payments', 'loans': 'loans', 'other': 'other' } if not subawards else { 'procurement': 'subcontracts', 'grant': 'subgrants' } category_name = 'category' if not subawards else 'award_type' # DB hit here for award in queryset: if award[category_name] is None: result_key = 'other' if not subawards else 'subcontracts' elif award[category_name] not in categories.keys(): result_key = 'other' else: result_key = categories[award[category_name]] results[result_key] += award['category_count'] return Response({"results": results})
def post(self, request): models = [ {'name': 'subawards', 'key': 'subawards', 'type': 'boolean', 'default': False}, {'name': 'scope', 'key': 'scope', 'type': 'enum', 'optional': False, 'enum_values': ['place_of_performance', 'recipient_location']}, {'name': 'geo_layer', 'key': 'geo_layer', 'type': 'enum', 'optional': False, 'enum_values': ['state', 'county', 'district']}, {'name': 'geo_layer_filters', 'key': 'geo_layer_filters', 'type': 'array', 'array_type': 'text', 'text_type': 'search'} ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) json_request = TinyShield(models).block(request.data) self.subawards = json_request["subawards"] self.scope = json_request["scope"] self.filters = json_request.get("filters", None) self.geo_layer = json_request["geo_layer"] self.geo_layer_filters = json_request.get("geo_layer_filters", None) fields_list = [] # fields to include in the aggregate query loc_dict = { 'state': 'state_code', 'county': 'county_code', 'district': 'congressional_code' } model_dict = { 'place_of_performance': 'pop', 'recipient_location': 'recipient_location', # 'subawards_place_of_performance': 'pop', # 'subawards_recipient_location': 'recipient_location' } # Build the query based on the scope fields and geo_layers # Fields not in the reference objects above then request is invalid scope_field_name = model_dict.get(self.scope) loc_field_name = loc_dict.get(self.geo_layer) loc_lookup = '{}_{}'.format(scope_field_name, loc_field_name) if self.subawards: # We do not use matviews for Subaward filtering, just the Subaward download filters self.queryset = subaward_filter(self.filters) self.model_name = SubawardView else: self.queryset, self.model_name = spending_by_geography(self.filters) if self.geo_layer == 'state': # State will have one field (state_code) containing letter A-Z column_isnull = 'generated_pragmatic_obligation__isnull' if self.subawards: column_isnull = 'amount__isnull' kwargs = { '{}_country_code'.format(scope_field_name): 'USA', column_isnull: False } # Only state scope will add its own state code # State codes are consistent in database i.e. AL, AK fields_list.append(loc_lookup) state_response = { 'scope': self.scope, 'geo_layer': self.geo_layer, 'results': self.state_results(kwargs, fields_list, loc_lookup) } return Response(state_response) else: # County and district scope will need to select multiple fields # State code is needed for county/district aggregation state_lookup = '{}_{}'.format(scope_field_name, loc_dict['state']) fields_list.append(state_lookup) # Adding regex to county/district codes to remove entries with letters since can't be surfaced by map kwargs = { '{}__isnull'.format('amount' if self.subawards else 'generated_pragmatic_obligation'): False } if self.geo_layer == 'county': # County name added to aggregation since consistent in db county_name_lookup = '{}_county_name'.format(scope_field_name) fields_list.append(county_name_lookup) self.county_district_queryset( kwargs, fields_list, loc_lookup, state_lookup, scope_field_name ) county_response = { 'scope': self.scope, 'geo_layer': self.geo_layer, 'results': self.county_results(state_lookup, county_name_lookup) } return Response(county_response) else: self.county_district_queryset( kwargs, fields_list, loc_lookup, state_lookup, scope_field_name ) district_response = { 'scope': self.scope, 'geo_layer': self.geo_layer, 'results': self.district_results(state_lookup) } return Response(district_response)
def test_any_rule(): models = [{ 'name': 'value', 'key': 'value', 'type': 'any', 'models': [{ 'type': 'integer' }, { 'type': 'text', 'text_type': 'search' }] }] # Test integer and random other key. ts = TinyShield(models).block({'value': 1, 'another_value': 2}) assert ts['value'] == 1 assert ts.get('another_value') is None # Test integer masquerading as a string. ts = TinyShield(models).block({'value': '1'}) assert ts['value'] == 1 # Test string. ts = TinyShield(models).block({'value': 'XYZ'}) assert ts['value'] == 'XYZ' # Test list (which should blow up). with pytest.raises(UnprocessableEntityException): TinyShield(models).block({'value': ['XYZ']}) # Test with optional 'value' missing. ts = TinyShield(models).block({'another_value': 2}) assert ts.get('value') is None assert ts.get('another_value') is None # Make 'value' required then run the same tests as above. models[0]['optional'] = False # Test integer and random other key. ts = TinyShield(models).block({'value': 1, 'another_value': 2}) assert ts['value'] == 1 assert ts.get('another_value') is None # Test integer masquerading as a string. ts = TinyShield(models).block({'value': '1'}) assert ts['value'] == 1 # Test string. ts = TinyShield(models).block({'value': 'XYZ'}) assert ts['value'] == 'XYZ' # Test list (which should blow up). with pytest.raises(UnprocessableEntityException): TinyShield(models).block({'value': ['XYZ']}) # Test with required 'value' missing. with pytest.raises(UnprocessableEntityException): TinyShield(models).block({'another_value': 2})
def post(self, request): """Return all awards matching the provided filters and limits""" models = [ {'name': 'fields', 'key': 'fields', 'type': 'array', 'array_type': 'text', 'text_type': 'search', 'min': 1}, {'name': 'subawards', 'key': 'subawards', 'type': 'boolean', 'default': False} ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) for m in models: if m['name'] in ('award_type_codes', 'fields'): m['optional'] = False json_request = TinyShield(models).block(request.data) fields = json_request["fields"] filters = json_request.get("filters", {}) subawards = json_request["subawards"] order = json_request["order"] limit = json_request["limit"] page = json_request["page"] if "no intersection" in filters["award_type_codes"]: # "Special case": there will never be results when the website provides this value return Response({ "limit": limit, "results": [], "page_metadata": {"page": page, "hasNext": False}, }) sort = json_request.get("sort", fields[0]) if sort not in fields: raise InvalidParameterException("Sort value '{}' not found in requested fields: {}".format(sort, fields)) subawards_values = list(contract_subaward_mapping.keys()) + list(grant_subaward_mapping.keys()) awards_values = list(award_contracts_mapping.keys()) + list(loan_award_mapping.keys()) + \ list(non_loan_assistance_award_mapping.keys()) + list(award_idv_mapping.keys()) msg = "Sort value '{}' not found in {{}} mappings: {{}}".format(sort) if not subawards and sort not in awards_values: raise InvalidParameterException(msg.format("award", awards_values)) elif subawards and sort not in subawards_values: raise InvalidParameterException(msg.format("subaward", subawards_values)) # build sql query filters if subawards: queryset = subaward_filter(filters) values = {'subaward_number', 'piid', 'fain', 'award_type'} for field in fields: if contract_subaward_mapping.get(field): values.add(contract_subaward_mapping.get(field)) if grant_subaward_mapping.get(field): values.add(grant_subaward_mapping.get(field)) else: queryset = matview_search_filter(filters, UniversalAwardView).values() values = {'award_id', 'piid', 'fain', 'uri', 'type'} for field in fields: if award_contracts_mapping.get(field): values.add(award_contracts_mapping.get(field)) if loan_award_mapping.get(field): values.add(loan_award_mapping.get(field)) if non_loan_assistance_award_mapping.get(field): values.add(non_loan_assistance_award_mapping.get(field)) if award_idv_mapping.get(field): values.add(award_idv_mapping.get(field)) # Modify queryset to be ordered by requested "sort" in the request or default value(s) if sort: if subawards: if set(filters["award_type_codes"]) <= set(contract_type_mapping): # Subaward contracts sort_filters = [contract_subaward_mapping[sort]] elif set(filters["award_type_codes"]) <= set(grant_type_mapping): # Subaward grants sort_filters = [grant_subaward_mapping[sort]] else: msg = 'Award Type codes limited for Subawards. Only contracts {} or grants {} are available' msg = msg.format(list(contract_type_mapping.keys()), list(grant_type_mapping.keys())) raise UnprocessableEntityException(msg) else: if set(filters["award_type_codes"]) <= set(contract_type_mapping): # contracts sort_filters = [award_contracts_mapping[sort]] elif set(filters["award_type_codes"]) <= set(loan_type_mapping): # loans sort_filters = [loan_award_mapping[sort]] elif set(filters["award_type_codes"]) <= set(idv_type_mapping): # idvs sort_filters = [award_idv_mapping[sort]] else: # assistance data sort_filters = [non_loan_assistance_award_mapping[sort]] # Explictly set NULLS LAST in the ordering to encourage the usage of the indexes if sort == "Award ID" and subawards: if order == "desc": queryset = queryset.order_by( F('award__piid').desc(nulls_last=True), F('award__fain').desc(nulls_last=True)).values(*list(values)) else: queryset = queryset.order_by( F('award__piid').asc(nulls_last=True), F('award__fain').asc(nulls_last=True)).values(*list(values)) elif sort == "Award ID": if order == "desc": queryset = queryset.order_by( F('piid').desc(nulls_last=True), F('fain').desc(nulls_last=True), F('uri').desc(nulls_last=True)).values(*list(values)) else: queryset = queryset.order_by( F('piid').asc(nulls_last=True), F('fain').asc(nulls_last=True), F('uri').asc(nulls_last=True)).values(*list(values)) elif order == "desc": queryset = queryset.order_by(F(sort_filters[0]).desc(nulls_last=True)).values(*list(values)) else: queryset = queryset.order_by(F(sort_filters[0]).asc(nulls_last=True)).values(*list(values)) limited_queryset = queryset[(page - 1) * limit:page * limit + 1] # lower limit : upper limit has_next = len(limited_queryset) > limit results = [] for award in limited_queryset[:limit]: if subawards: row = {"internal_id": award["subaward_number"]} if award['award_type'] == 'procurement': for field in fields: row[field] = award.get(contract_subaward_mapping[field]) elif award['award_type'] == 'grant': for field in fields: row[field] = award.get(grant_subaward_mapping[field]) else: row = {"internal_id": award["award_id"]} if award['type'] in loan_type_mapping: # loans for field in fields: row[field] = award.get(loan_award_mapping.get(field)) elif award['type'] in non_loan_assistance_type_mapping: # assistance data for field in fields: row[field] = award.get(non_loan_assistance_award_mapping.get(field)) elif award['type'] in idv_type_mapping: for field in fields: row[field] = award.get(award_idv_mapping.get(field)) elif (award['type'] is None and award['piid']) or award['type'] in contract_type_mapping: # IDV + contract for field in fields: row[field] = award.get(award_contracts_mapping.get(field)) if "Award ID" in fields: for id_type in ["piid", "fain", "uri"]: if award[id_type]: row["Award ID"] = award[id_type] break results.append(row) return Response({"limit": limit, "results": results, "page_metadata": {"page": page, "hasNext": has_next}})
def post(self, request): models = [{ 'name': 'subawards', 'key': 'subawards', 'type': 'boolean', 'default': False }, { 'name': 'scope', 'key': 'scope', 'type': 'enum', 'optional': False, 'enum_values': ['place_of_performance', 'recipient_location'] }, { 'name': 'geo_layer', 'key': 'geo_layer', 'type': 'enum', 'optional': False, 'enum_values': ['state', 'county', 'district'] }, { 'name': 'geo_layer_filters', 'key': 'geo_layer_filters', 'type': 'array', 'array_type': 'text', 'text_type': 'search' }] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) json_request = TinyShield(models).block(request.data) self.subawards = json_request["subawards"] self.scope = json_request["scope"] self.filters = json_request.get("filters", None) self.geo_layer = json_request["geo_layer"] self.geo_layer_filters = json_request.get("geo_layer_filters", None) fields_list = [] # fields to include in the aggregate query loc_dict = { 'state': 'state_code', 'county': 'county_code', 'district': 'congressional_code' } model_dict = { 'place_of_performance': 'pop', 'recipient_location': 'recipient_location', # 'subawards_place_of_performance': 'pop', # 'subawards_recipient_location': 'recipient_location' } # Build the query based on the scope fields and geo_layers # Fields not in the reference objects above then request is invalid scope_field_name = model_dict.get(self.scope) loc_field_name = loc_dict.get(self.geo_layer) loc_lookup = '{}_{}'.format(scope_field_name, loc_field_name) if self.subawards: # We do not use matviews for Subaward filtering, just the Subaward download filters self.queryset = subaward_filter(self.filters) self.model_name = SubawardView else: self.queryset, self.model_name = spending_by_geography( self.filters) if self.geo_layer == 'state': # State will have one field (state_code) containing letter A-Z column_isnull = 'generated_pragmatic_obligation__isnull' if self.subawards: column_isnull = 'amount__isnull' kwargs = { '{}_country_code'.format(scope_field_name): 'USA', column_isnull: False } # Only state scope will add its own state code # State codes are consistent in database i.e. AL, AK fields_list.append(loc_lookup) state_response = { 'scope': self.scope, 'geo_layer': self.geo_layer, 'results': self.state_results(kwargs, fields_list, loc_lookup) } return Response(state_response) else: # County and district scope will need to select multiple fields # State code is needed for county/district aggregation state_lookup = '{}_{}'.format(scope_field_name, loc_dict['state']) fields_list.append(state_lookup) # Adding regex to county/district codes to remove entries with letters since can't be surfaced by map kwargs = { '{}__isnull'.format('amount' if self.subawards else 'generated_pragmatic_obligation'): False } if self.geo_layer == 'county': # County name added to aggregation since consistent in db county_name_lookup = '{}_county_name'.format(scope_field_name) fields_list.append(county_name_lookup) self.county_district_queryset(kwargs, fields_list, loc_lookup, state_lookup, scope_field_name) county_response = { 'scope': self.scope, 'geo_layer': self.geo_layer, 'results': self.county_results(state_lookup, county_name_lookup) } return Response(county_response) else: self.county_district_queryset(kwargs, fields_list, loc_lookup, state_lookup, scope_field_name) district_response = { 'scope': self.scope, 'geo_layer': self.geo_layer, 'results': self.district_results(state_lookup) } return Response(district_response)