def validate_request_data(self, json_data): self.groupings = { "quarter": "quarter", "q": "quarter", "fiscal_year": "fiscal_year", "fy": "fiscal_year", "month": "month", "m": "month", } models = [ {"name": "subawards", "key": "subawards", "type": "boolean", "default": False}, { "name": "group", "key": "group", "type": "enum", "enum_values": list(self.groupings.keys()), "default": "fy", "optional": False, # allow to be optional in the future }, ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) validated_data = TinyShield(models).block(json_data) if validated_data.get("filters", None) is None: raise InvalidParameterException("Missing request parameters: filters") return validated_data
def post(self, request): """Returns boolean of whether a download request is greater than the max limit. """ models = [ {'name': 'subawards', 'key': 'subawards', 'type': 'boolean', 'default': False}, ] models.extend(copy.deepcopy(AWARD_FILTER)) json_request = TinyShield(models).block(request.data) # If no filters in request return empty object to return all transactions filters = json_request.get('filters', {}) is_over_limit = False if json_request['subawards']: total_count = subaward_filter(filters).count() else: queryset, model = download_transaction_count(filters) if model in ['UniversalTransactionView']: total_count = queryset.count() else: # "summary" materialized views are pre-aggregated and contain a counts col total_count = queryset.aggregate(total_count=Sum('counts'))['total_count'] if total_count and total_count > settings.MAX_DOWNLOAD_LIMIT: is_over_limit = True result = { "transaction_rows_gt_limit": is_over_limit } return Response(result)
def validate_request_data(self, json_data): self.groupings = { "quarter": "quarter", "q": "quarter", "fiscal_year": "fiscal_year", "fy": "fiscal_year", "month": "month", "m": "month", } models = [ { "name": "subawards", "key": "subawards", "type": "boolean", "default": False }, { "name": "group", "key": "group", "type": "enum", "enum_values": list(self.groupings.keys()), "default": "fy", "optional": False, # allow to be optional in the future }, ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) validated_data = TinyShield(models).block(json_data) if validated_data.get("filters", None) is None: raise InvalidParameterException( "Missing request parameters: filters") return validated_data
def validate_request_data(json_data: dict) -> dict: models = [ { "name": "subawards", "key": "subawards", "type": "boolean", "default": False }, { "name": "group", "key": "group", "type": "enum", "enum_values": list(GROUPING_LOOKUP.keys()), "default": "fy", "optional": False, # allow to be optional in the future }, ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) validated_data = TinyShield(models).block(json_data) if validated_data.get("filters", None) is None: raise InvalidParameterException( "Missing request parameters: filters") return validated_data
def post(self, request): models = [{'name': 'subawards', 'key': 'subawards', 'type': 'boolean', 'default': False}] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) json_request = TinyShield(models).block(request.data) filters = json_request.get("filters", None) subawards = json_request["subawards"] if filters is None: raise InvalidParameterException("Missing required request parameters: 'filters'") results = { "contracts": 0, "idvs": 0, "grants": 0, "direct_payments": 0, "loans": 0, "other": 0 } if not subawards else { "subcontracts": 0, "subgrants": 0 } if "award_type_codes" in filters and "no intersection" in filters["award_type_codes"]: # "Special case": there will never be results when the website provides this value return Response({"results": results}) if subawards: queryset = subaward_filter(filters) else: queryset, model = spending_by_award_count(filters) if subawards: queryset = queryset.values('award_type').annotate(category_count=Count('subaward_id')) elif model == 'SummaryAwardView': queryset = queryset.values('category').annotate(category_count=Sum('counts')) else: queryset = queryset.values('category').annotate(category_count=Count('category')) categories = { 'contract': 'contracts', 'idv': 'idvs', 'grant': 'grants', 'direct payment': 'direct_payments', 'loans': 'loans', 'other': 'other' } if not subawards else {'procurement': 'subcontracts', 'grant': 'subgrants'} category_name = 'category' if not subawards else 'award_type' # DB hit here for award in queryset: if award[category_name] is None: result_key = 'other' if not subawards else 'subcontracts' elif award[category_name] not in categories.keys(): result_key = 'other' else: result_key = categories[award[category_name]] results[result_key] += award['category_count'] return Response({"results": results})
def get(self, request): model = [ { "key": "fiscal_year", "name": "fiscal_year", "type": "integer", "min": 2017, "optional": True, "default": None, "allow_nulls": True, }, { "key": "fiscal_period", "name": "fiscal_period", "type": "integer", "min": 2, "max": 12, "optional": True, "default": None, "allow_nulls": True, }, ] validated = TinyShield(model).block(request.query_params) fiscal_year = validated.get("fiscal_year", None) fiscal_period = validated.get("fiscal_period", None) gtas_queryset = GTASSF133Balances.objects.values( "fiscal_year", "fiscal_period") if fiscal_period: if not fiscal_year: raise InvalidParameterException( "fiscal_period was provided without fiscal_year.") else: gtas_queryset = gtas_queryset.filter( fiscal_year=fiscal_year, fiscal_period=fiscal_period) elif fiscal_year: gtas_queryset = gtas_queryset.filter(fiscal_year=fiscal_year) results = gtas_queryset.annotate(total_budgetary_resources=Sum( "total_budgetary_resources_cpe")).order_by("-fiscal_year", "-fiscal_period") return Response({ "results": list(results), "messages": [get_account_data_time_period_message()] if not fiscal_year or fiscal_year < 2017 else [], })
def post(self, request): models = [ {"name": "subawards", "key": "subawards", "type": "boolean", "default": False}, { "name": "object_class", "key": "filter|object_class", "type": "array", "array_type": "text", "text_type": "search", }, { "name": "program_activity", "key": "filter|program_activity", "type": "array", "array_type": "integer", "array_max": maxsize, }, ] models.extend(copy.deepcopy(AWARD_FILTER_NO_RECIPIENT_ID)) models.extend(copy.deepcopy(PAGINATION)) self.original_filters = request.data.get("filters") json_request = TinyShield(models).block(request.data) subawards = json_request["subawards"] filters = add_date_range_comparison_types( json_request.get("filters", None), subawards, gte_date_type="action_date", lte_date_type="date_signed" ) if filters is None: raise InvalidParameterException("Missing required request parameters: 'filters'") if "award_type_codes" in filters and "no intersection" in filters["award_type_codes"]: # "Special case": there will never be results when the website provides this value empty_results = {"contracts": 0, "idvs": 0, "grants": 0, "direct_payments": 0, "loans": 0, "other": 0} if subawards: empty_results = {"subcontracts": 0, "subgrants": 0} results = empty_results elif subawards: results = self.handle_subawards(filters) else: results = self.query_elasticsearch_for_prime_awards(filters) return Response( { "results": results, "messages": get_generic_filters_message( self.original_filters.keys(), [elem["name"] for elem in AWARD_FILTER_NO_RECIPIENT_ID] ), } )
def post(self, request): models = [ {"name": "subawards", "key": "subawards", "type": "boolean", "default": False}, { "name": "object_class", "key": "filter|object_class", "type": "array", "array_type": "text", "text_type": "search", }, { "name": "program_activity", "key": "filter|program_activity", "type": "array", "array_type": "integer", "array_max": maxsize, }, ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) json_request = TinyShield(models).block(request.data) subawards = json_request["subawards"] filters = add_date_range_comparison_types( json_request.get("filters", None), subawards, gte_date_type="action_date", lte_date_type="date_signed" ) elasticsearch = is_experimental_elasticsearch_api(request) if elasticsearch and not subawards: logger.info("Using experimental Elasticsearch functionality for 'spending_by_award_count'") results = self.query_elasticsearch(filters) return Response({"results": results, "messages": [get_time_period_message()]}) if filters is None: raise InvalidParameterException("Missing required request parameters: 'filters'") empty_results = {"contracts": 0, "idvs": 0, "grants": 0, "direct_payments": 0, "loans": 0, "other": 0} if subawards: empty_results = {"subcontracts": 0, "subgrants": 0} if "award_type_codes" in filters and "no intersection" in filters["award_type_codes"]: # "Special case": there will never be results when the website provides this value return Response({"results": empty_results}) if subawards: results = self.handle_subawards(filters) else: results = self.handle_awards(filters, empty_results) return Response({"results": results, "messages": [get_time_period_message()]})
def get(self, request: Request) -> Response: """ Accepts only pagination-related query parameters """ models = [ {'name': 'page', 'key': 'page', 'type': 'integer', 'default': 1, 'min': 1}, {'name': 'limit', 'key': 'limit', 'type': 'integer', 'default': 500, 'min': 1, 'max': 500}, ] # Can't use the TinyShield decorator (yet) because this is a GET request only request_dict = request.query_params validated_request_data = TinyShield(models).block(request_dict) limit = validated_request_data["limit"] page = validated_request_data["page"] queryset = Definition.objects.all() queryset, pagination = get_pagination(queryset, int(limit), int(page)) serializer = DefinitionSerializer(queryset, many=True) response = { "page_metadata": pagination, "results": serializer.data } return Response(response)
def post(self, request): """ Returns a summary of transactions which match the award search filter Desired values: total number of transactions `award_count` The federal_action_obligation sum of all those transactions `award_spending` *Note* Only deals with prime awards, future plans to include sub-awards. """ models = [ { "name": "keywords", "key": "filters|keywords", "type": "array", "array_type": "text", "text_type": "search", "optional": False, "text_min": 3, } ] validated_payload = TinyShield(models).block(request.data) results = spending_by_transaction_sum_and_count(validated_payload) if not results: raise ElasticsearchConnectionException("Error generating the transaction sums and counts") return Response({"results": results})
def validate_request_data(request_data): models = [ {"name": "fields", "key": "fields", "type": "array", "array_type": "text", "text_type": "search", "min": 1}, {"name": "subawards", "key": "subawards", "type": "boolean", "default": False}, { "name": "object_class", "key": "filter|object_class", "type": "array", "array_type": "text", "text_type": "search", }, { "name": "program_activity", "key": "filter|program_activity", "type": "array", "array_type": "integer", "array_max": maxsize, }, ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) for m in models: if m["name"] in ("award_type_codes", "fields"): m["optional"] = False return TinyShield(models).block(request_data)
def test_check_models(): """We want this test to fail if either AWARD_FILTERS has an invalid model, OR if the logic of the check_models function has been corrupted. It will fail if an exception is raised. Otherwise it will define the global TS object so we can use it in the remaining tests.""" global TS TS = TinyShield(copy.deepcopy(AWARD_FILTER))
def post(self, request): models = [ { 'name': 'fields', 'key': 'fields', 'type': 'array', 'array_type': 'text', 'text_type': 'search', 'optional': False }, ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) for m in models: if m['name'] in ('keywords', 'award_type_codes', 'sort'): m['optional'] = False validated_payload = TinyShield(models).block(request.data) if validated_payload['sort'] not in validated_payload['fields']: raise InvalidParameterException( "Sort value not found in fields: {}".format( validated_payload['sort'])) if "filters" in validated_payload and "no intersection" in validated_payload[ "filters"]["award_type_codes"]: # "Special case": there will never be results when the website provides this value return Response({ "limit": validated_payload["limit"], "results": [], "page_metadata": { "page": validated_payload["page"], "next": None, "previous": None, "hasNext": False, "hasPrevious": False, }, }) lower_limit = (validated_payload['page'] - 1) * validated_payload['limit'] success, response, total = search_transactions( validated_payload, lower_limit, validated_payload['limit'] + 1) if not success: raise InvalidParameterException(response) metadata = get_simple_pagination_metadata(len(response), validated_payload['limit'], validated_payload['page']) results = [] for transaction in response[:validated_payload['limit']]: results.append(transaction) response = { 'limit': validated_payload['limit'], 'results': results, 'page_metadata': metadata } return Response(response)
def validate_api_request(self, json_payload): self.groupings = { 'quarter': 'quarter', 'q': 'quarter', 'fiscal_year': 'fiscal_year', 'fy': 'fiscal_year', 'month': 'month', 'm': 'month', } models = [{ 'name': 'group', 'key': 'group', 'type': 'enum', 'enum_values': list(self.groupings.keys()), 'default': 'fy', }] advanced_search_filters = [ model for model in copy.deepcopy(AWARD_FILTER) if model['name'] in ('time_period', 'recipient_id') ] for model in advanced_search_filters: if model['name'] in ('time_period', 'recipient_id'): model['optional'] = False models.extend(advanced_search_filters) return TinyShield(models).block(json_payload)
def validate_api_request(self, json_payload): self.groupings = { "quarter": "quarter", "q": "quarter", "fiscal_year": "fiscal_year", "fy": "fiscal_year", "month": "month", "m": "month", } models = [ { "name": "group", "key": "group", "type": "enum", "enum_values": list(self.groupings.keys()), "default": "fy", } ] advanced_search_filters = [ model for model in copy.deepcopy(AWARD_FILTER) if model["name"] in ("time_period", "recipient_id") ] for model in advanced_search_filters: if model["name"] in ("time_period", "recipient_id"): model["optional"] = False models.extend(advanced_search_filters) return TinyShield(models).block(json_payload)
def _parse_and_validate_request(self, request_dict): models = deepcopy(PAGINATION) models.append({ "key": "award_id", "name": "award_id", "type": "any", "models": [{ "type": "integer" }, { "type": "text", "text_type": "search" }], "optional": True, "default": None, "allow_nulls": True, }) for model in models: # Change sort to an enum of the desired values if model["name"] == "sort": model["type"] = "enum" model["enum_values"] = list(self.subaward_lookup.keys()) model["default"] = "subaward_number" validated_request_data = TinyShield(models).block(request_dict) return validated_request_data
def post(self, request): award_types = list(AWARD_TYPES.keys()) + ["all"] models = [ { "name": "keyword", "key": "keyword", "type": "text", "text_type": "search" }, { "name": "award_type", "key": "award_type", "type": "enum", "enum_values": award_types, "default": "all" }, ] models.extend(copy.deepcopy(PAGINATION)) # page, limit, sort, order new_sort = { "type": "enum", "enum_values": ["name", "duns", "amount"], "default": "amount" } models = update_model_in_list(models, "sort", new_sort) models = update_model_in_list(models, "limit", {"default": 50}) validated_payload = TinyShield(models).block(request.data) results, page_metadata = get_recipients(filters=validated_payload) return Response({"page_metadata": page_metadata, "results": results})
def post(self, request: dict): """Return all budget function/subfunction titles matching the provided search text""" categories = [ "awarding_agency", "awarding_subagency", "funding_agency", "funding_subagency", "recipient_duns", "recipient_parent_duns", "cfda", "psc", "naics", "county", "district", "country", "state_territory", "federal_account", ] models = [ {"name": "category", "key": "category", "type": "enum", "enum_values": categories, "optional": False}, {"name": "subawards", "key": "subawards", "type": "boolean", "default": False, "optional": True}, ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) # Apply/enforce POST body schema and data validation in request validated_payload = TinyShield(models).block(request.data) # Execute the business logic for the endpoint and return a python dict to be converted to a Django response return Response(BusinessLogic(validated_payload).results())
def _validate_tas_codes(filters, json_request): if "tas_codes" in filters: tas_codes = {"filters": {"tas_codes": filters["tas_codes"]}} models = [deepcopy(get_model_by_name(AWARD_FILTER, "tas_codes"))] tas_codes = TinyShield(models).block(tas_codes) if tas_codes: json_request["filters"]["tas_codes"] = tas_codes["filters"][ "tas_codes"]
def post(self, request): models = [{ "name": "fields", "key": "fields", "type": "array", "array_type": "text", "text_type": "search", "optional": False, }] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) for m in models: if m["name"] in ("keywords", "award_type_codes", "sort"): m["optional"] = False validated_payload = TinyShield(models).block(request.data) if validated_payload["sort"] not in validated_payload["fields"]: raise InvalidParameterException( "Sort value not found in fields: {}".format( validated_payload["sort"])) if "filters" in validated_payload and "no intersection" in validated_payload[ "filters"]["award_type_codes"]: # "Special case": there will never be results when the website provides this value return Response({ "limit": validated_payload["limit"], "results": [], "page_metadata": { "page": validated_payload["page"], "next": None, "previous": None, "hasNext": False, "hasPrevious": False, }, }) lower_limit = (validated_payload["page"] - 1) * validated_payload["limit"] success, response, total = search_transactions( validated_payload, lower_limit, validated_payload["limit"] + 1) if not success: raise InvalidParameterException(response) metadata = get_simple_pagination_metadata(len(response), validated_payload["limit"], validated_payload["page"]) results = [] for transaction in response[:validated_payload["limit"]]: results.append(transaction) response = { "limit": validated_payload["limit"], "results": results, "page_metadata": metadata } return Response(response)
def post(self, request): """Returns boolean of whether a download request is greater than the max limit. """ models = [{ "name": "subawards", "key": "subawards", "type": "boolean", "default": False }] models.extend(copy.deepcopy(AWARD_FILTER)) self.original_filters = request.data.get("filters") json_request = TinyShield(models).block(request.data) # If no filters in request return empty object to return all transactions filters = json_request.get("filters", {}) if json_request["subawards"]: total_count = subaward_filter(filters).count() else: queryset, model = download_transaction_count(filters) if model in ["UniversalTransactionView"]: total_count = queryset.count() else: # "summary" materialized views are pre-aggregated and contain a counts col total_count = queryset.aggregate( total_count=Sum("counts"))["total_count"] if total_count is None: total_count = 0 result = { "calculated_transaction_count": total_count, "maximum_transaction_limit": settings.MAX_DOWNLOAD_LIMIT, "transaction_rows_gt_limit": total_count > settings.MAX_DOWNLOAD_LIMIT, "messages": [ get_generic_filters_message( self.original_filters.keys(), [elem["name"] for elem in AWARD_FILTER]) ], } return Response(result)
def post(self, request): models = [{ "name": "subawards", "key": "subawards", "type": "boolean", "default": False }] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) json_request = TinyShield(models).block(request.data) subawards = json_request["subawards"] filters = add_date_range_comparison_types(json_request.get( "filters", None), subawards, gte_date_type="action_date", lte_date_type="date_signed") if filters is None: raise InvalidParameterException( "Missing required request parameters: 'filters'") empty_results = { "contracts": 0, "idvs": 0, "grants": 0, "direct_payments": 0, "loans": 0, "other": 0 } if subawards: empty_results = {"subcontracts": 0, "subgrants": 0} if "award_type_codes" in filters and "no intersection" in filters[ "award_type_codes"]: # "Special case": there will never be results when the website provides this value return Response({"results": empty_results}) if subawards: results = self.handle_subawards(filters) else: results = self.handle_awards(filters, empty_results) return Response({"results": results})
def test_non_default_overridden_pagination(): pagination = deepcopy(PAGINATION) r = TinyShield(pagination).block({ "page": 2, "limit": 11, "sort": "whatever", "order": "asc" }) assert r == {"page": 2, "limit": 11, "sort": "whatever", "order": "asc"}
def post(self, request: Request) -> Response: models = [ {"name": "subawards", "key": "subawards", "type": "boolean", "default": False, "optional": True}, ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) original_filters = request.data.get("filters") validated_payload = TinyShield(models).block(request.data) return Response(self.perform_search(validated_payload, original_filters))
def post(self, request): """Returns boolean of whether a download request is greater than the max limit. """ models = [{ "name": "subawards", "key": "subawards", "type": "boolean", "default": False }] models.extend(copy.deepcopy(AWARD_FILTER)) self.original_filters = request.data.get("filters") json_request = TinyShield(models).block(request.data) # If no filters in request return empty object to return all transactions filters = json_request.get("filters", {}) if json_request["subawards"]: total_count = subaward_filter(filters).count() else: filter_query = QueryWithFilters.generate_transactions_elasticsearch_query( filters) search = TransactionSearch().filter(filter_query) total_count = search.handle_count() if total_count is None: total_count = 0 result = { "calculated_transaction_count": total_count, "maximum_transaction_limit": settings.MAX_DOWNLOAD_LIMIT, "transaction_rows_gt_limit": total_count > settings.MAX_DOWNLOAD_LIMIT, "messages": [ get_generic_filters_message( self.original_filters.keys(), [elem["name"] for elem in AWARD_FILTER]) ], } return Response(result)
def get(self, request, toptier_code, fiscal_year, fiscal_period, type): my_request = { "type": type, "fiscal_year": fiscal_year, "fiscal_period": fiscal_period } validated = TinyShield(self.tinyshield_model).block(my_request) self.annotations = self.annotation_options[validated["type"]] self.fiscal_year = validated["fiscal_year"] self.fiscal_period = validated["fiscal_period"] return Response(self.get_unlinked_awards())
def post(self, request: Request) -> Response: """Return all budget function/subfunction titles matching the provided search text""" categories = [ "awarding_agency", "awarding_subagency", "funding_agency", "funding_subagency", "recipient_duns", "recipient_parent_duns", "cfda", "psc", "naics", "county", "district", "country", "state_territory", "federal_account", ] models = [ {"name": "category", "key": "category", "type": "enum", "enum_values": categories, "optional": False}, {"name": "subawards", "key": "subawards", "type": "boolean", "default": False, "optional": True}, ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) # Apply/enforce POST body schema and data validation in request original_filters = request.data.get("filters") validated_payload = TinyShield(models).block(request.data) # Execute the business logic for the endpoint and return a python dict to be converted to a Django response business_logic_lookup = { "awarding_agency": AwardingAgencyViewSet().perform_search, "awarding_subagency": AwardingSubagencyViewSet().perform_search, "cfda": CfdaViewSet().perform_search, "country": CountryViewSet().perform_search, "county": CountyViewSet().perform_search, "district": DistrictViewSet().perform_search, "federal_account": FederalAccountViewSet().perform_search, "funding_agency": FundingAgencyViewSet().perform_search, "funding_subagency": FundingSubagencyViewSet().perform_search, "naics": NAICSViewSet().perform_search, "psc": PSCViewSet().perform_search, "recipient_duns": RecipientDunsViewSet().perform_search, "state_territory": StateTerritoryViewSet().perform_search, } business_logic_func = business_logic_lookup.get(validated_payload["category"]) if business_logic_func: return Response(business_logic_func(validated_payload, original_filters)) else: self.raise_not_implemented(validated_payload)
def _parse_and_validate(self, request): all_def_codes = sorted(list(DisasterEmergencyFundCode.objects.values_list("code", flat=True))) models = [ { "key": "def_codes", "name": "def_codes", "type": "text", "text_type": "search", "allow_nulls": True, "optional": True, "default": ",".join(all_def_codes), }, ] return TinyShield(models).block(request)
def post(self, request): models = [{ "name": "keywords", "key": "filters|keywords", "type": "array", "array_type": "text", "text_type": "search", "optional": False, "text_min": 3, }] validated_payload = TinyShield(models).block(request.data) results = spending_by_transaction_count(validated_payload) return Response({"results": results})
def _parse_and_validate_request(requested_award: str, request_data: dict) -> dict: piid = request_data.get("piid") data = {"award_id": requested_award, "piid": piid} models = [ get_internal_or_generated_award_id_model(), { "key": "piid", "name": "piid", "allow_nulls": True, "optional": True, "type": "text", "text_type": "search", }, ] return TinyShield(models).block(data)
def _parse_and_validate_request(self, provided_award_id: str) -> dict: request_dict = {"generated_unique_award_id": provided_award_id} models = [ { "key": "generated_unique_award_id", "name": "generated_unique_award_id", "type": "text", "text_type": "search", "optional": False, } ] if str(provided_award_id).isdigit(): request_dict = {"id": int(provided_award_id)} models = [{"key": "id", "name": "id", "type": "integer", "optional": False}] validated_request_data = TinyShield(models).block(request_dict) return validated_request_data
def _parse_and_validate_request(self, requested_naics: str, request_data) -> dict: naics_filter = request_data.get("filter") data = {"code": requested_naics, "filter": naics_filter} models = [ {"key": "code", "name": "code", "type": "integer", "allow_nulls": True, "optional": True}, { "key": "filter", "name": "filter", "type": "text", "text_type": "search", "default": None, "optional": True, "allow_nulls": True, }, ] validated = TinyShield(models).block(data) return validated
def test_get_generate_award_id_rule(): models = [get_generated_award_id_model()] r = TinyShield(models).block({'award_id': 'abcd'}) assert r == {'award_id': 'abcd'} models = [get_generated_award_id_model()] r = TinyShield(models).block({'award_id': 'A' * MAX_ITEMS}) assert r == {'award_id': 'A' * MAX_ITEMS} models = [get_generated_award_id_model(key='my_award_id')] r = TinyShield(models).block({'my_award_id': 'abcd'}) assert r == {'my_award_id': 'abcd'} models = [get_generated_award_id_model(key='my_award_id', name='your_award_id')] r = TinyShield(models).block({'my_award_id': 'abcd'}) assert r == {'my_award_id': 'abcd'} models = [get_generated_award_id_model(key='my_award_id', name='your_award_id', optional=True)] r = TinyShield(models).block({'my_award_id': 'abcd'}) assert r == {'my_award_id': 'abcd'} models = [get_generated_award_id_model(key='my_award_id', name='your_award_id', optional=True)] r = TinyShield(models).block({}) assert r == {} # Rule violations. ts = TinyShield([get_generated_award_id_model()]) with pytest.raises(UnprocessableEntityException): ts.block({}) with pytest.raises(UnprocessableEntityException): ts.block({'award_id': 'A' * (MAX_ITEMS + 1)}) with pytest.raises(InvalidParameterException): ts.block({'award_id': 1.1}) with pytest.raises(InvalidParameterException): ts.block({'award_id': [1, 2]}) with pytest.raises(UnprocessableEntityException): ts.block({'id': 'abcd'})
def test_get_internal_award_id_rule(): models = [get_internal_award_id_model()] r = TinyShield(models).block({'award_id': 12345}) assert r == {'award_id': 12345} models = [get_internal_award_id_model()] r = TinyShield(models).block({'award_id': '12345'}) assert r == {'award_id': 12345} models = [get_internal_award_id_model()] r = TinyShield(models).block({'award_id': MAX_INT}) assert r == {'award_id': MAX_INT} models = [get_internal_award_id_model()] r = TinyShield(models).block({'award_id': MIN_INT}) assert r == {'award_id': MIN_INT} models = [get_internal_award_id_model(key='my_award_id')] r = TinyShield(models).block({'my_award_id': 12345}) assert r == {'my_award_id': 12345} models = [get_internal_award_id_model(key='my_award_id', name='your_award_id')] r = TinyShield(models).block({'my_award_id': 12345}) assert r == {'my_award_id': 12345} models = [get_internal_award_id_model(key='my_award_id', name='your_award_id', optional=True)] r = TinyShield(models).block({'my_award_id': 12345}) assert r == {'my_award_id': 12345} models = [get_internal_award_id_model(key='my_award_id', name='your_award_id', optional=True)] r = TinyShield(models).block({}) assert r == {} # Rule violations. ts = TinyShield([get_internal_award_id_model()]) with pytest.raises(UnprocessableEntityException): ts.block({}) with pytest.raises(UnprocessableEntityException): ts.block({'award_id': MAX_INT + 1}) with pytest.raises(UnprocessableEntityException): ts.block({'award_id': MIN_INT - 1}) with pytest.raises(InvalidParameterException): ts.block({'award_id': 1.1}) with pytest.raises(InvalidParameterException): ts.block({'award_id': [1, 2]}) with pytest.raises(UnprocessableEntityException): ts.block({'id': 'abcde'})
def post(self, request): models = [ {'name': 'subawards', 'key': 'subawards', 'type': 'boolean', 'default': False}, {'name': 'scope', 'key': 'scope', 'type': 'enum', 'optional': False, 'enum_values': ['place_of_performance', 'recipient_location']}, {'name': 'geo_layer', 'key': 'geo_layer', 'type': 'enum', 'optional': False, 'enum_values': ['state', 'county', 'district']}, {'name': 'geo_layer_filters', 'key': 'geo_layer_filters', 'type': 'array', 'array_type': 'text', 'text_type': 'search'} ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) json_request = TinyShield(models).block(request.data) self.subawards = json_request["subawards"] self.scope = json_request["scope"] self.filters = json_request.get("filters", None) self.geo_layer = json_request["geo_layer"] self.geo_layer_filters = json_request.get("geo_layer_filters", None) fields_list = [] # fields to include in the aggregate query loc_dict = { 'state': 'state_code', 'county': 'county_code', 'district': 'congressional_code' } model_dict = { 'place_of_performance': 'pop', 'recipient_location': 'recipient_location', # 'subawards_place_of_performance': 'pop', # 'subawards_recipient_location': 'recipient_location' } # Build the query based on the scope fields and geo_layers # Fields not in the reference objects above then request is invalid scope_field_name = model_dict.get(self.scope) loc_field_name = loc_dict.get(self.geo_layer) loc_lookup = '{}_{}'.format(scope_field_name, loc_field_name) if self.subawards: # We do not use matviews for Subaward filtering, just the Subaward download filters self.queryset = subaward_filter(self.filters) self.model_name = SubawardView else: self.queryset, self.model_name = spending_by_geography(self.filters) if self.geo_layer == 'state': # State will have one field (state_code) containing letter A-Z column_isnull = 'generated_pragmatic_obligation__isnull' if self.subawards: column_isnull = 'amount__isnull' kwargs = { '{}_country_code'.format(scope_field_name): 'USA', column_isnull: False } # Only state scope will add its own state code # State codes are consistent in database i.e. AL, AK fields_list.append(loc_lookup) state_response = { 'scope': self.scope, 'geo_layer': self.geo_layer, 'results': self.state_results(kwargs, fields_list, loc_lookup) } return Response(state_response) else: # County and district scope will need to select multiple fields # State code is needed for county/district aggregation state_lookup = '{}_{}'.format(scope_field_name, loc_dict['state']) fields_list.append(state_lookup) # Adding regex to county/district codes to remove entries with letters since can't be surfaced by map kwargs = { '{}__isnull'.format('amount' if self.subawards else 'generated_pragmatic_obligation'): False } if self.geo_layer == 'county': # County name added to aggregation since consistent in db county_name_lookup = '{}_county_name'.format(scope_field_name) fields_list.append(county_name_lookup) self.county_district_queryset( kwargs, fields_list, loc_lookup, state_lookup, scope_field_name ) county_response = { 'scope': self.scope, 'geo_layer': self.geo_layer, 'results': self.county_results(state_lookup, county_name_lookup) } return Response(county_response) else: self.county_district_queryset( kwargs, fields_list, loc_lookup, state_lookup, scope_field_name ) district_response = { 'scope': self.scope, 'geo_layer': self.geo_layer, 'results': self.district_results(state_lookup) } return Response(district_response)
def test_get_internal_or_generated_award_id_rule_bad(): # Rule violations. ts = TinyShield([get_internal_or_generated_award_id_model()]) with pytest.raises(UnprocessableEntityException): ts.block({}) with pytest.raises(UnprocessableEntityException): ts.block({'award_id': 'B' * (MAX_ITEMS + 1)}) with pytest.raises(UnprocessableEntityException): ts.block({'award_id': MAX_INT + 1}) with pytest.raises(UnprocessableEntityException): ts.block({'award_id': MIN_INT - 1}) with pytest.raises(UnprocessableEntityException): ts.block({'award_id': 1.1}) with pytest.raises(UnprocessableEntityException): ts.block({'award_id': [1, 2]}) with pytest.raises(UnprocessableEntityException): ts.block({'id': 'abcde'})
def post(self, request): """Return all awards matching the provided filters and limits""" models = [ {'name': 'fields', 'key': 'fields', 'type': 'array', 'array_type': 'text', 'text_type': 'search', 'min': 1}, {'name': 'subawards', 'key': 'subawards', 'type': 'boolean', 'default': False} ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) for m in models: if m['name'] in ('award_type_codes', 'fields'): m['optional'] = False json_request = TinyShield(models).block(request.data) fields = json_request["fields"] filters = json_request.get("filters", {}) subawards = json_request["subawards"] order = json_request["order"] limit = json_request["limit"] page = json_request["page"] if "no intersection" in filters["award_type_codes"]: # "Special case": there will never be results when the website provides this value return Response({ "limit": limit, "results": [], "page_metadata": {"page": page, "hasNext": False}, }) sort = json_request.get("sort", fields[0]) if sort not in fields: raise InvalidParameterException("Sort value '{}' not found in requested fields: {}".format(sort, fields)) subawards_values = list(contract_subaward_mapping.keys()) + list(grant_subaward_mapping.keys()) awards_values = list(award_contracts_mapping.keys()) + list(loan_award_mapping.keys()) + \ list(non_loan_assistance_award_mapping.keys()) + list(award_idv_mapping.keys()) msg = "Sort value '{}' not found in {{}} mappings: {{}}".format(sort) if not subawards and sort not in awards_values: raise InvalidParameterException(msg.format("award", awards_values)) elif subawards and sort not in subawards_values: raise InvalidParameterException(msg.format("subaward", subawards_values)) # build sql query filters if subawards: queryset = subaward_filter(filters) values = {'subaward_number', 'piid', 'fain', 'award_type'} for field in fields: if contract_subaward_mapping.get(field): values.add(contract_subaward_mapping.get(field)) if grant_subaward_mapping.get(field): values.add(grant_subaward_mapping.get(field)) else: queryset = matview_search_filter(filters, UniversalAwardView).values() values = {'award_id', 'piid', 'fain', 'uri', 'type'} for field in fields: if award_contracts_mapping.get(field): values.add(award_contracts_mapping.get(field)) if loan_award_mapping.get(field): values.add(loan_award_mapping.get(field)) if non_loan_assistance_award_mapping.get(field): values.add(non_loan_assistance_award_mapping.get(field)) if award_idv_mapping.get(field): values.add(award_idv_mapping.get(field)) # Modify queryset to be ordered by requested "sort" in the request or default value(s) if sort: if subawards: if set(filters["award_type_codes"]) <= set(contract_type_mapping): # Subaward contracts sort_filters = [contract_subaward_mapping[sort]] elif set(filters["award_type_codes"]) <= set(grant_type_mapping): # Subaward grants sort_filters = [grant_subaward_mapping[sort]] else: msg = 'Award Type codes limited for Subawards. Only contracts {} or grants {} are available' msg = msg.format(list(contract_type_mapping.keys()), list(grant_type_mapping.keys())) raise UnprocessableEntityException(msg) else: if set(filters["award_type_codes"]) <= set(contract_type_mapping): # contracts sort_filters = [award_contracts_mapping[sort]] elif set(filters["award_type_codes"]) <= set(loan_type_mapping): # loans sort_filters = [loan_award_mapping[sort]] elif set(filters["award_type_codes"]) <= set(idv_type_mapping): # idvs sort_filters = [award_idv_mapping[sort]] else: # assistance data sort_filters = [non_loan_assistance_award_mapping[sort]] # Explictly set NULLS LAST in the ordering to encourage the usage of the indexes if sort == "Award ID" and subawards: if order == "desc": queryset = queryset.order_by( F('award__piid').desc(nulls_last=True), F('award__fain').desc(nulls_last=True)).values(*list(values)) else: queryset = queryset.order_by( F('award__piid').asc(nulls_last=True), F('award__fain').asc(nulls_last=True)).values(*list(values)) elif sort == "Award ID": if order == "desc": queryset = queryset.order_by( F('piid').desc(nulls_last=True), F('fain').desc(nulls_last=True), F('uri').desc(nulls_last=True)).values(*list(values)) else: queryset = queryset.order_by( F('piid').asc(nulls_last=True), F('fain').asc(nulls_last=True), F('uri').asc(nulls_last=True)).values(*list(values)) elif order == "desc": queryset = queryset.order_by(F(sort_filters[0]).desc(nulls_last=True)).values(*list(values)) else: queryset = queryset.order_by(F(sort_filters[0]).asc(nulls_last=True)).values(*list(values)) limited_queryset = queryset[(page - 1) * limit:page * limit + 1] # lower limit : upper limit has_next = len(limited_queryset) > limit results = [] for award in limited_queryset[:limit]: if subawards: row = {"internal_id": award["subaward_number"]} if award['award_type'] == 'procurement': for field in fields: row[field] = award.get(contract_subaward_mapping[field]) elif award['award_type'] == 'grant': for field in fields: row[field] = award.get(grant_subaward_mapping[field]) else: row = {"internal_id": award["award_id"]} if award['type'] in loan_type_mapping: # loans for field in fields: row[field] = award.get(loan_award_mapping.get(field)) elif award['type'] in non_loan_assistance_type_mapping: # assistance data for field in fields: row[field] = award.get(non_loan_assistance_award_mapping.get(field)) elif award['type'] in idv_type_mapping: for field in fields: row[field] = award.get(award_idv_mapping.get(field)) elif (award['type'] is None and award['piid']) or award['type'] in contract_type_mapping: # IDV + contract for field in fields: row[field] = award.get(award_contracts_mapping.get(field)) if "Award ID" in fields: for id_type in ["piid", "fain", "uri"]: if award[id_type]: row["Award ID"] = award[id_type] break results.append(row) return Response({"limit": limit, "results": results, "page_metadata": {"page": page, "hasNext": has_next}})
def test_any_rule(): models = [{ 'name': 'value', 'key': 'value', 'type': 'any', 'models': [ {'type': 'integer'}, {'type': 'text', 'text_type': 'search'} ] }] # Test integer and random other key. ts = TinyShield(models).block({'value': 1, 'another_value': 2}) assert ts['value'] == 1 assert ts.get('another_value') is None # Test integer masquerading as a string. ts = TinyShield(models).block({'value': '1'}) assert ts['value'] == 1 # Test string. ts = TinyShield(models).block({'value': 'XYZ'}) assert ts['value'] == 'XYZ' # Test list (which should blow up). with pytest.raises(UnprocessableEntityException): TinyShield(models).block({'value': ['XYZ']}) # Test with optional 'value' missing. ts = TinyShield(models).block({'another_value': 2}) assert ts.get('value') is None assert ts.get('another_value') is None # Make 'value' required then run the same tests as above. models[0]['optional'] = False # Test integer and random other key. ts = TinyShield(models).block({'value': 1, 'another_value': 2}) assert ts['value'] == 1 assert ts.get('another_value') is None # Test integer masquerading as a string. ts = TinyShield(models).block({'value': '1'}) assert ts['value'] == 1 # Test string. ts = TinyShield(models).block({'value': 'XYZ'}) assert ts['value'] == 'XYZ' # Test list (which should blow up). with pytest.raises(UnprocessableEntityException): TinyShield(models).block({'value': ['XYZ']}) # Test with required 'value' missing. with pytest.raises(UnprocessableEntityException): TinyShield(models).block({'another_value': 2})