def validate_request_data(self, json_data):
self.groupings = {
"quarter": "quarter",
"q": "quarter",
"fiscal_year": "fiscal_year",
"fy": "fiscal_year",
"month": "month",
"m": "month",
}
models = [
{"name": "subawards", "key": "subawards", "type": "boolean", "default": False},
{
"name": "group",
"key": "group",
"type": "enum",
"enum_values": list(self.groupings.keys()),
"default": "fy",
"optional": False, # allow to be optional in the future
},
]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
validated_data = TinyShield(models).block(json_data)
if validated_data.get("filters", None) is None:
raise InvalidParameterException("Missing request parameters: filters")
return validated_data
def post(self, request):
"""Returns boolean of whether a download request is greater than the max limit. """
models = [
{'name': 'subawards', 'key': 'subawards', 'type': 'boolean', 'default': False},
]
models.extend(copy.deepcopy(AWARD_FILTER))
json_request = TinyShield(models).block(request.data)
# If no filters in request return empty object to return all transactions
filters = json_request.get('filters', {})
is_over_limit = False
if json_request['subawards']:
total_count = subaward_filter(filters).count()
else:
queryset, model = download_transaction_count(filters)
if model in ['UniversalTransactionView']:
total_count = queryset.count()
else:
# "summary" materialized views are pre-aggregated and contain a counts col
total_count = queryset.aggregate(total_count=Sum('counts'))['total_count']
if total_count and total_count > settings.MAX_DOWNLOAD_LIMIT:
is_over_limit = True
result = {
"transaction_rows_gt_limit": is_over_limit
}
return Response(result)
def validate_request_data(self, json_data):
self.groupings = {
"quarter": "quarter",
"q": "quarter",
"fiscal_year": "fiscal_year",
"fy": "fiscal_year",
"month": "month",
"m": "month",
}
models = [
{
"name": "subawards",
"key": "subawards",
"type": "boolean",
"default": False
},
{
"name": "group",
"key": "group",
"type": "enum",
"enum_values": list(self.groupings.keys()),
"default": "fy",
"optional": False, # allow to be optional in the future
},
]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
validated_data = TinyShield(models).block(json_data)
if validated_data.get("filters", None) is None:
raise InvalidParameterException(
"Missing request parameters: filters")
return validated_data
def validate_request_data(json_data: dict) -> dict:
models = [
{
"name": "subawards",
"key": "subawards",
"type": "boolean",
"default": False
},
{
"name": "group",
"key": "group",
"type": "enum",
"enum_values": list(GROUPING_LOOKUP.keys()),
"default": "fy",
"optional": False, # allow to be optional in the future
},
]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
validated_data = TinyShield(models).block(json_data)
if validated_data.get("filters", None) is None:
raise InvalidParameterException(
"Missing request parameters: filters")
return validated_data
def post(self, request):
models = [{'name': 'subawards', 'key': 'subawards', 'type': 'boolean', 'default': False}]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
json_request = TinyShield(models).block(request.data)
filters = json_request.get("filters", None)
subawards = json_request["subawards"]
if filters is None:
raise InvalidParameterException("Missing required request parameters: 'filters'")
results = {
"contracts": 0, "idvs": 0, "grants": 0, "direct_payments": 0, "loans": 0, "other": 0
} if not subawards else {
"subcontracts": 0, "subgrants": 0
}
if "award_type_codes" in filters and "no intersection" in filters["award_type_codes"]:
# "Special case": there will never be results when the website provides this value
return Response({"results": results})
if subawards:
queryset = subaward_filter(filters)
else:
queryset, model = spending_by_award_count(filters)
if subawards:
queryset = queryset.values('award_type').annotate(category_count=Count('subaward_id'))
elif model == 'SummaryAwardView':
queryset = queryset.values('category').annotate(category_count=Sum('counts'))
else:
queryset = queryset.values('category').annotate(category_count=Count('category'))
categories = {
'contract': 'contracts',
'idv': 'idvs',
'grant': 'grants',
'direct payment': 'direct_payments',
'loans': 'loans',
'other': 'other'
} if not subawards else {'procurement': 'subcontracts', 'grant': 'subgrants'}
category_name = 'category' if not subawards else 'award_type'
# DB hit here
for award in queryset:
if award[category_name] is None:
result_key = 'other' if not subawards else 'subcontracts'
elif award[category_name] not in categories.keys():
result_key = 'other'
else:
result_key = categories[award[category_name]]
results[result_key] += award['category_count']
return Response({"results": results})
def get(self, request):
model = [
{
"key": "fiscal_year",
"name": "fiscal_year",
"type": "integer",
"min": 2017,
"optional": True,
"default": None,
"allow_nulls": True,
},
{
"key": "fiscal_period",
"name": "fiscal_period",
"type": "integer",
"min": 2,
"max": 12,
"optional": True,
"default": None,
"allow_nulls": True,
},
]
validated = TinyShield(model).block(request.query_params)
fiscal_year = validated.get("fiscal_year", None)
fiscal_period = validated.get("fiscal_period", None)
gtas_queryset = GTASSF133Balances.objects.values(
"fiscal_year", "fiscal_period")
if fiscal_period:
if not fiscal_year:
raise InvalidParameterException(
"fiscal_period was provided without fiscal_year.")
else:
gtas_queryset = gtas_queryset.filter(
fiscal_year=fiscal_year, fiscal_period=fiscal_period)
elif fiscal_year:
gtas_queryset = gtas_queryset.filter(fiscal_year=fiscal_year)
results = gtas_queryset.annotate(total_budgetary_resources=Sum(
"total_budgetary_resources_cpe")).order_by("-fiscal_year",
"-fiscal_period")
return Response({
"results":
list(results),
"messages": [get_account_data_time_period_message()]
if not fiscal_year or fiscal_year < 2017 else [],
})
def post(self, request):
models = [
{"name": "subawards", "key": "subawards", "type": "boolean", "default": False},
{
"name": "object_class",
"key": "filter|object_class",
"type": "array",
"array_type": "text",
"text_type": "search",
},
{
"name": "program_activity",
"key": "filter|program_activity",
"type": "array",
"array_type": "integer",
"array_max": maxsize,
},
]
models.extend(copy.deepcopy(AWARD_FILTER_NO_RECIPIENT_ID))
models.extend(copy.deepcopy(PAGINATION))
self.original_filters = request.data.get("filters")
json_request = TinyShield(models).block(request.data)
subawards = json_request["subawards"]
filters = add_date_range_comparison_types(
json_request.get("filters", None), subawards, gte_date_type="action_date", lte_date_type="date_signed"
)
if filters is None:
raise InvalidParameterException("Missing required request parameters: 'filters'")
if "award_type_codes" in filters and "no intersection" in filters["award_type_codes"]:
# "Special case": there will never be results when the website provides this value
empty_results = {"contracts": 0, "idvs": 0, "grants": 0, "direct_payments": 0, "loans": 0, "other": 0}
if subawards:
empty_results = {"subcontracts": 0, "subgrants": 0}
results = empty_results
elif subawards:
results = self.handle_subawards(filters)
else:
results = self.query_elasticsearch_for_prime_awards(filters)
return Response(
{
"results": results,
"messages": get_generic_filters_message(
self.original_filters.keys(), [elem["name"] for elem in AWARD_FILTER_NO_RECIPIENT_ID]
),
}
)
def post(self, request):
models = [
{"name": "subawards", "key": "subawards", "type": "boolean", "default": False},
{
"name": "object_class",
"key": "filter|object_class",
"type": "array",
"array_type": "text",
"text_type": "search",
},
{
"name": "program_activity",
"key": "filter|program_activity",
"type": "array",
"array_type": "integer",
"array_max": maxsize,
},
]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
json_request = TinyShield(models).block(request.data)
subawards = json_request["subawards"]
filters = add_date_range_comparison_types(
json_request.get("filters", None), subawards, gte_date_type="action_date", lte_date_type="date_signed"
)
elasticsearch = is_experimental_elasticsearch_api(request)
if elasticsearch and not subawards:
logger.info("Using experimental Elasticsearch functionality for 'spending_by_award_count'")
results = self.query_elasticsearch(filters)
return Response({"results": results, "messages": [get_time_period_message()]})
if filters is None:
raise InvalidParameterException("Missing required request parameters: 'filters'")
empty_results = {"contracts": 0, "idvs": 0, "grants": 0, "direct_payments": 0, "loans": 0, "other": 0}
if subawards:
empty_results = {"subcontracts": 0, "subgrants": 0}
if "award_type_codes" in filters and "no intersection" in filters["award_type_codes"]:
# "Special case": there will never be results when the website provides this value
return Response({"results": empty_results})
if subawards:
results = self.handle_subawards(filters)
else:
results = self.handle_awards(filters, empty_results)
return Response({"results": results, "messages": [get_time_period_message()]})
def get(self, request: Request) -> Response:
"""
Accepts only pagination-related query parameters
"""
models = [
{'name': 'page', 'key': 'page', 'type': 'integer', 'default': 1, 'min': 1},
{'name': 'limit', 'key': 'limit', 'type': 'integer', 'default': 500, 'min': 1, 'max': 500},
]
# Can't use the TinyShield decorator (yet) because this is a GET request only
request_dict = request.query_params
validated_request_data = TinyShield(models).block(request_dict)
limit = validated_request_data["limit"]
page = validated_request_data["page"]
queryset = Definition.objects.all()
queryset, pagination = get_pagination(queryset, int(limit), int(page))
serializer = DefinitionSerializer(queryset, many=True)
response = {
"page_metadata": pagination,
"results": serializer.data
}
return Response(response)
def post(self, request):
"""
Returns a summary of transactions which match the award search filter
Desired values:
total number of transactions `award_count`
The federal_action_obligation sum of all those transactions `award_spending`
*Note* Only deals with prime awards, future plans to include sub-awards.
"""
models = [
{
"name": "keywords",
"key": "filters|keywords",
"type": "array",
"array_type": "text",
"text_type": "search",
"optional": False,
"text_min": 3,
}
]
validated_payload = TinyShield(models).block(request.data)
results = spending_by_transaction_sum_and_count(validated_payload)
if not results:
raise ElasticsearchConnectionException("Error generating the transaction sums and counts")
return Response({"results": results})
def validate_request_data(request_data):
models = [
{"name": "fields", "key": "fields", "type": "array", "array_type": "text", "text_type": "search", "min": 1},
{"name": "subawards", "key": "subawards", "type": "boolean", "default": False},
{
"name": "object_class",
"key": "filter|object_class",
"type": "array",
"array_type": "text",
"text_type": "search",
},
{
"name": "program_activity",
"key": "filter|program_activity",
"type": "array",
"array_type": "integer",
"array_max": maxsize,
},
]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
for m in models:
if m["name"] in ("award_type_codes", "fields"):
m["optional"] = False
return TinyShield(models).block(request_data)
def test_check_models():
"""We want this test to fail if either AWARD_FILTERS has an invalid model,
OR if the logic of the check_models function has been corrupted.
It will fail if an exception is raised. Otherwise it will define the global TS object
so we can use it in the remaining tests."""
global TS
TS = TinyShield(copy.deepcopy(AWARD_FILTER))
def post(self, request):
models = [
{
'name': 'fields',
'key': 'fields',
'type': 'array',
'array_type': 'text',
'text_type': 'search',
'optional': False
},
]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
for m in models:
if m['name'] in ('keywords', 'award_type_codes', 'sort'):
m['optional'] = False
validated_payload = TinyShield(models).block(request.data)
if validated_payload['sort'] not in validated_payload['fields']:
raise InvalidParameterException(
"Sort value not found in fields: {}".format(
validated_payload['sort']))
if "filters" in validated_payload and "no intersection" in validated_payload[
"filters"]["award_type_codes"]:
# "Special case": there will never be results when the website provides this value
return Response({
"limit": validated_payload["limit"],
"results": [],
"page_metadata": {
"page": validated_payload["page"],
"next": None,
"previous": None,
"hasNext": False,
"hasPrevious": False,
},
})
lower_limit = (validated_payload['page'] -
1) * validated_payload['limit']
success, response, total = search_transactions(
validated_payload, lower_limit, validated_payload['limit'] + 1)
if not success:
raise InvalidParameterException(response)
metadata = get_simple_pagination_metadata(len(response),
validated_payload['limit'],
validated_payload['page'])
results = []
for transaction in response[:validated_payload['limit']]:
results.append(transaction)
response = {
'limit': validated_payload['limit'],
'results': results,
'page_metadata': metadata
}
return Response(response)
def validate_api_request(self, json_payload):
self.groupings = {
'quarter': 'quarter',
'q': 'quarter',
'fiscal_year': 'fiscal_year',
'fy': 'fiscal_year',
'month': 'month',
'm': 'month',
}
models = [{
'name': 'group',
'key': 'group',
'type': 'enum',
'enum_values': list(self.groupings.keys()),
'default': 'fy',
}]
advanced_search_filters = [
model for model in copy.deepcopy(AWARD_FILTER)
if model['name'] in ('time_period', 'recipient_id')
]
for model in advanced_search_filters:
if model['name'] in ('time_period', 'recipient_id'):
model['optional'] = False
models.extend(advanced_search_filters)
return TinyShield(models).block(json_payload)
def validate_api_request(self, json_payload):
self.groupings = {
"quarter": "quarter",
"q": "quarter",
"fiscal_year": "fiscal_year",
"fy": "fiscal_year",
"month": "month",
"m": "month",
}
models = [
{
"name": "group",
"key": "group",
"type": "enum",
"enum_values": list(self.groupings.keys()),
"default": "fy",
}
]
advanced_search_filters = [
model for model in copy.deepcopy(AWARD_FILTER) if model["name"] in ("time_period", "recipient_id")
]
for model in advanced_search_filters:
if model["name"] in ("time_period", "recipient_id"):
model["optional"] = False
models.extend(advanced_search_filters)
return TinyShield(models).block(json_payload)
def _parse_and_validate_request(self, request_dict):
models = deepcopy(PAGINATION)
models.append({
"key":
"award_id",
"name":
"award_id",
"type":
"any",
"models": [{
"type": "integer"
}, {
"type": "text",
"text_type": "search"
}],
"optional":
True,
"default":
None,
"allow_nulls":
True,
})
for model in models:
# Change sort to an enum of the desired values
if model["name"] == "sort":
model["type"] = "enum"
model["enum_values"] = list(self.subaward_lookup.keys())
model["default"] = "subaward_number"
validated_request_data = TinyShield(models).block(request_dict)
return validated_request_data
def post(self, request):
award_types = list(AWARD_TYPES.keys()) + ["all"]
models = [
{
"name": "keyword",
"key": "keyword",
"type": "text",
"text_type": "search"
},
{
"name": "award_type",
"key": "award_type",
"type": "enum",
"enum_values": award_types,
"default": "all"
},
]
models.extend(copy.deepcopy(PAGINATION)) # page, limit, sort, order
new_sort = {
"type": "enum",
"enum_values": ["name", "duns", "amount"],
"default": "amount"
}
models = update_model_in_list(models, "sort", new_sort)
models = update_model_in_list(models, "limit", {"default": 50})
validated_payload = TinyShield(models).block(request.data)
results, page_metadata = get_recipients(filters=validated_payload)
return Response({"page_metadata": page_metadata, "results": results})
def post(self, request: dict):
"""Return all budget function/subfunction titles matching the provided search text"""
categories = [
"awarding_agency",
"awarding_subagency",
"funding_agency",
"funding_subagency",
"recipient_duns",
"recipient_parent_duns",
"cfda",
"psc",
"naics",
"county",
"district",
"country",
"state_territory",
"federal_account",
]
models = [
{"name": "category", "key": "category", "type": "enum", "enum_values": categories, "optional": False},
{"name": "subawards", "key": "subawards", "type": "boolean", "default": False, "optional": True},
]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
# Apply/enforce POST body schema and data validation in request
validated_payload = TinyShield(models).block(request.data)
# Execute the business logic for the endpoint and return a python dict to be converted to a Django response
return Response(BusinessLogic(validated_payload).results())
def _validate_tas_codes(filters, json_request):
if "tas_codes" in filters:
tas_codes = {"filters": {"tas_codes": filters["tas_codes"]}}
models = [deepcopy(get_model_by_name(AWARD_FILTER, "tas_codes"))]
tas_codes = TinyShield(models).block(tas_codes)
if tas_codes:
json_request["filters"]["tas_codes"] = tas_codes["filters"][
"tas_codes"]
def post(self, request):
models = [{
"name": "fields",
"key": "fields",
"type": "array",
"array_type": "text",
"text_type": "search",
"optional": False,
}]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
for m in models:
if m["name"] in ("keywords", "award_type_codes", "sort"):
m["optional"] = False
validated_payload = TinyShield(models).block(request.data)
if validated_payload["sort"] not in validated_payload["fields"]:
raise InvalidParameterException(
"Sort value not found in fields: {}".format(
validated_payload["sort"]))
if "filters" in validated_payload and "no intersection" in validated_payload[
"filters"]["award_type_codes"]:
# "Special case": there will never be results when the website provides this value
return Response({
"limit": validated_payload["limit"],
"results": [],
"page_metadata": {
"page": validated_payload["page"],
"next": None,
"previous": None,
"hasNext": False,
"hasPrevious": False,
},
})
lower_limit = (validated_payload["page"] -
1) * validated_payload["limit"]
success, response, total = search_transactions(
validated_payload, lower_limit, validated_payload["limit"] + 1)
if not success:
raise InvalidParameterException(response)
metadata = get_simple_pagination_metadata(len(response),
validated_payload["limit"],
validated_payload["page"])
results = []
for transaction in response[:validated_payload["limit"]]:
results.append(transaction)
response = {
"limit": validated_payload["limit"],
"results": results,
"page_metadata": metadata
}
return Response(response)
def post(self, request):
"""Returns boolean of whether a download request is greater than the max limit. """
models = [{
"name": "subawards",
"key": "subawards",
"type": "boolean",
"default": False
}]
models.extend(copy.deepcopy(AWARD_FILTER))
self.original_filters = request.data.get("filters")
json_request = TinyShield(models).block(request.data)
# If no filters in request return empty object to return all transactions
filters = json_request.get("filters", {})
if json_request["subawards"]:
total_count = subaward_filter(filters).count()
else:
queryset, model = download_transaction_count(filters)
if model in ["UniversalTransactionView"]:
total_count = queryset.count()
else:
# "summary" materialized views are pre-aggregated and contain a counts col
total_count = queryset.aggregate(
total_count=Sum("counts"))["total_count"]
if total_count is None:
total_count = 0
result = {
"calculated_transaction_count":
total_count,
"maximum_transaction_limit":
settings.MAX_DOWNLOAD_LIMIT,
"transaction_rows_gt_limit":
total_count > settings.MAX_DOWNLOAD_LIMIT,
"messages": [
get_generic_filters_message(
self.original_filters.keys(),
[elem["name"] for elem in AWARD_FILTER])
],
}
return Response(result)
def post(self, request):
models = [{
"name": "subawards",
"key": "subawards",
"type": "boolean",
"default": False
}]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
json_request = TinyShield(models).block(request.data)
subawards = json_request["subawards"]
filters = add_date_range_comparison_types(json_request.get(
"filters", None),
subawards,
gte_date_type="action_date",
lte_date_type="date_signed")
if filters is None:
raise InvalidParameterException(
"Missing required request parameters: 'filters'")
empty_results = {
"contracts": 0,
"idvs": 0,
"grants": 0,
"direct_payments": 0,
"loans": 0,
"other": 0
}
if subawards:
empty_results = {"subcontracts": 0, "subgrants": 0}
if "award_type_codes" in filters and "no intersection" in filters[
"award_type_codes"]:
# "Special case": there will never be results when the website provides this value
return Response({"results": empty_results})
if subawards:
results = self.handle_subawards(filters)
else:
results = self.handle_awards(filters, empty_results)
return Response({"results": results})
def test_non_default_overridden_pagination():
pagination = deepcopy(PAGINATION)
r = TinyShield(pagination).block({
"page": 2,
"limit": 11,
"sort": "whatever",
"order": "asc"
})
assert r == {"page": 2, "limit": 11, "sort": "whatever", "order": "asc"}
def post(self, request: Request) -> Response:
models = [
{"name": "subawards", "key": "subawards", "type": "boolean", "default": False, "optional": True},
]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
original_filters = request.data.get("filters")
validated_payload = TinyShield(models).block(request.data)
return Response(self.perform_search(validated_payload, original_filters))
def post(self, request):
"""Returns boolean of whether a download request is greater than the max limit. """
models = [{
"name": "subawards",
"key": "subawards",
"type": "boolean",
"default": False
}]
models.extend(copy.deepcopy(AWARD_FILTER))
self.original_filters = request.data.get("filters")
json_request = TinyShield(models).block(request.data)
# If no filters in request return empty object to return all transactions
filters = json_request.get("filters", {})
if json_request["subawards"]:
total_count = subaward_filter(filters).count()
else:
filter_query = QueryWithFilters.generate_transactions_elasticsearch_query(
filters)
search = TransactionSearch().filter(filter_query)
total_count = search.handle_count()
if total_count is None:
total_count = 0
result = {
"calculated_transaction_count":
total_count,
"maximum_transaction_limit":
settings.MAX_DOWNLOAD_LIMIT,
"transaction_rows_gt_limit":
total_count > settings.MAX_DOWNLOAD_LIMIT,
"messages": [
get_generic_filters_message(
self.original_filters.keys(),
[elem["name"] for elem in AWARD_FILTER])
],
}
return Response(result)
def get(self, request, toptier_code, fiscal_year, fiscal_period, type):
my_request = {
"type": type,
"fiscal_year": fiscal_year,
"fiscal_period": fiscal_period
}
validated = TinyShield(self.tinyshield_model).block(my_request)
self.annotations = self.annotation_options[validated["type"]]
self.fiscal_year = validated["fiscal_year"]
self.fiscal_period = validated["fiscal_period"]
return Response(self.get_unlinked_awards())
def post(self, request: Request) -> Response:
"""Return all budget function/subfunction titles matching the provided search text"""
categories = [
"awarding_agency",
"awarding_subagency",
"funding_agency",
"funding_subagency",
"recipient_duns",
"recipient_parent_duns",
"cfda",
"psc",
"naics",
"county",
"district",
"country",
"state_territory",
"federal_account",
]
models = [
{"name": "category", "key": "category", "type": "enum", "enum_values": categories, "optional": False},
{"name": "subawards", "key": "subawards", "type": "boolean", "default": False, "optional": True},
]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
# Apply/enforce POST body schema and data validation in request
original_filters = request.data.get("filters")
validated_payload = TinyShield(models).block(request.data)
# Execute the business logic for the endpoint and return a python dict to be converted to a Django response
business_logic_lookup = {
"awarding_agency": AwardingAgencyViewSet().perform_search,
"awarding_subagency": AwardingSubagencyViewSet().perform_search,
"cfda": CfdaViewSet().perform_search,
"country": CountryViewSet().perform_search,
"county": CountyViewSet().perform_search,
"district": DistrictViewSet().perform_search,
"federal_account": FederalAccountViewSet().perform_search,
"funding_agency": FundingAgencyViewSet().perform_search,
"funding_subagency": FundingSubagencyViewSet().perform_search,
"naics": NAICSViewSet().perform_search,
"psc": PSCViewSet().perform_search,
"recipient_duns": RecipientDunsViewSet().perform_search,
"state_territory": StateTerritoryViewSet().perform_search,
}
business_logic_func = business_logic_lookup.get(validated_payload["category"])
if business_logic_func:
return Response(business_logic_func(validated_payload, original_filters))
else:
self.raise_not_implemented(validated_payload)
def _parse_and_validate(self, request):
all_def_codes = sorted(list(DisasterEmergencyFundCode.objects.values_list("code", flat=True)))
models = [
{
"key": "def_codes",
"name": "def_codes",
"type": "text",
"text_type": "search",
"allow_nulls": True,
"optional": True,
"default": ",".join(all_def_codes),
},
]
return TinyShield(models).block(request)
def post(self, request):
models = [{
"name": "keywords",
"key": "filters|keywords",
"type": "array",
"array_type": "text",
"text_type": "search",
"optional": False,
"text_min": 3,
}]
validated_payload = TinyShield(models).block(request.data)
results = spending_by_transaction_count(validated_payload)
return Response({"results": results})
def _parse_and_validate_request(requested_award: str,
request_data: dict) -> dict:
piid = request_data.get("piid")
data = {"award_id": requested_award, "piid": piid}
models = [
get_internal_or_generated_award_id_model(),
{
"key": "piid",
"name": "piid",
"allow_nulls": True,
"optional": True,
"type": "text",
"text_type": "search",
},
]
return TinyShield(models).block(data)
def _parse_and_validate_request(self, provided_award_id: str) -> dict:
request_dict = {"generated_unique_award_id": provided_award_id}
models = [
{
"key": "generated_unique_award_id",
"name": "generated_unique_award_id",
"type": "text",
"text_type": "search",
"optional": False,
}
]
if str(provided_award_id).isdigit():
request_dict = {"id": int(provided_award_id)}
models = [{"key": "id", "name": "id", "type": "integer", "optional": False}]
validated_request_data = TinyShield(models).block(request_dict)
return validated_request_data
def _parse_and_validate_request(self, requested_naics: str, request_data) -> dict:
naics_filter = request_data.get("filter")
data = {"code": requested_naics, "filter": naics_filter}
models = [
{"key": "code", "name": "code", "type": "integer", "allow_nulls": True, "optional": True},
{
"key": "filter",
"name": "filter",
"type": "text",
"text_type": "search",
"default": None,
"optional": True,
"allow_nulls": True,
},
]
validated = TinyShield(models).block(data)
return validated
def test_get_generate_award_id_rule():
models = [get_generated_award_id_model()]
r = TinyShield(models).block({'award_id': 'abcd'})
assert r == {'award_id': 'abcd'}
models = [get_generated_award_id_model()]
r = TinyShield(models).block({'award_id': 'A' * MAX_ITEMS})
assert r == {'award_id': 'A' * MAX_ITEMS}
models = [get_generated_award_id_model(key='my_award_id')]
r = TinyShield(models).block({'my_award_id': 'abcd'})
assert r == {'my_award_id': 'abcd'}
models = [get_generated_award_id_model(key='my_award_id', name='your_award_id')]
r = TinyShield(models).block({'my_award_id': 'abcd'})
assert r == {'my_award_id': 'abcd'}
models = [get_generated_award_id_model(key='my_award_id', name='your_award_id', optional=True)]
r = TinyShield(models).block({'my_award_id': 'abcd'})
assert r == {'my_award_id': 'abcd'}
models = [get_generated_award_id_model(key='my_award_id', name='your_award_id', optional=True)]
r = TinyShield(models).block({})
assert r == {}
# Rule violations.
ts = TinyShield([get_generated_award_id_model()])
with pytest.raises(UnprocessableEntityException):
ts.block({})
with pytest.raises(UnprocessableEntityException):
ts.block({'award_id': 'A' * (MAX_ITEMS + 1)})
with pytest.raises(InvalidParameterException):
ts.block({'award_id': 1.1})
with pytest.raises(InvalidParameterException):
ts.block({'award_id': [1, 2]})
with pytest.raises(UnprocessableEntityException):
ts.block({'id': 'abcd'})
def test_get_internal_award_id_rule():
models = [get_internal_award_id_model()]
r = TinyShield(models).block({'award_id': 12345})
assert r == {'award_id': 12345}
models = [get_internal_award_id_model()]
r = TinyShield(models).block({'award_id': '12345'})
assert r == {'award_id': 12345}
models = [get_internal_award_id_model()]
r = TinyShield(models).block({'award_id': MAX_INT})
assert r == {'award_id': MAX_INT}
models = [get_internal_award_id_model()]
r = TinyShield(models).block({'award_id': MIN_INT})
assert r == {'award_id': MIN_INT}
models = [get_internal_award_id_model(key='my_award_id')]
r = TinyShield(models).block({'my_award_id': 12345})
assert r == {'my_award_id': 12345}
models = [get_internal_award_id_model(key='my_award_id', name='your_award_id')]
r = TinyShield(models).block({'my_award_id': 12345})
assert r == {'my_award_id': 12345}
models = [get_internal_award_id_model(key='my_award_id', name='your_award_id', optional=True)]
r = TinyShield(models).block({'my_award_id': 12345})
assert r == {'my_award_id': 12345}
models = [get_internal_award_id_model(key='my_award_id', name='your_award_id', optional=True)]
r = TinyShield(models).block({})
assert r == {}
# Rule violations.
ts = TinyShield([get_internal_award_id_model()])
with pytest.raises(UnprocessableEntityException):
ts.block({})
with pytest.raises(UnprocessableEntityException):
ts.block({'award_id': MAX_INT + 1})
with pytest.raises(UnprocessableEntityException):
ts.block({'award_id': MIN_INT - 1})
with pytest.raises(InvalidParameterException):
ts.block({'award_id': 1.1})
with pytest.raises(InvalidParameterException):
ts.block({'award_id': [1, 2]})
with pytest.raises(UnprocessableEntityException):
ts.block({'id': 'abcde'})
def post(self, request):
models = [
{'name': 'subawards', 'key': 'subawards', 'type': 'boolean', 'default': False},
{'name': 'scope', 'key': 'scope', 'type': 'enum', 'optional': False,
'enum_values': ['place_of_performance', 'recipient_location']},
{'name': 'geo_layer', 'key': 'geo_layer', 'type': 'enum', 'optional': False,
'enum_values': ['state', 'county', 'district']},
{'name': 'geo_layer_filters', 'key': 'geo_layer_filters', 'type': 'array', 'array_type': 'text',
'text_type': 'search'}
]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
json_request = TinyShield(models).block(request.data)
self.subawards = json_request["subawards"]
self.scope = json_request["scope"]
self.filters = json_request.get("filters", None)
self.geo_layer = json_request["geo_layer"]
self.geo_layer_filters = json_request.get("geo_layer_filters", None)
fields_list = [] # fields to include in the aggregate query
loc_dict = {
'state': 'state_code',
'county': 'county_code',
'district': 'congressional_code'
}
model_dict = {
'place_of_performance': 'pop',
'recipient_location': 'recipient_location',
# 'subawards_place_of_performance': 'pop',
# 'subawards_recipient_location': 'recipient_location'
}
# Build the query based on the scope fields and geo_layers
# Fields not in the reference objects above then request is invalid
scope_field_name = model_dict.get(self.scope)
loc_field_name = loc_dict.get(self.geo_layer)
loc_lookup = '{}_{}'.format(scope_field_name, loc_field_name)
if self.subawards:
# We do not use matviews for Subaward filtering, just the Subaward download filters
self.queryset = subaward_filter(self.filters)
self.model_name = SubawardView
else:
self.queryset, self.model_name = spending_by_geography(self.filters)
if self.geo_layer == 'state':
# State will have one field (state_code) containing letter A-Z
column_isnull = 'generated_pragmatic_obligation__isnull'
if self.subawards:
column_isnull = 'amount__isnull'
kwargs = {
'{}_country_code'.format(scope_field_name): 'USA',
column_isnull: False
}
# Only state scope will add its own state code
# State codes are consistent in database i.e. AL, AK
fields_list.append(loc_lookup)
state_response = {
'scope': self.scope,
'geo_layer': self.geo_layer,
'results': self.state_results(kwargs, fields_list, loc_lookup)
}
return Response(state_response)
else:
# County and district scope will need to select multiple fields
# State code is needed for county/district aggregation
state_lookup = '{}_{}'.format(scope_field_name, loc_dict['state'])
fields_list.append(state_lookup)
# Adding regex to county/district codes to remove entries with letters since can't be surfaced by map
kwargs = {
'{}__isnull'.format('amount' if self.subawards else 'generated_pragmatic_obligation'): False
}
if self.geo_layer == 'county':
# County name added to aggregation since consistent in db
county_name_lookup = '{}_county_name'.format(scope_field_name)
fields_list.append(county_name_lookup)
self.county_district_queryset(
kwargs,
fields_list,
loc_lookup,
state_lookup,
scope_field_name
)
county_response = {
'scope': self.scope,
'geo_layer': self.geo_layer,
'results': self.county_results(state_lookup, county_name_lookup)
}
return Response(county_response)
else:
self.county_district_queryset(
kwargs,
fields_list,
loc_lookup,
state_lookup,
scope_field_name
)
district_response = {
'scope': self.scope,
'geo_layer': self.geo_layer,
'results': self.district_results(state_lookup)
}
return Response(district_response)
def test_get_internal_or_generated_award_id_rule_bad():
# Rule violations.
ts = TinyShield([get_internal_or_generated_award_id_model()])
with pytest.raises(UnprocessableEntityException):
ts.block({})
with pytest.raises(UnprocessableEntityException):
ts.block({'award_id': 'B' * (MAX_ITEMS + 1)})
with pytest.raises(UnprocessableEntityException):
ts.block({'award_id': MAX_INT + 1})
with pytest.raises(UnprocessableEntityException):
ts.block({'award_id': MIN_INT - 1})
with pytest.raises(UnprocessableEntityException):
ts.block({'award_id': 1.1})
with pytest.raises(UnprocessableEntityException):
ts.block({'award_id': [1, 2]})
with pytest.raises(UnprocessableEntityException):
ts.block({'id': 'abcde'})
def post(self, request):
"""Return all awards matching the provided filters and limits"""
models = [
{'name': 'fields', 'key': 'fields', 'type': 'array', 'array_type': 'text', 'text_type': 'search', 'min': 1},
{'name': 'subawards', 'key': 'subawards', 'type': 'boolean', 'default': False}
]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
for m in models:
if m['name'] in ('award_type_codes', 'fields'):
m['optional'] = False
json_request = TinyShield(models).block(request.data)
fields = json_request["fields"]
filters = json_request.get("filters", {})
subawards = json_request["subawards"]
order = json_request["order"]
limit = json_request["limit"]
page = json_request["page"]
if "no intersection" in filters["award_type_codes"]:
# "Special case": there will never be results when the website provides this value
return Response({
"limit": limit,
"results": [],
"page_metadata": {"page": page, "hasNext": False},
})
sort = json_request.get("sort", fields[0])
if sort not in fields:
raise InvalidParameterException("Sort value '{}' not found in requested fields: {}".format(sort, fields))
subawards_values = list(contract_subaward_mapping.keys()) + list(grant_subaward_mapping.keys())
awards_values = list(award_contracts_mapping.keys()) + list(loan_award_mapping.keys()) + \
list(non_loan_assistance_award_mapping.keys()) + list(award_idv_mapping.keys())
msg = "Sort value '{}' not found in {{}} mappings: {{}}".format(sort)
if not subawards and sort not in awards_values:
raise InvalidParameterException(msg.format("award", awards_values))
elif subawards and sort not in subawards_values:
raise InvalidParameterException(msg.format("subaward", subawards_values))
# build sql query filters
if subawards:
queryset = subaward_filter(filters)
values = {'subaward_number', 'piid', 'fain', 'award_type'}
for field in fields:
if contract_subaward_mapping.get(field):
values.add(contract_subaward_mapping.get(field))
if grant_subaward_mapping.get(field):
values.add(grant_subaward_mapping.get(field))
else:
queryset = matview_search_filter(filters, UniversalAwardView).values()
values = {'award_id', 'piid', 'fain', 'uri', 'type'}
for field in fields:
if award_contracts_mapping.get(field):
values.add(award_contracts_mapping.get(field))
if loan_award_mapping.get(field):
values.add(loan_award_mapping.get(field))
if non_loan_assistance_award_mapping.get(field):
values.add(non_loan_assistance_award_mapping.get(field))
if award_idv_mapping.get(field):
values.add(award_idv_mapping.get(field))
# Modify queryset to be ordered by requested "sort" in the request or default value(s)
if sort:
if subawards:
if set(filters["award_type_codes"]) <= set(contract_type_mapping): # Subaward contracts
sort_filters = [contract_subaward_mapping[sort]]
elif set(filters["award_type_codes"]) <= set(grant_type_mapping): # Subaward grants
sort_filters = [grant_subaward_mapping[sort]]
else:
msg = 'Award Type codes limited for Subawards. Only contracts {} or grants {} are available'
msg = msg.format(list(contract_type_mapping.keys()), list(grant_type_mapping.keys()))
raise UnprocessableEntityException(msg)
else:
if set(filters["award_type_codes"]) <= set(contract_type_mapping): # contracts
sort_filters = [award_contracts_mapping[sort]]
elif set(filters["award_type_codes"]) <= set(loan_type_mapping): # loans
sort_filters = [loan_award_mapping[sort]]
elif set(filters["award_type_codes"]) <= set(idv_type_mapping): # idvs
sort_filters = [award_idv_mapping[sort]]
else: # assistance data
sort_filters = [non_loan_assistance_award_mapping[sort]]
# Explictly set NULLS LAST in the ordering to encourage the usage of the indexes
if sort == "Award ID" and subawards:
if order == "desc":
queryset = queryset.order_by(
F('award__piid').desc(nulls_last=True),
F('award__fain').desc(nulls_last=True)).values(*list(values))
else:
queryset = queryset.order_by(
F('award__piid').asc(nulls_last=True),
F('award__fain').asc(nulls_last=True)).values(*list(values))
elif sort == "Award ID":
if order == "desc":
queryset = queryset.order_by(
F('piid').desc(nulls_last=True),
F('fain').desc(nulls_last=True),
F('uri').desc(nulls_last=True)).values(*list(values))
else:
queryset = queryset.order_by(
F('piid').asc(nulls_last=True),
F('fain').asc(nulls_last=True),
F('uri').asc(nulls_last=True)).values(*list(values))
elif order == "desc":
queryset = queryset.order_by(F(sort_filters[0]).desc(nulls_last=True)).values(*list(values))
else:
queryset = queryset.order_by(F(sort_filters[0]).asc(nulls_last=True)).values(*list(values))
limited_queryset = queryset[(page - 1) * limit:page * limit + 1] # lower limit : upper limit
has_next = len(limited_queryset) > limit
results = []
for award in limited_queryset[:limit]:
if subawards:
row = {"internal_id": award["subaward_number"]}
if award['award_type'] == 'procurement':
for field in fields:
row[field] = award.get(contract_subaward_mapping[field])
elif award['award_type'] == 'grant':
for field in fields:
row[field] = award.get(grant_subaward_mapping[field])
else:
row = {"internal_id": award["award_id"]}
if award['type'] in loan_type_mapping: # loans
for field in fields:
row[field] = award.get(loan_award_mapping.get(field))
elif award['type'] in non_loan_assistance_type_mapping: # assistance data
for field in fields:
row[field] = award.get(non_loan_assistance_award_mapping.get(field))
elif award['type'] in idv_type_mapping:
for field in fields:
row[field] = award.get(award_idv_mapping.get(field))
elif (award['type'] is None and award['piid']) or award['type'] in contract_type_mapping:
# IDV + contract
for field in fields:
row[field] = award.get(award_contracts_mapping.get(field))
if "Award ID" in fields:
for id_type in ["piid", "fain", "uri"]:
if award[id_type]:
row["Award ID"] = award[id_type]
break
results.append(row)
return Response({"limit": limit, "results": results, "page_metadata": {"page": page, "hasNext": has_next}})
def test_any_rule():
models = [{
'name': 'value',
'key': 'value',
'type': 'any',
'models': [
{'type': 'integer'},
{'type': 'text', 'text_type': 'search'}
]
}]
# Test integer and random other key.
ts = TinyShield(models).block({'value': 1, 'another_value': 2})
assert ts['value'] == 1
assert ts.get('another_value') is None
# Test integer masquerading as a string.
ts = TinyShield(models).block({'value': '1'})
assert ts['value'] == 1
# Test string.
ts = TinyShield(models).block({'value': 'XYZ'})
assert ts['value'] == 'XYZ'
# Test list (which should blow up).
with pytest.raises(UnprocessableEntityException):
TinyShield(models).block({'value': ['XYZ']})
# Test with optional 'value' missing.
ts = TinyShield(models).block({'another_value': 2})
assert ts.get('value') is None
assert ts.get('another_value') is None
# Make 'value' required then run the same tests as above.
models[0]['optional'] = False
# Test integer and random other key.
ts = TinyShield(models).block({'value': 1, 'another_value': 2})
assert ts['value'] == 1
assert ts.get('another_value') is None
# Test integer masquerading as a string.
ts = TinyShield(models).block({'value': '1'})
assert ts['value'] == 1
# Test string.
ts = TinyShield(models).block({'value': 'XYZ'})
assert ts['value'] == 'XYZ'
# Test list (which should blow up).
with pytest.raises(UnprocessableEntityException):
TinyShield(models).block({'value': ['XYZ']})
# Test with required 'value' missing.
with pytest.raises(UnprocessableEntityException):
TinyShield(models).block({'another_value': 2})
