def _check_max(rule):
value = rule["value"]
if rule["type"] in ("integer", "float"):
if value > rule["max"]:
raise UnprocessableEntityException(ABOVE_MAXIMUM_MSG.format(**rule))
if rule["type"] in ("text", "enum", "array", "object"):
if len(value) > rule["max"]:
raise UnprocessableEntityException(ABOVE_MAXIMUM_MSG.format(**rule) + " items")
def _check_min(rule):
value = rule["value"]
if rule["type"] in ("integer", "float"):
if value < rule["min"]:
raise UnprocessableEntityException(BELOW_MINIMUM_MSG.format(**rule))
if rule["type"] in ("text", "enum", "array", "object"):
if len(value) < rule["min"]:
raise UnprocessableEntityException(BELOW_MINIMUM_MSG.format(**rule) + " items")
def _check_max(rule):
value = rule['value']
if rule['type'] in ('integer', 'float'):
if value > rule['max']:
raise UnprocessableEntityException(
ABOVE_MAXIMUM_MSG.format(**rule))
if rule['type'] in ('text', 'enum', 'array', 'object'):
if len(value) > rule['max']:
raise UnprocessableEntityException(
ABOVE_MAXIMUM_MSG.format(**rule) + ' items')
def _check_min(rule):
value = rule['value']
if rule['type'] in ('integer', 'float'):
if value < rule['min']:
raise UnprocessableEntityException(
BELOW_MINIMUM_MSG.format(**rule))
if rule['type'] in ('text', 'enum', 'array', 'object'):
if len(value) < rule['min']:
raise UnprocessableEntityException(
BELOW_MINIMUM_MSG.format(**rule) + ' items')
def validate_object(rule):
provided_object = rule["value"]
if type(provided_object) is not dict:
raise InvalidParameterException(INVALID_TYPE_MSG.format(**rule))
if not provided_object:
raise InvalidParameterException(
"'{}' is empty. Please populate object".format(rule["key"]))
for field in provided_object.keys():
if field not in rule["object_keys"].keys():
raise InvalidParameterException(
"Unexpected field '{}' in parameter {}".format(
field, rule["key"]))
for key, value in rule["object_keys"].items():
if key not in provided_object:
if "optional" in value and value["optional"] is False:
raise UnprocessableEntityException(
"Required object fields: {}".format(
rule["object_keys"].keys()))
else:
continue
return provided_object
def search_regex_of(v):
if isinstance(v, str):
v = string_to_dictionary(v, "aid")
code_lookup = {
"ata": v["ata"] if v.get("ata") else None,
"aid": v["aid"] if v.get("aid") else ".*",
"main": v["main"] if v.get("main") else ".*",
"sub": v["sub"] if v.get("sub") else ".*",
"bpoa": v["bpoa"] if v.get("bpoa") else ".*",
"epoa": v["epoa"] if v.get("epoa") else ".*",
"a": v["a"] if v.get("a") else ".*",
}
# This is NOT the order of elements as displayed in the tas rendering label, but instead the order in the award_delta_view and transaction_delta_view
search_regex = TreasuryAppropriationAccount.generate_tas_rendering_label(
code_lookup["ata"],
code_lookup["aid"],
code_lookup["a"],
code_lookup["bpoa"],
code_lookup["epoa"],
code_lookup["main"],
code_lookup["sub"],
)
# TODO: move this to a Tinyshield filter
if not re.match(r"^(\d|\w|-|\*|\.)+$", search_regex):
raise UnprocessableEntityException(
f"Unable to parse TAS filter {search_regex}")
return search_regex
def validate_object(rule):
provided_object = rule['value']
if type(provided_object) is not dict:
raise InvalidParameterException(INVALID_TYPE_MSG.format(**rule))
if not provided_object:
raise InvalidParameterException(
'\'{}\' is empty. Please populate object'.format(rule['key']))
for field in provided_object.keys():
if field not in rule['object_keys'].keys():
raise InvalidParameterException(
'Unexpected field \'{}\' in parameter {}'.format(
field, rule['key']))
for key, value in rule['object_keys'].items():
if key not in provided_object:
if 'optional' in value and value['optional'] is False:
raise UnprocessableEntityException(
'Required object fields: {}'.format(
rule['object_keys'].keys()))
else:
continue
return provided_object
def search_regex_of(v):
if isinstance(v, str):
v = string_to_dictionary(v, "agency")
code_lookup = {
"agency": f"agency={v['agency']}" if v.get("agency") else "*",
"faaid": f"faaid={v['faaid']}" if v.get("faaid") else "*",
"famain": f"famain={v['famain']}" if v.get("famain") else "*",
"aid": f"aid={v['aid']}" if v.get("aid") else "*",
"main": f"main={v['main']}" if v.get("main") else "*",
"ata": f"ata={v['ata']}" if v.get("ata") else "*",
"sub": f"sub={v['sub']}" if v.get("sub") else "*",
"bpoa": f"bpoa={v['bpoa']}" if v.get("bpoa") else "*",
"epoa": f"epoa={v['epoa']}" if v.get("epoa") else "*",
"a": f"a={v['a']}" if v.get("a") else "*",
}
# This is NOT the order of elements as displayed in the tas rendering label, but instead the order in the award_delta_view and transaction_delta_view
search_regex = (code_lookup["agency"] + code_lookup["faaid"] +
code_lookup["famain"] + code_lookup["aid"] +
code_lookup["main"] + code_lookup["ata"] +
code_lookup["sub"] + code_lookup["bpoa"] +
code_lookup["epoa"] + code_lookup["a"])
# TODO: move this to a Tinyshield filter
if not re.match(r"^(\d|\w|-|\*|=)+$", search_regex):
raise UnprocessableEntityException(f"Unable to parse TAS filter")
return search_regex
def tas_rendering_label_to_component_dictionary(
tas_rendering_label) -> dict:
try:
components = tas_rendering_label.split("-")
if len(components) < 4 or len(components) > 5:
raise Exception # don't have to be specific here since this is being swallowed and replaced
retval = {}
# we go in reverse, since the first component is the only optional one
retval["sub"] = components[-1]
retval["main"] = components[-2]
# the third component from the back can either be two years, or one character
if len(components[-3]) > 1:
dates = components[-3].split("/")
retval["bpoa"] = dates[0]
retval["epoa"] = dates[1]
else:
retval["a"] = components[-3]
retval["aid"] = components[-4]
# ata may or may not be present
if len(components) > 4:
retval["ata"] = components[-5]
return retval
except Exception:
raise UnprocessableEntityException(
f"Cannot parse provided TAS: {tas_rendering_label}. Valid examples: 000-2010/2010-0400-000, 009-X-1701-000, 019-011-X-1071-000"
)
def fiscal_year(self):
fiscal_year = str(
self.request.query_params.get("fiscal_year",
current_fiscal_year()))
if not fullmatch("[0-9]{4}", fiscal_year):
raise UnprocessableEntityException(
"Unrecognized fiscal_year format. Should be YYYY.")
min_fiscal_year = fy(settings.API_SEARCH_MIN_DATE)
fiscal_year = int(fiscal_year)
if fiscal_year < min_fiscal_year:
raise UnprocessableEntityException(
f"fiscal_year is currently limited to an earliest year of {min_fiscal_year}."
)
if fiscal_year > current_fiscal_year():
raise UnprocessableEntityException(
f"fiscal_year may not exceed current fiscal year of {current_fiscal_year()}."
)
return fiscal_year
def _check_datetime_min_max(rule, value, dt_format):
# DEV-4097 introduces new behavior whereby minimum/maximum date violations should raise an error. To
# implement this, we added a min/max_exception property to the rule. If that property exists, we will
# raise an exception, otherwise we will retain the old behavior for backward compatibility.
if "min" in rule:
min_cap = datetime.datetime.strptime(rule["min"], dt_format)
if value < min_cap:
if "min_exception" in rule:
message = rule["min_exception"].format(**rule)
raise UnprocessableEntityException(message)
logger.info("{}. Setting to {}".format(BELOW_MINIMUM_MSG.format(**rule), min_cap))
value = min_cap
if "max" in rule:
max_cap = datetime.datetime.strptime(rule["max"], dt_format)
if value > max_cap:
if "max_exception" in rule:
message = rule["max_exception"].format(**rule)
raise UnprocessableEntityException(message)
logger.info(ABOVE_MAXIMUM_MSG.format(**rule))
value = max_cap
def raise_if_award_types_not_valid_subset(award_type_codes, is_subaward=False):
"""Test to ensure the award types are a subset of only one group.
For Awards: contracts, idvs, direct_payments, loans, grants, other assistance
For Sub-Awards: procurement, assistance
If the types are not a valid subset:
Raise API exception with a JSON response describing the award type groupings
"""
msg_head = '"message": "\'award_type_codes\' must only contain types from one group.","award_type_groups": '
if is_subaward:
if not subaward_types_are_valid_groups(award_type_codes):
# Return a JSON response describing the award type groupings
error_msg = ('{{{} {{ "procurement": {}, "assistance": {}}}}}').format(
msg_head, json.dumps(procurement_type_mapping), json.dumps(assistance_type_mapping)
)
raise UnprocessableEntityException(json.loads(error_msg))
else:
if not award_types_are_valid_groups(award_type_codes):
error_msg = (
"{{{} {{"
'"contracts": {},'
'"loans": {},'
'"idvs": {},'
'"grants": {},'
'"other_financial_assistance": {},'
'"direct_payments": {}}}}}'
).format(
msg_head,
json.dumps(contract_type_mapping),
json.dumps(loan_type_mapping),
json.dumps(idv_type_mapping),
json.dumps(grant_type_mapping),
json.dumps(direct_payment_type_mapping),
json.dumps(other_type_mapping),
)
raise UnprocessableEntityException(json.loads(error_msg))
def split_filter_values(cls, filter_values):
""" Here we assume that filter_values has already been run through validate_filter_values. """
if isinstance(filter_values, list):
# Legacy is treated as a "require" filter.
require = [[f] for f in filter_values]
exclude = []
elif isinstance(filter_values, dict):
# PSCCodeObject
require = filter_values.get("require") or []
exclude = filter_values.get("exclude") or []
else:
raise UnprocessableEntityException(
f"psc_codes must be an array or object")
return require, exclude
def apply_rule(self, rule):
if rule.get('allow_nulls', False) and rule['value'] is None:
return rule['value']
elif rule['type'] not in ('array', 'object'):
if rule['type'] in VALIDATORS:
return VALIDATORS[rule['type']]['func'](rule)
else:
raise Exception('Invalid Type {} in rule'.format(rule['type']))
# Array is a "special" type since it is a list of other types which need to be validated
elif rule['type'] == 'array':
rule['array_min'] = rule.get('array_min', 1)
rule['array_max'] = rule.get('array_max', MAX_ITEMS)
value = VALIDATORS[rule['type']]['func'](rule)
child_rule = copy.copy(rule)
child_rule['type'] = rule['array_type']
child_rule['min'] = rule.get('array_min')
child_rule['max'] = rule.get('array_max')
child_rule = self.promote_subrules(child_rule, child_rule)
array_result = []
for v in value:
child_rule['value'] = v
array_result.append(self.apply_rule(child_rule))
return array_result
# Object is a "special" type since it is comprised of other types which need to be validated
elif rule['type'] == 'object':
rule['object_min'] = rule.get('object_min', 1)
rule['object_max'] = rule.get('object_max', MAX_ITEMS)
provided_object = VALIDATORS[rule['type']]['func'](rule)
object_result = {}
for k, v in rule['object_keys'].items():
try:
value = provided_object[k]
except KeyError as e:
if "optional" in v and v['optional'] is False:
raise UnprocessableEntityException(
'Required object fields: {}'.format(k))
else:
continue
# Start with the sub-rule definition and supplement with parent's key-values as needed
child_rule = copy.copy(v)
child_rule['key'] = rule['key']
child_rule['value'] = value
child_rule = self.promote_subrules(child_rule, v)
object_result[k] = self.apply_rule(child_rule)
return object_result
def query_elasticsearch(self) -> list:
filter_query = QueryWithFilters.generate_awards_elasticsearch_query(
self.filters)
sort_field = self.get_elastic_sort_by_fields()
sorts = [{
field: self.pagination["sort_order"]
} for field in sort_field]
record_num = (self.pagination["page"] - 1) * self.pagination["limit"]
# random page jumping was removed due to performance concerns
if (self.last_record_sort_value is None and self.last_record_unique_id
is not None) or (self.last_record_sort_value is not None
and self.last_record_unique_id is None):
# malformed request
raise Exception(
"Using search_after functionality in Elasticsearch requires both last_record_sort_value and last_record_unique_id."
)
if record_num >= settings.ES_AWARDS_MAX_RESULT_WINDOW and (
self.last_record_unique_id is None
and self.last_record_sort_value is None):
raise UnprocessableEntityException(
"Page #{page} with limit {limit} is over the maximum result limit {es_limit}. Please provide the 'last_record_sort_value' and 'last_record_unique_id' to paginate sequentially."
.format(
page=self.pagination["page"],
limit=self.pagination["limit"],
es_limit=settings.ES_AWARDS_MAX_RESULT_WINDOW,
))
# Search_after values are provided in the API request - use search after
if self.last_record_sort_value is not None and self.last_record_unique_id is not None:
search = (
AwardSearch().filter(filter_query).sort(*sorts).extra(
search_after=[
self.last_record_sort_value, self.last_record_unique_id
])[:self.pagination["limit"] +
1] # add extra result to check for next page
)
# no values, within result window, use regular elasticsearch
else:
search = AwardSearch().filter(filter_query).sort(
*sorts)[record_num:record_num + self.pagination["limit"]]
response = search.handle_execute()
return response
def parse_request(self, request):
for item in self.rules:
# Loop through the request to find the expected key
value = request
for subkey in item["key"].split(TINY_SHIELD_SEPARATOR):
value = value.get(subkey, {})
if value != {}:
# Key found in provided request dictionary, use the value
item["value"] = value
elif item["optional"] is False:
# If the value is required, raise exception since key wasn't found
raise UnprocessableEntityException("Missing value: '{}' is a required field".format(item["key"]))
elif "default" in item:
# If value wasn't found, and this is optional, use the default
item["value"] = item["default"]
else:
# This model/field is optional, no value provided, and no default value.
# Use the "hidden" feature Ellipsis since None can be a valid value provided in the request
item["value"] = ...
def post(self, request):
additional_models = [
{
"key": "filter|award_type",
"name": "award_type",
"type": "enum",
"enum_values": ("assistance", "procurement"),
"allow_nulls": False,
"optional": True,
}
]
f = TinyShield(additional_models).block(self.request.data).get("filter")
if f:
self.filters["award_type"] = f.get("award_type")
if all(x in self.filters for x in ["award_type_codes", "award_type"]):
raise UnprocessableEntityException("Cannot provide both 'award_type_codes' and 'award_type'")
if self.count_only:
return Response({"count": self.aggregation["award_count"]})
else:
return Response(self.aggregation)
def validate_filter_values(cls, filter_values):
""" This is validation on top of whatever TinyShield performs. """
if isinstance(filter_values, list):
# Legacy.
for code in filter_values:
if not isinstance(
code,
str) or not cls.validation_pattern.fullmatch(code):
raise UnprocessableEntityException(
f"PSC codes must be one to four character uppercased alphanumeric strings. "
f"Offending code: '{code}'.")
elif isinstance(filter_values, dict):
# PSCCodeObject
for key in ("require", "exclude"):
code_lists = filter_values.get(key) or []
if not isinstance(code_lists, list):
raise UnprocessableEntityException(
f"require and exclude properties must be arrays of arrays."
)
for code_list in code_lists:
if not isinstance(code_list, list):
raise UnprocessableEntityException(
f"require and exclude properties must be arrays of arrays."
)
for seq, code in enumerate(code_list):
if seq == 0 and code not in PSC_GROUPS:
raise UnprocessableEntityException(
f"Tier1 PSC filter values must be one of: {tuple(PSC_GROUPS)}. "
f"Offending code: '{code}'.")
elif seq > 0 and (
not isinstance(code, str)
or not cls.validation_pattern.fullmatch(code)):
raise UnprocessableEntityException(
f"PSC codes must be one to four character uppercased alphanumeric strings. "
f"Offending code: '{code}'.")
else:
raise UnprocessableEntityException(
f"psc_codes must be an array or object")
def validate_fiscal_period(request_data):
fiscal_period = request_data["fiscal_period"]
if fiscal_period < 2 or fiscal_period > 12:
raise UnprocessableEntityException(
f"fiscal_period must be in the range 2-12")
def post(self, request):
models = [{
"name": "fields",
"key": "fields",
"type": "array",
"array_type": "text",
"text_type": "search",
"optional": False,
}]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
for m in models:
if m["name"] in ("keywords", "award_type_codes", "sort"):
m["optional"] = False
validated_payload = TinyShield(models).block(request.data)
record_num = (validated_payload["page"] -
1) * validated_payload["limit"]
if record_num >= settings.ES_TRANSACTIONS_MAX_RESULT_WINDOW:
raise UnprocessableEntityException(
"Page #{page} of size {limit} is over the maximum result limit ({es_limit}). Consider using custom data downloads to obtain large data sets."
.format(
page=validated_payload["page"],
limit=validated_payload["limit"],
es_limit=settings.ES_TRANSACTIONS_MAX_RESULT_WINDOW,
))
if validated_payload["sort"] not in validated_payload["fields"]:
raise InvalidParameterException(
"Sort value not found in fields: {}".format(
validated_payload["sort"]))
if "filters" in validated_payload and "no intersection" in validated_payload[
"filters"]["award_type_codes"]:
# "Special case": there will never be results when the website provides this value
return Response({
"limit": validated_payload["limit"],
"results": [],
"page_metadata": {
"page": validated_payload["page"],
"next": None,
"previous": None,
"hasNext": False,
"hasPrevious": False,
},
})
sorts = {
TRANSACTIONS_LOOKUP[validated_payload["sort"]]:
validated_payload["order"]
}
lower_limit = (validated_payload["page"] -
1) * validated_payload["limit"]
upper_limit = (
validated_payload["page"]) * validated_payload["limit"] + 1
validated_payload["filters"]["keyword_search"] = [
es_minimal_sanitize(x)
for x in validated_payload["filters"]["keywords"]
]
validated_payload["filters"].pop("keywords")
filter_query = QueryWithFilters.generate_transactions_elasticsearch_query(
validated_payload["filters"])
search = TransactionSearch().filter(filter_query).sort(
sorts)[lower_limit:upper_limit]
response = search.handle_execute()
return Response(
self.build_elasticsearch_result(validated_payload, response))
def post(self, request):
"""Return all awards matching the provided filters and limits"""
models = [{
'name': 'fields',
'key': 'fields',
'type': 'array',
'array_type': 'text',
'text_type': 'search',
'min': 1
}, {
'name': 'subawards',
'key': 'subawards',
'type': 'boolean',
'default': False
}]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
for m in models:
if m['name'] in ('award_type_codes', 'fields'):
m['optional'] = False
json_request = TinyShield(models).block(request.data)
fields = json_request["fields"]
filters = json_request.get("filters", {})
subawards = json_request["subawards"]
order = json_request["order"]
limit = json_request["limit"]
page = json_request["page"]
if "no intersection" in filters["award_type_codes"]:
# "Special case": there will never be results when the website provides this value
return Response({
"limit": limit,
"results": [],
"page_metadata": {
"page": page,
"hasNext": False
},
})
sort = json_request.get("sort", fields[0])
if sort not in fields:
raise InvalidParameterException(
"Sort value '{}' not found in requested fields: {}".format(
sort, fields))
subawards_values = list(contract_subaward_mapping.keys()) + list(
grant_subaward_mapping.keys())
awards_values = list(award_contracts_mapping.keys()) + list(loan_award_mapping.keys()) + \
list(non_loan_assistance_award_mapping.keys()) + list(award_idv_mapping.keys())
msg = "Sort value '{}' not found in {{}} mappings: {{}}".format(sort)
if not subawards and sort not in awards_values:
raise InvalidParameterException(msg.format("award", awards_values))
elif subawards and sort not in subawards_values:
raise InvalidParameterException(
msg.format("subaward", subawards_values))
# build sql query filters
if subawards:
queryset = subaward_filter(filters)
values = {'subaward_number', 'piid', 'fain', 'award_type'}
for field in fields:
if contract_subaward_mapping.get(field):
values.add(contract_subaward_mapping.get(field))
if grant_subaward_mapping.get(field):
values.add(grant_subaward_mapping.get(field))
else:
queryset = matview_search_filter(filters,
UniversalAwardView).values()
values = {'award_id', 'piid', 'fain', 'uri', 'type'}
for field in fields:
if award_contracts_mapping.get(field):
values.add(award_contracts_mapping.get(field))
if loan_award_mapping.get(field):
values.add(loan_award_mapping.get(field))
if non_loan_assistance_award_mapping.get(field):
values.add(non_loan_assistance_award_mapping.get(field))
if award_idv_mapping.get(field):
values.add(award_idv_mapping.get(field))
# Modify queryset to be ordered by requested "sort" in the request or default value(s)
if sort:
if subawards:
if set(filters["award_type_codes"]) <= set(
contract_type_mapping): # Subaward contracts
sort_filters = [contract_subaward_mapping[sort]]
elif set(filters["award_type_codes"]) <= set(
grant_type_mapping): # Subaward grants
sort_filters = [grant_subaward_mapping[sort]]
else:
msg = 'Award Type codes limited for Subawards. Only contracts {} or grants {} are available'
msg = msg.format(list(contract_type_mapping.keys()),
list(grant_type_mapping.keys()))
raise UnprocessableEntityException(msg)
else:
if set(filters["award_type_codes"]) <= set(
contract_type_mapping): # contracts
sort_filters = [award_contracts_mapping[sort]]
elif set(filters["award_type_codes"]) <= set(
loan_type_mapping): # loans
sort_filters = [loan_award_mapping[sort]]
elif set(filters["award_type_codes"]) <= set(
idv_type_mapping): # idvs
sort_filters = [award_idv_mapping[sort]]
else: # assistance data
sort_filters = [non_loan_assistance_award_mapping[sort]]
# Explictly set NULLS LAST in the ordering to encourage the usage of the indexes
if sort == "Award ID" and subawards:
if order == "desc":
queryset = queryset.order_by(
F('award__piid').desc(nulls_last=True),
F('award__fain').desc(nulls_last=True)).values(
*list(values))
else:
queryset = queryset.order_by(
F('award__piid').asc(nulls_last=True),
F('award__fain').asc(nulls_last=True)).values(
*list(values))
elif sort == "Award ID":
if order == "desc":
queryset = queryset.order_by(
F('piid').desc(nulls_last=True),
F('fain').desc(nulls_last=True),
F('uri').desc(nulls_last=True)).values(*list(values))
else:
queryset = queryset.order_by(
F('piid').asc(nulls_last=True),
F('fain').asc(nulls_last=True),
F('uri').asc(nulls_last=True)).values(*list(values))
elif order == "desc":
queryset = queryset.order_by(
F(sort_filters[0]).desc(nulls_last=True)).values(
*list(values))
else:
queryset = queryset.order_by(
F(sort_filters[0]).asc(nulls_last=True)).values(
*list(values))
limited_queryset = queryset[(page - 1) * limit:page * limit +
1] # lower limit : upper limit
has_next = len(limited_queryset) > limit
results = []
for award in limited_queryset[:limit]:
if subawards:
row = {"internal_id": award["subaward_number"]}
if award['award_type'] == 'procurement':
for field in fields:
row[field] = award.get(
contract_subaward_mapping[field])
elif award['award_type'] == 'grant':
for field in fields:
row[field] = award.get(grant_subaward_mapping[field])
else:
row = {"internal_id": award["award_id"]}
if award['type'] in loan_type_mapping: # loans
for field in fields:
row[field] = award.get(loan_award_mapping.get(field))
elif award[
'type'] in non_loan_assistance_type_mapping: # assistance data
for field in fields:
row[field] = award.get(
non_loan_assistance_award_mapping.get(field))
elif award['type'] in idv_type_mapping:
for field in fields:
row[field] = award.get(award_idv_mapping.get(field))
elif (award['type'] is None and
award['piid']) or award['type'] in contract_type_mapping:
# IDV + contract
for field in fields:
row[field] = award.get(
award_contracts_mapping.get(field))
if "Award ID" in fields:
for id_type in ["piid", "fain", "uri"]:
if award[id_type]:
row["Award ID"] = award[id_type]
break
results.append(row)
return Response({
"limit": limit,
"results": results,
"page_metadata": {
"page": page,
"hasNext": has_next
}
})
def apply_rule(self, rule):
_return = None
if rule.get('allow_nulls', False) and rule['value'] is None:
_return = rule['value']
elif rule['type'] not in ('array', 'object', 'any'):
if rule['type'] in VALIDATORS:
_return = VALIDATORS[rule['type']]['func'](rule)
else:
raise Exception('Invalid Type {} in rule'.format(rule['type']))
# Array is a "special" type since it is a list of other types which need to be validated
elif rule['type'] == 'array':
rule['array_min'] = rule.get('array_min', 1)
rule['array_max'] = rule.get('array_max', MAX_ITEMS)
value = VALIDATORS[rule['type']]['func'](rule)
child_rule = copy.copy(rule)
child_rule['type'] = rule['array_type']
child_rule['min'] = rule.get('array_min')
child_rule['max'] = rule.get('array_max')
child_rule = self.promote_subrules(child_rule, child_rule)
array_result = []
for v in value:
child_rule['value'] = v
array_result.append(self.apply_rule(child_rule))
_return = array_result
# Object is a "special" type since it is comprised of other types which need to be validated
elif rule['type'] == 'object':
rule['object_min'] = rule.get('object_min', 1)
rule['object_max'] = rule.get('object_max', MAX_ITEMS)
provided_object = VALIDATORS[rule['type']]['func'](rule)
object_result = {}
for k, v in rule['object_keys'].items():
try:
value = provided_object[k]
except KeyError as e:
if "optional" in v and v['optional'] is False:
raise UnprocessableEntityException('Required object fields: {}'.format(k))
else:
continue
# Start with the sub-rule definition and supplement with parent's key-values as needed
child_rule = copy.copy(v)
child_rule['key'] = rule['key']
child_rule['value'] = value
child_rule = self.promote_subrules(child_rule, v)
object_result[k] = self.apply_rule(child_rule)
_return = object_result
# Any is a "special" type since it is is really a collection of other rules.
elif rule['type'] == 'any':
for child_rule in rule['models']:
child_rule['value'] = rule['value']
try:
# First successful rule wins.
return self.apply_rule(child_rule)
except Exception:
pass
# No rules succeeded.
raise UnprocessableEntityException(INVALID_TYPE_MSG.format(
key=rule['key'],
value=rule['value'],
type=', '.join(sorted([m['type'] for m in rule['models']]))
))
return _return
def validate_publication_sort(self, sort_key):
regex_string = r"publication_date,([2-9]|1[0-2])"
if not re.match(regex_string, sort_key):
raise UnprocessableEntityException(
"publication_date sort param must be in the format 'publication_date,<fiscal_period>' where <fiscal_period> is in the range 2-12"
)
def apply_rule(self, rule):
_return = None
if rule.get("allow_nulls", False) and rule["value"] is None:
_return = rule["value"]
elif rule["type"] not in ("array", "object", "any"):
if rule["type"] in VALIDATORS:
_return = VALIDATORS[rule["type"]]["func"](rule)
else:
raise Exception("Invalid Type {} in rule".format(rule["type"]))
# Array is a "special" type since it is a list of other types which need to be validated
elif rule["type"] == "array":
rule["array_min"] = rule.get("array_min", 1)
rule["array_max"] = rule.get("array_max", MAX_ITEMS)
value = VALIDATORS[rule["type"]]["func"](rule)
child_rule = copy.copy(rule)
child_rule["type"] = rule["array_type"]
child_rule["min"] = rule.get("array_min")
child_rule["max"] = rule.get("array_max")
child_rule = self.promote_subrules(child_rule, child_rule)
array_result = []
for v in value:
child_rule["value"] = v
array_result.append(self.apply_rule(child_rule))
_return = array_result
# Object is a "special" type since it is comprised of other types which need to be validated
elif rule["type"] == "object":
rule["object_min"] = rule.get("object_min", 1)
rule["object_max"] = rule.get("object_max", MAX_ITEMS)
provided_object = VALIDATORS[rule["type"]]["func"](rule)
object_result = {}
for k, v in rule["object_keys"].items():
try:
value = provided_object[k]
except KeyError:
if "optional" in v and v["optional"] is False:
raise UnprocessableEntityException(
"Required object fields: {}".format(k))
else:
continue
# Start with the sub-rule definition and supplement with parent's key-values as needed
child_rule = copy.copy(v)
child_rule["key"] = rule["key"]
child_rule["value"] = value
child_rule = self.promote_subrules(child_rule, v)
object_result[k] = self.apply_rule(child_rule)
_return = object_result
# Any is a "special" type since it is is really a collection of other rules.
elif rule["type"] == "any":
for child_rule in rule["models"]:
child_rule["value"] = rule["value"]
try:
# First successful rule wins.
return self.apply_rule(child_rule)
except Exception:
pass
# No rules succeeded.
raise UnprocessableEntityException(
INVALID_TYPE_MSG.format(
key=rule["key"],
value=rule["value"],
type=", ".join(sorted([m["type"]
for m in rule["models"]]))))
return _return
def post(self, request: Request) -> Response:
models = [
{
"key":
"geo_layer",
"name":
"geo_layer",
"type":
"enum",
"enum_values":
sorted([geo_layer.value for geo_layer in list(GeoLayer)]),
"text_type":
"search",
"allow_nulls":
False,
"optional":
False,
},
{
"key": "geo_layer_filters",
"name": "geo_layer_filters",
"type": "array",
"array_type": "text",
"text_type": "search",
},
{
"key": "spending_type",
"name": "spending_type",
"type": "enum",
"enum_values": ["obligation", "outlay", "face_value_of_loan"],
"allow_nulls": False,
"optional": False,
},
{
"name": "scope",
"key": "scope",
"type": "enum",
"optional": True,
"enum_values": ["place_of_performance", "recipient_location"],
"default": "recipient_location",
},
]
# NOTE: filter object in request handled in base class: see self.filters
json_request = TinyShield(models).block(request.data)
agg_key_dict = {
"county": "county_agg_key",
"district": "congressional_agg_key",
"state": "state_agg_key",
}
scope_dict = {
"place_of_performance": "pop",
"recipient_location": "recipient_location"
}
location_dict = {
"county": "county_code",
"district": "congressional_code",
"state": "state_code"
}
self.geo_layer = GeoLayer(json_request["geo_layer"])
scope_field_name = scope_dict[json_request["scope"]]
loc_field_name = location_dict[self.geo_layer.value]
self.agg_key = f"{scope_field_name}_{agg_key_dict[json_request['geo_layer']]}"
self.geo_layer_filters = json_request.get("geo_layer_filters")
self.spending_type = json_request.get("spending_type")
self.loc_lookup = f"{scope_field_name}_{loc_field_name}"
# Set which field will be the aggregation amount
if self.spending_type == "obligation":
self.metric_field = "total_covid_obligation"
elif self.spending_type == "outlay":
self.metric_field = "total_covid_outlay"
elif self.spending_type == "face_value_of_loan":
self.metric_field = "total_loan_value"
else:
raise UnprocessableEntityException(
f"Unrecognized value '{self.spending_type}' for field "
f"'spending_type'")
filter_query = QueryWithFilters.generate_awards_elasticsearch_query(
self.filters)
result = self.query_elasticsearch(filter_query)
return Response({
"geo_layer": self.geo_layer.value,
"spending_type": self.spending_type,
"scope": json_request["scope"],
"results": result,
})
def post(self, request):
"""Return all budget function/subfunction titles matching the provided search text"""
models = [
{'name': 'fields', 'key': 'fields', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'subawards', 'key': 'subawards', 'type': 'boolean', 'default': False}
]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
for m in models:
if m['name'] in ('award_type_codes', 'fields'):
m['optional'] = False
json_request = TinyShield(models).block(request.data)
fields = json_request.get("fields", None)
filters = json_request.get("filters", None)
subawards = json_request["subawards"]
order = json_request["order"]
limit = json_request["limit"]
page = json_request["page"]
lower_limit = (page - 1) * limit
upper_limit = page * limit
sort = json_request.get("sort", fields[0])
if sort not in fields:
raise InvalidParameterException("Sort value not found in fields: {}".format(sort))
subawards_values = list(contract_subaward_mapping.keys()) + list(grant_subaward_mapping.keys())
awards_values = list(award_contracts_mapping.keys()) + list(loan_award_mapping) + \
list(non_loan_assistance_award_mapping.keys())
if (subawards and sort not in subawards_values) or (not subawards and sort not in awards_values):
raise InvalidParameterException("Sort value not found in award mappings: {}".format(sort))
# build sql query filters
if subawards:
# We do not use matviews for Subaward filtering, just the Subaward download filters
queryset = subaward_filter(filters)
values = {'subaward_number', 'award__piid', 'award__fain', 'award_type'}
for field in fields:
if contract_subaward_mapping.get(field):
values.add(contract_subaward_mapping.get(field))
if grant_subaward_mapping.get(field):
values.add(grant_subaward_mapping.get(field))
else:
queryset = matview_search_filter(filters, UniversalAwardView).values()
values = {'award_id', 'piid', 'fain', 'uri', 'type'}
for field in fields:
if award_contracts_mapping.get(field):
values.add(award_contracts_mapping.get(field))
if loan_award_mapping.get(field):
values.add(loan_award_mapping.get(field))
if non_loan_assistance_award_mapping.get(field):
values.add(non_loan_assistance_award_mapping.get(field))
# Modify queryset to be ordered if we specify "sort" in the request
if sort and "no intersection" not in filters["award_type_codes"]:
if subawards:
if set(filters["award_type_codes"]) <= set(contract_type_mapping): # Subaward contracts
sort_filters = [contract_subaward_mapping[sort]]
elif set(filters["award_type_codes"]) <= set(grant_type_mapping): # Subaward grants
sort_filters = [grant_subaward_mapping[sort]]
else:
msg = 'Award Type codes limited for Subawards. Only contracts {} or grants {} are available'
msg = msg.format(list(contract_type_mapping.keys()), list(grant_type_mapping.keys()))
raise UnprocessableEntityException(msg)
else:
if set(filters["award_type_codes"]) <= set(contract_type_mapping): # contracts
sort_filters = [award_contracts_mapping[sort]]
elif set(filters["award_type_codes"]) <= set(loan_type_mapping): # loans
sort_filters = [loan_award_mapping[sort]]
else: # assistance data
sort_filters = [non_loan_assistance_award_mapping[sort]]
# Explictly set NULLS LAST in the ordering to encourage the usage of the indexes
if sort == "Award ID" and subawards:
if order == "desc":
queryset = queryset.order_by(
F('award__piid').desc(nulls_last=True),
F('award__fain').desc(nulls_last=True)).values(*list(values))
else:
queryset = queryset.order_by(
F('award__piid').asc(nulls_last=True),
F('award__fain').asc(nulls_last=True)).values(*list(values))
elif sort == "Award ID":
if order == "desc":
queryset = queryset.order_by(
F('piid').desc(nulls_last=True),
F('fain').desc(nulls_last=True),
F('uri').desc(nulls_last=True)).values(*list(values))
else:
queryset = queryset.order_by(
F('piid').asc(nulls_last=True),
F('fain').asc(nulls_last=True),
F('uri').asc(nulls_last=True)).values(*list(values))
elif order == "desc":
queryset = queryset.order_by(F(sort_filters[0]).desc(nulls_last=True)).values(*list(values))
else:
queryset = queryset.order_by(F(sort_filters[0]).asc(nulls_last=True)).values(*list(values))
limited_queryset = queryset[lower_limit:upper_limit + 1]
has_next = len(limited_queryset) > limit
results = []
for award in limited_queryset[:limit]:
if subawards:
row = {"internal_id": award["subaward_number"]}
if award['award_type'] == 'procurement':
for field in fields:
row[field] = award.get(contract_subaward_mapping[field])
elif award['award_type'] == 'grant':
for field in fields:
row[field] = award.get(grant_subaward_mapping[field])
else:
row = {"internal_id": award["award_id"]}
if award['type'] in loan_type_mapping: # loans
for field in fields:
row[field] = award.get(loan_award_mapping.get(field))
elif award['type'] in non_loan_assistance_type_mapping: # assistance data
for field in fields:
row[field] = award.get(non_loan_assistance_award_mapping.get(field))
elif (award['type'] is None and award['piid']) or award['type'] in contract_type_mapping:
# IDV + contract
for field in fields:
row[field] = award.get(award_contracts_mapping.get(field))
if "Award ID" in fields:
for id_type in ["piid", "fain", "uri"]:
if award[id_type]:
row["Award ID"] = award[id_type]
break
results.append(row)
# build response
response = {
'limit': limit,
'results': results,
'page_metadata': {
'page': page,
'hasNext': has_next
}
}
return Response(response)