def test_download_awards_status(client, download_test_data, monkeypatch,
elasticsearch_award_index):
setup_elasticsearch_test(monkeypatch, elasticsearch_award_index)
download_generation.retrieve_db_string = Mock(
return_value=generate_test_db_connection_string())
# Test without columns specified
dl_resp = client.post(
"/api/v2/download/awards/",
content_type="application/json",
data=json.dumps({
"filters": {
"award_type_codes": list(award_type_mapping.keys())
},
"columns": []
}),
)
resp = client.get("/api/v2/download/status/?file_name={}".format(
dl_resp.json()["file_name"]))
expected_number_of_columns = get_number_of_columns_for_query_paths(
("award", "d1"), ("award", "d2"), ("subaward", "d1"),
("subaward", "d2"))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 3
assert resp.json()["total_columns"] == expected_number_of_columns
# Test with columns specified
dl_resp = client.post(
"/api/v2/download/awards/",
content_type="application/json",
data=json.dumps({
"filters": {
"award_type_codes": list(award_type_mapping.keys())
},
"columns": [
"total_obligated_amount",
"product_or_service_code",
"product_or_service_code_description",
"naics_code",
"naics_description",
],
}),
)
resp = client.get("/api/v2/download/status/?file_name={}".format(
dl_resp.json()["file_name"]))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 3
assert resp.json()["total_columns"] == 6
def test_download_awards_with_all_prime_awards(client, award_data):
download_generation.retrieve_db_string = Mock(
return_value=generate_test_db_connection_string())
filters = {
"agency": "all",
"prime_award_types": list(award_type_mapping.keys()),
"date_type": "action_date",
"date_range": {
"start_date": "2016-10-01",
"end_date": "2017-09-30"
},
}
dl_resp = client.post(
"/api/v2/bulk_download/awards",
content_type="application/json",
data=json.dumps({
"filters": filters,
"columns": []
}),
)
assert dl_resp.status_code == status.HTTP_200_OK
resp = client.get("/api/v2/download/status/?file_name={}".format(
dl_resp.json()["file_name"]))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 6
assert resp.json()["total_columns"] == 377
def test_download_transactions_limit(client, download_test_data, monkeypatch,
elasticsearch_transaction_index):
setup_elasticsearch_test(monkeypatch, elasticsearch_transaction_index)
download_generation.retrieve_db_string = Mock(
return_value=generate_test_db_connection_string())
dl_resp = client.post(
"/api/v2/download/transactions/",
content_type="application/json",
data=json.dumps({
"limit": 1,
"filters": {
"award_type_codes": list(award_type_mapping.keys())
},
"columns": []
}),
)
resp = client.get("/api/v2/download/status/?file_name={}".format(
dl_resp.json()["file_name"]))
expected_number_of_columns = get_number_of_columns_for_query_paths(
("transaction", "d1"), ("transaction", "d2"), ("subaward", "d1"),
("subaward", "d2"))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 2
assert resp.json()["total_columns"] == expected_number_of_columns
def test_download_awards_with_foreign_scope(client, award_data):
# Recipient Location Scope
download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string())
filters = {
"agency": "all",
"prime_award_types": [*list(award_type_mapping.keys())],
"sub_award_types": [*all_subaward_types],
"date_type": "action_date",
"date_range": {"start_date": "2016-10-01", "end_date": "2017-09-30"},
"recipient_scope": "foreign",
}
dl_resp = client.post(
"/api/v2/bulk_download/awards",
content_type="application/json",
data=json.dumps({"filters": filters, "columns": []}),
)
assert dl_resp.status_code == status.HTTP_200_OK
resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"]))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 5
assert resp.json()["total_columns"] == 550
# Place of Performance Scope
download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string())
filters = {
"agency": "all",
"prime_award_types": [*list(award_type_mapping.keys())],
"sub_award_types": [*all_subaward_types],
"date_type": "action_date",
"date_range": {"start_date": "2016-10-01", "end_date": "2017-09-30"},
"place_of_performance_scope": "foreign",
}
dl_resp = client.post(
"/api/v2/bulk_download/awards",
content_type="application/json",
data=json.dumps({"filters": filters, "columns": []}),
)
assert dl_resp.status_code == status.HTTP_200_OK
resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"]))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 5
assert resp.json()["total_columns"] == 550
def validate_award_request(request_data):
"""Analyze request and raise any formatting errors as Exceptions"""
_validate_required_parameters(request_data, ["award_levels", "filters"])
filters = _validate_filters(request_data)
award_levels = _validate_award_levels(request_data)
json_request = {"download_types": award_levels, "filters": {}}
# Overriding all other filters if the keyword filter is provided in year-constraint download
# Make sure this is after checking the award_levels
constraint_type = request_data.get("constraint_type")
if constraint_type == "year" and "elasticsearch_keyword" in filters:
json_request["filters"] = {
"elasticsearch_keyword": filters["elasticsearch_keyword"],
"award_type_codes": list(award_type_mapping.keys()),
}
json_request["limit"] = settings.MAX_DOWNLOAD_LIMIT
return json_request
# Set defaults of non-required parameters
json_request["columns"] = request_data.get("columns", [])
json_request["file_format"] = request_data.get("file_format", "csv")
check_types_and_assign_defaults(filters, json_request["filters"],
SHARED_AWARD_FILTER_DEFAULTS)
json_request["filters"]["award_type_codes"] = _validate_award_type_codes(
filters)
_validate_and_update_locations(filters, json_request)
_validate_tas_codes(filters, json_request)
# Validate time periods
total_range_count = validate_time_periods(filters, json_request)
if constraint_type == "row_count":
# Validate limit exists and is below MAX_DOWNLOAD_LIMIT
json_request["limit"] = parse_limit(request_data)
# Validate row_count-constrained filter types and assign defaults
check_types_and_assign_defaults(filters, json_request["filters"],
ROW_CONSTRAINT_FILTER_DEFAULTS)
elif constraint_type == "year":
# Validate combined total dates within one year (allow for leap years)
if total_range_count > 366:
raise InvalidParameterException(
"Invalid Parameter: time_period total days must be within a year"
)
# Validate year-constrained filter types and assign defaults
check_types_and_assign_defaults(filters, json_request["filters"],
YEAR_CONSTRAINT_FILTER_DEFAULTS)
else:
raise InvalidParameterException(
'Invalid parameter: constraint_type must be "row_count" or "year"')
return json_request
def filters(self):
all_def_codes = sorted(
DisasterEmergencyFundCode.objects.values_list("code", flat=True))
object_keys_lookup = {
"def_codes": {
"key": "filter|def_codes",
"name": "def_codes",
"type": "array",
"array_type": "enum",
"enum_values": all_def_codes,
"allow_nulls": False,
"optional": False,
},
"query": {
"key": "filter|query",
"name": "query",
"type": "text",
"text_type": "search",
"allow_nulls": True,
"optional": True,
},
"award_type_codes": {
"key": "filter|award_type_codes",
"name": "award_type_codes",
"type": "array",
"array_type": "enum",
"enum_values": sorted(award_type_mapping.keys()),
"allow_nulls": True,
"optional": True,
},
"_loan_award_type_codes": {
"key": "filter|award_type_codes",
"name": "award_type_codes",
"type": "array",
"array_type": "enum",
"enum_values": sorted(loan_type_mapping.keys()),
"allow_nulls": True,
"optional": True,
"default": list(loan_type_mapping.keys()),
},
"_assistance_award_type_codes": {
"key": "filter|award_type_codes",
"name": "award_type_codes",
"type": "array",
"array_type": "enum",
"enum_values": sorted(assistance_type_mapping.keys()),
"allow_nulls": True,
"optional": True,
"default": list(assistance_type_mapping.keys()),
},
}
model = [object_keys_lookup[key] for key in self.required_filters]
json_request = TinyShield(model).block(self.request.data)
return json_request["filter"]
def test_get_awards_csv_sources():
original = VALUE_MAPPINGS["awards"]["filter_function"]
VALUE_MAPPINGS["awards"]["filter_function"] = MagicMock(returned_value="")
csv_sources = download_generation.get_download_sources(
{"download_types": ["awards"], "filters": {"award_type_codes": list(award_type_mapping.keys())}}
)
assert len(csv_sources) == 2
VALUE_MAPPINGS["awards"]["filter_function"] = original
assert csv_sources[0].file_type == "d1"
assert csv_sources[0].source_type == "awards"
assert csv_sources[1].file_type == "d2"
assert csv_sources[1].source_type == "awards"
def test_get_awards_csv_sources():
original = VALUE_MAPPINGS['awards']['filter_function']
VALUE_MAPPINGS['awards']['filter_function'] = MagicMock(returned_value='')
csv_sources = csv_generation.get_csv_sources({
"download_types": ["awards"],
"filters": {'award_type_codes': list(award_type_mapping.keys())}
})
assert len(csv_sources) == 2
VALUE_MAPPINGS['awards']['filter_function'] = original
assert csv_sources[0].file_type == 'd1'
assert csv_sources[0].source_type == 'awards'
assert csv_sources[1].file_type == 'd2'
assert csv_sources[1].source_type == 'awards'
def test_get_awards_csv_sources():
original = VALUE_MAPPINGS['awards']['filter_function']
VALUE_MAPPINGS['awards']['filter_function'] = MagicMock(returned_value='')
csv_sources = csv_generation.get_csv_sources({
"download_types": ["awards"],
"filters": {
'award_type_codes': list(award_type_mapping.keys())
}
})
assert len(csv_sources) == 2
VALUE_MAPPINGS['awards']['filter_function'] = original
assert csv_sources[0].file_type == 'd1'
assert csv_sources[0].source_type == 'awards'
assert csv_sources[1].file_type == 'd2'
assert csv_sources[1].source_type == 'awards'
def test_empty_array_filter_fail(client, award_data):
filters = {
"agency": "all",
"prime_award_types": [*list(award_type_mapping.keys())],
"sub_award_types": [],
"date_type": "action_date",
"date_range": {"start_date": "2016-10-01", "end_date": "2017-09-30"},
"recipient_scope": "foreign",
}
resp = client.post(
"/api/v2/bulk_download/awards", content_type="application/json", data=json.dumps({"filters": filters})
)
assert resp.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY
assert (
"Field 'filters|sub_award_types' value '[]' is below min '1' items" in resp.json()["detail"]
), "Incorrect error message"
def __init__(self, request_data: dict):
super().__init__(request_data)
self.request_data = request_data
self._json_request["download_types"] = self.request_data.get(
"award_levels")
self._json_request["filters"] = _validate_filters_exist(request_data)
self.set_filter_defaults(
{"award_type_codes": list(award_type_mapping.keys())})
constraint_type = self.request_data.get("constraint_type")
if constraint_type == "year" and sorted(
self._json_request["filters"]) == [
"award_type_codes", "keywords"
]:
self._handle_keyword_search_download()
elif constraint_type == "year":
self._handle_custom_award_download()
elif constraint_type == "row_count":
self._handle_advanced_search_download()
else:
raise InvalidParameterException(
'Invalid parameter: constraint_type must be "row_count" or "year"'
)
def test_download_transactions_with_columns(client, monkeypatch,
download_test_data,
elasticsearch_transaction_index):
setup_elasticsearch_test(monkeypatch, elasticsearch_transaction_index)
resp = client.post(
"/api/v2/download/transactions/",
content_type="application/json",
data=json.dumps({
"filters": {
"award_type_codes": list(award_type_mapping.keys())
},
"columns": [
"assistance_transaction_unique_key",
"award_id_fain",
"modification_number",
"sai_number",
"contract_transaction_unique_key",
],
}),
)
assert resp.status_code == status.HTTP_200_OK
assert ".zip" in resp.json()["file_url"]
def validate_disaster_recipient_request(request_data):
_validate_required_parameters(request_data, ["filters"])
model = [
{
"key":
"filters|def_codes",
"name":
"def_codes",
"type":
"array",
"array_type":
"enum",
"enum_values":
sorted(
DisasterEmergencyFundCode.objects.values_list("code",
flat=True)),
"allow_nulls":
False,
"optional":
False,
},
{
"key": "filters|query",
"name": "query",
"type": "text",
"text_type": "search",
"allow_nulls": False,
"optional": True,
},
{
"key": "filters|award_type_codes",
"name": "award_type_codes",
"type": "array",
"array_type": "enum",
"enum_values": sorted(award_type_mapping.keys()),
"allow_nulls": False,
"optional": True,
},
]
filters = TinyShield(model).block(request_data)["filters"]
# Determine what to use in the filename based on "award_type_codes" filter;
# Also add "face_value_of_loans" column if only loan types
award_category = "All-Awards"
award_type_codes = set(
filters.get("award_type_codes", award_type_mapping.keys()))
columns = [
"recipient", "award_obligations", "award_outlays", "number_of_awards"
]
if award_type_codes <= set(contract_type_mapping.keys()):
award_category = "Contracts"
elif award_type_codes <= set(idv_type_mapping.keys()):
award_category = "Contract-IDVs"
elif award_type_codes <= set(grant_type_mapping.keys()):
award_category = "Grants"
elif award_type_codes <= set(loan_type_mapping.keys()):
award_category = "Loans"
columns.insert(3, "face_value_of_loans")
elif award_type_codes <= set(direct_payment_type_mapping.keys()):
award_category = "Direct-Payments"
elif award_type_codes <= set(other_type_mapping.keys()):
award_category = "Other-Financial-Assistance"
# Need to specify the field to use "query" filter on if present
query_text = filters.pop("query", None)
if query_text:
filters["query"] = {"text": query_text, "fields": ["recipient_name"]}
json_request = {
"award_category": award_category,
"columns": tuple(columns),
"download_types": ["disaster_recipient"],
"file_format": str(request_data.get("file_format", "csv")).lower(),
"filters": filters,
}
_validate_file_format(json_request)
return json_request
def validate_award_request(self, request_data):
"""Analyze request and raise any formatting errors as Exceptions"""
json_request = {}
constraint_type = request_data.get('constraint_type', None)
# Validate required parameters
for required_param in ['award_levels', 'filters']:
if required_param not in request_data:
raise InvalidParameterException(
'Missing one or more required query parameters: {}'.format(required_param)
)
if not isinstance(request_data['award_levels'], list):
raise InvalidParameterException('Award levels parameter not provided as a list')
elif len(request_data['award_levels']) == 0:
raise InvalidParameterException('At least one award level is required.')
for award_level in request_data['award_levels']:
if award_level not in VALUE_MAPPINGS:
raise InvalidParameterException('Invalid award_level: {}'.format(award_level))
json_request['download_types'] = request_data['award_levels']
# Overriding all other filters if the keyword filter is provided in year-constraint download
# Make sure this is after checking the award_levels
if constraint_type == 'year' and 'elasticsearch_keyword' in request_data['filters']:
json_request['filters'] = {
'elasticsearch_keyword': request_data['filters']['elasticsearch_keyword'],
'award_type_codes': list(award_type_mapping.keys()),
}
json_request['limit'] = settings.MAX_DOWNLOAD_LIMIT
return json_request
if not isinstance(request_data['filters'], dict):
raise InvalidParameterException('Filters parameter not provided as a dict')
elif len(request_data['filters']) == 0:
raise InvalidParameterException('At least one filter is required.')
json_request['filters'] = {}
# Set defaults of non-required parameters
json_request['columns'] = request_data.get('columns', [])
json_request['file_format'] = request_data.get('file_format', 'csv')
# Validate shared filter types and assign defaults
filters = request_data['filters']
check_types_and_assign_defaults(filters, json_request['filters'], SHARED_AWARD_FILTER_DEFAULTS)
# Validate award type types
if not filters.get('award_type_codes', None) or len(filters['award_type_codes']) < 1:
filters['award_type_codes'] = list(award_type_mapping.keys())
for award_type_code in filters['award_type_codes']:
if award_type_code not in award_type_mapping:
raise InvalidParameterException('Invalid award_type: {}'.format(award_type_code))
json_request['filters']['award_type_codes'] = filters['award_type_codes']
# Validate locations
for location_filter in ['place_of_performance_locations', 'recipient_locations']:
if filters.get(location_filter):
for location_dict in filters[location_filter]:
if not isinstance(location_dict, dict):
raise InvalidParameterException('Location is not a dictionary: {}'.format(location_dict))
location_error_handling(location_dict.keys())
json_request['filters'][location_filter] = filters[location_filter]
# Validate time periods
total_range_count = validate_time_periods(filters, json_request)
if constraint_type == 'row_count':
# Validate limit exists and is below MAX_DOWNLOAD_LIMIT
json_request['limit'] = parse_limit(request_data)
# Validate row_count-constrainted filter types and assign defaults
check_types_and_assign_defaults(filters, json_request['filters'], ROW_CONSTRAINT_FILTER_DEFAULTS)
elif constraint_type == 'year':
# Validate combined total dates within one year (allow for leap years)
if total_range_count > 366:
raise InvalidParameterException('Invalid Parameter: time_period total days must be within a year')
# Validate year-constrainted filter types and assign defaults
check_types_and_assign_defaults(filters, json_request['filters'], YEAR_CONSTRAINT_FILTER_DEFAULTS)
else:
raise InvalidParameterException('Invalid parameter: constraint_type must be "row_count" or "year"')
return json_request
def _handle_custom_award_download(self):
"""
Custom Award Download allows different filters than other Award Download Endpoints
and thus it needs to be normalized before moving forward
# TODO: Refactor to use similar filters as Advanced Search download
"""
self.tinyshield_models.extend([
{
"name": "agencies",
"key": "filters|agencies",
"type": "array",
"array_type": "object",
"object_keys": {
"type": {
"type": "enum",
"enum_values": ["funding", "awarding"],
"optional": False
},
"tier": {
"type": "enum",
"enum_values": ["toptier", "subtier"],
"optional": False
},
"toptier_name": {
"type": "text",
"text_type": "search",
"optional": True
},
"name": {
"type": "text",
"text_type": "search",
"optional": False
},
},
},
{
"name": "agency",
"key": "filters|agency",
"type": "integer"
},
{
"name": "date_range",
"key": "filters|date_range",
"type": "object",
"optional": False,
"object_keys": {
"start_date": {
"type": "date",
"default": "1000-01-01"
},
"end_date": {
"type": "date",
"default": datetime.strftime(datetime.utcnow(),
"%Y-%m-%d")
},
},
},
{
"name": "date_type",
"key": "filters|date_type",
"type": "enum",
"enum_values": ["action_date", "last_modified_date"],
"default": "action_date",
},
{
"name": "place_of_performance_locations",
"key": "filters|place_of_performance_locations",
"type": "array",
"array_type": "object",
"object_keys": {
"country": {
"type": "text",
"text_type": "search",
"optional": False
},
"state": {
"type": "text",
"text_type": "search",
"optional": True
},
"zip": {
"type": "text",
"text_type": "search",
"optional": True
},
"district": {
"type": "text",
"text_type": "search",
"optional": True
},
"county": {
"type": "text",
"text_type": "search",
"optional": True
},
"city": {
"type": "text",
"text_type": "search",
"optional": True
},
},
},
{
"name": "place_of_performance_scope",
"key": "filters|place_of_performance_scope",
"type": "enum",
"enum_values": ["domestic", "foreign"],
},
{
"name": "prime_award_types",
"key": "filters|prime_award_types",
"type": "array",
"array_type": "enum",
"min": 0,
"enum_values": list(award_type_mapping.keys()),
},
{
"name": "recipient_locations",
"key": "filters|recipient_locations",
"type": "array",
"array_type": "object",
"object_keys": {
"country": {
"type": "text",
"text_type": "search",
"optional": False
},
"state": {
"type": "text",
"text_type": "search",
"optional": True
},
"zip": {
"type": "text",
"text_type": "search",
"optional": True
},
"district": {
"type": "text",
"text_type": "search",
"optional": True
},
"county": {
"type": "text",
"text_type": "search",
"optional": True
},
"city": {
"type": "text",
"text_type": "search",
"optional": True
},
},
},
{
"name": "recipient_scope",
"key": "filters|recipient_scope",
"type": "enum",
"enum_values": ("domestic", "foreign"),
},
{
"name": "sub_agency",
"key": "filters|sub_agency",
"type": "text",
"text_type": "search"
},
{
"name": "sub_award_types",
"key": "filters|sub_award_types",
"type": "array",
"array_type": "enum",
"min": 0,
"enum_values": all_subaward_types,
},
])
filter_all_agencies = False
if str(self._json_request["filters"].get("agency",
"")).lower() == "all":
filter_all_agencies = True
self._json_request["filters"].pop("agency")
self._json_request = self.get_validated_request()
custom_award_filters = self._json_request["filters"]
final_award_filters = {}
# These filters do not need any normalization
for key, value in custom_award_filters.items():
if key in [
"recipient_locations",
"recipient_scope",
"place_of_performance_location",
"place_of_performance_scope",
]:
final_award_filters[key] = value
if get_date_range_length(custom_award_filters["date_range"]) > 366:
raise InvalidParameterException(
"Invalid Parameter: date_range total days must be within a year"
)
final_award_filters["time_period"] = [{
**custom_award_filters["date_range"], "date_type":
custom_award_filters["date_type"]
}]
if (custom_award_filters.get("prime_award_types") is None
and custom_award_filters.get("sub_award_types") is None):
raise InvalidParameterException(
"Missing one or more required body parameters: prime_award_types or sub_award_types"
)
self._json_request["download_types"] = []
final_award_filters["prime_and_sub_award_types"] = {}
if custom_award_filters.get("prime_award_types"):
self._json_request["download_types"].append("prime_awards")
final_award_filters["prime_and_sub_award_types"][
"prime_awards"] = custom_award_filters["prime_award_types"]
if custom_award_filters.get("sub_award_types"):
self._json_request["download_types"].append("sub_awards")
final_award_filters["prime_and_sub_award_types"][
"sub_awards"] = custom_award_filters["sub_award_types"]
if "agency" in custom_award_filters:
if "agencies" not in custom_award_filters:
final_award_filters["agencies"] = []
if filter_all_agencies:
toptier_name = "all"
else:
toptier_name = (ToptierAgency.objects.filter(
toptier_agency_id=custom_award_filters["agency"]).values(
"name").first())
if toptier_name is None:
raise InvalidParameterException(
f"Toptier ID not found: {custom_award_filters['agency']}"
)
toptier_name = toptier_name["name"]
if "sub_agency" in custom_award_filters:
final_award_filters["agencies"].append({
"type":
"awarding",
"tier":
"subtier",
"name":
custom_award_filters["sub_agency"],
"toptier_name":
toptier_name,
})
else:
final_award_filters["agencies"].append({
"type": "awarding",
"tier": "toptier",
"name": toptier_name
})
if "agencies" in custom_award_filters:
final_award_filters["agencies"] = [
val for val in custom_award_filters["agencies"]
if val.get("name", "").lower() != "all"
]
self._json_request["filters"] = final_award_filters
from usaspending_api.awards.v2.lookups.lookups import award_type_mapping
from usaspending_api.core.validator.helpers import TINY_SHIELD_SEPARATOR
AWARD_FILTER = [
{
'name': 'award_ids',
'type': 'array',
'array_type': 'integer'
},
{
'name': 'award_type_codes',
'type': 'array',
'array_type': 'enum',
'enum_values': list(award_type_mapping.keys())
},
{
'name': 'contract_pricing_type_codes',
'type': 'array',
'array_type': 'text',
'text_type': 'search'
},
{
'name': 'extent_competed_type_codes',
'type': 'array',
'array_type': 'text',
'text_type': 'search'
},
{
'name': 'keyword',
'type': 'text',
'text_type': 'search',
def validate_award_request(self, request_data):
"""Analyze request and raise any formatting errors as Exceptions"""
json_request = {}
constraint_type = request_data.get('constraint_type', None)
# Validate required parameters
for required_param in ['award_levels', 'filters']:
if required_param not in request_data:
raise InvalidParameterException(
'Missing one or more required query parameters: {}'.format(
required_param))
if not isinstance(request_data['award_levels'], list):
raise InvalidParameterException(
'Award levels parameter not provided as a list')
elif len(request_data['award_levels']) == 0:
raise InvalidParameterException(
'At least one award level is required.')
for award_level in request_data['award_levels']:
if award_level not in VALUE_MAPPINGS:
raise InvalidParameterException(
'Invalid award_level: {}'.format(award_level))
json_request['download_types'] = request_data['award_levels']
# Overriding all other filters if the keyword filter is provided in year-constraint download
# Make sure this is after checking the award_levels
if constraint_type == 'year' and 'elasticsearch_keyword' in request_data[
'filters']:
json_request['filters'] = {
'elasticsearch_keyword':
request_data['filters']['elasticsearch_keyword'],
'award_type_codes':
list(award_type_mapping.keys())
}
json_request['limit'] = settings.MAX_DOWNLOAD_LIMIT
return json_request
if not isinstance(request_data['filters'], dict):
raise InvalidParameterException(
'Filters parameter not provided as a dict')
elif len(request_data['filters']) == 0:
raise InvalidParameterException('At least one filter is required.')
json_request['filters'] = {}
# Set defaults of non-required parameters
json_request['columns'] = request_data.get('columns', [])
json_request['file_format'] = request_data.get('file_format', 'csv')
# Validate shared filter types and assign defaults
filters = request_data['filters']
check_types_and_assign_defaults(filters, json_request['filters'],
SHARED_AWARD_FILTER_DEFAULTS)
# Validate award type types
if not filters.get('award_type_codes',
None) or len(filters['award_type_codes']) < 1:
filters['award_type_codes'] = list(award_type_mapping.keys())
for award_type_code in filters['award_type_codes']:
if award_type_code not in award_type_mapping:
raise InvalidParameterException(
'Invalid award_type: {}'.format(award_type_code))
json_request['filters']['award_type_codes'] = filters[
'award_type_codes']
# Validate locations
for location_filter in [
'place_of_performance_locations', 'recipient_locations'
]:
if filters.get(location_filter):
for location_dict in filters[location_filter]:
if not isinstance(location_dict, dict):
raise InvalidParameterException(
'Location is not a dictionary: {}'.format(
location_dict))
location_error_handling(location_dict.keys())
json_request['filters'][location_filter] = filters[
location_filter]
# Validate time periods
total_range_count = validate_time_periods(filters, json_request)
if constraint_type == 'row_count':
# Validate limit exists and is below MAX_DOWNLOAD_LIMIT
json_request['limit'] = parse_limit(request_data)
# Validate row_count-constrainted filter types and assign defaults
check_types_and_assign_defaults(filters, json_request['filters'],
ROW_CONSTRAINT_FILTER_DEFAULTS)
elif constraint_type == 'year':
# Validate combined total dates within one year (allow for leap years)
if total_range_count > 366:
raise InvalidParameterException(
'Invalid Parameter: time_period total days must be within a year'
)
# Validate year-constrainted filter types and assign defaults
check_types_and_assign_defaults(filters, json_request['filters'],
YEAR_CONSTRAINT_FILTER_DEFAULTS)
else:
raise InvalidParameterException(
'Invalid parameter: constraint_type must be "row_count" or "year"'
)
return json_request
'filter_function': account_download_filter
},
'award_financial': {
'source_type': 'account',
'table': FinancialAccountsByAwards,
'table_name': 'award_financial',
'download_name': 'account_breakdown_by_award',
'filter_function': account_download_filter
}
}
# Bulk Download still uses "prime awards" instead of "transactions"
VALUE_MAPPINGS['prime_awards'] = VALUE_MAPPINGS['transactions']
SHARED_AWARD_FILTER_DEFAULTS = {
'award_type_codes': list(award_type_mapping.keys()),
'agencies': [],
'time_period': [],
'place_of_performance_locations': [],
'recipient_locations': []
}
YEAR_CONSTRAINT_FILTER_DEFAULTS = {'elasticsearch_keyword': ''}
ROW_CONSTRAINT_FILTER_DEFAULTS = {
'keywords': [],
'legal_entities': [],
'recipient_search_text': [],
'recipient_scope': '',
'recipient_type_names': [],
'place_of_performance_scope': '',
'award_amounts': [],
'award_ids': [],
def test_get_csv_sources():
mappings = csv_generation.VALUE_MAPPINGS
original = mappings['awards']['filter_function']
mappings['awards']['filter_function'] = MagicMock(returned_value='')
csv_sources = csv_generation.get_csv_sources({
"download_types": ["awards"],
"filters": {
'award_type_codes': list(award_type_mapping.keys())
}
})
mappings['awards']['filter_function'] = original
assert len(csv_sources) == 2
assert csv_sources[0].file_type == 'd1'
assert csv_sources[0].source_type == 'awards'
assert csv_sources[1].file_type == 'd2'
assert csv_sources[1].source_type == 'awards'
original = mappings['transactions']['filter_function']
mappings['transactions']['filter_function'] = MagicMock(returned_value='')
csv_sources = csv_generation.get_csv_sources({
"download_types": ["transactions"],
"filters": {
'award_type_codes': list(award_type_mapping.keys())
}
})
mappings['transactions']['filter_function'] = original
assert len(csv_sources) == 2
assert csv_sources[0].file_type == 'd1'
assert csv_sources[0].source_type == 'transactions'
assert csv_sources[1].file_type == 'd2'
assert csv_sources[1].source_type == 'transactions'
original = mappings['sub_awards']['filter_function']
mappings['sub_awards']['filter_function'] = MagicMock(returned_value='')
csv_sources = csv_generation.get_csv_sources({
"download_types": ["sub_awards"],
"filters": {
'award_type_codes': list(award_type_mapping.keys())
}
})
mappings['sub_awards']['filter_function'] = original
assert len(csv_sources) == 2
assert csv_sources[0].file_type == 'd1'
assert csv_sources[0].source_type == 'sub_awards'
assert csv_sources[1].file_type == 'd2'
assert csv_sources[1].source_type == 'sub_awards'
original = mappings['account_balances']['filter_function']
mappings['account_balances']['filter_function'] = MagicMock(
returned_value='')
csv_sources = csv_generation.get_csv_sources({
"download_types": ["account_balances"],
"account_level":
"treasury_account",
"filters": {}
})
mappings['account_balances']['filter_function'] = original
assert len(csv_sources) == 1
assert csv_sources[0].file_type == 'treasury_account'
assert csv_sources[0].source_type == 'account_balances'
original = mappings['object_class_program_activity']['filter_function']
mappings['object_class_program_activity']['filter_function'] = MagicMock(
returned_value='')
csv_sources = csv_generation.get_csv_sources({
"download_types": ["object_class_program_activity"],
"account_level":
"treasury_account",
"filters": {}
})
mappings['object_class_program_activity']['filter_function'] = original
assert len(csv_sources) == 1
assert csv_sources[0].file_type == 'treasury_account'
assert csv_sources[0].source_type == 'object_class_program_activity'
def __init__(self, request_data: dict):
super().__init__(request_data)
self.tinyshield_models.extend([
{
"key":
"filters|def_codes",
"name":
"def_codes",
"type":
"array",
"array_type":
"enum",
"enum_values":
sorted(
DisasterEmergencyFundCode.objects.values_list("code",
flat=True)),
"allow_nulls":
False,
"optional":
False,
},
{
"key": "filters|query",
"name": "query",
"type": "text",
"text_type": "search",
"allow_nulls": False,
"optional": True,
},
{
"key": "filters|award_type_codes",
"name": "award_type_codes",
"type": "array",
"array_type": "enum",
"enum_values": sorted(award_type_mapping.keys()),
"allow_nulls": False,
"optional": True,
},
])
self._json_request["filters"] = request_data.get("filters")
self._json_request = self.get_validated_request()
self._json_request["download_types"] = [self.name]
# Determine what to use in the filename based on "award_type_codes" filter;
# Also add "face_value_of_loans" column if only loan types
award_category = "All-Awards"
award_type_codes = set(self._json_request["filters"].get(
"award_type_codes", award_type_mapping.keys()))
columns = [
"recipient", "award_obligations", "award_outlays",
"number_of_awards"
]
if award_type_codes <= set(contract_type_mapping.keys()):
award_category = "Contracts"
elif award_type_codes <= set(idv_type_mapping.keys()):
award_category = "Contract-IDVs"
elif award_type_codes <= set(grant_type_mapping.keys()):
award_category = "Grants"
elif award_type_codes <= set(loan_type_mapping.keys()):
award_category = "Loans"
columns.insert(3, "face_value_of_loans")
elif award_type_codes <= set(direct_payment_type_mapping.keys()):
award_category = "Direct-Payments"
elif award_type_codes <= set(other_type_mapping.keys()):
award_category = "Other-Financial-Assistance"
self._json_request["award_category"] = award_category
self._json_request["columns"] = self._json_request.get(
"columns") or tuple(columns)
# Need to specify the field to use "query" filter on if present
query_text = self._json_request["filters"].pop("query", None)
if query_text:
self._json_request["filters"]["query"] = {
"text": query_text,
"fields": ["recipient_name"]
}
"exclude": {
"type": "array",
"array_type": "any",
"models": [{"type": "array", "array_type": "text", "text_type": "search"}],
"min": 0,
},
},
}
AWARD_FILTER = [
{"name": "award_ids", "type": "array", "array_type": "text", "text_type": "search"},
{
"name": "award_type_codes",
"type": "array",
"array_type": "enum",
"enum_values": list(award_type_mapping.keys()) + ["no intersection"],
},
{"name": "contract_pricing_type_codes", "type": "array", "array_type": "text", "text_type": "search"},
{"name": "extent_competed_type_codes", "type": "array", "array_type": "text", "text_type": "search"},
{"name": "keywords", "type": "array", "array_type": "text", "text_type": "search", "text_min": 3},
{"name": "legal_entities", "type": "array", "array_type": "integer", "array_max": maxsize},
{
"name": "naics_codes",
"type": "any",
"models": [
{
"name": "naics_codes",
"type": "object",
"min": 0,
"object_keys": {
"require": {"type": "array", "array_type": "integer", "text_type": "search", "min": 0},
from usaspending_api.awards.v2.lookups.lookups import award_type_mapping
from usaspending_api.common.validator.helpers import TINY_SHIELD_SEPARATOR
AWARD_FILTER = [
{
'name': 'award_ids',
'type': 'array',
'array_type': 'text',
'text_type': 'search'
},
{
'name': 'award_type_codes',
'type': 'array',
'array_type': 'enum',
'enum_values': list(award_type_mapping.keys()) + ['no intersection']
},
{
'name': 'contract_pricing_type_codes',
'type': 'array',
'array_type': 'text',
'text_type': 'search'
},
{
'name': 'extent_competed_type_codes',
'type': 'array',
'array_type': 'text',
'text_type': 'search'
},
{
'name': 'keywords',
