def test_download_awards_status(client, download_test_data, monkeypatch, elasticsearch_award_index): setup_elasticsearch_test(monkeypatch, elasticsearch_award_index) download_generation.retrieve_db_string = Mock( return_value=generate_test_db_connection_string()) # Test without columns specified dl_resp = client.post( "/api/v2/download/awards/", content_type="application/json", data=json.dumps({ "filters": { "award_type_codes": list(award_type_mapping.keys()) }, "columns": [] }), ) resp = client.get("/api/v2/download/status/?file_name={}".format( dl_resp.json()["file_name"])) expected_number_of_columns = get_number_of_columns_for_query_paths( ("award", "d1"), ("award", "d2"), ("subaward", "d1"), ("subaward", "d2")) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 3 assert resp.json()["total_columns"] == expected_number_of_columns # Test with columns specified dl_resp = client.post( "/api/v2/download/awards/", content_type="application/json", data=json.dumps({ "filters": { "award_type_codes": list(award_type_mapping.keys()) }, "columns": [ "total_obligated_amount", "product_or_service_code", "product_or_service_code_description", "naics_code", "naics_description", ], }), ) resp = client.get("/api/v2/download/status/?file_name={}".format( dl_resp.json()["file_name"])) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 3 assert resp.json()["total_columns"] == 6
def test_download_awards_with_all_prime_awards(client, award_data): download_generation.retrieve_db_string = Mock( return_value=generate_test_db_connection_string()) filters = { "agency": "all", "prime_award_types": list(award_type_mapping.keys()), "date_type": "action_date", "date_range": { "start_date": "2016-10-01", "end_date": "2017-09-30" }, } dl_resp = client.post( "/api/v2/bulk_download/awards", content_type="application/json", data=json.dumps({ "filters": filters, "columns": [] }), ) assert dl_resp.status_code == status.HTTP_200_OK resp = client.get("/api/v2/download/status/?file_name={}".format( dl_resp.json()["file_name"])) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 6 assert resp.json()["total_columns"] == 377
def test_download_transactions_limit(client, download_test_data, monkeypatch, elasticsearch_transaction_index): setup_elasticsearch_test(monkeypatch, elasticsearch_transaction_index) download_generation.retrieve_db_string = Mock( return_value=generate_test_db_connection_string()) dl_resp = client.post( "/api/v2/download/transactions/", content_type="application/json", data=json.dumps({ "limit": 1, "filters": { "award_type_codes": list(award_type_mapping.keys()) }, "columns": [] }), ) resp = client.get("/api/v2/download/status/?file_name={}".format( dl_resp.json()["file_name"])) expected_number_of_columns = get_number_of_columns_for_query_paths( ("transaction", "d1"), ("transaction", "d2"), ("subaward", "d1"), ("subaward", "d2")) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 2 assert resp.json()["total_columns"] == expected_number_of_columns
def test_download_awards_with_foreign_scope(client, award_data): # Recipient Location Scope download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string()) filters = { "agency": "all", "prime_award_types": [*list(award_type_mapping.keys())], "sub_award_types": [*all_subaward_types], "date_type": "action_date", "date_range": {"start_date": "2016-10-01", "end_date": "2017-09-30"}, "recipient_scope": "foreign", } dl_resp = client.post( "/api/v2/bulk_download/awards", content_type="application/json", data=json.dumps({"filters": filters, "columns": []}), ) assert dl_resp.status_code == status.HTTP_200_OK resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"])) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 5 assert resp.json()["total_columns"] == 550 # Place of Performance Scope download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string()) filters = { "agency": "all", "prime_award_types": [*list(award_type_mapping.keys())], "sub_award_types": [*all_subaward_types], "date_type": "action_date", "date_range": {"start_date": "2016-10-01", "end_date": "2017-09-30"}, "place_of_performance_scope": "foreign", } dl_resp = client.post( "/api/v2/bulk_download/awards", content_type="application/json", data=json.dumps({"filters": filters, "columns": []}), ) assert dl_resp.status_code == status.HTTP_200_OK resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"])) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 5 assert resp.json()["total_columns"] == 550
def validate_award_request(request_data): """Analyze request and raise any formatting errors as Exceptions""" _validate_required_parameters(request_data, ["award_levels", "filters"]) filters = _validate_filters(request_data) award_levels = _validate_award_levels(request_data) json_request = {"download_types": award_levels, "filters": {}} # Overriding all other filters if the keyword filter is provided in year-constraint download # Make sure this is after checking the award_levels constraint_type = request_data.get("constraint_type") if constraint_type == "year" and "elasticsearch_keyword" in filters: json_request["filters"] = { "elasticsearch_keyword": filters["elasticsearch_keyword"], "award_type_codes": list(award_type_mapping.keys()), } json_request["limit"] = settings.MAX_DOWNLOAD_LIMIT return json_request # Set defaults of non-required parameters json_request["columns"] = request_data.get("columns", []) json_request["file_format"] = request_data.get("file_format", "csv") check_types_and_assign_defaults(filters, json_request["filters"], SHARED_AWARD_FILTER_DEFAULTS) json_request["filters"]["award_type_codes"] = _validate_award_type_codes( filters) _validate_and_update_locations(filters, json_request) _validate_tas_codes(filters, json_request) # Validate time periods total_range_count = validate_time_periods(filters, json_request) if constraint_type == "row_count": # Validate limit exists and is below MAX_DOWNLOAD_LIMIT json_request["limit"] = parse_limit(request_data) # Validate row_count-constrained filter types and assign defaults check_types_and_assign_defaults(filters, json_request["filters"], ROW_CONSTRAINT_FILTER_DEFAULTS) elif constraint_type == "year": # Validate combined total dates within one year (allow for leap years) if total_range_count > 366: raise InvalidParameterException( "Invalid Parameter: time_period total days must be within a year" ) # Validate year-constrained filter types and assign defaults check_types_and_assign_defaults(filters, json_request["filters"], YEAR_CONSTRAINT_FILTER_DEFAULTS) else: raise InvalidParameterException( 'Invalid parameter: constraint_type must be "row_count" or "year"') return json_request
def filters(self): all_def_codes = sorted( DisasterEmergencyFundCode.objects.values_list("code", flat=True)) object_keys_lookup = { "def_codes": { "key": "filter|def_codes", "name": "def_codes", "type": "array", "array_type": "enum", "enum_values": all_def_codes, "allow_nulls": False, "optional": False, }, "query": { "key": "filter|query", "name": "query", "type": "text", "text_type": "search", "allow_nulls": True, "optional": True, }, "award_type_codes": { "key": "filter|award_type_codes", "name": "award_type_codes", "type": "array", "array_type": "enum", "enum_values": sorted(award_type_mapping.keys()), "allow_nulls": True, "optional": True, }, "_loan_award_type_codes": { "key": "filter|award_type_codes", "name": "award_type_codes", "type": "array", "array_type": "enum", "enum_values": sorted(loan_type_mapping.keys()), "allow_nulls": True, "optional": True, "default": list(loan_type_mapping.keys()), }, "_assistance_award_type_codes": { "key": "filter|award_type_codes", "name": "award_type_codes", "type": "array", "array_type": "enum", "enum_values": sorted(assistance_type_mapping.keys()), "allow_nulls": True, "optional": True, "default": list(assistance_type_mapping.keys()), }, } model = [object_keys_lookup[key] for key in self.required_filters] json_request = TinyShield(model).block(self.request.data) return json_request["filter"]
def test_get_awards_csv_sources(): original = VALUE_MAPPINGS["awards"]["filter_function"] VALUE_MAPPINGS["awards"]["filter_function"] = MagicMock(returned_value="") csv_sources = download_generation.get_download_sources( {"download_types": ["awards"], "filters": {"award_type_codes": list(award_type_mapping.keys())}} ) assert len(csv_sources) == 2 VALUE_MAPPINGS["awards"]["filter_function"] = original assert csv_sources[0].file_type == "d1" assert csv_sources[0].source_type == "awards" assert csv_sources[1].file_type == "d2" assert csv_sources[1].source_type == "awards"
def test_get_awards_csv_sources(): original = VALUE_MAPPINGS['awards']['filter_function'] VALUE_MAPPINGS['awards']['filter_function'] = MagicMock(returned_value='') csv_sources = csv_generation.get_csv_sources({ "download_types": ["awards"], "filters": {'award_type_codes': list(award_type_mapping.keys())} }) assert len(csv_sources) == 2 VALUE_MAPPINGS['awards']['filter_function'] = original assert csv_sources[0].file_type == 'd1' assert csv_sources[0].source_type == 'awards' assert csv_sources[1].file_type == 'd2' assert csv_sources[1].source_type == 'awards'
def test_get_awards_csv_sources(): original = VALUE_MAPPINGS['awards']['filter_function'] VALUE_MAPPINGS['awards']['filter_function'] = MagicMock(returned_value='') csv_sources = csv_generation.get_csv_sources({ "download_types": ["awards"], "filters": { 'award_type_codes': list(award_type_mapping.keys()) } }) assert len(csv_sources) == 2 VALUE_MAPPINGS['awards']['filter_function'] = original assert csv_sources[0].file_type == 'd1' assert csv_sources[0].source_type == 'awards' assert csv_sources[1].file_type == 'd2' assert csv_sources[1].source_type == 'awards'
def test_empty_array_filter_fail(client, award_data): filters = { "agency": "all", "prime_award_types": [*list(award_type_mapping.keys())], "sub_award_types": [], "date_type": "action_date", "date_range": {"start_date": "2016-10-01", "end_date": "2017-09-30"}, "recipient_scope": "foreign", } resp = client.post( "/api/v2/bulk_download/awards", content_type="application/json", data=json.dumps({"filters": filters}) ) assert resp.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY assert ( "Field 'filters|sub_award_types' value '[]' is below min '1' items" in resp.json()["detail"] ), "Incorrect error message"
def __init__(self, request_data: dict): super().__init__(request_data) self.request_data = request_data self._json_request["download_types"] = self.request_data.get( "award_levels") self._json_request["filters"] = _validate_filters_exist(request_data) self.set_filter_defaults( {"award_type_codes": list(award_type_mapping.keys())}) constraint_type = self.request_data.get("constraint_type") if constraint_type == "year" and sorted( self._json_request["filters"]) == [ "award_type_codes", "keywords" ]: self._handle_keyword_search_download() elif constraint_type == "year": self._handle_custom_award_download() elif constraint_type == "row_count": self._handle_advanced_search_download() else: raise InvalidParameterException( 'Invalid parameter: constraint_type must be "row_count" or "year"' )
def test_download_transactions_with_columns(client, monkeypatch, download_test_data, elasticsearch_transaction_index): setup_elasticsearch_test(monkeypatch, elasticsearch_transaction_index) resp = client.post( "/api/v2/download/transactions/", content_type="application/json", data=json.dumps({ "filters": { "award_type_codes": list(award_type_mapping.keys()) }, "columns": [ "assistance_transaction_unique_key", "award_id_fain", "modification_number", "sai_number", "contract_transaction_unique_key", ], }), ) assert resp.status_code == status.HTTP_200_OK assert ".zip" in resp.json()["file_url"]
def validate_disaster_recipient_request(request_data): _validate_required_parameters(request_data, ["filters"]) model = [ { "key": "filters|def_codes", "name": "def_codes", "type": "array", "array_type": "enum", "enum_values": sorted( DisasterEmergencyFundCode.objects.values_list("code", flat=True)), "allow_nulls": False, "optional": False, }, { "key": "filters|query", "name": "query", "type": "text", "text_type": "search", "allow_nulls": False, "optional": True, }, { "key": "filters|award_type_codes", "name": "award_type_codes", "type": "array", "array_type": "enum", "enum_values": sorted(award_type_mapping.keys()), "allow_nulls": False, "optional": True, }, ] filters = TinyShield(model).block(request_data)["filters"] # Determine what to use in the filename based on "award_type_codes" filter; # Also add "face_value_of_loans" column if only loan types award_category = "All-Awards" award_type_codes = set( filters.get("award_type_codes", award_type_mapping.keys())) columns = [ "recipient", "award_obligations", "award_outlays", "number_of_awards" ] if award_type_codes <= set(contract_type_mapping.keys()): award_category = "Contracts" elif award_type_codes <= set(idv_type_mapping.keys()): award_category = "Contract-IDVs" elif award_type_codes <= set(grant_type_mapping.keys()): award_category = "Grants" elif award_type_codes <= set(loan_type_mapping.keys()): award_category = "Loans" columns.insert(3, "face_value_of_loans") elif award_type_codes <= set(direct_payment_type_mapping.keys()): award_category = "Direct-Payments" elif award_type_codes <= set(other_type_mapping.keys()): award_category = "Other-Financial-Assistance" # Need to specify the field to use "query" filter on if present query_text = filters.pop("query", None) if query_text: filters["query"] = {"text": query_text, "fields": ["recipient_name"]} json_request = { "award_category": award_category, "columns": tuple(columns), "download_types": ["disaster_recipient"], "file_format": str(request_data.get("file_format", "csv")).lower(), "filters": filters, } _validate_file_format(json_request) return json_request
def validate_award_request(self, request_data): """Analyze request and raise any formatting errors as Exceptions""" json_request = {} constraint_type = request_data.get('constraint_type', None) # Validate required parameters for required_param in ['award_levels', 'filters']: if required_param not in request_data: raise InvalidParameterException( 'Missing one or more required query parameters: {}'.format(required_param) ) if not isinstance(request_data['award_levels'], list): raise InvalidParameterException('Award levels parameter not provided as a list') elif len(request_data['award_levels']) == 0: raise InvalidParameterException('At least one award level is required.') for award_level in request_data['award_levels']: if award_level not in VALUE_MAPPINGS: raise InvalidParameterException('Invalid award_level: {}'.format(award_level)) json_request['download_types'] = request_data['award_levels'] # Overriding all other filters if the keyword filter is provided in year-constraint download # Make sure this is after checking the award_levels if constraint_type == 'year' and 'elasticsearch_keyword' in request_data['filters']: json_request['filters'] = { 'elasticsearch_keyword': request_data['filters']['elasticsearch_keyword'], 'award_type_codes': list(award_type_mapping.keys()), } json_request['limit'] = settings.MAX_DOWNLOAD_LIMIT return json_request if not isinstance(request_data['filters'], dict): raise InvalidParameterException('Filters parameter not provided as a dict') elif len(request_data['filters']) == 0: raise InvalidParameterException('At least one filter is required.') json_request['filters'] = {} # Set defaults of non-required parameters json_request['columns'] = request_data.get('columns', []) json_request['file_format'] = request_data.get('file_format', 'csv') # Validate shared filter types and assign defaults filters = request_data['filters'] check_types_and_assign_defaults(filters, json_request['filters'], SHARED_AWARD_FILTER_DEFAULTS) # Validate award type types if not filters.get('award_type_codes', None) or len(filters['award_type_codes']) < 1: filters['award_type_codes'] = list(award_type_mapping.keys()) for award_type_code in filters['award_type_codes']: if award_type_code not in award_type_mapping: raise InvalidParameterException('Invalid award_type: {}'.format(award_type_code)) json_request['filters']['award_type_codes'] = filters['award_type_codes'] # Validate locations for location_filter in ['place_of_performance_locations', 'recipient_locations']: if filters.get(location_filter): for location_dict in filters[location_filter]: if not isinstance(location_dict, dict): raise InvalidParameterException('Location is not a dictionary: {}'.format(location_dict)) location_error_handling(location_dict.keys()) json_request['filters'][location_filter] = filters[location_filter] # Validate time periods total_range_count = validate_time_periods(filters, json_request) if constraint_type == 'row_count': # Validate limit exists and is below MAX_DOWNLOAD_LIMIT json_request['limit'] = parse_limit(request_data) # Validate row_count-constrainted filter types and assign defaults check_types_and_assign_defaults(filters, json_request['filters'], ROW_CONSTRAINT_FILTER_DEFAULTS) elif constraint_type == 'year': # Validate combined total dates within one year (allow for leap years) if total_range_count > 366: raise InvalidParameterException('Invalid Parameter: time_period total days must be within a year') # Validate year-constrainted filter types and assign defaults check_types_and_assign_defaults(filters, json_request['filters'], YEAR_CONSTRAINT_FILTER_DEFAULTS) else: raise InvalidParameterException('Invalid parameter: constraint_type must be "row_count" or "year"') return json_request
def _handle_custom_award_download(self): """ Custom Award Download allows different filters than other Award Download Endpoints and thus it needs to be normalized before moving forward # TODO: Refactor to use similar filters as Advanced Search download """ self.tinyshield_models.extend([ { "name": "agencies", "key": "filters|agencies", "type": "array", "array_type": "object", "object_keys": { "type": { "type": "enum", "enum_values": ["funding", "awarding"], "optional": False }, "tier": { "type": "enum", "enum_values": ["toptier", "subtier"], "optional": False }, "toptier_name": { "type": "text", "text_type": "search", "optional": True }, "name": { "type": "text", "text_type": "search", "optional": False }, }, }, { "name": "agency", "key": "filters|agency", "type": "integer" }, { "name": "date_range", "key": "filters|date_range", "type": "object", "optional": False, "object_keys": { "start_date": { "type": "date", "default": "1000-01-01" }, "end_date": { "type": "date", "default": datetime.strftime(datetime.utcnow(), "%Y-%m-%d") }, }, }, { "name": "date_type", "key": "filters|date_type", "type": "enum", "enum_values": ["action_date", "last_modified_date"], "default": "action_date", }, { "name": "place_of_performance_locations", "key": "filters|place_of_performance_locations", "type": "array", "array_type": "object", "object_keys": { "country": { "type": "text", "text_type": "search", "optional": False }, "state": { "type": "text", "text_type": "search", "optional": True }, "zip": { "type": "text", "text_type": "search", "optional": True }, "district": { "type": "text", "text_type": "search", "optional": True }, "county": { "type": "text", "text_type": "search", "optional": True }, "city": { "type": "text", "text_type": "search", "optional": True }, }, }, { "name": "place_of_performance_scope", "key": "filters|place_of_performance_scope", "type": "enum", "enum_values": ["domestic", "foreign"], }, { "name": "prime_award_types", "key": "filters|prime_award_types", "type": "array", "array_type": "enum", "min": 0, "enum_values": list(award_type_mapping.keys()), }, { "name": "recipient_locations", "key": "filters|recipient_locations", "type": "array", "array_type": "object", "object_keys": { "country": { "type": "text", "text_type": "search", "optional": False }, "state": { "type": "text", "text_type": "search", "optional": True }, "zip": { "type": "text", "text_type": "search", "optional": True }, "district": { "type": "text", "text_type": "search", "optional": True }, "county": { "type": "text", "text_type": "search", "optional": True }, "city": { "type": "text", "text_type": "search", "optional": True }, }, }, { "name": "recipient_scope", "key": "filters|recipient_scope", "type": "enum", "enum_values": ("domestic", "foreign"), }, { "name": "sub_agency", "key": "filters|sub_agency", "type": "text", "text_type": "search" }, { "name": "sub_award_types", "key": "filters|sub_award_types", "type": "array", "array_type": "enum", "min": 0, "enum_values": all_subaward_types, }, ]) filter_all_agencies = False if str(self._json_request["filters"].get("agency", "")).lower() == "all": filter_all_agencies = True self._json_request["filters"].pop("agency") self._json_request = self.get_validated_request() custom_award_filters = self._json_request["filters"] final_award_filters = {} # These filters do not need any normalization for key, value in custom_award_filters.items(): if key in [ "recipient_locations", "recipient_scope", "place_of_performance_location", "place_of_performance_scope", ]: final_award_filters[key] = value if get_date_range_length(custom_award_filters["date_range"]) > 366: raise InvalidParameterException( "Invalid Parameter: date_range total days must be within a year" ) final_award_filters["time_period"] = [{ **custom_award_filters["date_range"], "date_type": custom_award_filters["date_type"] }] if (custom_award_filters.get("prime_award_types") is None and custom_award_filters.get("sub_award_types") is None): raise InvalidParameterException( "Missing one or more required body parameters: prime_award_types or sub_award_types" ) self._json_request["download_types"] = [] final_award_filters["prime_and_sub_award_types"] = {} if custom_award_filters.get("prime_award_types"): self._json_request["download_types"].append("prime_awards") final_award_filters["prime_and_sub_award_types"][ "prime_awards"] = custom_award_filters["prime_award_types"] if custom_award_filters.get("sub_award_types"): self._json_request["download_types"].append("sub_awards") final_award_filters["prime_and_sub_award_types"][ "sub_awards"] = custom_award_filters["sub_award_types"] if "agency" in custom_award_filters: if "agencies" not in custom_award_filters: final_award_filters["agencies"] = [] if filter_all_agencies: toptier_name = "all" else: toptier_name = (ToptierAgency.objects.filter( toptier_agency_id=custom_award_filters["agency"]).values( "name").first()) if toptier_name is None: raise InvalidParameterException( f"Toptier ID not found: {custom_award_filters['agency']}" ) toptier_name = toptier_name["name"] if "sub_agency" in custom_award_filters: final_award_filters["agencies"].append({ "type": "awarding", "tier": "subtier", "name": custom_award_filters["sub_agency"], "toptier_name": toptier_name, }) else: final_award_filters["agencies"].append({ "type": "awarding", "tier": "toptier", "name": toptier_name }) if "agencies" in custom_award_filters: final_award_filters["agencies"] = [ val for val in custom_award_filters["agencies"] if val.get("name", "").lower() != "all" ] self._json_request["filters"] = final_award_filters
from usaspending_api.awards.v2.lookups.lookups import award_type_mapping from usaspending_api.core.validator.helpers import TINY_SHIELD_SEPARATOR AWARD_FILTER = [ { 'name': 'award_ids', 'type': 'array', 'array_type': 'integer' }, { 'name': 'award_type_codes', 'type': 'array', 'array_type': 'enum', 'enum_values': list(award_type_mapping.keys()) }, { 'name': 'contract_pricing_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search' }, { 'name': 'extent_competed_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search' }, { 'name': 'keyword', 'type': 'text', 'text_type': 'search',
def validate_award_request(self, request_data): """Analyze request and raise any formatting errors as Exceptions""" json_request = {} constraint_type = request_data.get('constraint_type', None) # Validate required parameters for required_param in ['award_levels', 'filters']: if required_param not in request_data: raise InvalidParameterException( 'Missing one or more required query parameters: {}'.format( required_param)) if not isinstance(request_data['award_levels'], list): raise InvalidParameterException( 'Award levels parameter not provided as a list') elif len(request_data['award_levels']) == 0: raise InvalidParameterException( 'At least one award level is required.') for award_level in request_data['award_levels']: if award_level not in VALUE_MAPPINGS: raise InvalidParameterException( 'Invalid award_level: {}'.format(award_level)) json_request['download_types'] = request_data['award_levels'] # Overriding all other filters if the keyword filter is provided in year-constraint download # Make sure this is after checking the award_levels if constraint_type == 'year' and 'elasticsearch_keyword' in request_data[ 'filters']: json_request['filters'] = { 'elasticsearch_keyword': request_data['filters']['elasticsearch_keyword'], 'award_type_codes': list(award_type_mapping.keys()) } json_request['limit'] = settings.MAX_DOWNLOAD_LIMIT return json_request if not isinstance(request_data['filters'], dict): raise InvalidParameterException( 'Filters parameter not provided as a dict') elif len(request_data['filters']) == 0: raise InvalidParameterException('At least one filter is required.') json_request['filters'] = {} # Set defaults of non-required parameters json_request['columns'] = request_data.get('columns', []) json_request['file_format'] = request_data.get('file_format', 'csv') # Validate shared filter types and assign defaults filters = request_data['filters'] check_types_and_assign_defaults(filters, json_request['filters'], SHARED_AWARD_FILTER_DEFAULTS) # Validate award type types if not filters.get('award_type_codes', None) or len(filters['award_type_codes']) < 1: filters['award_type_codes'] = list(award_type_mapping.keys()) for award_type_code in filters['award_type_codes']: if award_type_code not in award_type_mapping: raise InvalidParameterException( 'Invalid award_type: {}'.format(award_type_code)) json_request['filters']['award_type_codes'] = filters[ 'award_type_codes'] # Validate locations for location_filter in [ 'place_of_performance_locations', 'recipient_locations' ]: if filters.get(location_filter): for location_dict in filters[location_filter]: if not isinstance(location_dict, dict): raise InvalidParameterException( 'Location is not a dictionary: {}'.format( location_dict)) location_error_handling(location_dict.keys()) json_request['filters'][location_filter] = filters[ location_filter] # Validate time periods total_range_count = validate_time_periods(filters, json_request) if constraint_type == 'row_count': # Validate limit exists and is below MAX_DOWNLOAD_LIMIT json_request['limit'] = parse_limit(request_data) # Validate row_count-constrainted filter types and assign defaults check_types_and_assign_defaults(filters, json_request['filters'], ROW_CONSTRAINT_FILTER_DEFAULTS) elif constraint_type == 'year': # Validate combined total dates within one year (allow for leap years) if total_range_count > 366: raise InvalidParameterException( 'Invalid Parameter: time_period total days must be within a year' ) # Validate year-constrainted filter types and assign defaults check_types_and_assign_defaults(filters, json_request['filters'], YEAR_CONSTRAINT_FILTER_DEFAULTS) else: raise InvalidParameterException( 'Invalid parameter: constraint_type must be "row_count" or "year"' ) return json_request
'filter_function': account_download_filter }, 'award_financial': { 'source_type': 'account', 'table': FinancialAccountsByAwards, 'table_name': 'award_financial', 'download_name': 'account_breakdown_by_award', 'filter_function': account_download_filter } } # Bulk Download still uses "prime awards" instead of "transactions" VALUE_MAPPINGS['prime_awards'] = VALUE_MAPPINGS['transactions'] SHARED_AWARD_FILTER_DEFAULTS = { 'award_type_codes': list(award_type_mapping.keys()), 'agencies': [], 'time_period': [], 'place_of_performance_locations': [], 'recipient_locations': [] } YEAR_CONSTRAINT_FILTER_DEFAULTS = {'elasticsearch_keyword': ''} ROW_CONSTRAINT_FILTER_DEFAULTS = { 'keywords': [], 'legal_entities': [], 'recipient_search_text': [], 'recipient_scope': '', 'recipient_type_names': [], 'place_of_performance_scope': '', 'award_amounts': [], 'award_ids': [],
def test_get_csv_sources(): mappings = csv_generation.VALUE_MAPPINGS original = mappings['awards']['filter_function'] mappings['awards']['filter_function'] = MagicMock(returned_value='') csv_sources = csv_generation.get_csv_sources({ "download_types": ["awards"], "filters": { 'award_type_codes': list(award_type_mapping.keys()) } }) mappings['awards']['filter_function'] = original assert len(csv_sources) == 2 assert csv_sources[0].file_type == 'd1' assert csv_sources[0].source_type == 'awards' assert csv_sources[1].file_type == 'd2' assert csv_sources[1].source_type == 'awards' original = mappings['transactions']['filter_function'] mappings['transactions']['filter_function'] = MagicMock(returned_value='') csv_sources = csv_generation.get_csv_sources({ "download_types": ["transactions"], "filters": { 'award_type_codes': list(award_type_mapping.keys()) } }) mappings['transactions']['filter_function'] = original assert len(csv_sources) == 2 assert csv_sources[0].file_type == 'd1' assert csv_sources[0].source_type == 'transactions' assert csv_sources[1].file_type == 'd2' assert csv_sources[1].source_type == 'transactions' original = mappings['sub_awards']['filter_function'] mappings['sub_awards']['filter_function'] = MagicMock(returned_value='') csv_sources = csv_generation.get_csv_sources({ "download_types": ["sub_awards"], "filters": { 'award_type_codes': list(award_type_mapping.keys()) } }) mappings['sub_awards']['filter_function'] = original assert len(csv_sources) == 2 assert csv_sources[0].file_type == 'd1' assert csv_sources[0].source_type == 'sub_awards' assert csv_sources[1].file_type == 'd2' assert csv_sources[1].source_type == 'sub_awards' original = mappings['account_balances']['filter_function'] mappings['account_balances']['filter_function'] = MagicMock( returned_value='') csv_sources = csv_generation.get_csv_sources({ "download_types": ["account_balances"], "account_level": "treasury_account", "filters": {} }) mappings['account_balances']['filter_function'] = original assert len(csv_sources) == 1 assert csv_sources[0].file_type == 'treasury_account' assert csv_sources[0].source_type == 'account_balances' original = mappings['object_class_program_activity']['filter_function'] mappings['object_class_program_activity']['filter_function'] = MagicMock( returned_value='') csv_sources = csv_generation.get_csv_sources({ "download_types": ["object_class_program_activity"], "account_level": "treasury_account", "filters": {} }) mappings['object_class_program_activity']['filter_function'] = original assert len(csv_sources) == 1 assert csv_sources[0].file_type == 'treasury_account' assert csv_sources[0].source_type == 'object_class_program_activity'
def __init__(self, request_data: dict): super().__init__(request_data) self.tinyshield_models.extend([ { "key": "filters|def_codes", "name": "def_codes", "type": "array", "array_type": "enum", "enum_values": sorted( DisasterEmergencyFundCode.objects.values_list("code", flat=True)), "allow_nulls": False, "optional": False, }, { "key": "filters|query", "name": "query", "type": "text", "text_type": "search", "allow_nulls": False, "optional": True, }, { "key": "filters|award_type_codes", "name": "award_type_codes", "type": "array", "array_type": "enum", "enum_values": sorted(award_type_mapping.keys()), "allow_nulls": False, "optional": True, }, ]) self._json_request["filters"] = request_data.get("filters") self._json_request = self.get_validated_request() self._json_request["download_types"] = [self.name] # Determine what to use in the filename based on "award_type_codes" filter; # Also add "face_value_of_loans" column if only loan types award_category = "All-Awards" award_type_codes = set(self._json_request["filters"].get( "award_type_codes", award_type_mapping.keys())) columns = [ "recipient", "award_obligations", "award_outlays", "number_of_awards" ] if award_type_codes <= set(contract_type_mapping.keys()): award_category = "Contracts" elif award_type_codes <= set(idv_type_mapping.keys()): award_category = "Contract-IDVs" elif award_type_codes <= set(grant_type_mapping.keys()): award_category = "Grants" elif award_type_codes <= set(loan_type_mapping.keys()): award_category = "Loans" columns.insert(3, "face_value_of_loans") elif award_type_codes <= set(direct_payment_type_mapping.keys()): award_category = "Direct-Payments" elif award_type_codes <= set(other_type_mapping.keys()): award_category = "Other-Financial-Assistance" self._json_request["award_category"] = award_category self._json_request["columns"] = self._json_request.get( "columns") or tuple(columns) # Need to specify the field to use "query" filter on if present query_text = self._json_request["filters"].pop("query", None) if query_text: self._json_request["filters"]["query"] = { "text": query_text, "fields": ["recipient_name"] }
"exclude": { "type": "array", "array_type": "any", "models": [{"type": "array", "array_type": "text", "text_type": "search"}], "min": 0, }, }, } AWARD_FILTER = [ {"name": "award_ids", "type": "array", "array_type": "text", "text_type": "search"}, { "name": "award_type_codes", "type": "array", "array_type": "enum", "enum_values": list(award_type_mapping.keys()) + ["no intersection"], }, {"name": "contract_pricing_type_codes", "type": "array", "array_type": "text", "text_type": "search"}, {"name": "extent_competed_type_codes", "type": "array", "array_type": "text", "text_type": "search"}, {"name": "keywords", "type": "array", "array_type": "text", "text_type": "search", "text_min": 3}, {"name": "legal_entities", "type": "array", "array_type": "integer", "array_max": maxsize}, { "name": "naics_codes", "type": "any", "models": [ { "name": "naics_codes", "type": "object", "min": 0, "object_keys": { "require": {"type": "array", "array_type": "integer", "text_type": "search", "min": 0},
from usaspending_api.awards.v2.lookups.lookups import award_type_mapping from usaspending_api.common.validator.helpers import TINY_SHIELD_SEPARATOR AWARD_FILTER = [ { 'name': 'award_ids', 'type': 'array', 'array_type': 'text', 'text_type': 'search' }, { 'name': 'award_type_codes', 'type': 'array', 'array_type': 'enum', 'enum_values': list(award_type_mapping.keys()) + ['no intersection'] }, { 'name': 'contract_pricing_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search' }, { 'name': 'extent_competed_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search' }, { 'name': 'keywords',