def test_download_assistance_status(client, download_test_data, refresh_matviews): download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string()) # Test without columns specified dl_resp = client.post( "/api/v2/download/assistance/", content_type="application/json", data=json.dumps({"award_id": 789, "columns": []}), ) resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"])) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 1 assert resp.json()["total_columns"] == 88 # Test with columns specified dl_resp = client.post( "/api/v2/download/assistance/", content_type="application/json", data=json.dumps( {"award_id": 789, "columns": ["prime_award_unique_key", "prime_award_amount", "program_activity_name"]} ), ) resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"])) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 1 assert resp.json()["total_columns"] == 2
def test_es_download_awards_bad_filter_type_raises(client, monkeypatch, download_test_data, elasticsearch_award_index): logging_statements = [] monkeypatch.setattr( "usaspending_api.download.v2.views.logger.info", lambda message: logging_statements.append(message), ) monkeypatch.setattr( "usaspending_api.common.elasticsearch.search_wrappers.AwardSearch._index_name", settings.ES_AWARDS_QUERY_ALIAS_PREFIX, ) elasticsearch_award_index.update_index() download_generation.retrieve_db_string = Mock( return_value=generate_test_db_connection_string()) payload = {"filters": "01", "columns": []} resp = client.post( "/api/v2/download/awards/", content_type="application/json", data=json.dumps(payload), **{EXPERIMENTAL_API_HEADER: ELASTICSEARCH_HEADER_VALUE}, ) assert resp.status_code == status.HTTP_400_BAD_REQUEST assert len(logging_statements) == 1, "Expected one logging statement" assert ( logging_statements[0] == "Using experimental Elasticsearch functionality for '/download/awards'" ), "Expected a different logging statement" assert resp.json()["detail"] == "Filters parameter not provided as a dict"
def test_download_awards_with_some_sub_awards(client, award_data): download_generation.retrieve_db_string = Mock( return_value=generate_test_db_connection_string()) filters = { "agency": "all", "sub_award_types": ["grant"], "date_type": "action_date", "date_range": { "start_date": "2017-10-01", "end_date": "2018-09-30" }, } dl_resp = client.post( "/api/v2/bulk_download/awards", content_type="application/json", data=json.dumps({ "filters": filters, "columns": [] }), ) assert dl_resp.status_code == status.HTTP_200_OK resp = client.get("/api/v2/download/status/?file_name={}".format( dl_resp.json()["file_name"])) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 2 assert resp.json()["total_columns"] == 99
def test_download_count(client, download_test_data, monkeypatch, elasticsearch_transaction_index): setup_elasticsearch_test(monkeypatch, elasticsearch_transaction_index) download_generation.retrieve_db_string = Mock( return_value=generate_test_db_connection_string()) resp = client.post( "/api/v2/download/count/", content_type="application/json", data=json.dumps({ "filters": { "agencies": [{ "type": "awarding", "tier": "toptier", "name": "Bureau of Things" }] } }), ) resp_json = resp.json() assert resp.status_code == status.HTTP_200_OK, "Failed to return 200 Response" assert resp_json["calculated_transaction_count"] == 1 assert resp_json[ "maximum_transaction_limit"] == settings.MAX_DOWNLOAD_LIMIT assert resp_json["transaction_rows_gt_limit"] is False
def test_download_idv_status(client, download_test_data): download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string()) # Test without columns specified dl_resp = client.post( "/api/v2/download/idv/", content_type="application/json", data=json.dumps({"award_id": 123, "columns": []}) ) resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"])) expected_number_of_columns = get_number_of_columns_for_query_paths( ("award_financial", "treasury_account"), ("idv_orders", "d1"), ("idv_transaction_history", "d1") ) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 1 assert resp.json()["total_columns"] == expected_number_of_columns # Test with columns specified dl_resp = client.post( "/api/v2/download/idv/", content_type="application/json", data=json.dumps( { "award_id": 123, "columns": ["current_total_value_of_award", "contract_award_unique_key", "program_activity_name"], } ), ) resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"])) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 1 assert resp.json()["total_columns"] == 5
def test_es_download_transactions_excessive_limit( client, monkeypatch, download_test_data, elasticsearch_transaction_index): logging_statements = [] monkeypatch.setattr( "usaspending_api.download.v2.views.logger.info", lambda message: logging_statements.append(message), ) monkeypatch.setattr( "usaspending_api.common.elasticsearch.search_wrappers.TransactionSearch._index_name", settings.ES_TRANSACTIONS_QUERY_ALIAS_PREFIX, ) elasticsearch_transaction_index.update_index() download_generation.retrieve_db_string = Mock( return_value=generate_test_db_connection_string()) resp = client.post( "/api/v2/download/transactions/", content_type="application/json", data=json.dumps({ "limit": settings.MAX_DOWNLOAD_LIMIT + 1, "filters": { "award_type_codes": [] }, "columns": [] }), **{EXPERIMENTAL_API_HEADER: ELASTICSEARCH_HEADER_VALUE}, ) assert resp.status_code == status.HTTP_400_BAD_REQUEST assert len(logging_statements) == 1, "Expected one logging statement" assert ( logging_statements[0] == "Using experimental Elasticsearch functionality for '/download/transactions'" ), "Expected a different logging statement"
def test_download_transactions_limit(client, download_test_data, monkeypatch, elasticsearch_transaction_index): setup_elasticsearch_test(monkeypatch, elasticsearch_transaction_index) download_generation.retrieve_db_string = Mock( return_value=generate_test_db_connection_string()) dl_resp = client.post( "/api/v2/download/transactions/", content_type="application/json", data=json.dumps({ "limit": 1, "filters": { "award_type_codes": [] }, "columns": [] }), ) resp = client.get("/api/v2/download/status/?file_name={}".format( dl_resp.json()["file_name"])) expected_number_of_columns = get_number_of_columns_for_query_paths( ("transaction", "d1"), ("transaction", "d2"), ("subaward", "d1"), ("subaward", "d2")) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 2 assert resp.json()["total_columns"] == expected_number_of_columns
def test_download_idv_status(client, download_test_data, refresh_matviews): download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string()) # Test without columns specified dl_resp = client.post( "/api/v2/download/idv/", content_type="application/json", data=json.dumps({"award_id": 123, "columns": []}) ) resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"])) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 1 assert resp.json()["total_columns"] == len(query_paths["transaction"]["d1"]) # Test with columns specified dl_resp = client.post( "/api/v2/download/idv/", content_type="application/json", data=json.dumps( { "award_id": 123, "columns": ["current_total_value_of_award", "contract_award_unique_key", "program_activity_name"], } ), ) resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"])) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 1 assert resp.json()["total_columns"] == 2
def test_download_idv_without_columns(client, download_test_data): download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string()) resp = client.post( "/api/v2/download/idv/", content_type="application/json", data=json.dumps({"award_id": 123, "columns": []}) ) assert resp.status_code == status.HTTP_200_OK assert ".zip" in resp.json()["url"]
def test_download_transactions_excessive_limit(client, download_test_data): download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string()) resp = client.post( "/api/v2/download/transactions/", content_type="application/json", data=json.dumps({"limit": settings.MAX_DOWNLOAD_LIMIT + 1, "filters": {"award_type_codes": []}, "columns": []}), ) assert resp.status_code == status.HTTP_400_BAD_REQUEST
def test_download_transactions_bad_column_list_raises(client, download_test_data): download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string()) payload = {"filters": {"award_type_codes": []}, "columns": ["modification_number", "bogus_column"]} resp = client.post("/api/v2/download/transactions/", content_type="application/json", data=json.dumps(payload)) assert resp.status_code == status.HTTP_400_BAD_REQUEST assert "Unknown columns" in resp.json()["detail"] assert "bogus_column" in resp.json()["detail"] assert "modification_number" not in resp.json()["detail"]
def test_download_awards_with_foreign_scope(client, award_data): # Recipient Location Scope download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string()) filters = { "agency": "all", "prime_award_types": [*list(award_type_mapping.keys())], "sub_award_types": [*all_subaward_types], "date_type": "action_date", "date_range": {"start_date": "2016-10-01", "end_date": "2017-09-30"}, "recipient_scope": "foreign", } dl_resp = client.post( "/api/v2/bulk_download/awards", content_type="application/json", data=json.dumps({"filters": filters, "columns": []}), ) assert dl_resp.status_code == status.HTTP_200_OK resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"])) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 5 assert resp.json()["total_columns"] == 550 # Place of Performance Scope download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string()) filters = { "agency": "all", "prime_award_types": [*list(award_type_mapping.keys())], "sub_award_types": [*all_subaward_types], "date_type": "action_date", "date_range": {"start_date": "2016-10-01", "end_date": "2017-09-30"}, "place_of_performance_scope": "foreign", } dl_resp = client.post( "/api/v2/bulk_download/awards", content_type="application/json", data=json.dumps({"filters": filters, "columns": []}), ) assert dl_resp.status_code == status.HTTP_200_OK resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"])) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 5 assert resp.json()["total_columns"] == 550
def test_download_awards_bad_filter_type_raises(client, download_test_data): download_generation.retrieve_db_string = Mock( return_value=generate_test_db_connection_string()) payload = {"filters": "01", "columns": []} resp = client.post("/api/v2/download/awards/", content_type="application/json", data=json.dumps(payload)) assert resp.status_code == status.HTTP_400_BAD_REQUEST assert resp.json()["detail"] == "Filters parameter not provided as a dict"
def test_download_transactions_status(client, download_test_data): download_generation.retrieve_db_string = Mock( return_value=generate_test_db_connection_string()) # Test without columns specified dl_resp = client.post( "/api/v2/download/transactions/", content_type="application/json", data=json.dumps({ "filters": { "agencies": [{ "type": "awarding", "tier": "toptier", "name": "Bureau of Stuff" }] }, "columns": [], }), ) resp = client.get("/api/v2/download/status/?file_name={}".format( dl_resp.json()["file_name"])) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 2 assert resp.json()["total_columns"] == len( query_paths["transaction"]["d1"]) # Test with columns specified dl_resp = client.post( "/api/v2/download/transactions/", content_type="application/json", data=json.dumps({ "filters": { "agencies": [ { "type": "awarding", "tier": "toptier", "name": "Bureau of Stuff" }, { "type": "awarding", "tier": "toptier", "name": "Bureau of Things" }, ] }, "columns": ["award_id_piid", "modification_number"], }), ) resp = client.get("/api/v2/download/status/?file_name={}".format( dl_resp.json()["file_name"])) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 3 assert resp.json()["total_columns"] == 2
def test_download_assistance_bad_award_id_raises(client, download_test_data): download_generation.retrieve_db_string = Mock( return_value=generate_test_db_connection_string()) payload = {"award_id": -1, "columns": []} resp = client.post("/api/v2/download/assistance/", content_type="application/json", data=json.dumps(payload)) assert resp.status_code == status.HTTP_400_BAD_REQUEST assert resp.json( )["detail"] == "Unable to find award matching the provided award id"
def test_download_assistance_with_columns(client, download_test_data): download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string()) resp = client.post( "/api/v2/download/assistance/", content_type="application/json", data=json.dumps( {"award_id": 789, "columns": ["prime_award_unique_key", "prime_award_amount", "program_activity_name"]} ), ) assert resp.status_code == status.HTTP_200_OK assert ".zip" in resp.json()["file_url"]
def test_download_transactions_limit(client, download_test_data, refresh_matviews): download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string()) dl_resp = client.post( "/api/v2/download/transactions/", content_type="application/json", data=json.dumps({"limit": 1, "filters": {"award_type_codes": []}, "columns": []}), ) resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"])) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 2 assert resp.json()["total_columns"] == len(query_paths["transaction"]["d1"])
def test_download_awards_status(client, download_test_data, monkeypatch, elasticsearch_award_index): setup_elasticsearch_test(monkeypatch, elasticsearch_award_index) download_generation.retrieve_db_string = Mock( return_value=generate_test_db_connection_string()) # Test without columns specified dl_resp = client.post( "/api/v2/download/awards/", content_type="application/json", data=json.dumps({ "filters": { "award_type_codes": [] }, "columns": [] }), ) resp = client.get("/api/v2/download/status/?file_name={}".format( dl_resp.json()["file_name"])) expected_number_of_columns = get_number_of_columns_for_query_paths( ("award", "d1"), ("award", "d2"), ("subaward", "d1"), ("subaward", "d2")) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 3 assert resp.json()["total_columns"] == expected_number_of_columns # Test with columns specified dl_resp = client.post( "/api/v2/download/awards/", content_type="application/json", data=json.dumps({ "filters": { "award_type_codes": [] }, "columns": [ "total_obligated_amount", "product_or_service_code", "product_or_service_code_description", "naics_code", "naics_description", ], }), ) resp = client.get("/api/v2/download/status/?file_name={}".format( dl_resp.json()["file_name"])) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 3 assert resp.json()["total_columns"] == 6
def test_agency_filter_success(client, download_test_data): download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string()) resp = client.post( "/api/v2/download/accounts/", content_type="application/json", data=json.dumps( { "account_level": "federal_account", "filters": {"submission_types": ["account_balances"], "fy": "2017", "quarter": "4", "agency": "100"}, "file_format": "csv", } ), ) assert resp.status_code == status.HTTP_200_OK
def test_quarter_failure(client, download_test_data): download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string()) resp = client.post( "/api/v2/download/accounts/", content_type="application/json", data=json.dumps( { "account_level": "treasury_account", "filters": {"submission_types": ["award_financial"], "fy": "2017", "quarter": "string_not_int"}, "file_format": "csv", } ), ) assert resp.status_code == status.HTTP_400_BAD_REQUEST
def test_download_idv_with_columns(client, download_test_data): download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string()) resp = client.post( "/api/v2/download/idv/", content_type="application/json", data=json.dumps( { "award_id": 123, "columns": ["current_total_value_of_award", "contract_award_unique_key", "program_activity_name"], } ), ) assert resp.status_code == status.HTTP_200_OK assert ".zip" in resp.json()["url"]
def test_federal_account_c_defaults_success(client, download_test_data): download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string()) resp = client.post( "/api/v2/download/accounts/", content_type="application/json", data=json.dumps( { "account_level": "federal_account", "filters": {"submission_types": ["award_financial"], "fy": "2016", "quarter": "4"}, "file_format": "csv", } ), ) assert resp.status_code == status.HTTP_200_OK assert ".zip" in resp.json()["file_url"]
def test_tas_b_defaults_success(client, download_test_data): download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string()) resp = client.post( "/api/v2/download/accounts/", content_type="application/json", data=json.dumps( { "account_level": "treasury_account", "filters": {"submission_types": ["object_class_program_activity"], "fy": "2018", "quarter": "1"}, "file_format": "csv", } ), ) assert resp.status_code == status.HTTP_200_OK assert ".zip" in resp.json()["file_url"]
def test_psv_download_awards_without_columns(client, download_test_data): download_generation.retrieve_db_string = Mock( return_value=generate_test_db_connection_string()) resp = client.post( "/api/v2/download/awards/", content_type="application/json", data=json.dumps({ "filters": { "award_type_codes": [] }, "columns": [], "file_format": "pstxt" }), ) assert resp.status_code == status.HTTP_200_OK assert ".zip" in resp.json()["file_url"]
def test_empty_submission_types_fail(client, download_test_data): download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string()) resp = client.post( "/api/v2/download/accounts/", content_type="application/json", data=json.dumps( { "account_level": "treasury_account", "filters": {"submission_types": [], "fy": "2017", "quarter": "3"}, "file_format": "tsv", } ), ) assert resp.status_code == status.HTTP_400_BAD_REQUEST assert "Provide at least one value" in resp.json()["detail"], "Incorrect error message"
def test_download_awards_status(client, download_test_data): download_generation.retrieve_db_string = Mock( return_value=generate_test_db_connection_string()) # Test without columns specified dl_resp = client.post( "/api/v2/download/awards/", content_type="application/json", data=json.dumps({ "filters": { "award_type_codes": [] }, "columns": [] }), ) resp = client.get("/api/v2/download/status/?file_name={}".format( dl_resp.json()["file_name"])) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 3 assert resp.json()["total_columns"] == len(query_paths["award"]["d1"]) # Test with columns specified dl_resp = client.post( "/api/v2/download/awards/", content_type="application/json", data=json.dumps({ "filters": { "award_type_codes": [] }, "columns": [ "total_obligated_amount", "product_or_service_code", "product_or_service_code_description", "naics_code", "naics_description", ], }), ) resp = client.get("/api/v2/download/status/?file_name={}".format( dl_resp.json()["file_name"])) assert resp.status_code == status.HTTP_200_OK assert resp.json()["total_rows"] == 3 assert resp.json()["total_columns"] == 5
def test_download_count(client, download_test_data): download_generation.retrieve_db_string = Mock( return_value=generate_test_db_connection_string()) resp = client.post( "/api/v2/download/count/", content_type="application/json", data=json.dumps({ "filters": { "agencies": [{ "type": "awarding", "tier": "toptier", "name": "Bureau of Things" }] } }), ) assert resp.json()["transaction_rows_gt_limit"] is False
def test_es_download_awards_with_columns(client, monkeypatch, download_test_data, elasticsearch_award_index): logging_statements = [] monkeypatch.setattr( "usaspending_api.download.v2.views.logger.info", lambda message: logging_statements.append(message), ) monkeypatch.setattr( "usaspending_api.common.elasticsearch.search_wrappers.AwardSearch._index_name", settings.ES_AWARDS_QUERY_ALIAS_PREFIX, ) elasticsearch_award_index.update_index() download_generation.retrieve_db_string = Mock( return_value=generate_test_db_connection_string()) resp = client.post( "/api/v2/download/awards/", content_type="application/json", data=json.dumps({ "filters": { "award_type_codes": [] }, "columns": [ "total_obligated_amount", "product_or_service_code", "product_or_service_code_description", "naics_code", "naics_description", ], }), **{EXPERIMENTAL_API_HEADER: ELASTICSEARCH_HEADER_VALUE}, ) assert resp.status_code == status.HTTP_200_OK assert len(logging_statements) == 1, "Expected one logging statement" assert ( logging_statements[0] == "Using experimental Elasticsearch functionality for '/download/awards'" ), "Expected a different logging statement" assert ".zip" in resp.json()["file_url"]
def test_es_download_transactions_with_columns( client, monkeypatch, download_test_data, elasticsearch_transaction_index): logging_statements = [] monkeypatch.setattr( "usaspending_api.download.v2.views.logger.info", lambda message: logging_statements.append(message), ) monkeypatch.setattr( "usaspending_api.common.elasticsearch.search_wrappers.TransactionSearch._index_name", settings.ES_TRANSACTIONS_QUERY_ALIAS_PREFIX, ) elasticsearch_transaction_index.update_index() download_generation.retrieve_db_string = Mock( return_value=generate_test_db_connection_string()) resp = client.post( "/api/v2/download/transactions/", content_type="application/json", data=json.dumps({ "filters": { "award_type_codes": [] }, "columns": [ "assistance_transaction_unique_key", "award_id_fain", "modification_number", "sai_number", "contract_transaction_unique_key", ], }), **{EXPERIMENTAL_API_HEADER: ELASTICSEARCH_HEADER_VALUE}, ) assert resp.status_code == status.HTTP_200_OK assert len(logging_statements) == 1, "Expected one logging statement" assert ( logging_statements[0] == "Using experimental Elasticsearch functionality for '/download/transactions'" ), "Expected a different logging statement" assert ".zip" in resp.json()["file_url"]
def test_agency_filter_failure(client, download_test_data): download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string()) resp = client.post( "/api/v2/download/accounts/", content_type="application/json", data=json.dumps( { "account_level": "treasury_account", "filters": { "submission_type": "object_class_program_activity", "fy": "2017", "quarter": "4", "agency": "-2", }, "file_format": "csv", } ), ) assert resp.status_code == status.HTTP_400_BAD_REQUEST