def test_download_assistance_status(client, download_test_data, refresh_matviews):
download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string())
# Test without columns specified
dl_resp = client.post(
"/api/v2/download/assistance/",
content_type="application/json",
data=json.dumps({"award_id": 789, "columns": []}),
)
resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"]))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 1
assert resp.json()["total_columns"] == 88
# Test with columns specified
dl_resp = client.post(
"/api/v2/download/assistance/",
content_type="application/json",
data=json.dumps(
{"award_id": 789, "columns": ["prime_award_unique_key", "prime_award_amount", "program_activity_name"]}
),
)
resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"]))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 1
assert resp.json()["total_columns"] == 2
def test_es_download_awards_bad_filter_type_raises(client, monkeypatch,
download_test_data,
elasticsearch_award_index):
logging_statements = []
monkeypatch.setattr(
"usaspending_api.download.v2.views.logger.info",
lambda message: logging_statements.append(message),
)
monkeypatch.setattr(
"usaspending_api.common.elasticsearch.search_wrappers.AwardSearch._index_name",
settings.ES_AWARDS_QUERY_ALIAS_PREFIX,
)
elasticsearch_award_index.update_index()
download_generation.retrieve_db_string = Mock(
return_value=generate_test_db_connection_string())
payload = {"filters": "01", "columns": []}
resp = client.post(
"/api/v2/download/awards/",
content_type="application/json",
data=json.dumps(payload),
**{EXPERIMENTAL_API_HEADER: ELASTICSEARCH_HEADER_VALUE},
)
assert resp.status_code == status.HTTP_400_BAD_REQUEST
assert len(logging_statements) == 1, "Expected one logging statement"
assert (
logging_statements[0] ==
"Using experimental Elasticsearch functionality for '/download/awards'"
), "Expected a different logging statement"
assert resp.json()["detail"] == "Filters parameter not provided as a dict"
def test_download_awards_with_some_sub_awards(client, award_data):
download_generation.retrieve_db_string = Mock(
return_value=generate_test_db_connection_string())
filters = {
"agency": "all",
"sub_award_types": ["grant"],
"date_type": "action_date",
"date_range": {
"start_date": "2017-10-01",
"end_date": "2018-09-30"
},
}
dl_resp = client.post(
"/api/v2/bulk_download/awards",
content_type="application/json",
data=json.dumps({
"filters": filters,
"columns": []
}),
)
assert dl_resp.status_code == status.HTTP_200_OK
resp = client.get("/api/v2/download/status/?file_name={}".format(
dl_resp.json()["file_name"]))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 2
assert resp.json()["total_columns"] == 99
def test_download_count(client, download_test_data, monkeypatch,
elasticsearch_transaction_index):
setup_elasticsearch_test(monkeypatch, elasticsearch_transaction_index)
download_generation.retrieve_db_string = Mock(
return_value=generate_test_db_connection_string())
resp = client.post(
"/api/v2/download/count/",
content_type="application/json",
data=json.dumps({
"filters": {
"agencies": [{
"type": "awarding",
"tier": "toptier",
"name": "Bureau of Things"
}]
}
}),
)
resp_json = resp.json()
assert resp.status_code == status.HTTP_200_OK, "Failed to return 200 Response"
assert resp_json["calculated_transaction_count"] == 1
assert resp_json[
"maximum_transaction_limit"] == settings.MAX_DOWNLOAD_LIMIT
assert resp_json["transaction_rows_gt_limit"] is False
def test_download_idv_status(client, download_test_data):
download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string())
# Test without columns specified
dl_resp = client.post(
"/api/v2/download/idv/", content_type="application/json", data=json.dumps({"award_id": 123, "columns": []})
)
resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"]))
expected_number_of_columns = get_number_of_columns_for_query_paths(
("award_financial", "treasury_account"), ("idv_orders", "d1"), ("idv_transaction_history", "d1")
)
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 1
assert resp.json()["total_columns"] == expected_number_of_columns
# Test with columns specified
dl_resp = client.post(
"/api/v2/download/idv/",
content_type="application/json",
data=json.dumps(
{
"award_id": 123,
"columns": ["current_total_value_of_award", "contract_award_unique_key", "program_activity_name"],
}
),
)
resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"]))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 1
assert resp.json()["total_columns"] == 5
def test_es_download_transactions_excessive_limit(
client, monkeypatch, download_test_data,
elasticsearch_transaction_index):
logging_statements = []
monkeypatch.setattr(
"usaspending_api.download.v2.views.logger.info",
lambda message: logging_statements.append(message),
)
monkeypatch.setattr(
"usaspending_api.common.elasticsearch.search_wrappers.TransactionSearch._index_name",
settings.ES_TRANSACTIONS_QUERY_ALIAS_PREFIX,
)
elasticsearch_transaction_index.update_index()
download_generation.retrieve_db_string = Mock(
return_value=generate_test_db_connection_string())
resp = client.post(
"/api/v2/download/transactions/",
content_type="application/json",
data=json.dumps({
"limit": settings.MAX_DOWNLOAD_LIMIT + 1,
"filters": {
"award_type_codes": []
},
"columns": []
}),
**{EXPERIMENTAL_API_HEADER: ELASTICSEARCH_HEADER_VALUE},
)
assert resp.status_code == status.HTTP_400_BAD_REQUEST
assert len(logging_statements) == 1, "Expected one logging statement"
assert (
logging_statements[0] ==
"Using experimental Elasticsearch functionality for '/download/transactions'"
), "Expected a different logging statement"
def test_download_transactions_limit(client, download_test_data, monkeypatch,
elasticsearch_transaction_index):
setup_elasticsearch_test(monkeypatch, elasticsearch_transaction_index)
download_generation.retrieve_db_string = Mock(
return_value=generate_test_db_connection_string())
dl_resp = client.post(
"/api/v2/download/transactions/",
content_type="application/json",
data=json.dumps({
"limit": 1,
"filters": {
"award_type_codes": []
},
"columns": []
}),
)
resp = client.get("/api/v2/download/status/?file_name={}".format(
dl_resp.json()["file_name"]))
expected_number_of_columns = get_number_of_columns_for_query_paths(
("transaction", "d1"), ("transaction", "d2"), ("subaward", "d1"),
("subaward", "d2"))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 2
assert resp.json()["total_columns"] == expected_number_of_columns
def test_download_idv_status(client, download_test_data, refresh_matviews):
download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string())
# Test without columns specified
dl_resp = client.post(
"/api/v2/download/idv/", content_type="application/json", data=json.dumps({"award_id": 123, "columns": []})
)
resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"]))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 1
assert resp.json()["total_columns"] == len(query_paths["transaction"]["d1"])
# Test with columns specified
dl_resp = client.post(
"/api/v2/download/idv/",
content_type="application/json",
data=json.dumps(
{
"award_id": 123,
"columns": ["current_total_value_of_award", "contract_award_unique_key", "program_activity_name"],
}
),
)
resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"]))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 1
assert resp.json()["total_columns"] == 2
def test_download_idv_without_columns(client, download_test_data):
download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string())
resp = client.post(
"/api/v2/download/idv/", content_type="application/json", data=json.dumps({"award_id": 123, "columns": []})
)
assert resp.status_code == status.HTTP_200_OK
assert ".zip" in resp.json()["url"]
def test_download_transactions_excessive_limit(client, download_test_data):
download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string())
resp = client.post(
"/api/v2/download/transactions/",
content_type="application/json",
data=json.dumps({"limit": settings.MAX_DOWNLOAD_LIMIT + 1, "filters": {"award_type_codes": []}, "columns": []}),
)
assert resp.status_code == status.HTTP_400_BAD_REQUEST
def test_download_transactions_bad_column_list_raises(client, download_test_data):
download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string())
payload = {"filters": {"award_type_codes": []}, "columns": ["modification_number", "bogus_column"]}
resp = client.post("/api/v2/download/transactions/", content_type="application/json", data=json.dumps(payload))
assert resp.status_code == status.HTTP_400_BAD_REQUEST
assert "Unknown columns" in resp.json()["detail"]
assert "bogus_column" in resp.json()["detail"]
assert "modification_number" not in resp.json()["detail"]
def test_download_awards_with_foreign_scope(client, award_data):
# Recipient Location Scope
download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string())
filters = {
"agency": "all",
"prime_award_types": [*list(award_type_mapping.keys())],
"sub_award_types": [*all_subaward_types],
"date_type": "action_date",
"date_range": {"start_date": "2016-10-01", "end_date": "2017-09-30"},
"recipient_scope": "foreign",
}
dl_resp = client.post(
"/api/v2/bulk_download/awards",
content_type="application/json",
data=json.dumps({"filters": filters, "columns": []}),
)
assert dl_resp.status_code == status.HTTP_200_OK
resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"]))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 5
assert resp.json()["total_columns"] == 550
# Place of Performance Scope
download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string())
filters = {
"agency": "all",
"prime_award_types": [*list(award_type_mapping.keys())],
"sub_award_types": [*all_subaward_types],
"date_type": "action_date",
"date_range": {"start_date": "2016-10-01", "end_date": "2017-09-30"},
"place_of_performance_scope": "foreign",
}
dl_resp = client.post(
"/api/v2/bulk_download/awards",
content_type="application/json",
data=json.dumps({"filters": filters, "columns": []}),
)
assert dl_resp.status_code == status.HTTP_200_OK
resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"]))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 5
assert resp.json()["total_columns"] == 550
def test_download_awards_bad_filter_type_raises(client, download_test_data):
download_generation.retrieve_db_string = Mock(
return_value=generate_test_db_connection_string())
payload = {"filters": "01", "columns": []}
resp = client.post("/api/v2/download/awards/",
content_type="application/json",
data=json.dumps(payload))
assert resp.status_code == status.HTTP_400_BAD_REQUEST
assert resp.json()["detail"] == "Filters parameter not provided as a dict"
def test_download_transactions_status(client, download_test_data):
download_generation.retrieve_db_string = Mock(
return_value=generate_test_db_connection_string())
# Test without columns specified
dl_resp = client.post(
"/api/v2/download/transactions/",
content_type="application/json",
data=json.dumps({
"filters": {
"agencies": [{
"type": "awarding",
"tier": "toptier",
"name": "Bureau of Stuff"
}]
},
"columns": [],
}),
)
resp = client.get("/api/v2/download/status/?file_name={}".format(
dl_resp.json()["file_name"]))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 2
assert resp.json()["total_columns"] == len(
query_paths["transaction"]["d1"])
# Test with columns specified
dl_resp = client.post(
"/api/v2/download/transactions/",
content_type="application/json",
data=json.dumps({
"filters": {
"agencies": [
{
"type": "awarding",
"tier": "toptier",
"name": "Bureau of Stuff"
},
{
"type": "awarding",
"tier": "toptier",
"name": "Bureau of Things"
},
]
},
"columns": ["award_id_piid", "modification_number"],
}),
)
resp = client.get("/api/v2/download/status/?file_name={}".format(
dl_resp.json()["file_name"]))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 3
assert resp.json()["total_columns"] == 2
def test_download_assistance_bad_award_id_raises(client, download_test_data):
download_generation.retrieve_db_string = Mock(
return_value=generate_test_db_connection_string())
payload = {"award_id": -1, "columns": []}
resp = client.post("/api/v2/download/assistance/",
content_type="application/json",
data=json.dumps(payload))
assert resp.status_code == status.HTTP_400_BAD_REQUEST
assert resp.json(
)["detail"] == "Unable to find award matching the provided award id"
def test_download_assistance_with_columns(client, download_test_data):
download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string())
resp = client.post(
"/api/v2/download/assistance/",
content_type="application/json",
data=json.dumps(
{"award_id": 789, "columns": ["prime_award_unique_key", "prime_award_amount", "program_activity_name"]}
),
)
assert resp.status_code == status.HTTP_200_OK
assert ".zip" in resp.json()["file_url"]
def test_download_transactions_limit(client, download_test_data, refresh_matviews):
download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string())
dl_resp = client.post(
"/api/v2/download/transactions/",
content_type="application/json",
data=json.dumps({"limit": 1, "filters": {"award_type_codes": []}, "columns": []}),
)
resp = client.get("/api/v2/download/status/?file_name={}".format(dl_resp.json()["file_name"]))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 2
assert resp.json()["total_columns"] == len(query_paths["transaction"]["d1"])
def test_download_awards_status(client, download_test_data, monkeypatch,
elasticsearch_award_index):
setup_elasticsearch_test(monkeypatch, elasticsearch_award_index)
download_generation.retrieve_db_string = Mock(
return_value=generate_test_db_connection_string())
# Test without columns specified
dl_resp = client.post(
"/api/v2/download/awards/",
content_type="application/json",
data=json.dumps({
"filters": {
"award_type_codes": []
},
"columns": []
}),
)
resp = client.get("/api/v2/download/status/?file_name={}".format(
dl_resp.json()["file_name"]))
expected_number_of_columns = get_number_of_columns_for_query_paths(
("award", "d1"), ("award", "d2"), ("subaward", "d1"),
("subaward", "d2"))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 3
assert resp.json()["total_columns"] == expected_number_of_columns
# Test with columns specified
dl_resp = client.post(
"/api/v2/download/awards/",
content_type="application/json",
data=json.dumps({
"filters": {
"award_type_codes": []
},
"columns": [
"total_obligated_amount",
"product_or_service_code",
"product_or_service_code_description",
"naics_code",
"naics_description",
],
}),
)
resp = client.get("/api/v2/download/status/?file_name={}".format(
dl_resp.json()["file_name"]))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 3
assert resp.json()["total_columns"] == 6
def test_agency_filter_success(client, download_test_data):
download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string())
resp = client.post(
"/api/v2/download/accounts/",
content_type="application/json",
data=json.dumps(
{
"account_level": "federal_account",
"filters": {"submission_types": ["account_balances"], "fy": "2017", "quarter": "4", "agency": "100"},
"file_format": "csv",
}
),
)
assert resp.status_code == status.HTTP_200_OK
def test_quarter_failure(client, download_test_data):
download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string())
resp = client.post(
"/api/v2/download/accounts/",
content_type="application/json",
data=json.dumps(
{
"account_level": "treasury_account",
"filters": {"submission_types": ["award_financial"], "fy": "2017", "quarter": "string_not_int"},
"file_format": "csv",
}
),
)
assert resp.status_code == status.HTTP_400_BAD_REQUEST
def test_download_idv_with_columns(client, download_test_data):
download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string())
resp = client.post(
"/api/v2/download/idv/",
content_type="application/json",
data=json.dumps(
{
"award_id": 123,
"columns": ["current_total_value_of_award", "contract_award_unique_key", "program_activity_name"],
}
),
)
assert resp.status_code == status.HTTP_200_OK
assert ".zip" in resp.json()["url"]
def test_federal_account_c_defaults_success(client, download_test_data):
download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string())
resp = client.post(
"/api/v2/download/accounts/",
content_type="application/json",
data=json.dumps(
{
"account_level": "federal_account",
"filters": {"submission_types": ["award_financial"], "fy": "2016", "quarter": "4"},
"file_format": "csv",
}
),
)
assert resp.status_code == status.HTTP_200_OK
assert ".zip" in resp.json()["file_url"]
def test_tas_b_defaults_success(client, download_test_data):
download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string())
resp = client.post(
"/api/v2/download/accounts/",
content_type="application/json",
data=json.dumps(
{
"account_level": "treasury_account",
"filters": {"submission_types": ["object_class_program_activity"], "fy": "2018", "quarter": "1"},
"file_format": "csv",
}
),
)
assert resp.status_code == status.HTTP_200_OK
assert ".zip" in resp.json()["file_url"]
def test_psv_download_awards_without_columns(client, download_test_data):
download_generation.retrieve_db_string = Mock(
return_value=generate_test_db_connection_string())
resp = client.post(
"/api/v2/download/awards/",
content_type="application/json",
data=json.dumps({
"filters": {
"award_type_codes": []
},
"columns": [],
"file_format": "pstxt"
}),
)
assert resp.status_code == status.HTTP_200_OK
assert ".zip" in resp.json()["file_url"]
def test_empty_submission_types_fail(client, download_test_data):
download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string())
resp = client.post(
"/api/v2/download/accounts/",
content_type="application/json",
data=json.dumps(
{
"account_level": "treasury_account",
"filters": {"submission_types": [], "fy": "2017", "quarter": "3"},
"file_format": "tsv",
}
),
)
assert resp.status_code == status.HTTP_400_BAD_REQUEST
assert "Provide at least one value" in resp.json()["detail"], "Incorrect error message"
def test_download_awards_status(client, download_test_data):
download_generation.retrieve_db_string = Mock(
return_value=generate_test_db_connection_string())
# Test without columns specified
dl_resp = client.post(
"/api/v2/download/awards/",
content_type="application/json",
data=json.dumps({
"filters": {
"award_type_codes": []
},
"columns": []
}),
)
resp = client.get("/api/v2/download/status/?file_name={}".format(
dl_resp.json()["file_name"]))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 3
assert resp.json()["total_columns"] == len(query_paths["award"]["d1"])
# Test with columns specified
dl_resp = client.post(
"/api/v2/download/awards/",
content_type="application/json",
data=json.dumps({
"filters": {
"award_type_codes": []
},
"columns": [
"total_obligated_amount",
"product_or_service_code",
"product_or_service_code_description",
"naics_code",
"naics_description",
],
}),
)
resp = client.get("/api/v2/download/status/?file_name={}".format(
dl_resp.json()["file_name"]))
assert resp.status_code == status.HTTP_200_OK
assert resp.json()["total_rows"] == 3
assert resp.json()["total_columns"] == 5
def test_download_count(client, download_test_data):
download_generation.retrieve_db_string = Mock(
return_value=generate_test_db_connection_string())
resp = client.post(
"/api/v2/download/count/",
content_type="application/json",
data=json.dumps({
"filters": {
"agencies": [{
"type": "awarding",
"tier": "toptier",
"name": "Bureau of Things"
}]
}
}),
)
assert resp.json()["transaction_rows_gt_limit"] is False
def test_es_download_awards_with_columns(client, monkeypatch,
download_test_data,
elasticsearch_award_index):
logging_statements = []
monkeypatch.setattr(
"usaspending_api.download.v2.views.logger.info",
lambda message: logging_statements.append(message),
)
monkeypatch.setattr(
"usaspending_api.common.elasticsearch.search_wrappers.AwardSearch._index_name",
settings.ES_AWARDS_QUERY_ALIAS_PREFIX,
)
elasticsearch_award_index.update_index()
download_generation.retrieve_db_string = Mock(
return_value=generate_test_db_connection_string())
resp = client.post(
"/api/v2/download/awards/",
content_type="application/json",
data=json.dumps({
"filters": {
"award_type_codes": []
},
"columns": [
"total_obligated_amount",
"product_or_service_code",
"product_or_service_code_description",
"naics_code",
"naics_description",
],
}),
**{EXPERIMENTAL_API_HEADER: ELASTICSEARCH_HEADER_VALUE},
)
assert resp.status_code == status.HTTP_200_OK
assert len(logging_statements) == 1, "Expected one logging statement"
assert (
logging_statements[0] ==
"Using experimental Elasticsearch functionality for '/download/awards'"
), "Expected a different logging statement"
assert ".zip" in resp.json()["file_url"]
def test_es_download_transactions_with_columns(
client, monkeypatch, download_test_data,
elasticsearch_transaction_index):
logging_statements = []
monkeypatch.setattr(
"usaspending_api.download.v2.views.logger.info",
lambda message: logging_statements.append(message),
)
monkeypatch.setattr(
"usaspending_api.common.elasticsearch.search_wrappers.TransactionSearch._index_name",
settings.ES_TRANSACTIONS_QUERY_ALIAS_PREFIX,
)
elasticsearch_transaction_index.update_index()
download_generation.retrieve_db_string = Mock(
return_value=generate_test_db_connection_string())
resp = client.post(
"/api/v2/download/transactions/",
content_type="application/json",
data=json.dumps({
"filters": {
"award_type_codes": []
},
"columns": [
"assistance_transaction_unique_key",
"award_id_fain",
"modification_number",
"sai_number",
"contract_transaction_unique_key",
],
}),
**{EXPERIMENTAL_API_HEADER: ELASTICSEARCH_HEADER_VALUE},
)
assert resp.status_code == status.HTTP_200_OK
assert len(logging_statements) == 1, "Expected one logging statement"
assert (
logging_statements[0] ==
"Using experimental Elasticsearch functionality for '/download/transactions'"
), "Expected a different logging statement"
assert ".zip" in resp.json()["file_url"]
def test_agency_filter_failure(client, download_test_data):
download_generation.retrieve_db_string = Mock(return_value=generate_test_db_connection_string())
resp = client.post(
"/api/v2/download/accounts/",
content_type="application/json",
data=json.dumps(
{
"account_level": "treasury_account",
"filters": {
"submission_type": "object_class_program_activity",
"fy": "2017",
"quarter": "4",
"agency": "-2",
},
"file_format": "csv",
}
),
)
assert resp.status_code == status.HTTP_400_BAD_REQUEST
