def state_view_data(db, monkeypatch):
monkeypatch.setattr("usaspending_api.recipient.v2.views.states.VALID_FIPS",
{"01": {
"code": "AB"
}})
award_old = mommy.make("awards.Award", type="A")
award_cur = mommy.make("awards.Award", type="B")
trans_old = mommy.make(
"awards.TransactionNormalized",
award=award_old,
type="A",
assistance_data__place_of_perfor_state_code="AB",
assistance_data__place_of_perform_country_c="USA",
federal_action_obligation=10,
fiscal_year=generate_fiscal_year(OUTSIDE_OF_LATEST),
action_date=OUTSIDE_OF_LATEST.strftime("%Y-%m-%d"),
)
trans_cur = mommy.make(
"awards.TransactionNormalized",
award=award_cur,
type="B",
assistance_data__place_of_perfor_state_code="AB",
assistance_data__place_of_perform_country_c="USA",
federal_action_obligation=15,
fiscal_year=generate_fiscal_year(TODAY),
action_date=TODAY.strftime("%Y-%m-%d"),
)
mommy.make("awards.TransactionFPDS", transaction=trans_old)
mommy.make("awards.TransactionFPDS", transaction=trans_cur)
def test_state_breakdown_success_year(client, state_view_data, state_breakdown_result):
resp = client.get("/api/v2/recipient/state/awards/01/?year={}".format(str(generate_fiscal_year(TODAY))))
sorted_resp = sort_breakdown_response(resp.data)
expected = state_breakdown_result
expected[0] = {"type": "contracts", "amount": 15, "count": 1}
assert resp.status_code == status.HTTP_200_OK
assert sorted_resp == expected
def test_obtain_state_totals_loan_agg(state_view_loan_data):
result = obtain_state_totals("01", str(generate_fiscal_year(OUTSIDE_OF_LATEST)))
expected = {
"pop_state_code": "AB",
"total": 25,
"count": 2,
"total_face_value_loan_amount": 1511,
}
assert result == expected
def post(self, request: Request) -> Response:
self.original_filters = request.data.get("filters")
json_request = self.validate_request_data(request.data)
self.group = GROUPING_LOOKUP[json_request["group"]]
self.subawards = json_request["subawards"]
self.filters = json_request["filters"]
# time_period is optional so we're setting a default window from API_SEARCH_MIN_DATE to end of the current FY.
# Otherwise, users will see blank results for years
current_fy = generate_fiscal_year(datetime.now(timezone.utc))
if self.group == "fiscal_year":
end_date = "{}-09-30".format(current_fy)
else:
current_fiscal_month = generate_fiscal_month(
datetime.now(timezone.utc))
days_in_month = monthrange(current_fy, current_fiscal_month)[1]
end_date = f"{current_fy}-{current_fiscal_month}-{days_in_month}"
default_time_period = {
"start_date": settings.API_SEARCH_MIN_DATE,
"end_date": end_date
}
time_periods = self.filters.get("time_period", [default_time_period])
if self.subawards:
db_results, values = self.database_data_layer_for_subawards()
results = bolster_missing_time_periods(
filter_time_periods=time_periods,
queryset=db_results,
date_range_type=values[-1],
columns={"aggregated_amount": "aggregated_amount"},
)
else:
results = self.query_elasticsearch_for_prime_awards(time_periods)
return Response(
OrderedDict([
("group", self.group),
("results", results),
(
"messages",
get_generic_filters_message(
self.original_filters.keys(),
[elem["name"] for elem in AWARD_FILTER]),
),
]))
def test_calculate_last_completed_fiscal_quarter():
now = datetime.now(timezone.utc)
yesterday = now + timedelta(days=-1)
tomorrow = now + timedelta(days=1)
current_fy = fyh.generate_fiscal_year(now)
mommy.make(
"submissions.DABSSubmissionWindowSchedule",
submission_fiscal_year=2000,
submission_reveal_date=now,
submission_fiscal_quarter=1,
is_quarter=True,
)
mommy.make(
"submissions.DABSSubmissionWindowSchedule",
submission_fiscal_year=2000,
submission_reveal_date=now,
submission_fiscal_quarter=2,
is_quarter=False,
)
mommy.make(
"submissions.DABSSubmissionWindowSchedule",
submission_fiscal_year=2010,
submission_reveal_date=tomorrow,
submission_fiscal_quarter=2,
is_quarter=True,
)
mommy.make(
"submissions.DABSSubmissionWindowSchedule",
submission_fiscal_year=current_fy,
submission_reveal_date=yesterday,
submission_fiscal_quarter=3,
is_quarter=True,
)
mommy.make(
"submissions.DABSSubmissionWindowSchedule",
submission_fiscal_year=current_fy,
submission_reveal_date=now,
submission_fiscal_quarter=4,
is_quarter=True,
)
assert fyh.calculate_last_completed_fiscal_quarter(2000) == 1 # not 2, since is_quarter=False
assert fyh.calculate_last_completed_fiscal_quarter(2001) is None # no row in table for 2001
assert fyh.calculate_last_completed_fiscal_quarter(2010) is None # not revealed yet
assert fyh.calculate_last_completed_fiscal_quarter(current_fy) == 4 # not 3, since both are revealed & quarters
def get(self, request, fips):
get_request = request.query_params
year = validate_year(get_request.get("year", "latest"))
fips = validate_fips(fips)
state_data_qs = StateData.objects.filter(fips=fips)
state_data_results = state_data_qs.values()
general_state_data = state_data_results[0]
state_pop_data = self.get_state_data(state_data_results, "population",
year)
state_mhi_data = self.get_state_data(state_data_results,
"median_household_income", year)
state_aggregates = obtain_state_totals(fips, year=year)
if year == "all" or (year and year.isdigit() and int(year)
== generate_fiscal_year(datetime.now())):
amt_per_capita = None
else:
amt_per_capita = (round(
state_aggregates["total"] / state_pop_data["population"], 2)
if state_aggregates["count"] else 0)
result = {
"name": general_state_data["name"],
"code": general_state_data["code"],
"fips": general_state_data["fips"],
"type": general_state_data["type"],
"population": state_pop_data["population"],
"pop_year": state_pop_data["year"],
"pop_source": state_pop_data["pop_source"],
"median_household_income":
state_mhi_data["median_household_income"],
"mhi_year": state_mhi_data["year"],
"mhi_source": state_mhi_data["mhi_source"],
"total_prime_amount": state_aggregates["total"],
"total_prime_awards": state_aggregates["count"],
"award_amount_per_capita": amt_per_capita,
# Commented out for now
# 'total_subaward_amount': total_subaward_amount,
# 'total_subawards': total_subaward_count,
}
return Response(result)
def test_malformed_date_month_year():
date = datetime.strptime("10/2018", "%m/%Y").date
with pytest.raises(Exception):
generate_fiscal_year(date)
def test_incorrect_data_type_int():
with pytest.raises(TypeError):
generate_fiscal_year(2019)
def test_incorrect_data_type_string():
with pytest.raises(TypeError):
generate_fiscal_year("2019")
def test_middle_of_fiscal_year():
date = datetime.strptime("01/01/2019", "%m/%d/%Y")
expected = 2019
actual = generate_fiscal_year(date)
assert actual == expected
def test_beginning_of_fiscal_year():
date = datetime.strptime("10/01/2018", "%m/%d/%Y")
expected = 2019
actual = generate_fiscal_year(date)
assert actual == expected
def test_calculate_fiscal_years():
assert fyh.generate_fiscal_year(date(2000, 9, 30)) == 2000
assert fyh.generate_fiscal_year(date(2001, 10, 1)) == 2002
assert fyh.generate_fiscal_year(date(2020, 3, 2)) == 2020
assert fyh.generate_fiscal_year(date(2017, 5, 30)) == 2017
assert fyh.generate_fiscal_year(date(2019, 10, 30)) == 2020
def handle(self, *args, **options):
"""Run the application."""
# Make sure
# settings.BULK_DOWNLOAD_S3_BUCKET_NAME
# settings.BULK_DOWNLOAD_SQS_QUEUE_NAME
# settings.USASPENDING_AWS_REGION
# are properly configured!
local = options["local"]
clobber = options["clobber"]
use_modified_list = options["use_modified_list"]
agencies = options["agencies"]
award_types = options["award_types"]
for award_type in award_types:
if award_type not in ["contracts", "assistance"]:
raise Exception(
"Unacceptable award type: {}".format(award_type))
fiscal_years = options["fiscal_years"]
placeholders = options["placeholders"]
cleanup = options["cleanup"]
empty_assistance_file = options["empty_assistance_file"]
empty_contracts_file = options["empty_contracts_file"]
if placeholders and (not empty_assistance_file
or not empty_contracts_file):
raise Exception(
"Placeholder arg provided but empty files not provided")
current_date = datetime.date.today()
updated_date_timestamp = datetime.datetime.strftime(
current_date, "%Y%m%d")
toptier_agencies = ToptierAgency.objects.all()
include_all = True
if use_modified_list:
used_cgacs = set(pull_modified_agencies_cgacs())
toptier_agencies = ToptierAgency.objects.filter(
toptier_code__in=used_cgacs)
if agencies:
if "all" in agencies:
agencies.remove("all")
else:
include_all = False
toptier_agencies = ToptierAgency.objects.filter(
toptier_agency_id__in=agencies)
toptier_agencies = list(
toptier_agencies.values("name", "toptier_agency_id",
"toptier_code"))
# Adding 'all' to prevent duplication of code
if include_all:
toptier_agencies.append({
"name": "All",
"toptier_agency_id": "all",
"toptier_code": "All"
})
if not fiscal_years:
fiscal_years = range(2001, generate_fiscal_year(current_date) + 1)
# moving it to self.bucket as it may be used in different cases
bucket_name = settings.MONTHLY_DOWNLOAD_S3_BUCKET_NAME
region_name = settings.USASPENDING_AWS_REGION
self.bucket = boto3.resource(
"s3", region_name=region_name).Bucket(bucket_name)
if not clobber:
reuploads = []
for key in self.bucket.objects.all():
re_match = re.findall(
"(.*)_Full_{}.zip".format(updated_date_timestamp), key.key)
if re_match:
reuploads.append(re_match[0])
logger.info("Generating {} files...".format(
len(toptier_agencies) * len(fiscal_years) * 2))
for agency in toptier_agencies:
for fiscal_year in fiscal_years:
start_date = "{}-10-01".format(fiscal_year - 1)
end_date = "{}-09-30".format(fiscal_year)
for award_type in award_types:
file_name = f"FY{fiscal_year}_{agency['toptier_code']}_{award_type.capitalize()}"
full_file_name = f"{file_name}_Full_{updated_date_timestamp}.zip"
if not clobber and file_name in reuploads:
logger.info(
f"Skipping already uploaded: {full_file_name}")
continue
if placeholders:
empty_file = empty_contracts_file if award_type == "contracts" else empty_assistance_file
self.upload_placeholder(file_name=full_file_name,
empty_file=empty_file)
else:
self.download(
file_name=full_file_name,
prime_award_types=award_mappings[award_type],
agency=agency["toptier_agency_id"],
date_type="action_date",
start_date=start_date,
end_date=end_date,
monthly_download=True,
cleanup=cleanup,
use_sqs=(not local),
)
logger.info("Populate Monthly Files complete")
def test_obtain_state_totals(state_view_data):
result = obtain_state_totals("01",
str(generate_fiscal_year(OUTSIDE_OF_LATEST)),
["A"])
expected = {"pop_state_code": "AB", "total": 10, "count": 1}
assert result == expected
# Core Django imports
# Third-party app imports
from rest_framework import status
from model_mommy import mommy
import pytest
# Imports from your apps
from usaspending_api.common.helpers.fiscal_year_helpers import generate_fiscal_year
from usaspending_api.recipient.v2.views.states import obtain_state_totals
# Getting relative dates as the 'latest'/default argument returns results relative to when it gets called
TODAY = datetime.datetime.now()
OUTSIDE_OF_LATEST = TODAY - datetime.timedelta(365 * 2)
CURRENT_FISCAL_YEAR = generate_fiscal_year(TODAY)
EXPECTED_STATE = {
"name": "Test State",
"code": "TS",
"fips": "01",
"type": "state",
"population": 100000,
"pop_year": CURRENT_FISCAL_YEAR,
"pop_source": "Census 2010 Pop",
"median_household_income": 50000,
"mhi_year": CURRENT_FISCAL_YEAR - 2,
"mhi_source": "Census 2010 MHI",
"total_prime_amount": 100000,
"total_prime_awards": 1,
"award_amount_per_capita": 1,
