def state_view_data(db, monkeypatch): monkeypatch.setattr("usaspending_api.recipient.v2.views.states.VALID_FIPS", {"01": { "code": "AB" }}) award_old = mommy.make("awards.Award", type="A") award_cur = mommy.make("awards.Award", type="B") trans_old = mommy.make( "awards.TransactionNormalized", award=award_old, type="A", assistance_data__place_of_perfor_state_code="AB", assistance_data__place_of_perform_country_c="USA", federal_action_obligation=10, fiscal_year=generate_fiscal_year(OUTSIDE_OF_LATEST), action_date=OUTSIDE_OF_LATEST.strftime("%Y-%m-%d"), ) trans_cur = mommy.make( "awards.TransactionNormalized", award=award_cur, type="B", assistance_data__place_of_perfor_state_code="AB", assistance_data__place_of_perform_country_c="USA", federal_action_obligation=15, fiscal_year=generate_fiscal_year(TODAY), action_date=TODAY.strftime("%Y-%m-%d"), ) mommy.make("awards.TransactionFPDS", transaction=trans_old) mommy.make("awards.TransactionFPDS", transaction=trans_cur)
def test_state_breakdown_success_year(client, state_view_data, state_breakdown_result): resp = client.get("/api/v2/recipient/state/awards/01/?year={}".format(str(generate_fiscal_year(TODAY)))) sorted_resp = sort_breakdown_response(resp.data) expected = state_breakdown_result expected[0] = {"type": "contracts", "amount": 15, "count": 1} assert resp.status_code == status.HTTP_200_OK assert sorted_resp == expected
def test_obtain_state_totals_loan_agg(state_view_loan_data): result = obtain_state_totals("01", str(generate_fiscal_year(OUTSIDE_OF_LATEST))) expected = { "pop_state_code": "AB", "total": 25, "count": 2, "total_face_value_loan_amount": 1511, } assert result == expected
def post(self, request: Request) -> Response: self.original_filters = request.data.get("filters") json_request = self.validate_request_data(request.data) self.group = GROUPING_LOOKUP[json_request["group"]] self.subawards = json_request["subawards"] self.filters = json_request["filters"] # time_period is optional so we're setting a default window from API_SEARCH_MIN_DATE to end of the current FY. # Otherwise, users will see blank results for years current_fy = generate_fiscal_year(datetime.now(timezone.utc)) if self.group == "fiscal_year": end_date = "{}-09-30".format(current_fy) else: current_fiscal_month = generate_fiscal_month( datetime.now(timezone.utc)) days_in_month = monthrange(current_fy, current_fiscal_month)[1] end_date = f"{current_fy}-{current_fiscal_month}-{days_in_month}" default_time_period = { "start_date": settings.API_SEARCH_MIN_DATE, "end_date": end_date } time_periods = self.filters.get("time_period", [default_time_period]) if self.subawards: db_results, values = self.database_data_layer_for_subawards() results = bolster_missing_time_periods( filter_time_periods=time_periods, queryset=db_results, date_range_type=values[-1], columns={"aggregated_amount": "aggregated_amount"}, ) else: results = self.query_elasticsearch_for_prime_awards(time_periods) return Response( OrderedDict([ ("group", self.group), ("results", results), ( "messages", get_generic_filters_message( self.original_filters.keys(), [elem["name"] for elem in AWARD_FILTER]), ), ]))
def test_calculate_last_completed_fiscal_quarter(): now = datetime.now(timezone.utc) yesterday = now + timedelta(days=-1) tomorrow = now + timedelta(days=1) current_fy = fyh.generate_fiscal_year(now) mommy.make( "submissions.DABSSubmissionWindowSchedule", submission_fiscal_year=2000, submission_reveal_date=now, submission_fiscal_quarter=1, is_quarter=True, ) mommy.make( "submissions.DABSSubmissionWindowSchedule", submission_fiscal_year=2000, submission_reveal_date=now, submission_fiscal_quarter=2, is_quarter=False, ) mommy.make( "submissions.DABSSubmissionWindowSchedule", submission_fiscal_year=2010, submission_reveal_date=tomorrow, submission_fiscal_quarter=2, is_quarter=True, ) mommy.make( "submissions.DABSSubmissionWindowSchedule", submission_fiscal_year=current_fy, submission_reveal_date=yesterday, submission_fiscal_quarter=3, is_quarter=True, ) mommy.make( "submissions.DABSSubmissionWindowSchedule", submission_fiscal_year=current_fy, submission_reveal_date=now, submission_fiscal_quarter=4, is_quarter=True, ) assert fyh.calculate_last_completed_fiscal_quarter(2000) == 1 # not 2, since is_quarter=False assert fyh.calculate_last_completed_fiscal_quarter(2001) is None # no row in table for 2001 assert fyh.calculate_last_completed_fiscal_quarter(2010) is None # not revealed yet assert fyh.calculate_last_completed_fiscal_quarter(current_fy) == 4 # not 3, since both are revealed & quarters
def get(self, request, fips): get_request = request.query_params year = validate_year(get_request.get("year", "latest")) fips = validate_fips(fips) state_data_qs = StateData.objects.filter(fips=fips) state_data_results = state_data_qs.values() general_state_data = state_data_results[0] state_pop_data = self.get_state_data(state_data_results, "population", year) state_mhi_data = self.get_state_data(state_data_results, "median_household_income", year) state_aggregates = obtain_state_totals(fips, year=year) if year == "all" or (year and year.isdigit() and int(year) == generate_fiscal_year(datetime.now())): amt_per_capita = None else: amt_per_capita = (round( state_aggregates["total"] / state_pop_data["population"], 2) if state_aggregates["count"] else 0) result = { "name": general_state_data["name"], "code": general_state_data["code"], "fips": general_state_data["fips"], "type": general_state_data["type"], "population": state_pop_data["population"], "pop_year": state_pop_data["year"], "pop_source": state_pop_data["pop_source"], "median_household_income": state_mhi_data["median_household_income"], "mhi_year": state_mhi_data["year"], "mhi_source": state_mhi_data["mhi_source"], "total_prime_amount": state_aggregates["total"], "total_prime_awards": state_aggregates["count"], "award_amount_per_capita": amt_per_capita, # Commented out for now # 'total_subaward_amount': total_subaward_amount, # 'total_subawards': total_subaward_count, } return Response(result)
def test_malformed_date_month_year(): date = datetime.strptime("10/2018", "%m/%Y").date with pytest.raises(Exception): generate_fiscal_year(date)
def test_incorrect_data_type_int(): with pytest.raises(TypeError): generate_fiscal_year(2019)
def test_incorrect_data_type_string(): with pytest.raises(TypeError): generate_fiscal_year("2019")
def test_middle_of_fiscal_year(): date = datetime.strptime("01/01/2019", "%m/%d/%Y") expected = 2019 actual = generate_fiscal_year(date) assert actual == expected
def test_beginning_of_fiscal_year(): date = datetime.strptime("10/01/2018", "%m/%d/%Y") expected = 2019 actual = generate_fiscal_year(date) assert actual == expected
def test_calculate_fiscal_years(): assert fyh.generate_fiscal_year(date(2000, 9, 30)) == 2000 assert fyh.generate_fiscal_year(date(2001, 10, 1)) == 2002 assert fyh.generate_fiscal_year(date(2020, 3, 2)) == 2020 assert fyh.generate_fiscal_year(date(2017, 5, 30)) == 2017 assert fyh.generate_fiscal_year(date(2019, 10, 30)) == 2020
def handle(self, *args, **options): """Run the application.""" # Make sure # settings.BULK_DOWNLOAD_S3_BUCKET_NAME # settings.BULK_DOWNLOAD_SQS_QUEUE_NAME # settings.USASPENDING_AWS_REGION # are properly configured! local = options["local"] clobber = options["clobber"] use_modified_list = options["use_modified_list"] agencies = options["agencies"] award_types = options["award_types"] for award_type in award_types: if award_type not in ["contracts", "assistance"]: raise Exception( "Unacceptable award type: {}".format(award_type)) fiscal_years = options["fiscal_years"] placeholders = options["placeholders"] cleanup = options["cleanup"] empty_assistance_file = options["empty_assistance_file"] empty_contracts_file = options["empty_contracts_file"] if placeholders and (not empty_assistance_file or not empty_contracts_file): raise Exception( "Placeholder arg provided but empty files not provided") current_date = datetime.date.today() updated_date_timestamp = datetime.datetime.strftime( current_date, "%Y%m%d") toptier_agencies = ToptierAgency.objects.all() include_all = True if use_modified_list: used_cgacs = set(pull_modified_agencies_cgacs()) toptier_agencies = ToptierAgency.objects.filter( toptier_code__in=used_cgacs) if agencies: if "all" in agencies: agencies.remove("all") else: include_all = False toptier_agencies = ToptierAgency.objects.filter( toptier_agency_id__in=agencies) toptier_agencies = list( toptier_agencies.values("name", "toptier_agency_id", "toptier_code")) # Adding 'all' to prevent duplication of code if include_all: toptier_agencies.append({ "name": "All", "toptier_agency_id": "all", "toptier_code": "All" }) if not fiscal_years: fiscal_years = range(2001, generate_fiscal_year(current_date) + 1) # moving it to self.bucket as it may be used in different cases bucket_name = settings.MONTHLY_DOWNLOAD_S3_BUCKET_NAME region_name = settings.USASPENDING_AWS_REGION self.bucket = boto3.resource( "s3", region_name=region_name).Bucket(bucket_name) if not clobber: reuploads = [] for key in self.bucket.objects.all(): re_match = re.findall( "(.*)_Full_{}.zip".format(updated_date_timestamp), key.key) if re_match: reuploads.append(re_match[0]) logger.info("Generating {} files...".format( len(toptier_agencies) * len(fiscal_years) * 2)) for agency in toptier_agencies: for fiscal_year in fiscal_years: start_date = "{}-10-01".format(fiscal_year - 1) end_date = "{}-09-30".format(fiscal_year) for award_type in award_types: file_name = f"FY{fiscal_year}_{agency['toptier_code']}_{award_type.capitalize()}" full_file_name = f"{file_name}_Full_{updated_date_timestamp}.zip" if not clobber and file_name in reuploads: logger.info( f"Skipping already uploaded: {full_file_name}") continue if placeholders: empty_file = empty_contracts_file if award_type == "contracts" else empty_assistance_file self.upload_placeholder(file_name=full_file_name, empty_file=empty_file) else: self.download( file_name=full_file_name, prime_award_types=award_mappings[award_type], agency=agency["toptier_agency_id"], date_type="action_date", start_date=start_date, end_date=end_date, monthly_download=True, cleanup=cleanup, use_sqs=(not local), ) logger.info("Populate Monthly Files complete")
def test_obtain_state_totals(state_view_data): result = obtain_state_totals("01", str(generate_fiscal_year(OUTSIDE_OF_LATEST)), ["A"]) expected = {"pop_state_code": "AB", "total": 10, "count": 1} assert result == expected
# Core Django imports # Third-party app imports from rest_framework import status from model_mommy import mommy import pytest # Imports from your apps from usaspending_api.common.helpers.fiscal_year_helpers import generate_fiscal_year from usaspending_api.recipient.v2.views.states import obtain_state_totals # Getting relative dates as the 'latest'/default argument returns results relative to when it gets called TODAY = datetime.datetime.now() OUTSIDE_OF_LATEST = TODAY - datetime.timedelta(365 * 2) CURRENT_FISCAL_YEAR = generate_fiscal_year(TODAY) EXPECTED_STATE = { "name": "Test State", "code": "TS", "fips": "01", "type": "state", "population": 100000, "pop_year": CURRENT_FISCAL_YEAR, "pop_source": "Census 2010 Pop", "median_household_income": 50000, "mhi_year": CURRENT_FISCAL_YEAR - 2, "mhi_source": "Census 2010 MHI", "total_prime_amount": 100000, "total_prime_awards": 1, "award_amount_per_capita": 1,