def state_view_data(db, monkeypatch):
monkeypatch.setattr("usaspending_api.recipient.v2.views.states.VALID_FIPS",
{"01": {
"code": "AB"
}})
award_old = mommy.make("awards.Award", type="A")
award_cur = mommy.make("awards.Award", type="B")
trans_old = mommy.make(
"awards.TransactionNormalized",
award=award_old,
type="A",
assistance_data__place_of_perfor_state_code="AB",
assistance_data__place_of_perform_country_c="USA",
federal_action_obligation=10,
fiscal_year=generate_fiscal_year(OUTSIDE_OF_LATEST),
action_date=OUTSIDE_OF_LATEST.strftime("%Y-%m-%d"),
)
trans_cur = mommy.make(
"awards.TransactionNormalized",
award=award_cur,
type="B",
assistance_data__place_of_perfor_state_code="AB",
assistance_data__place_of_perform_country_c="USA",
federal_action_obligation=15,
fiscal_year=generate_fiscal_year(TODAY),
action_date=TODAY.strftime("%Y-%m-%d"),
)
mommy.make("awards.TransactionFPDS", transaction=trans_old)
mommy.make("awards.TransactionFPDS", transaction=trans_cur)
def state_view_data(db, monkeypatch):
monkeypatch.setattr('usaspending_api.recipient.v2.views.states.VALID_FIPS',
{'01': {
'code': 'AB'
}})
location = mommy.make('references.Location',
location_country_code='USA',
state_code='AB')
award_old = mommy.make('awards.Award', type='A')
award_cur = mommy.make('awards.Award', type='B')
trans_old = mommy.make('awards.TransactionNormalized',
award=award_old,
type='A',
place_of_performance=location,
federal_action_obligation=10,
fiscal_year=generate_fiscal_year(OUTSIDE_OF_LATEST),
action_date=OUTSIDE_OF_LATEST.strftime('%Y-%m-%d'))
trans_cur = mommy.make('awards.TransactionNormalized',
award=award_cur,
type='B',
place_of_performance=location,
federal_action_obligation=15,
fiscal_year=generate_fiscal_year(TODAY),
action_date=TODAY.strftime('%Y-%m-%d'))
mommy.make('awards.TransactionFPDS', transaction=trans_old)
mommy.make('awards.TransactionFPDS', transaction=trans_cur)
def post(self, request):
json_request = self.validate_request_data(request.data)
self.group = json_request["group"]
self.subawards = json_request["subawards"]
self.filters = json_request["filters"]
db_results, values = self.database_data_layer()
# time_period is optional so we're setting a default window from API_SEARCH_MIN_DATE to end of the current FY.
# Otherwise, users will see blank results for years
current_fy = generate_fiscal_year(datetime.now(timezone.utc))
if self.groupings[self.group] == "fiscal_year":
end_date = "{}-09-30".format(current_fy)
else:
end_date = "{}-{}-30".format(current_fy,
datetime.now(timezone.utc).month)
default_time_period = {
"start_date": settings.API_SEARCH_MIN_DATE,
"end_date": end_date
}
time_periods = self.filters.get("time_period", [default_time_period])
results = bolster_missing_time_periods(
filter_time_periods=time_periods,
queryset=db_results,
date_range_type=values[-1],
columns={"aggregated_amount": "aggregated_amount"},
)
return Response({
"group": self.groupings[self.group],
"results": results
})
def post(self, request):
json_request = self.validate_request_data(request.data)
self.group = json_request["group"]
self.subawards = json_request["subawards"]
self.filters = json_request["filters"]
db_results, values = self.database_data_layer()
# time_period is optional so we're setting a default window from API_SEARCH_MIN_DATE to end of the current FY.
# Otherwise, users will see blank results for years
current_fy = generate_fiscal_year(datetime.now(timezone.utc))
if self.groupings[self.group] == "fiscal_year":
end_date = "{}-09-30".format(current_fy)
else:
end_date = "{}-{}-30".format(current_fy, datetime.now(timezone.utc).month)
default_time_period = {"start_date": settings.API_SEARCH_MIN_DATE, "end_date": end_date}
time_periods = self.filters.get("time_period", [default_time_period])
results = bolster_missing_time_periods(
filter_time_periods=time_periods,
queryset=db_results,
date_range_type=values[-1],
columns={"aggregated_amount": "aggregated_amount"},
)
return Response({"group": self.groupings[self.group], "results": results})
def post(self, request):
json_request = self.validate_request_data(request.data)
self.group = json_request["group"]
self.subawards = json_request["subawards"]
self.filters = json_request["filters"]
db_results, values = self.database_data_layer()
# time_period is optional so we're setting a default window from API_SEARCH_MIN_DATE to end of the current FY.
# Otherwise, users will see blank results for years
default_time_period = {
'start_date': settings.API_SEARCH_MIN_DATE,
'end_date':
'{}-09-30'.format(generate_fiscal_year(datetime.utcnow()))
}
time_periods = self.filters.get('time_period', [default_time_period])
results = generate_date_ranged_results_from_queryset(
filter_time_periods=time_periods,
queryset=db_results,
date_range_type=values[-1],
columns={'aggregated_amount': 'aggregated_amount'})
return Response({
"group": self.groupings[self.group],
"results": results
})
def test_state_breakdown_success_year(client, state_view_data, state_breakdown_result, refresh_matviews):
resp = client.get('/api/v2/recipient/state/awards/01/?year={}'.format(str(generate_fiscal_year(TODAY))))
sorted_resp = sort_breakdown_response(resp.data)
expected = state_breakdown_result
expected[0] = {'type': 'contracts', 'amount': 15, 'count': 1}
assert resp.status_code == status.HTTP_200_OK
assert sorted_resp == expected
def test_state_current_fy_capita_success(client, state_data, refresh_matviews):
# making sure amount per capita is null for current fiscal year
expected_response = EXPECTED_STATE.copy()
expected_response.update({'award_amount_per_capita': None})
resp = client.get(
state_metadata_endpoint('01',
generate_fiscal_year(datetime.date.today())))
assert resp.status_code == status.HTTP_200_OK
assert resp.data == expected_response
def test_state_breakdown_success_year(client, state_view_data,
state_breakdown_result):
resp = client.get("/api/v2/recipient/state/awards/01/?year={}".format(
str(generate_fiscal_year(TODAY))))
sorted_resp = sort_breakdown_response(resp.data)
expected = state_breakdown_result
expected[0] = {"type": "contracts", "amount": 15, "count": 1}
assert resp.status_code == status.HTTP_200_OK
assert sorted_resp == expected
def post(self, request: Request) -> Response:
self.original_filters = request.data.get("filters")
json_request = self.validate_request_data(request.data)
self.group = GROUPING_LOOKUP[json_request["group"]]
self.subawards = json_request["subawards"]
self.filters = json_request["filters"]
self.elasticsearch = is_experimental_elasticsearch_api(request)
if not self.elasticsearch:
mirror_request_to_elasticsearch(request)
# time_period is optional so we're setting a default window from API_SEARCH_MIN_DATE to end of the current FY.
# Otherwise, users will see blank results for years
current_fy = generate_fiscal_year(datetime.now(timezone.utc))
if self.group == "fiscal_year":
end_date = "{}-09-30".format(current_fy)
else:
current_fiscal_month = generate_fiscal_month(
datetime.now(timezone.utc))
days_in_month = monthrange(current_fy, current_fiscal_month)[1]
end_date = f"{current_fy}-{current_fiscal_month}-{days_in_month}"
default_time_period = {
"start_date": settings.API_SEARCH_MIN_DATE,
"end_date": end_date
}
time_periods = self.filters.get("time_period", [default_time_period])
if self.elasticsearch and not self.subawards:
logger.info(
"Using experimental Elasticsearch functionality for 'spending_over_time'"
)
results = self.query_elasticsearch(time_periods)
else:
db_results, values = self.database_data_layer()
results = bolster_missing_time_periods(
filter_time_periods=time_periods,
queryset=db_results,
date_range_type=values[-1],
columns={"aggregated_amount": "aggregated_amount"},
)
return Response(
OrderedDict([
("group", self.group),
("results", results),
(
"messages",
get_generic_filters_message(
self.original_filters.keys(),
[elem["name"] for elem in AWARD_FILTER]),
),
]))
def state_view_data(db, monkeypatch):
monkeypatch.setattr('usaspending_api.recipient.v2.views.states.VALID_FIPS', {'01': {'code': 'AB'}})
location = mommy.make(
'references.Location',
location_country_code='USA',
state_code='AB'
)
award_old = mommy.make('awards.Award',
type='A')
award_cur = mommy.make('awards.Award',
type='B')
trans_old = mommy.make(
'awards.TransactionNormalized',
award=award_old,
type='A',
place_of_performance=location,
federal_action_obligation=10,
fiscal_year=generate_fiscal_year(OUTSIDE_OF_LATEST),
action_date=OUTSIDE_OF_LATEST.strftime('%Y-%m-%d')
)
trans_cur = mommy.make(
'awards.TransactionNormalized',
award=award_cur,
type='B',
place_of_performance=location,
federal_action_obligation=15,
fiscal_year=generate_fiscal_year(TODAY),
action_date=TODAY.strftime('%Y-%m-%d')
)
mommy.make('awards.TransactionFPDS', transaction=trans_old)
mommy.make('awards.TransactionFPDS', transaction=trans_cur)
def get(self, request, fips):
get_request = request.query_params
year = validate_year(get_request.get('year', 'latest'))
fips = validate_fips(fips)
state_data_qs = StateData.objects.filter(fips=fips)
state_data_results = state_data_qs.values()
general_state_data = state_data_results[0]
state_pop_data = self.get_state_data(state_data_results, 'population',
year)
state_mhi_data = self.get_state_data(state_data_results,
'median_household_income', year)
state_aggregates = obtain_state_totals(fips, year=year)
if year == 'all' or (year and year.isdigit() and int(year)
== generate_fiscal_year(datetime.now())):
amt_per_capita = None
else:
amt_per_capita = (round(
state_aggregates['total'] / state_pop_data['population'], 2)
if state_aggregates['count'] else 0)
result = {
'name': general_state_data['name'],
'code': general_state_data['code'],
'fips': general_state_data['fips'],
'type': general_state_data['type'],
'population': state_pop_data['population'],
'pop_year': state_pop_data['year'],
'pop_source': state_pop_data['pop_source'],
'median_household_income':
state_mhi_data['median_household_income'],
'mhi_year': state_mhi_data['year'],
'mhi_source': state_mhi_data['mhi_source'],
'total_prime_amount': state_aggregates['total'],
'total_prime_awards': state_aggregates['count'],
'award_amount_per_capita': amt_per_capita,
# Commented out for now
# 'total_subaward_amount': total_subaward_amount,
# 'total_subawards': total_subaward_count,
}
return Response(result)
def get(self, request, fips):
get_request = request.query_params
year = validate_year(get_request.get("year", "latest"))
fips = validate_fips(fips)
state_data_qs = StateData.objects.filter(fips=fips)
state_data_results = state_data_qs.values()
general_state_data = state_data_results[0]
state_pop_data = self.get_state_data(state_data_results, "population",
year)
state_mhi_data = self.get_state_data(state_data_results,
"median_household_income", year)
state_aggregates = obtain_state_totals(fips, year=year)
if year == "all" or (year and year.isdigit() and int(year)
== generate_fiscal_year(datetime.now())):
amt_per_capita = None
else:
amt_per_capita = (round(
state_aggregates["total"] / state_pop_data["population"], 2)
if state_aggregates["count"] else 0)
result = {
"name": general_state_data["name"],
"code": general_state_data["code"],
"fips": general_state_data["fips"],
"type": general_state_data["type"],
"population": state_pop_data["population"],
"pop_year": state_pop_data["year"],
"pop_source": state_pop_data["pop_source"],
"median_household_income":
state_mhi_data["median_household_income"],
"mhi_year": state_mhi_data["year"],
"mhi_source": state_mhi_data["mhi_source"],
"total_prime_amount": state_aggregates["total"],
"total_prime_awards": state_aggregates["count"],
"award_amount_per_capita": amt_per_capita,
# Commented out for now
# 'total_subaward_amount': total_subaward_amount,
# 'total_subawards': total_subaward_count,
}
return Response(result)
def test_incorrect_data_type_string():
with pytest.raises(TypeError):
generate_fiscal_year('2019')
def test_middle_of_fiscal_year():
date = datetime.strptime('01/01/2019', '%m/%d/%Y')
expected = 2019
actual = generate_fiscal_year(date)
assert actual == expected
def test_beginning_of_fiscal_year():
date = datetime.strptime('10/01/2018', '%m/%d/%Y')
expected = 2019
actual = generate_fiscal_year(date)
assert actual == expected
def test_malformed_date_month_year():
date = datetime.strptime('10/2018', '%m/%Y').date
with pytest.raises(Exception):
generate_fiscal_year(date)
def test_malformed_date_month_year():
date = datetime.strptime("10/2018", "%m/%Y").date
with pytest.raises(Exception):
generate_fiscal_year(date)
def test_obtain_state_totals(state_view_data, refresh_matviews):
result = obtain_state_totals('01',
str(generate_fiscal_year(OUTSIDE_OF_LATEST)),
['A'])
expected = {'pop_state_code': 'AB', 'total': 10, 'count': 1}
assert result == expected
# Core Django imports
# Third-party app imports
from rest_framework import status
from model_mommy import mommy
import pytest
# Imports from your apps
from usaspending_api.common.helpers.generic_helper import generate_fiscal_year
from usaspending_api.recipient.v2.views.states import obtain_state_totals
# Getting relative dates as the 'latest'/default argument returns results relative to when it gets called
TODAY = datetime.datetime.now()
OUTSIDE_OF_LATEST = (TODAY - datetime.timedelta(365 * 2))
CURRENT_FISCAL_YEAR = generate_fiscal_year(TODAY)
EXPECTED_STATE = {
'name': 'Test State',
'code': 'TS',
'fips': '01',
'type': 'state',
'population': 100000,
'pop_year': CURRENT_FISCAL_YEAR,
'pop_source': 'Census 2010 Pop',
'median_household_income': 50000,
'mhi_year': CURRENT_FISCAL_YEAR-2,
'mhi_source': 'Census 2010 MHI',
'total_prime_amount': 100000,
'total_prime_awards': 1,
'award_amount_per_capita': 1
def test_obtain_state_totals(state_view_data):
result = obtain_state_totals("01",
str(generate_fiscal_year(OUTSIDE_OF_LATEST)),
["A"])
expected = {"pop_state_code": "AB", "total": 10, "count": 1}
assert result == expected
def handle(self, *args, **options):
"""Run the application."""
# Make sure
# settings.BULK_DOWNLOAD_S3_BUCKET_NAME
# settings.BULK_DOWNLOAD_SQS_QUEUE_NAME
# settings.BULK_DOWNLOAD_AWS_REGION
# are properly configured!
local = options['local']
clobber = options['clobber']
use_modified_list = options['use_modified_list']
agencies = options['agencies']
award_types = options['award_types']
for award_type in award_types:
if award_type not in ['contracts', 'assistance']:
raise Exception(
'Unacceptable award type: {}'.format(award_type))
fiscal_years = options['fiscal_years']
placeholders = options['placeholders']
cleanup = options['cleanup']
empty_asssistance_file = options['empty_asssistance_file']
empty_contracts_file = options['empty_contracts_file']
if placeholders and (not empty_asssistance_file
or not empty_contracts_file):
raise Exception(
'Placeholder arg provided but empty files not provided')
current_date = datetime.date.today()
updated_date_timestamp = datetime.datetime.strftime(
current_date, '%Y%m%d')
toptier_agencies = ToptierAgency.objects.all()
include_all = True
if use_modified_list:
used_cgacs = set(pull_modified_agencies_cgacs())
toptier_agencies = ToptierAgency.objects.filter(
cgac_code__in=used_cgacs)
if agencies:
if 'all' in agencies:
agencies.remove('all')
else:
include_all = False
toptier_agencies = ToptierAgency.objects.filter(
toptier_agency_id__in=agencies)
toptier_agencies = list(
toptier_agencies.values('name', 'toptier_agency_id', 'cgac_code'))
# Adding 'all' to prevent duplication of code
if include_all:
toptier_agencies.append({
'name': 'All',
'toptier_agency_id': 'all',
'cgac_code': 'all'
})
if not fiscal_years:
fiscal_years = range(2001, generate_fiscal_year(current_date) + 1)
# moving it to self.bucket as it may be used in different cases
bucket_name = settings.MONTHLY_DOWNLOAD_S3_BUCKET_NAME
region_name = settings.BULK_DOWNLOAD_AWS_REGION
self.bucket = boto.s3.connect_to_region(region_name).get_bucket(
bucket_name)
if not clobber:
reuploads = []
for key in self.bucket.list():
re_match = re.findall(
'(.*)_Full_{}.zip'.format(updated_date_timestamp),
key.name)
if re_match:
reuploads.append(re_match[0])
logger.info('Generating {} files...'.format(
len(toptier_agencies) * len(fiscal_years) * 2))
for agency in toptier_agencies:
for fiscal_year in fiscal_years:
start_date = '{}-10-01'.format(fiscal_year - 1)
end_date = '{}-09-30'.format(fiscal_year)
for award_type in award_types:
file_name = '{}_{}_{}'.format(fiscal_year,
agency['cgac_code'],
award_type.capitalize())
full_file_name = '{}_Full_{}.zip'.format(
file_name, updated_date_timestamp)
if not clobber and file_name in reuploads:
logger.info('Skipping already uploaded: {}'.format(
full_file_name))
continue
if placeholders:
empty_file = empty_contracts_file if award_type == 'contracts' else empty_asssistance_file
self.upload_placeholder(file_name=full_file_name,
empty_file=empty_file)
else:
self.download(full_file_name, ['prime_awards'],
award_types=award_mappings[award_type],
agency=agency['toptier_agency_id'],
date_type='action_date',
start_date=start_date,
end_date=end_date,
monthly_download=True,
cleanup=cleanup,
use_sqs=(not local))
logger.info('Populate Monthly Files complete')
def test_middle_of_fiscal_year():
date = datetime.strptime("01/01/2019", "%m/%d/%Y")
expected = 2019
actual = generate_fiscal_year(date)
assert actual == expected
def test_obtain_state_totals(state_view_data, refresh_matviews):
result = obtain_state_totals('01', str(generate_fiscal_year(OUTSIDE_OF_LATEST)), ['A'])
expected = {'pop_state_code': 'AB', 'total': 10, 'count': 1}
assert result == expected
def test_incorrect_data_type_string():
with pytest.raises(TypeError):
generate_fiscal_year("2019")
def test_incorrect_data_type_int():
with pytest.raises(TypeError):
generate_fiscal_year(2019)
def test_beginning_of_fiscal_year():
date = datetime.strptime("10/01/2018", "%m/%d/%Y")
expected = 2019
actual = generate_fiscal_year(date)
assert actual == expected
def test_end_of_fiscal_year():
date = datetime.strptime('09/30/2019', '%m/%d/%Y')
expected = 2019
actual = generate_fiscal_year(date)
assert actual == expected
def test_incorrect_data_type_int():
with pytest.raises(TypeError):
generate_fiscal_year(2019)
def handle(self, *args, **options):
"""Run the application."""
# Make sure
# settings.BULK_DOWNLOAD_S3_BUCKET_NAME
# settings.BULK_DOWNLOAD_SQS_QUEUE_NAME
# settings.USASPENDING_AWS_REGION
# are properly configured!
local = options["local"]
clobber = options["clobber"]
use_modified_list = options["use_modified_list"]
agencies = options["agencies"]
award_types = options["award_types"]
for award_type in award_types:
if award_type not in ["contracts", "assistance"]:
raise Exception(
"Unacceptable award type: {}".format(award_type))
fiscal_years = options["fiscal_years"]
placeholders = options["placeholders"]
cleanup = options["cleanup"]
empty_asssistance_file = options["empty_asssistance_file"]
empty_contracts_file = options["empty_contracts_file"]
if placeholders and (not empty_asssistance_file
or not empty_contracts_file):
raise Exception(
"Placeholder arg provided but empty files not provided")
current_date = datetime.date.today()
updated_date_timestamp = datetime.datetime.strftime(
current_date, "%Y%m%d")
toptier_agencies = ToptierAgency.objects.all()
include_all = True
if use_modified_list:
used_cgacs = set(pull_modified_agencies_cgacs())
toptier_agencies = ToptierAgency.objects.filter(
toptier_code__in=used_cgacs)
if agencies:
if "all" in agencies:
agencies.remove("all")
else:
include_all = False
toptier_agencies = ToptierAgency.objects.filter(
toptier_agency_id__in=agencies)
toptier_agencies = list(
toptier_agencies.values("name", "toptier_agency_id",
"toptier_code"))
# Adding 'all' to prevent duplication of code
if include_all:
toptier_agencies.append({
"name": "All",
"toptier_agency_id": "all",
"toptier_code": "All"
})
if not fiscal_years:
fiscal_years = range(2001, generate_fiscal_year(current_date) + 1)
# moving it to self.bucket as it may be used in different cases
bucket_name = settings.MONTHLY_DOWNLOAD_S3_BUCKET_NAME
region_name = settings.USASPENDING_AWS_REGION
self.bucket = boto3.resource(
"s3", region_name=region_name).Bucket(bucket_name)
if not clobber:
reuploads = []
for key in self.bucket.objects.all():
re_match = re.findall(
"(.*)_Full_{}.zip".format(updated_date_timestamp), key.key)
if re_match:
reuploads.append(re_match[0])
logger.info("Generating {} files...".format(
len(toptier_agencies) * len(fiscal_years) * 2))
for agency in toptier_agencies:
for fiscal_year in fiscal_years:
start_date = "{}-10-01".format(fiscal_year - 1)
end_date = "{}-09-30".format(fiscal_year)
for award_type in award_types:
file_name = f"FY{fiscal_year}_{agency['toptier_code']}_{award_type.capitalize()}"
full_file_name = f"{file_name}_Full_{updated_date_timestamp}.zip"
if not clobber and file_name in reuploads:
logger.info(
f"Skipping already uploaded: {full_file_name}")
continue
if placeholders:
empty_file = empty_contracts_file if award_type == "contracts" else empty_asssistance_file
self.upload_placeholder(file_name=full_file_name,
empty_file=empty_file)
else:
self.download(
file_name=full_file_name,
prime_award_types=award_mappings[award_type],
agency=agency["toptier_agency_id"],
date_type="action_date",
start_date=start_date,
end_date=end_date,
monthly_download=True,
cleanup=cleanup,
use_sqs=(not local),
)
logger.info("Populate Monthly Files complete")
# Core Django imports
# Third-party app imports
from rest_framework import status
from model_mommy import mommy
import pytest
# Imports from your apps
from usaspending_api.common.helpers.generic_helper import generate_fiscal_year
from usaspending_api.recipient.v2.views.states import obtain_state_totals
# Getting relative dates as the 'latest'/default argument returns results relative to when it gets called
TODAY = datetime.datetime.now()
OUTSIDE_OF_LATEST = (TODAY - datetime.timedelta(365 + 2))
CURRENT_FISCAL_YEAR = generate_fiscal_year(TODAY)
EXPECTED_STATE = {
'name': 'Test State',
'code': 'TS',
'fips': '01',
'type': 'state',
'population': 100000,
'pop_year': CURRENT_FISCAL_YEAR,
'pop_source': 'Census 2010 Pop',
'median_household_income': 50000,
'mhi_year': CURRENT_FISCAL_YEAR - 1,
'mhi_source': 'Census 2010 MHI',
'total_prime_amount': 100000,
'total_prime_awards': 1,
'award_amount_per_capita': 1
def get(self, request, fips):
get_request = request.query_params
year = get_request.get('year')
fips = fips.zfill(2)
state_data_qs = StateData.objects.filter(fips=fips)
if not state_data_qs.count():
raise InvalidParameterException('Invalid FIPS ({}) or data unavailable.'.format(fips))
state_data_results = state_data_qs.values()
general_state_data = state_data_results[0]
# recreate filters
filters = {'place_of_performance_locations': [{'country': 'USA', 'state': general_state_data['code']}]}
today = datetime.now()
if year and year.isdigit():
time_period = [{
'start_date': '{}-10-01'.format(int(year)-1),
'end_date': '{}-09-30'.format(year)
}]
elif year == 'all':
time_period = [{
'start_date': '2008-10-01',
'end_date': datetime.strftime(today, '%Y-%m-%d')
}]
elif year == 'latest' or not year:
last_year = today - relativedelta(years=1)
time_period = [{
'start_date': datetime.strftime(last_year, '%Y-%m-%d'),
'end_date': datetime.strftime(today, '%Y-%m-%d')
}]
else:
raise InvalidParameterException('Invalid year: {}.'.format(year))
filters['time_period'] = time_period
state_pop_data = self.get_state_data(state_data_results, 'population', year)
state_mhi_data = self.get_state_data(state_data_results, 'median_household_income', year)
# calculate award total filtered by state
total_award_qs = universal_transaction_matview_filter(filters)
total_award_qs = sum_transaction_amount(total_award_qs.values('award_id'))
total_award_count = total_award_qs.values('award_id').distinct().count()
total_award_amount = total_award_qs.aggregate(total=Sum('transaction_amount'))['total'] \
if total_award_count else 0
if year == 'all' or (year and year.isdigit() and int(year) == generate_fiscal_year(today)):
amt_per_capita = None
else:
amt_per_capita = round(total_award_amount/state_pop_data['population'], 2) if total_award_count else 0
# calculate subaward total filtered by state - COMMENTED OUT FOR NOW
# total_subaward_qs = subaward_filter(filters)
# total_subaward_count = total_subaward_qs.count()
# total_subaward_amount = total_subaward_qs.aggregate(total=Sum('amount'))['total'] \
# if total_subaward_count else 0
result = {
'name': general_state_data['name'],
'code': general_state_data['code'],
'fips': general_state_data['fips'],
'type': general_state_data['type'],
'population': state_pop_data['population'],
'pop_year': state_pop_data['year'],
'pop_source': state_pop_data['pop_source'],
'median_household_income': state_mhi_data['median_household_income'],
'mhi_year': state_mhi_data['year'],
'mhi_source': state_mhi_data['mhi_source'],
'total_prime_amount': total_award_amount,
'total_prime_awards': total_award_count,
'award_amount_per_capita': amt_per_capita,
# Commented out for now
# 'total_subaward_amount': total_subaward_amount,
# 'total_subawards': total_subaward_count,
}
return Response(result)
def handle(self, *args, **options):
"""Run the application."""
# Make sure
# settings.BULK_DOWNLOAD_S3_BUCKET_NAME
# settings.BULK_DOWNLOAD_SQS_QUEUE_NAME
# settings.USASPENDING_AWS_REGION
# are properly configured!
local = options['local']
clobber = options['clobber']
use_modified_list = options['use_modified_list']
agencies = options['agencies']
award_types = options['award_types']
for award_type in award_types:
if award_type not in ['contracts', 'assistance']:
raise Exception('Unacceptable award type: {}'.format(award_type))
fiscal_years = options['fiscal_years']
placeholders = options['placeholders']
cleanup = options['cleanup']
empty_asssistance_file = options['empty_asssistance_file']
empty_contracts_file = options['empty_contracts_file']
if placeholders and (not empty_asssistance_file or not empty_contracts_file):
raise Exception('Placeholder arg provided but empty files not provided')
current_date = datetime.date.today()
updated_date_timestamp = datetime.datetime.strftime(current_date, '%Y%m%d')
toptier_agencies = ToptierAgency.objects.all()
include_all = True
if use_modified_list:
used_cgacs = set(pull_modified_agencies_cgacs())
toptier_agencies = ToptierAgency.objects.filter(cgac_code__in=used_cgacs)
if agencies:
if 'all' in agencies:
agencies.remove('all')
else:
include_all = False
toptier_agencies = ToptierAgency.objects.filter(toptier_agency_id__in=agencies)
toptier_agencies = list(toptier_agencies.values('name', 'toptier_agency_id', 'cgac_code'))
# Adding 'all' to prevent duplication of code
if include_all:
toptier_agencies.append({'name': 'All', 'toptier_agency_id': 'all', 'cgac_code': 'all'})
if not fiscal_years:
fiscal_years = range(2001, generate_fiscal_year(current_date) + 1)
# moving it to self.bucket as it may be used in different cases
bucket_name = settings.MONTHLY_DOWNLOAD_S3_BUCKET_NAME
region_name = settings.USASPENDING_AWS_REGION
self.bucket = boto3.resource('s3', region_name=region_name).Bucket(bucket_name)
if not clobber:
reuploads = []
for key in self.bucket.objects.all():
re_match = re.findall('(.*)_Full_{}.zip'.format(updated_date_timestamp), key.key)
if re_match:
reuploads.append(re_match[0])
logger.info('Generating {} files...'.format(len(toptier_agencies) * len(fiscal_years) * 2))
for agency in toptier_agencies:
for fiscal_year in fiscal_years:
start_date = '{}-10-01'.format(fiscal_year - 1)
end_date = '{}-09-30'.format(fiscal_year)
for award_type in award_types:
file_name = '{}_{}_{}'.format(fiscal_year, agency['cgac_code'], award_type.capitalize())
full_file_name = '{}_Full_{}.zip'.format(file_name, updated_date_timestamp)
if not clobber and file_name in reuploads:
logger.info('Skipping already uploaded: {}'.format(full_file_name))
continue
if placeholders:
empty_file = empty_contracts_file if award_type == 'contracts' else empty_asssistance_file
self.upload_placeholder(file_name=full_file_name, empty_file=empty_file)
else:
self.download(full_file_name, ['prime_awards'], award_types=award_mappings[award_type],
agency=agency['toptier_agency_id'], date_type='action_date',
start_date=start_date, end_date=end_date, monthly_download=True, cleanup=cleanup,
use_sqs=(not local))
logger.info('Populate Monthly Files complete')
