def test_federal_account_update():
"""Test federal account updates from underlying TAS records."""
fa = mommy.make(FederalAccount,
id=1,
agency_identifier="abc",
main_account_code="0987",
account_title="Fancy duck")
tas1 = mommy.make(
TreasuryAppropriationAccount,
federal_account_id=1,
agency_id="abc",
main_account_code="0987",
account_title="Fancy duck",
)
tas2 = mommy.make(
TreasuryAppropriationAccount,
agency_id="abc",
main_account_code="0987",
account_title="Fancy goose",
ending_period_of_availability="2020",
)
# run the federal account update process and check results
update_federal_accounts()
# federal_account fk of tas2 has been updated
assert (TreasuryAppropriationAccount.objects.get(
treasury_account_identifier=tas2.treasury_account_identifier).
federal_account == fa)
# federal_account fk of tas1 is unchanged
assert (TreasuryAppropriationAccount.objects.get(
treasury_account_identifier=tas1.treasury_account_identifier).
federal_account_id == 1)
def test_federal_account_update_subset():
"""Test ability to update federal_account FK on a subset of TAS records."""
fa = mommy.make(FederalAccount,
id=1,
agency_identifier="abc",
main_account_code="0987",
account_title="Fancy duck")
mommy.make(TreasuryAppropriationAccount,
agency_id="abc",
main_account_code="0987",
_quantity=4)
# send two TAS records to the federal account update process and check results
update_federal_accounts((
TreasuryAppropriationAccount.objects.first(
).treasury_account_identifier,
TreasuryAppropriationAccount.objects.last().
treasury_account_identifier,
))
# only two of the four TAS records were updated with a foreign key, even
# though all four map back to the same federal account
assert TreasuryAppropriationAccount.objects.filter(
federal_account__isnull=True).count() == 2
# the other two records have federal account FKs
assert TreasuryAppropriationAccount.objects.filter(
federal_account=fa).count() == 2
def handle(self, *args, **options):
field_map = {
"treasury_account_identifier": "ACCT_NUM",
"account_title": "GWA_TAS NAME",
"reporting_agency_id": "Agency AID",
"reporting_agency_name": "Agency Name",
"budget_bureau_code": "ADMIN_ORG",
"budget_bureau_name": "Admin Org Name",
"fr_entity_code": "FR Entity Type Code",
"fr_entity_description": "FR Entity Description",
"budget_function_code": "Function Code",
"budget_function_title": "Function Description",
"budget_subfunction_code": "Sub Function Code",
"budget_subfunction_title": "Sub Function Description"
}
value_map = {
"data_source":
"USA",
"tas_rendering_label":
self.generate_tas_rendering_label,
"allocation_transfer_agency_id":
lambda row: row["ATA"].strip(),
"agency_id":
lambda row: row["AID"].strip(),
"beginning_period_of_availability":
lambda row: row["BPOA"].strip(),
"ending_period_of_availability":
lambda row: row["EPOA"].strip(),
"availability_type_code":
lambda row: row["A"].strip(),
"main_account_code":
lambda row: row["MAIN"].strip(),
"sub_account_code":
lambda row: row["SUB"].strip(),
"awarding_toptier_agency":
lambda row: ToptierAgency.objects.filter(cgac_code=row[
"ATA"].strip()).order_by("fpds_code").first(),
"funding_toptier_agency":
lambda row: ToptierAgency.objects.filter(cgac_code=row[
"AID"].strip()).order_by("fpds_code").first()
}
loader = ThreadedDataLoader(
model_class=TreasuryAppropriationAccount,
field_map=field_map,
value_map=value_map,
collision_field='treasury_account_identifier',
collision_behavior='update',
pre_row_function=self.skip_and_remove_financing_tas)
loader.load_from_file(options['file'][0])
# update TAS fk relationships to federal accounts
remove_empty_federal_accounts()
update_federal_accounts()
insert_federal_accounts()
def handle(self, *args, **options):
try:
with Timer("Loading TAS from {}".format(options["location"]
or "Broker")):
if options["location"]:
self.csv_tas_loader(options["location"])
else:
call_command("run_sql", "-f", TAS_SQL_PATH)
# Match funding toptiers by FREC if they didn't match by AID
with Timer(
"Matching Funding Toptier Agency where AID didn't match"):
unmapped_funding_agencies = TreasuryAppropriationAccount.objects.filter(
funding_toptier_agency=None)
match_count = 0
msg_str = "=== Found {} unmatched funding agencies across all TAS objects. ==="
logger.info(msg_str.format(unmapped_funding_agencies.count()))
for next_tas in unmapped_funding_agencies:
frec_match = ToptierAgency.objects.filter(
toptier_code=next_tas.fr_entity_code).first()
if frec_match:
match_count += 1
logger.info(
" Matched unknown funding agency for TAS {} with FREC {}"
.format(next_tas.tas_rendering_label,
next_tas.fr_entity_code))
next_tas.funding_toptier_agency = frec_match
next_tas.save()
logger.info(
"=== Updated {} Funding Toptier Agencies with a FREC agency. ==="
.format(match_count))
# update TAS fk relationships to federal accounts
with Timer("Updated TAS FK relationships to Federal Accounts"):
logger.info(
"=== Updating TAS FK relationships to Federal Accounts ==="
)
updates = update_federal_accounts()
logger.info(
" Updated {} Federal Account Rows".format(updates))
inserts = insert_federal_accounts()
logger.info(
" Created {} Federal Account Rows".format(inserts))
deletes = remove_empty_federal_accounts()
logger.info(
" Removed {} Federal Account Rows".format(deletes))
logger.info("=== TAS loader finished successfully! ===")
except Exception as e:
logger.error(e)
logger.error("=== TAS loader failed ===")
sys.exit(1)
def handle(self, *args, **options):
try:
call_command("run_sql", "-f", TAS_SQL_PATH)
# Match funding toptiers by FREC if they didn't match by AID
unmapped_funding_agencies = TreasuryAppropriationAccount.objects.filter(
funding_toptier_agency=None)
match_count = 0
msg_str = "\n=== Found {} unmatched funding agencies across all TAS objects. ==="
logger.info(msg_str.format(unmapped_funding_agencies.count()))
for next_tas in unmapped_funding_agencies:
# CGAC code is a combination of FRECs and CGACs. It will never be empty and it will always
# be unique in ToptierAgencies; this should be safe to do.
frec_match = ToptierAgency.objects.filter(
cgac_code=next_tas.fr_entity_code).first()
if frec_match:
match_count += 1
logger.info(
" Matched unknown funding agency for TAS {} with FREC {}"
.format(next_tas.tas_rendering_label,
next_tas.fr_entity_code))
next_tas.funding_toptier_agency = frec_match
next_tas.save()
logger.info(
"\n=== Updated {} funding toptiers with a FREC agency. ===".
format(match_count))
# update TAS fk relationships to federal accounts
logger.info(
"\n=== Updating TAS FK relationships to Federal Accounts ===")
updates = update_federal_accounts()
logger.info(" Updated {} Federal Account Rows".format(updates))
inserts = insert_federal_accounts()
logger.info(" Created {} Federal Account Rows".format(inserts))
deletes = remove_empty_federal_accounts()
logger.info(" Removed {} Federal Account Rows".format(deletes))
logger.info("\n=== TAS loader finished successfully! ===")
except Exception as e:
logger.error(e)
logger.error("\n=== TAS loader failed ===")
sys.exit(1)
def handle(self, *args, **options):
try:
call_command("run_sql", "-f", TAS_SQL_PATH)
# Match funding toptiers by FREC if they didn't match by AID
unmapped_funding_agencies = TreasuryAppropriationAccount.objects.filter(funding_toptier_agency=None)
match_count = 0
msg_str = "\n=== Found {} unmatched funding agencies across all TAS objects. ==="
logger.info(msg_str.format(unmapped_funding_agencies.count()))
for next_tas in unmapped_funding_agencies:
# CGAC code is a combination of FRECs and CGACs. It will never be empty and it will always
# be unique in ToptierAgencies; this should be safe to do.
frec_match = ToptierAgency.objects.filter(cgac_code=next_tas.fr_entity_code).first()
if frec_match:
match_count += 1
logger.info(" Matched unknown funding agency for TAS {} with FREC {}".format(
next_tas.tas_rendering_label, next_tas.fr_entity_code))
next_tas.funding_toptier_agency = frec_match
next_tas.save()
logger.info("\n=== Updated {} funding toptiers with a FREC agency. ===".format(match_count))
# update TAS fk relationships to federal accounts
logger.info("\n=== Updating TAS FK relationships to Federal Accounts ===")
updates = update_federal_accounts()
logger.info(" Updated {} Federal Account Rows".format(updates))
inserts = insert_federal_accounts()
logger.info(" Created {} Federal Account Rows".format(inserts))
deletes = remove_empty_federal_accounts()
logger.info(" Removed {} Federal Account Rows".format(deletes))
logger.info("\n=== TAS loader finished successfully! ===")
except Exception as e:
logger.error(e)
logger.error("\n=== TAS loader failed ===")
sys.exit(1)
def handle(self, *args, **options):
is_remote_file = len(options['location'][0].split('.')) == 1
if is_remote_file:
s3connection = boto.s3.connect_to_region(
os.environ.get('USASPENDING_AWS_REGION'))
s3bucket = s3connection.lookup(options['location'][0])
file_path = s3bucket.get_key('cars_tas.csv')
else:
file_path = options['location'][0]
field_map = {
"treasury_account_identifier": "ACCT_NUM",
"account_title": "GWA_TAS_NAME",
"reporting_agency_id": "Agency AID",
"reporting_agency_name": "Agency Name",
"budget_bureau_code": "ADMIN_ORG",
"budget_bureau_name": "Admin Org Name",
"fr_entity_code": "FR Entity Type",
"fr_entity_description": "FR Entity Description",
"budget_function_code": "Function Code",
"budget_function_title": "Function Description",
"budget_subfunction_code": "Sub Function Code",
"budget_subfunction_title": "Sub Function Description"
}
value_map = {
"data_source":
"USA",
"tas_rendering_label":
self.generate_tas_rendering_label,
"allocation_transfer_agency_id":
lambda row: row["ATA"].strip(),
"agency_id":
lambda row: row["AID"].strip(),
"beginning_period_of_availability":
lambda row: row["BPOA"].strip(),
"ending_period_of_availability":
lambda row: row["EPOA"].strip(),
"availability_type_code":
lambda row: row["A"].strip(),
"main_account_code":
lambda row: row["MAIN"].strip(),
"sub_account_code":
lambda row: row["SUB"].strip(),
"awarding_toptier_agency":
lambda row: ToptierAgency.objects.filter(cgac_code=row[
"ATA"].strip()).order_by("fpds_code").first(),
"funding_toptier_agency":
lambda row: ToptierAgency.objects.filter(cgac_code=row[
"AID"].strip()).order_by("fpds_code").first()
}
loader = ThreadedDataLoader(
model_class=TreasuryAppropriationAccount,
field_map=field_map,
value_map=value_map,
collision_field='treasury_account_identifier',
collision_behavior='update',
pre_row_function=self.skip_and_remove_financing_tas)
loader.load_from_file(filepath=file_path, remote_file=is_remote_file)
# Match funding toptiers by FREC if they didn't match by AID
unmapped_funding_agencies = TreasuryAppropriationAccount.objects.filter(
funding_toptier_agency=None)
match_count = 0
self.logger.info(
'Found {} unmatched funding agencies across all TAS objects. '
'Attempting to match on FREC.'.format(
unmapped_funding_agencies.count()))
for next_tas in unmapped_funding_agencies:
# CGAC code is a combination of FRECs and CGACs. It will never be empty and it will always
# be unique in ToptierAgencies; this should be safe to do.
frec_match = ToptierAgency.objects.filter(
cgac_code=next_tas.fr_entity_code).first()
if frec_match:
match_count += 1
self.logger.info(
'Matched unknown funding agency for TAS {} with FREC {}'.
format(next_tas.tas_rendering_label,
next_tas.fr_entity_code))
next_tas.funding_toptier_agency = frec_match
next_tas.save()
self.logger.info(
'Updated {} funding toptiers with a FREC agency.'.format(
match_count))
# update TAS fk relationships to federal accounts
remove_empty_federal_accounts()
update_federal_accounts()
insert_federal_accounts()