def test_get_quarterly_numbers(tas_balances_data):
"""Test the function that generates quarterly tas balances."""
# retrieve all quarterly numbers and test results
quarters = AppropriationAccountBalances.get_quarterly_numbers()
# number of quarterly adjusted records should = number of records
# in AppropriationAccountBalances
assert len(list(quarters)) == 2
# submission 1: has no previous subission
# submission 2: its previous submission is submission 1
sub1 = SubmissionAttributes.objects.get(previous_submission__isnull=True)
sub2 = SubmissionAttributes.objects.get(previous_submission__isnull=False)
for q in quarters:
if q.submission == sub1:
# qtrly values for year's first submission should remain unchanged
assert q.budget_authority_appropriated_amount_cpe == Decimal('10.00')
assert q.unobligated_balance_cpe == Decimal('99.99')
else:
# qtrly values for year's 2nd submission should be equal to 2nd
# submission values - first submission values
assert q.budget_authority_appropriated_amount_cpe == Decimal('0.00')
assert q.unobligated_balance_cpe == Decimal('.01')
# test getting quarterly results for a specific submission
quarters = AppropriationAccountBalances.get_quarterly_numbers(
sub2.submission_id)
# number of quarterly adjusted records should = number of records
# in AppropriationAccountBalances
assert len(list(quarters)) == 1
# requesting data for non-existent submission returns zero records
quarters = AppropriationAccountBalances.get_quarterly_numbers(-888)
assert len(list(quarters)) == 0
def load_file_a(submission_attributes, appropriation_data, db_cursor):
""" Process and load file A broker data (aka TAS balances, aka appropriation account balances). """
reverse = re.compile("gross_outlay_amount_by_tas_cpe")
# dictionary to capture TAS that were skipped and some metadata
# tas = top-level key
# count = number of rows skipped
# rows = row numbers skipped, corresponding to the original row numbers in the file that was submitted
skipped_tas = {}
# Create account objects
for row in appropriation_data:
# Check and see if there is an entry for this TAS
treasury_account, tas_rendering_label = get_treasury_appropriation_account_tas_lookup(
row.get("tas_id"), db_cursor)
if treasury_account is None:
update_skipped_tas(row, tas_rendering_label, skipped_tas)
continue
# Now that we have the account, we can load the appropriation balances
# TODO: Figure out how we want to determine what row is overriden by what row
# If we want to correlate, the following attributes are available in the data broker data that might be useful:
# appropriation_id, row_number appropriation_balances = somethingsomething get appropriation balances...
appropriation_balances = AppropriationAccountBalances()
value_map = {
"treasury_account_identifier": treasury_account,
"submission": submission_attributes,
"reporting_period_start":
submission_attributes.reporting_period_start,
"reporting_period_end": submission_attributes.reporting_period_end,
}
field_map = {}
load_data_into_model(appropriation_balances,
row,
field_map=field_map,
value_map=value_map,
save=True,
reverse=reverse)
AppropriationAccountBalances.populate_final_of_fy()
for key in skipped_tas:
logger.info(
f"Skipped {skipped_tas[key]['count']:,} rows due to missing TAS: {key}"
)
total_tas_skipped = 0
for key in skipped_tas:
total_tas_skipped += skipped_tas[key]["count"]
logger.info(
f"Skipped a total of {total_tas_skipped:,} TAS rows for File A")
def load_file_a(submission_attributes, appropriation_data, db_cursor):
"""
Process and load file A broker data (aka TAS balances, aka appropriation account balances).
"""
reverse = re.compile('gross_outlay_amount_by_tas_cpe')
# dictionary to capture TAS that were skipped and some metadata
# tas = top-level key
# count = number of rows skipped
# rows = row numbers skipped, corresponding to the original row numbers in the file that was submitted
skipped_tas = {}
# Create account objects
for row in appropriation_data:
# Check and see if there is an entry for this TAS
treasury_account, tas_rendering_label = get_treasury_appropriation_account_tas_lookup(
row.get('tas_id'), db_cursor)
if treasury_account is None:
update_skipped_tas(row, tas_rendering_label, skipped_tas)
continue
# Now that we have the account, we can load the appropriation balances
# TODO: Figure out how we want to determine what row is overriden by what row
# If we want to correlate, the following attributes are available in the data broker data that might be useful:
# appropriation_id, row_number appropriation_balances = somethingsomething get appropriation balances...
appropriation_balances = AppropriationAccountBalances()
value_map = {
'treasury_account_identifier': treasury_account,
'submission': submission_attributes,
'reporting_period_start': submission_attributes.reporting_period_start,
'reporting_period_end': submission_attributes.reporting_period_end
}
field_map = {}
load_data_into_model(appropriation_balances, row, field_map=field_map, value_map=value_map, save=True,
reverse=reverse)
AppropriationAccountBalances.populate_final_of_fy()
# Insert File A quarterly numbers for this submission
AppropriationAccountBalancesQuarterly.insert_quarterly_numbers(submission_attributes.submission_id)
for key in skipped_tas:
logger.info('Skipped %d rows due to missing TAS: %s', skipped_tas[key]['count'], key)
total_tas_skipped = 0
for key in skipped_tas:
total_tas_skipped += skipped_tas[key]['count']
logger.info('Skipped a total of {} TAS rows for File A'.format(total_tas_skipped))
def app_acc_bal_models():
sub_16_1 = mommy.make("submissions.SubmissionAttributes",
reporting_period_start=date(2016, 1, 1))
sub_16_2 = mommy.make("submissions.SubmissionAttributes",
reporting_period_start=date(2016, 4, 1))
sub_17_1 = mommy.make("submissions.SubmissionAttributes",
reporting_period_start=date(2016, 10, 1))
sub_17_2 = mommy.make("submissions.SubmissionAttributes",
reporting_period_start=date(2017, 2, 1))
tas_1 = mommy.make("accounts.TreasuryAppropriationAccount",
tas_rendering_label="ABC",
_fill_optional=True)
tas_2 = mommy.make("accounts.TreasuryAppropriationAccount",
tas_rendering_label="XYZ",
_fill_optional=True)
mommy.make(
"accounts.AppropriationAccountBalances",
treasury_account_identifier=tas_1,
submission=sub_16_1,
_fill_optional=True,
)
mommy.make(
"accounts.AppropriationAccountBalances",
treasury_account_identifier=tas_1,
submission=sub_16_2,
_fill_optional=True,
)
mommy.make(
"accounts.AppropriationAccountBalances",
treasury_account_identifier=tas_1,
submission=sub_17_1,
_fill_optional=True,
)
mommy.make(
"accounts.AppropriationAccountBalances",
treasury_account_identifier=tas_1,
submission=sub_17_2,
_fill_optional=True,
)
mommy.make(
"accounts.AppropriationAccountBalances",
treasury_account_identifier=tas_2,
submission=sub_16_1,
_fill_optional=True,
)
mommy.make(
"accounts.AppropriationAccountBalances",
treasury_account_identifier=tas_2,
submission=sub_16_2,
_fill_optional=True,
)
AppropriationAccountBalances.populate_final_of_fy()
def account_models():
# Add submission data
subm_2015_1 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2014, 10, 1))
subm_2015_2 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2015, 8, 1))
subm_2016_1 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2016, 1, 1))
subm_2016_2 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2016, 6, 1))
# add object classes
obj_clas_1 = mommy.make('references.ObjectClass', object_class=1)
obj_clas_2 = mommy.make('references.ObjectClass', object_class=2)
# add program activity
prg_atvy_1 = mommy.make('references.RefProgramActivity', id=1)
prg_atvy_2 = mommy.make('references.RefProgramActivity', id=2)
# add tas data
tas_1 = mommy.make('accounts.TreasuryAppropriationAccount', tas_rendering_label="ABC", _fill_optional=True)
tas_2 = mommy.make('accounts.TreasuryAppropriationAccount', tas_rendering_label="XYZ", _fill_optional=True)
# add file A data
mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_1,
budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True,
submission=subm_2015_1)
mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_2,
budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True,
submission=subm_2015_2)
mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_1,
budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True,
submission=subm_2016_1)
mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_2,
budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True,
submission=subm_2016_2)
AppropriationAccountBalances.populate_final_of_fy()
# add file B data
mommy.make('financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_1,
program_activity=prg_atvy_1, treasury_account=tas_1,
obligations_undelivered_orders_unpaid_total_cpe=8000, _fill_optional=True,
submission=subm_2015_1) # ignored, superseded by the next submission in the FY
mommy.make('financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_2,
program_activity=prg_atvy_2, treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=1000, _fill_optional=True,
submission=subm_2015_2)
mommy.make('financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_1,
program_activity=prg_atvy_1, treasury_account=tas_1,
obligations_undelivered_orders_unpaid_total_cpe=9000, _fill_optional=True,
submission=subm_2016_1) # ignored, superseded by the next submission in the FY
mommy.make('financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_2,
program_activity=prg_atvy_2, treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=2000, _fill_optional=True,
submission=subm_2016_2)
FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy()
def load_file_a(submission_attributes, appropriation_data, db_cursor):
"""
Process and load file A broker data (aka TAS balances,
aka appropriation account balances).
"""
reverse = re.compile('gross_outlay_amount_by_tas_cpe')
# Create account objects
for row in appropriation_data:
# Check and see if there is an entry for this TAS
treasury_account = get_treasury_appropriation_account_tas_lookup(
row.get('tas_id'), db_cursor)
if treasury_account is None:
raise Exception('Could not find appropriation account for TAS: ' +
row['tas'])
# Now that we have the account, we can load the appropriation balances
# TODO: Figure out how we want to determine what row is overriden by what row
# If we want to correlate, the following attributes are available in the
# data broker data that might be useful: appropriation_id, row_number
# appropriation_balances = somethingsomething get appropriation balances...
appropriation_balances = AppropriationAccountBalances()
value_map = {
'treasury_account_identifier': treasury_account,
'submission': submission_attributes,
'reporting_period_start':
submission_attributes.reporting_period_start,
'reporting_period_end': submission_attributes.reporting_period_end
}
field_map = {}
load_data_into_model(appropriation_balances,
row,
field_map=field_map,
value_map=value_map,
save=True,
reverse=reverse)
AppropriationAccountBalances.populate_final_of_fy()
# Insert File A quarterly numbers for this submission
AppropriationAccountBalancesQuarterly.insert_quarterly_numbers(
submission_attributes.submission_id)
def app_acc_bal_models():
sub_16_1 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2016, 1, 1))
sub_16_2 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2016, 4, 1))
sub_17_1 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2016, 10, 1))
sub_17_2 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2017, 2, 1))
tas_1 = mommy.make('accounts.TreasuryAppropriationAccount', tas_rendering_label="ABC", _fill_optional=True)
tas_2 = mommy.make('accounts.TreasuryAppropriationAccount', tas_rendering_label="XYZ", _fill_optional=True)
mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_1, submission=sub_16_1,
_fill_optional=True)
mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_1, submission=sub_16_2,
_fill_optional=True)
mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_1, submission=sub_17_1,
_fill_optional=True)
mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_1, submission=sub_17_2,
_fill_optional=True)
mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_2, submission=sub_16_1,
_fill_optional=True)
mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_2, submission=sub_16_2,
_fill_optional=True)
AppropriationAccountBalances.populate_final_of_fy()
def test_get_quarterly_numbers(tas_balances_data):
"""Test the function that generates quarterly tas balances."""
# retrieve all quarterly numbers and test results
quarters = AppropriationAccountBalances.get_quarterly_numbers()
# number of quarterly adjusted records should = number of records
# in AppropriationAccountBalances
assert len(list(quarters)) == 2
# submission 1: has no previous subission
# submission 2: its previous submission is submission 1
sub1 = SubmissionAttributes.objects.get(previous_submission__isnull=True)
sub2 = SubmissionAttributes.objects.get(previous_submission__isnull=False)
for q in quarters:
if q.submission == sub1:
# qtrly values for year's first submission should remain unchanged
assert q.budget_authority_appropriated_amount_cpe == Decimal(
"10.00")
assert q.unobligated_balance_cpe == Decimal("99.99")
else:
# qtrly values for year's 2nd submission should be equal to 2nd
# submission values - first submission values
assert q.budget_authority_appropriated_amount_cpe == Decimal(
"0.00")
assert q.unobligated_balance_cpe == Decimal(".01")
# test getting quarterly results for a specific submission
quarters = AppropriationAccountBalances.get_quarterly_numbers(
sub2.submission_id)
# number of quarterly adjusted records should = number of records
# in AppropriationAccountBalances
assert len(list(quarters)) == 1
# requesting data for non-existent submission returns zero records
quarters = AppropriationAccountBalances.get_quarterly_numbers(-888)
assert len(list(quarters)) == 0
def load_file_a(submission_attributes, appropriation_data, db_cursor):
""" Process and load file A broker data (aka TAS balances, aka appropriation account balances). """
reverse = re.compile("gross_outlay_amount_by_tas_cpe")
skipped_tas = defaultdict(int) # tracks count of rows skipped due to "missing" TAS
bulk_treasury_appropriation_account_tas_lookup(appropriation_data, db_cursor)
# Create account objects
save_manager = BulkCreateManager(AppropriationAccountBalances)
for row in appropriation_data:
# Check and see if there is an entry for this TAS
treasury_account, tas_rendering_label = get_treasury_appropriation_account_tas_lookup(row.get("tas_id"))
if treasury_account is None:
skipped_tas[tas_rendering_label] += 1
continue
# Now that we have the account, we can load the appropriation balances
# TODO: Figure out how we want to determine what row is overridden by what row
# If we want to correlate, the following attributes are available in the data broker data that might be useful:
# appropriation_id, row_number appropriation_balances = something something get appropriation balances...
appropriation_balances = AppropriationAccountBalances()
value_map = {
"treasury_account_identifier": treasury_account,
"submission": submission_attributes,
"reporting_period_start": submission_attributes.reporting_period_start,
"reporting_period_end": submission_attributes.reporting_period_end,
}
field_map = {}
save_manager.append(
load_data_into_model(
appropriation_balances, row, field_map=field_map, value_map=value_map, save=False, reverse=reverse
)
)
save_manager.save_stragglers()
for tas, count in skipped_tas.items():
logger.info(f"Skipped {count:,} rows due to {tas}")
total_tas_skipped = sum([count for count in skipped_tas.values()])
if total_tas_skipped > 0:
logger.info(f"SKIPPED {total_tas_skipped:,} ROWS of File A (missing TAS)")
else:
logger.info("All File A records in Broker loaded into USAspending")
def test_get_quarterly_null_previous_submission(tas_balances_data):
"""
Test case when a field value is not in the current submission but
was null in the previous submission.
"""
sub1 = SubmissionAttributes.objects.get(previous_submission__isnull=True)
quarters = AppropriationAccountBalances.get_quarterly_numbers()
for q in quarters:
if q.submission == sub1:
# qtrly values for year's first submission should remain unchanged
# (null values become 0 due to COALESCE)
assert q.budget_authority_unobligated_balance_brought_forward_fyb == Decimal('0.0')
else:
# if the equivalent value in the first submisison is NULL, quarterly
# numbers should equal the number on the 2nd submission
assert q.budget_authority_unobligated_balance_brought_forward_fyb == Decimal('6.99')
def test_get_quarterly_null_previous_submission(tas_balances_data):
"""
Test case when a field value is not in the current submission but
was null in the previous submission.
"""
sub1 = SubmissionAttributes.objects.get(previous_submission__isnull=True)
quarters = AppropriationAccountBalances.get_quarterly_numbers()
for q in quarters:
if q.submission == sub1:
# qtrly values for year's first submission should remain unchanged
# (null values become 0 due to COALESCE)
assert q.budget_authority_unobligated_balance_brought_forward_fyb == Decimal(
"0.0")
else:
# if the equivalent value in the first submisison is NULL, quarterly
# numbers should equal the number on the 2nd submission
assert q.budget_authority_unobligated_balance_brought_forward_fyb == Decimal(
"6.99")
def populate_final_of_fy():
AppropriationAccountBalances.populate_final_of_fy()
FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy()
def account_models():
subm_2015_1 = mommy.make('submissions.SubmissionAttributes',
reporting_period_start=date(2014, 10, 1))
subm_2015_2 = mommy.make('submissions.SubmissionAttributes',
reporting_period_start=date(2015, 8, 1))
subm_2016_1 = mommy.make('submissions.SubmissionAttributes',
reporting_period_start=date(2016, 1, 1))
subm_2016_2 = mommy.make('submissions.SubmissionAttributes',
reporting_period_start=date(2016, 6, 1))
obj_clas_1 = mommy.make('references.ObjectClass', object_class=1)
obj_clas_2 = mommy.make('references.ObjectClass', object_class=2)
prg_atvy_1 = mommy.make('references.RefProgramActivity', id=1)
prg_atvy_2 = mommy.make('references.RefProgramActivity', id=2)
tas_1 = mommy.make('accounts.TreasuryAppropriationAccount',
tas_rendering_label="ABC",
_fill_optional=True)
tas_2 = mommy.make('accounts.TreasuryAppropriationAccount',
tas_rendering_label="XYZ",
_fill_optional=True)
mommy.make('accounts.AppropriationAccountBalances',
treasury_account_identifier=tas_1,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_quantity=2,
_fill_optional=True)
mommy.make('accounts.AppropriationAccountBalances',
treasury_account_identifier=tas_2,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_quantity=3,
_fill_optional=True)
AppropriationAccountBalances.populate_final_of_fy()
mommy.make('accounts.AppropriationAccountBalancesQuarterly',
treasury_account_identifier=tas_1,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_fill_optional=True)
mommy.make('accounts.AppropriationAccountBalancesQuarterly',
treasury_account_identifier=tas_2,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_fill_optional=True)
mommy.make(
'financial_activities.FinancialAccountsByProgramActivityObjectClass',
object_class=obj_clas_1,
program_activity=prg_atvy_1,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=8000,
_quantity=2,
_fill_optional=True,
submission=subm_2015_1
) # ignored, superseded by the next submission in the FY
mommy.make(
'financial_activities.FinancialAccountsByProgramActivityObjectClass',
object_class=obj_clas_1,
program_activity=prg_atvy_1,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=1000,
_quantity=2,
_fill_optional=True,
submission=subm_2015_2)
mommy.make(
'financial_activities.FinancialAccountsByProgramActivityObjectClass',
object_class=obj_clas_2,
program_activity=prg_atvy_2,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=9000,
_quantity=2,
_fill_optional=True,
submission=subm_2016_1
) # ignored, superseded by the next submission in the FY
mommy.make(
'financial_activities.FinancialAccountsByProgramActivityObjectClass',
object_class=obj_clas_2,
program_activity=prg_atvy_2,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=2000,
_quantity=2,
_fill_optional=True,
submission=subm_2016_2)
mommy.make(
'financial_activities.FinancialAccountsByProgramActivityObjectClass',
object_class=obj_clas_2,
program_activity=prg_atvy_1,
treasury_account=tas_1,
obligations_undelivered_orders_unpaid_total_cpe=50,
_fill_optional=True,
submission=subm_2016_2)
FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy()
mommy.make('financial_activities.TasProgramActivityObjectClassQuarterly',
object_class=obj_clas_1,
program_activity=prg_atvy_1,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=5000,
_quantity=2,
_fill_optional=True)
mommy.make('financial_activities.TasProgramActivityObjectClassQuarterly',
object_class=obj_clas_2,
program_activity=prg_atvy_2,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=3000,
_quantity=2,
_fill_optional=True)
mommy.make('financial_activities.TasProgramActivityObjectClassQuarterly',
object_class=obj_clas_2,
program_activity=prg_atvy_1,
treasury_account=tas_1,
obligations_undelivered_orders_unpaid_total_cpe=100,
_fill_optional=True)
def account_models():
# Add submission data
subm_2015_1 = mommy.make("submissions.SubmissionAttributes",
reporting_period_start=date(2014, 10, 1))
subm_2015_2 = mommy.make("submissions.SubmissionAttributes",
reporting_period_start=date(2015, 8, 1))
subm_2016_1 = mommy.make("submissions.SubmissionAttributes",
reporting_period_start=date(2016, 1, 1))
subm_2016_2 = mommy.make("submissions.SubmissionAttributes",
reporting_period_start=date(2016, 6, 1))
# add object classes
obj_clas_1 = mommy.make("references.ObjectClass", object_class=1)
obj_clas_2 = mommy.make("references.ObjectClass", object_class=2)
# add program activity
prg_atvy_1 = mommy.make("references.RefProgramActivity", id=1)
prg_atvy_2 = mommy.make("references.RefProgramActivity", id=2)
# add tas data
tas_1 = mommy.make("accounts.TreasuryAppropriationAccount",
tas_rendering_label="ABC",
_fill_optional=True)
tas_2 = mommy.make("accounts.TreasuryAppropriationAccount",
tas_rendering_label="XYZ",
_fill_optional=True)
# add file A data
mommy.make(
"accounts.AppropriationAccountBalances",
treasury_account_identifier=tas_1,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_fill_optional=True,
submission=subm_2015_1,
)
mommy.make(
"accounts.AppropriationAccountBalances",
treasury_account_identifier=tas_2,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_fill_optional=True,
submission=subm_2015_2,
)
mommy.make(
"accounts.AppropriationAccountBalances",
treasury_account_identifier=tas_1,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_fill_optional=True,
submission=subm_2016_1,
)
mommy.make(
"accounts.AppropriationAccountBalances",
treasury_account_identifier=tas_2,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_fill_optional=True,
submission=subm_2016_2,
)
AppropriationAccountBalances.populate_final_of_fy()
# add file B data
mommy.make(
"financial_activities.FinancialAccountsByProgramActivityObjectClass",
object_class=obj_clas_1,
program_activity=prg_atvy_1,
treasury_account=tas_1,
obligations_undelivered_orders_unpaid_total_cpe=8000,
_fill_optional=True,
submission=subm_2015_1,
) # ignored, superseded by the next submission in the FY
mommy.make(
"financial_activities.FinancialAccountsByProgramActivityObjectClass",
object_class=obj_clas_2,
program_activity=prg_atvy_2,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=1000,
_fill_optional=True,
submission=subm_2015_2,
)
mommy.make(
"financial_activities.FinancialAccountsByProgramActivityObjectClass",
object_class=obj_clas_1,
program_activity=prg_atvy_1,
treasury_account=tas_1,
obligations_undelivered_orders_unpaid_total_cpe=9000,
_fill_optional=True,
submission=subm_2016_1,
) # ignored, superseded by the next submission in the FY
mommy.make(
"financial_activities.FinancialAccountsByProgramActivityObjectClass",
object_class=obj_clas_2,
program_activity=prg_atvy_2,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=2000,
_fill_optional=True,
submission=subm_2016_2,
)
FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy()
def load_file_a(submission_attributes, appropriation_data, db_cursor):
"""
Process and load file A broker data (aka TAS balances, aka appropriation account balances).
"""
reverse = re.compile('gross_outlay_amount_by_tas_cpe')
# dictionary to capture TAS that were skipped and some metadata
# tas = top-level key
# count = number of rows skipped
# rows = row numbers skipped, corresponding to the original row numbers in the file that was submitted
skipped_tas = {}
# Create account objects
for row in appropriation_data:
# Check and see if there is an entry for this TAS
treasury_account = get_treasury_appropriation_account_tas_lookup(
row.get('tas_id'), db_cursor)
if treasury_account is None:
if row['tas'] not in skipped_tas:
skipped_tas[row['tas']] = {}
skipped_tas[row['tas']]['count'] = 1
skipped_tas[row['tas']]['rows'] = [row['row_number']]
else:
skipped_tas[row['tas']]['count'] += 1
skipped_tas[row['tas']]['rows'] += row['row_number']
continue
# Now that we have the account, we can load the appropriation balances
# TODO: Figure out how we want to determine what row is overriden by what row
# If we want to correlate, the following attributes are available in the data broker data that might be useful:
# appropriation_id, row_number appropriation_balances = somethingsomething get appropriation balances...
appropriation_balances = AppropriationAccountBalances()
value_map = {
'treasury_account_identifier': treasury_account,
'submission': submission_attributes,
'reporting_period_start':
submission_attributes.reporting_period_start,
'reporting_period_end': submission_attributes.reporting_period_end
}
field_map = {}
load_data_into_model(appropriation_balances,
row,
field_map=field_map,
value_map=value_map,
save=True,
reverse=reverse)
AppropriationAccountBalances.populate_final_of_fy()
# Insert File A quarterly numbers for this submission
AppropriationAccountBalancesQuarterly.insert_quarterly_numbers(
submission_attributes.submission_id)
for key in skipped_tas:
logger.info('Skipped %d rows due to missing TAS: %s',
skipped_tas[key]['count'], key)
total_tas_skipped = 0
for key in skipped_tas:
total_tas_skipped += skipped_tas[key]['count']
logger.info(
'Skipped a total of {} TAS rows for File A'.format(total_tas_skipped))
def account_models():
obj_clas_1 = mommy.make('references.ObjectClass', object_class=1)
obj_clas_2 = mommy.make('references.ObjectClass', object_class=2)
prg_atvy_1 = mommy.make('references.RefProgramActivity', id=1)
prg_atvy_2 = mommy.make('references.RefProgramActivity', id=2)
tas_1 = mommy.make('accounts.TreasuryAppropriationAccount',
tas_rendering_label="ABC",
_fill_optional=True)
tas_2 = mommy.make('accounts.TreasuryAppropriationAccount',
tas_rendering_label="XYZ",
_fill_optional=True)
mommy.make('accounts.AppropriationAccountBalances',
treasury_account_identifier=tas_1,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_quantity=2,
_fill_optional=True)
mommy.make('accounts.AppropriationAccountBalances',
treasury_account_identifier=tas_2,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_quantity=3,
_fill_optional=True)
AppropriationAccountBalances.populate_final_of_fy()
mommy.make('accounts.AppropriationAccountBalancesQuarterly',
treasury_account_identifier=tas_1,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_fill_optional=True)
mommy.make('accounts.AppropriationAccountBalancesQuarterly',
treasury_account_identifier=tas_2,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_fill_optional=True)
mommy.make(
'financial_activities.FinancialAccountsByProgramActivityObjectClass',
object_class=obj_clas_1,
program_activity=prg_atvy_1,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=1000,
_quantity=2,
_fill_optional=True)
mommy.make(
'financial_activities.FinancialAccountsByProgramActivityObjectClass',
object_class=obj_clas_2,
program_activity=prg_atvy_2,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=2000,
_quantity=2,
_fill_optional=True)
mommy.make(
'financial_activities.FinancialAccountsByProgramActivityObjectClass',
object_class=obj_clas_2,
program_activity=prg_atvy_1,
treasury_account=tas_1,
obligations_undelivered_orders_unpaid_total_cpe=50,
_fill_optional=True),
mommy.make('financial_activities.TasProgramActivityObjectClassQuarterly',
object_class=obj_clas_1,
program_activity=prg_atvy_1,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=5000,
_quantity=2,
_fill_optional=True)
mommy.make('financial_activities.TasProgramActivityObjectClassQuarterly',
object_class=obj_clas_2,
program_activity=prg_atvy_2,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=3000,
_quantity=2,
_fill_optional=True)
mommy.make('financial_activities.TasProgramActivityObjectClassQuarterly',
object_class=obj_clas_2,
program_activity=prg_atvy_1,
treasury_account=tas_1,
obligations_undelivered_orders_unpaid_total_cpe=100,
_fill_optional=True)
