def load_file_b(submission_attributes, prg_act_obj_cls_data, db_cursor):
"""
Process and load file B broker data (aka TAS balances by program
activity and object class).
"""
reverse = re.compile(r'(_(cpe|fyb)$)|^transaction_obligated_amount$')
test_counter = 0
for row in prg_act_obj_cls_data:
test_counter += 1
account_balances = None
try:
# Check and see if there is an entry for this TAS
treasury_account = get_treasury_appropriation_account_tas_lookup(
row.get('tas_id'), db_cursor)
if treasury_account is None:
raise Exception(
'Could not find appropriation account for TAS: ' +
row['tas'])
except:
continue
# get the corresponding account balances row (aka "File A" record)
account_balances = AppropriationAccountBalances.objects.get(
treasury_account_identifier=treasury_account,
submission_id=submission_attributes.submission_id)
financial_by_prg_act_obj_cls = FinancialAccountsByProgramActivityObjectClass(
)
value_map = {
'submission':
submission_attributes,
'reporting_period_start':
submission_attributes.reporting_period_start,
'reporting_period_end':
submission_attributes.reporting_period_end,
'treasury_account':
treasury_account,
'appropriation_account_balances':
account_balances,
'object_class':
get_or_create_object_class(row['object_class'],
row['by_direct_reimbursable_fun'],
logger),
'program_activity':
get_or_create_program_activity(row, submission_attributes)
}
load_data_into_model(financial_by_prg_act_obj_cls,
row,
value_map=value_map,
save=True,
reverse=reverse)
# Insert File B quarterly numbers for this submission
TasProgramActivityObjectClassQuarterly.insert_quarterly_numbers(
submission_attributes.submission_id)
FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy()
def load_file_b(submission_attributes, prg_act_obj_cls_data, db_cursor):
"""
Process and load file B broker data (aka TAS balances by program activity and object class).
"""
reverse = re.compile(r'(_(cpe|fyb)$)|^transaction_obligated_amount$')
# dictionary to capture TAS that were skipped and some metadata
# tas = top-level key
# count = number of rows skipped
# rows = row numbers skipped, corresponding to the original row numbers in the file that was submitted
skipped_tas = {}
test_counter = 0
for row in prg_act_obj_cls_data:
test_counter += 1
account_balances = None
try:
# Check and see if there is an entry for this TAS
treasury_account, tas_rendering_label = get_treasury_appropriation_account_tas_lookup(row.get('tas_id'),
db_cursor)
if treasury_account is None:
update_skipped_tas(row, tas_rendering_label, skipped_tas)
continue
except Exception: # TODO: What is this trying to catch, actually?
continue
# get the corresponding account balances row (aka "File A" record)
account_balances = AppropriationAccountBalances.objects.get(
treasury_account_identifier=treasury_account,
submission_id=submission_attributes.submission_id
)
financial_by_prg_act_obj_cls = FinancialAccountsByProgramActivityObjectClass()
value_map = {
'submission': submission_attributes,
'reporting_period_start': submission_attributes.reporting_period_start,
'reporting_period_end': submission_attributes.reporting_period_end,
'treasury_account': treasury_account,
'appropriation_account_balances': account_balances,
'object_class': get_or_create_object_class(row['object_class'], row['by_direct_reimbursable_fun'], logger),
'program_activity': get_or_create_program_activity(row, submission_attributes)
}
load_data_into_model(financial_by_prg_act_obj_cls, row, value_map=value_map, save=True, reverse=reverse)
# Insert File B quarterly numbers for this submission
TasProgramActivityObjectClassQuarterly.insert_quarterly_numbers(submission_attributes.submission_id)
FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy()
for key in skipped_tas:
logger.info('Skipped %d rows due to missing TAS: %s', skipped_tas[key]['count'], key)
total_tas_skipped = 0
for key in skipped_tas:
total_tas_skipped += skipped_tas[key]['count']
logger.info('Skipped a total of {} TAS rows for File B'.format(total_tas_skipped))
def account_models():
# Add submission data
subm_2015_1 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2014, 10, 1))
subm_2015_2 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2015, 8, 1))
subm_2016_1 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2016, 1, 1))
subm_2016_2 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2016, 6, 1))
# add object classes
obj_clas_1 = mommy.make('references.ObjectClass', object_class=1)
obj_clas_2 = mommy.make('references.ObjectClass', object_class=2)
# add program activity
prg_atvy_1 = mommy.make('references.RefProgramActivity', id=1)
prg_atvy_2 = mommy.make('references.RefProgramActivity', id=2)
# add tas data
tas_1 = mommy.make('accounts.TreasuryAppropriationAccount', tas_rendering_label="ABC", _fill_optional=True)
tas_2 = mommy.make('accounts.TreasuryAppropriationAccount', tas_rendering_label="XYZ", _fill_optional=True)
# add file A data
mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_1,
budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True,
submission=subm_2015_1)
mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_2,
budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True,
submission=subm_2015_2)
mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_1,
budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True,
submission=subm_2016_1)
mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_2,
budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True,
submission=subm_2016_2)
AppropriationAccountBalances.populate_final_of_fy()
# add file B data
mommy.make('financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_1,
program_activity=prg_atvy_1, treasury_account=tas_1,
obligations_undelivered_orders_unpaid_total_cpe=8000, _fill_optional=True,
submission=subm_2015_1) # ignored, superseded by the next submission in the FY
mommy.make('financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_2,
program_activity=prg_atvy_2, treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=1000, _fill_optional=True,
submission=subm_2015_2)
mommy.make('financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_1,
program_activity=prg_atvy_1, treasury_account=tas_1,
obligations_undelivered_orders_unpaid_total_cpe=9000, _fill_optional=True,
submission=subm_2016_1) # ignored, superseded by the next submission in the FY
mommy.make('financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_2,
program_activity=prg_atvy_2, treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=2000, _fill_optional=True,
submission=subm_2016_2)
FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy()
def populate_final_of_fy():
AppropriationAccountBalances.populate_final_of_fy()
FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy()
def account_models():
subm_2015_1 = mommy.make('submissions.SubmissionAttributes',
reporting_period_start=date(2014, 10, 1))
subm_2015_2 = mommy.make('submissions.SubmissionAttributes',
reporting_period_start=date(2015, 8, 1))
subm_2016_1 = mommy.make('submissions.SubmissionAttributes',
reporting_period_start=date(2016, 1, 1))
subm_2016_2 = mommy.make('submissions.SubmissionAttributes',
reporting_period_start=date(2016, 6, 1))
obj_clas_1 = mommy.make('references.ObjectClass', object_class=1)
obj_clas_2 = mommy.make('references.ObjectClass', object_class=2)
prg_atvy_1 = mommy.make('references.RefProgramActivity', id=1)
prg_atvy_2 = mommy.make('references.RefProgramActivity', id=2)
tas_1 = mommy.make('accounts.TreasuryAppropriationAccount',
tas_rendering_label="ABC",
_fill_optional=True)
tas_2 = mommy.make('accounts.TreasuryAppropriationAccount',
tas_rendering_label="XYZ",
_fill_optional=True)
mommy.make('accounts.AppropriationAccountBalances',
treasury_account_identifier=tas_1,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_quantity=2,
_fill_optional=True)
mommy.make('accounts.AppropriationAccountBalances',
treasury_account_identifier=tas_2,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_quantity=3,
_fill_optional=True)
AppropriationAccountBalances.populate_final_of_fy()
mommy.make('accounts.AppropriationAccountBalancesQuarterly',
treasury_account_identifier=tas_1,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_fill_optional=True)
mommy.make('accounts.AppropriationAccountBalancesQuarterly',
treasury_account_identifier=tas_2,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_fill_optional=True)
mommy.make(
'financial_activities.FinancialAccountsByProgramActivityObjectClass',
object_class=obj_clas_1,
program_activity=prg_atvy_1,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=8000,
_quantity=2,
_fill_optional=True,
submission=subm_2015_1
) # ignored, superseded by the next submission in the FY
mommy.make(
'financial_activities.FinancialAccountsByProgramActivityObjectClass',
object_class=obj_clas_1,
program_activity=prg_atvy_1,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=1000,
_quantity=2,
_fill_optional=True,
submission=subm_2015_2)
mommy.make(
'financial_activities.FinancialAccountsByProgramActivityObjectClass',
object_class=obj_clas_2,
program_activity=prg_atvy_2,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=9000,
_quantity=2,
_fill_optional=True,
submission=subm_2016_1
) # ignored, superseded by the next submission in the FY
mommy.make(
'financial_activities.FinancialAccountsByProgramActivityObjectClass',
object_class=obj_clas_2,
program_activity=prg_atvy_2,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=2000,
_quantity=2,
_fill_optional=True,
submission=subm_2016_2)
mommy.make(
'financial_activities.FinancialAccountsByProgramActivityObjectClass',
object_class=obj_clas_2,
program_activity=prg_atvy_1,
treasury_account=tas_1,
obligations_undelivered_orders_unpaid_total_cpe=50,
_fill_optional=True,
submission=subm_2016_2)
FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy()
mommy.make('financial_activities.TasProgramActivityObjectClassQuarterly',
object_class=obj_clas_1,
program_activity=prg_atvy_1,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=5000,
_quantity=2,
_fill_optional=True)
mommy.make('financial_activities.TasProgramActivityObjectClassQuarterly',
object_class=obj_clas_2,
program_activity=prg_atvy_2,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=3000,
_quantity=2,
_fill_optional=True)
mommy.make('financial_activities.TasProgramActivityObjectClassQuarterly',
object_class=obj_clas_2,
program_activity=prg_atvy_1,
treasury_account=tas_1,
obligations_undelivered_orders_unpaid_total_cpe=100,
_fill_optional=True)
def account_models():
# Add submission data
subm_2015_1 = mommy.make("submissions.SubmissionAttributes",
reporting_period_start=date(2014, 10, 1))
subm_2015_2 = mommy.make("submissions.SubmissionAttributes",
reporting_period_start=date(2015, 8, 1))
subm_2016_1 = mommy.make("submissions.SubmissionAttributes",
reporting_period_start=date(2016, 1, 1))
subm_2016_2 = mommy.make("submissions.SubmissionAttributes",
reporting_period_start=date(2016, 6, 1))
# add object classes
obj_clas_1 = mommy.make("references.ObjectClass", object_class=1)
obj_clas_2 = mommy.make("references.ObjectClass", object_class=2)
# add program activity
prg_atvy_1 = mommy.make("references.RefProgramActivity", id=1)
prg_atvy_2 = mommy.make("references.RefProgramActivity", id=2)
# add tas data
tas_1 = mommy.make("accounts.TreasuryAppropriationAccount",
tas_rendering_label="ABC",
_fill_optional=True)
tas_2 = mommy.make("accounts.TreasuryAppropriationAccount",
tas_rendering_label="XYZ",
_fill_optional=True)
# add file A data
mommy.make(
"accounts.AppropriationAccountBalances",
treasury_account_identifier=tas_1,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_fill_optional=True,
submission=subm_2015_1,
)
mommy.make(
"accounts.AppropriationAccountBalances",
treasury_account_identifier=tas_2,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_fill_optional=True,
submission=subm_2015_2,
)
mommy.make(
"accounts.AppropriationAccountBalances",
treasury_account_identifier=tas_1,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_fill_optional=True,
submission=subm_2016_1,
)
mommy.make(
"accounts.AppropriationAccountBalances",
treasury_account_identifier=tas_2,
budget_authority_unobligated_balance_brought_forward_fyb=10,
_fill_optional=True,
submission=subm_2016_2,
)
AppropriationAccountBalances.populate_final_of_fy()
# add file B data
mommy.make(
"financial_activities.FinancialAccountsByProgramActivityObjectClass",
object_class=obj_clas_1,
program_activity=prg_atvy_1,
treasury_account=tas_1,
obligations_undelivered_orders_unpaid_total_cpe=8000,
_fill_optional=True,
submission=subm_2015_1,
) # ignored, superseded by the next submission in the FY
mommy.make(
"financial_activities.FinancialAccountsByProgramActivityObjectClass",
object_class=obj_clas_2,
program_activity=prg_atvy_2,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=1000,
_fill_optional=True,
submission=subm_2015_2,
)
mommy.make(
"financial_activities.FinancialAccountsByProgramActivityObjectClass",
object_class=obj_clas_1,
program_activity=prg_atvy_1,
treasury_account=tas_1,
obligations_undelivered_orders_unpaid_total_cpe=9000,
_fill_optional=True,
submission=subm_2016_1,
) # ignored, superseded by the next submission in the FY
mommy.make(
"financial_activities.FinancialAccountsByProgramActivityObjectClass",
object_class=obj_clas_2,
program_activity=prg_atvy_2,
treasury_account=tas_2,
obligations_undelivered_orders_unpaid_total_cpe=2000,
_fill_optional=True,
submission=subm_2016_2,
)
FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy()
def load_file_b(submission_attributes, prg_act_obj_cls_data, db_cursor):
""" Process and load file B broker data (aka TAS balances by program activity and object class). """
reverse = re.compile(r"(_(cpe|fyb)$)|^transaction_obligated_amount$")
# dictionary to capture TAS that were skipped and some metadata
# tas = top-level key
# count = number of rows skipped
# rows = row numbers skipped, corresponding to the original row numbers in the file that was submitted
skipped_tas = {}
test_counter = 0
for row in prg_act_obj_cls_data:
test_counter += 1
try:
# Check and see if there is an entry for this TAS
treasury_account, tas_rendering_label = get_treasury_appropriation_account_tas_lookup(
row.get("tas_id"), db_cursor)
if treasury_account is None:
update_skipped_tas(row, tas_rendering_label, skipped_tas)
continue
except Exception: # TODO: What is this trying to catch, actually?
continue
# get the corresponding account balances row (aka "File A" record)
account_balances = AppropriationAccountBalances.objects.get(
treasury_account_identifier=treasury_account,
submission_id=submission_attributes.submission_id)
financial_by_prg_act_obj_cls = FinancialAccountsByProgramActivityObjectClass(
)
value_map = {
"submission":
submission_attributes,
"reporting_period_start":
submission_attributes.reporting_period_start,
"reporting_period_end":
submission_attributes.reporting_period_end,
"treasury_account":
treasury_account,
"appropriation_account_balances":
account_balances,
"object_class":
get_object_class(row["object_class"],
row["by_direct_reimbursable_fun"]),
"program_activity":
get_or_create_program_activity(row, submission_attributes),
"disaster_emergency_fund":
get_disaster_emergency_fund(row),
}
load_data_into_model(financial_by_prg_act_obj_cls,
row,
value_map=value_map,
save=True,
reverse=reverse)
FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy()
for key in skipped_tas:
logger.info(
f"Skipped {skipped_tas[key]['count']:,} rows due to missing TAS: {key}"
)
total_tas_skipped = 0
for key in skipped_tas:
total_tas_skipped += skipped_tas[key]["count"]
logger.info(
f"Skipped a total of {total_tas_skipped:,} TAS rows for File B")