def load_file_b(submission_attributes, prg_act_obj_cls_data, db_cursor): """ Process and load file B broker data (aka TAS balances by program activity and object class). """ reverse = re.compile(r'(_(cpe|fyb)$)|^transaction_obligated_amount$') test_counter = 0 for row in prg_act_obj_cls_data: test_counter += 1 account_balances = None try: # Check and see if there is an entry for this TAS treasury_account = get_treasury_appropriation_account_tas_lookup( row.get('tas_id'), db_cursor) if treasury_account is None: raise Exception( 'Could not find appropriation account for TAS: ' + row['tas']) except: continue # get the corresponding account balances row (aka "File A" record) account_balances = AppropriationAccountBalances.objects.get( treasury_account_identifier=treasury_account, submission_id=submission_attributes.submission_id) financial_by_prg_act_obj_cls = FinancialAccountsByProgramActivityObjectClass( ) value_map = { 'submission': submission_attributes, 'reporting_period_start': submission_attributes.reporting_period_start, 'reporting_period_end': submission_attributes.reporting_period_end, 'treasury_account': treasury_account, 'appropriation_account_balances': account_balances, 'object_class': get_or_create_object_class(row['object_class'], row['by_direct_reimbursable_fun'], logger), 'program_activity': get_or_create_program_activity(row, submission_attributes) } load_data_into_model(financial_by_prg_act_obj_cls, row, value_map=value_map, save=True, reverse=reverse) # Insert File B quarterly numbers for this submission TasProgramActivityObjectClassQuarterly.insert_quarterly_numbers( submission_attributes.submission_id) FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy()
def load_file_b(submission_attributes, prg_act_obj_cls_data, db_cursor): """ Process and load file B broker data (aka TAS balances by program activity and object class). """ reverse = re.compile(r'(_(cpe|fyb)$)|^transaction_obligated_amount$') # dictionary to capture TAS that were skipped and some metadata # tas = top-level key # count = number of rows skipped # rows = row numbers skipped, corresponding to the original row numbers in the file that was submitted skipped_tas = {} test_counter = 0 for row in prg_act_obj_cls_data: test_counter += 1 account_balances = None try: # Check and see if there is an entry for this TAS treasury_account, tas_rendering_label = get_treasury_appropriation_account_tas_lookup(row.get('tas_id'), db_cursor) if treasury_account is None: update_skipped_tas(row, tas_rendering_label, skipped_tas) continue except Exception: # TODO: What is this trying to catch, actually? continue # get the corresponding account balances row (aka "File A" record) account_balances = AppropriationAccountBalances.objects.get( treasury_account_identifier=treasury_account, submission_id=submission_attributes.submission_id ) financial_by_prg_act_obj_cls = FinancialAccountsByProgramActivityObjectClass() value_map = { 'submission': submission_attributes, 'reporting_period_start': submission_attributes.reporting_period_start, 'reporting_period_end': submission_attributes.reporting_period_end, 'treasury_account': treasury_account, 'appropriation_account_balances': account_balances, 'object_class': get_or_create_object_class(row['object_class'], row['by_direct_reimbursable_fun'], logger), 'program_activity': get_or_create_program_activity(row, submission_attributes) } load_data_into_model(financial_by_prg_act_obj_cls, row, value_map=value_map, save=True, reverse=reverse) # Insert File B quarterly numbers for this submission TasProgramActivityObjectClassQuarterly.insert_quarterly_numbers(submission_attributes.submission_id) FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy() for key in skipped_tas: logger.info('Skipped %d rows due to missing TAS: %s', skipped_tas[key]['count'], key) total_tas_skipped = 0 for key in skipped_tas: total_tas_skipped += skipped_tas[key]['count'] logger.info('Skipped a total of {} TAS rows for File B'.format(total_tas_skipped))
def account_models(): # Add submission data subm_2015_1 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2014, 10, 1)) subm_2015_2 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2015, 8, 1)) subm_2016_1 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2016, 1, 1)) subm_2016_2 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2016, 6, 1)) # add object classes obj_clas_1 = mommy.make('references.ObjectClass', object_class=1) obj_clas_2 = mommy.make('references.ObjectClass', object_class=2) # add program activity prg_atvy_1 = mommy.make('references.RefProgramActivity', id=1) prg_atvy_2 = mommy.make('references.RefProgramActivity', id=2) # add tas data tas_1 = mommy.make('accounts.TreasuryAppropriationAccount', tas_rendering_label="ABC", _fill_optional=True) tas_2 = mommy.make('accounts.TreasuryAppropriationAccount', tas_rendering_label="XYZ", _fill_optional=True) # add file A data mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_1, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True, submission=subm_2015_1) mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_2, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True, submission=subm_2015_2) mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_1, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True, submission=subm_2016_1) mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_2, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True, submission=subm_2016_2) AppropriationAccountBalances.populate_final_of_fy() # add file B data mommy.make('financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_1, program_activity=prg_atvy_1, treasury_account=tas_1, obligations_undelivered_orders_unpaid_total_cpe=8000, _fill_optional=True, submission=subm_2015_1) # ignored, superseded by the next submission in the FY mommy.make('financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_2, program_activity=prg_atvy_2, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=1000, _fill_optional=True, submission=subm_2015_2) mommy.make('financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_1, program_activity=prg_atvy_1, treasury_account=tas_1, obligations_undelivered_orders_unpaid_total_cpe=9000, _fill_optional=True, submission=subm_2016_1) # ignored, superseded by the next submission in the FY mommy.make('financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_2, program_activity=prg_atvy_2, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=2000, _fill_optional=True, submission=subm_2016_2) FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy()
def test_get_quarterly_numbers(tas_oc_pa_data): """ Test the function that generates quarterly tas/obj class/pgm activity numbers. """ # retrieve all quarterly numbers and test results quarters = FinancialAccountsByProgramActivityObjectClass.get_quarterly_numbers() # number of quarterly adjusted records should = number of records # in FinancialAccountsByProgramActivityObjectClass assert len(list(quarters)) == 2 # submission 1: has no previous subission # submission 2: its previous submission is submission 1 sub1 = SubmissionAttributes.objects.get(previous_submission__isnull=True) sub2 = SubmissionAttributes.objects.get(previous_submission__isnull=False) for q in quarters: if q.submission == sub1: # qtrly values for year's first submission should remain unchanged assert q.gross_outlays_delivered_orders_paid_total_cpe == Decimal('10.10') assert q.obligations_incurred_by_program_object_class_cpe == Decimal('10.00') assert q.obligations_undelivered_orders_unpaid_total_fyb == Decimal('99.99') assert q.ussgl490800_authority_outlayed_not_yet_disbursed_fyb == Decimal('-30.00') else: # qtrly values for year's 2nd submission should be equal to 2nd # submission values - first submission values assert q.gross_outlays_delivered_orders_paid_total_cpe == Decimal('10.50') assert q.obligations_incurred_by_program_object_class_cpe == Decimal('-5.00') assert q.obligations_undelivered_orders_unpaid_total_fyb == Decimal('0.00') # TODO: check on correct logic where previous submission is negative assert q.ussgl490800_authority_outlayed_not_yet_disbursed_fyb == Decimal('60.00') # test getting quarterly results for a specific submission quarters = FinancialAccountsByProgramActivityObjectClass.get_quarterly_numbers( sub2.submission_id) # number of quarterly adjusted records should = number of records # in FinancialAccountsByProgramActivityObjectClass assert len(list(quarters)) == 1 q = quarters[0] # qtrly values for year's 2nd submission should return the same # values as the test above assert q.gross_outlays_delivered_orders_paid_total_cpe == Decimal('10.50') assert q.obligations_incurred_by_program_object_class_cpe == Decimal('-5.00') assert q.obligations_undelivered_orders_unpaid_total_fyb == Decimal('0.00') assert q.ussgl490800_authority_outlayed_not_yet_disbursed_fyb == Decimal('60.00') # requesting data for non-existent submission returns zero records quarters = FinancialAccountsByProgramActivityObjectClass.get_quarterly_numbers(-888) assert len(list(quarters)) == 0
def populate_final_of_fy(): AppropriationAccountBalances.populate_final_of_fy() FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy()
def load_file_b(submission_attributes, prg_act_obj_cls_data, db_cursor): """ Process and load file B broker data (aka TAS balances by program activity and object class). """ reverse = re.compile(r"(_(cpe|fyb)$)|^transaction_obligated_amount$") # dictionary to capture TAS that were skipped and some metadata # tas = top-level key # count = number of rows skipped # rows = row numbers skipped, corresponding to the original row numbers in the file that was submitted skipped_tas = {} bulk_treasury_appropriation_account_tas_lookup(prg_act_obj_cls_data, db_cursor) save_manager = BulkCreateManager( FinancialAccountsByProgramActivityObjectClass) for row in prg_act_obj_cls_data: # Check and see if there is an entry for this TAS treasury_account, tas_rendering_label = get_treasury_appropriation_account_tas_lookup( row.get("tas_id")) if treasury_account is None: update_skipped_tas(row, tas_rendering_label, skipped_tas) continue # get the corresponding account balances row (aka "File A" record) account_balances = AppropriationAccountBalances.objects.get( treasury_account_identifier=treasury_account, submission_id=submission_attributes.submission_id) financial_by_prg_act_obj_cls = FinancialAccountsByProgramActivityObjectClass( ) value_map = { "submission": submission_attributes, "reporting_period_start": submission_attributes.reporting_period_start, "reporting_period_end": submission_attributes.reporting_period_end, "treasury_account": treasury_account, "appropriation_account_balances": account_balances, "object_class": get_object_class(row["object_class"], row["by_direct_reimbursable_fun"]), "program_activity": get_program_activity(row, submission_attributes), "disaster_emergency_fund": get_disaster_emergency_fund(row), } save_manager.append( load_data_into_model(financial_by_prg_act_obj_cls, row, value_map=value_map, save=False, reverse=reverse)) save_manager.save_stragglers() for key in skipped_tas: logger.info( f"Skipped {skipped_tas[key]['count']:,} rows due to missing TAS: {key}" ) total_tas_skipped = 0 for key in skipped_tas: total_tas_skipped += skipped_tas[key]["count"] logger.info( f"Skipped a total of {total_tas_skipped:,} TAS rows for File B")
def account_models(): subm_2015_1 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2014, 10, 1)) subm_2015_2 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2015, 8, 1)) subm_2016_1 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2016, 1, 1)) subm_2016_2 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2016, 6, 1)) obj_clas_1 = mommy.make('references.ObjectClass', object_class=1) obj_clas_2 = mommy.make('references.ObjectClass', object_class=2) prg_atvy_1 = mommy.make('references.RefProgramActivity', id=1) prg_atvy_2 = mommy.make('references.RefProgramActivity', id=2) tas_1 = mommy.make('accounts.TreasuryAppropriationAccount', tas_rendering_label="ABC", _fill_optional=True) tas_2 = mommy.make('accounts.TreasuryAppropriationAccount', tas_rendering_label="XYZ", _fill_optional=True) mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_1, budget_authority_unobligated_balance_brought_forward_fyb=10, _quantity=2, _fill_optional=True) mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_2, budget_authority_unobligated_balance_brought_forward_fyb=10, _quantity=3, _fill_optional=True) AppropriationAccountBalances.populate_final_of_fy() mommy.make('accounts.AppropriationAccountBalancesQuarterly', treasury_account_identifier=tas_1, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True) mommy.make('accounts.AppropriationAccountBalancesQuarterly', treasury_account_identifier=tas_2, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True) mommy.make( 'financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_1, program_activity=prg_atvy_1, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=8000, _quantity=2, _fill_optional=True, submission=subm_2015_1 ) # ignored, superseded by the next submission in the FY mommy.make( 'financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_1, program_activity=prg_atvy_1, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=1000, _quantity=2, _fill_optional=True, submission=subm_2015_2) mommy.make( 'financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_2, program_activity=prg_atvy_2, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=9000, _quantity=2, _fill_optional=True, submission=subm_2016_1 ) # ignored, superseded by the next submission in the FY mommy.make( 'financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_2, program_activity=prg_atvy_2, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=2000, _quantity=2, _fill_optional=True, submission=subm_2016_2) mommy.make( 'financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_2, program_activity=prg_atvy_1, treasury_account=tas_1, obligations_undelivered_orders_unpaid_total_cpe=50, _fill_optional=True, submission=subm_2016_2) FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy() mommy.make('financial_activities.TasProgramActivityObjectClassQuarterly', object_class=obj_clas_1, program_activity=prg_atvy_1, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=5000, _quantity=2, _fill_optional=True) mommy.make('financial_activities.TasProgramActivityObjectClassQuarterly', object_class=obj_clas_2, program_activity=prg_atvy_2, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=3000, _quantity=2, _fill_optional=True) mommy.make('financial_activities.TasProgramActivityObjectClassQuarterly', object_class=obj_clas_2, program_activity=prg_atvy_1, treasury_account=tas_1, obligations_undelivered_orders_unpaid_total_cpe=100, _fill_optional=True)
def account_models(): # Add submission data subm_2015_1 = mommy.make("submissions.SubmissionAttributes", reporting_period_start=date(2014, 10, 1)) subm_2015_2 = mommy.make("submissions.SubmissionAttributes", reporting_period_start=date(2015, 8, 1)) subm_2016_1 = mommy.make("submissions.SubmissionAttributes", reporting_period_start=date(2016, 1, 1)) subm_2016_2 = mommy.make("submissions.SubmissionAttributes", reporting_period_start=date(2016, 6, 1)) # add object classes obj_clas_1 = mommy.make("references.ObjectClass", object_class=1) obj_clas_2 = mommy.make("references.ObjectClass", object_class=2) # add program activity prg_atvy_1 = mommy.make("references.RefProgramActivity", id=1) prg_atvy_2 = mommy.make("references.RefProgramActivity", id=2) # add tas data tas_1 = mommy.make("accounts.TreasuryAppropriationAccount", tas_rendering_label="ABC", _fill_optional=True) tas_2 = mommy.make("accounts.TreasuryAppropriationAccount", tas_rendering_label="XYZ", _fill_optional=True) # add file A data mommy.make( "accounts.AppropriationAccountBalances", treasury_account_identifier=tas_1, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True, submission=subm_2015_1, ) mommy.make( "accounts.AppropriationAccountBalances", treasury_account_identifier=tas_2, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True, submission=subm_2015_2, ) mommy.make( "accounts.AppropriationAccountBalances", treasury_account_identifier=tas_1, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True, submission=subm_2016_1, ) mommy.make( "accounts.AppropriationAccountBalances", treasury_account_identifier=tas_2, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True, submission=subm_2016_2, ) AppropriationAccountBalances.populate_final_of_fy() # add file B data mommy.make( "financial_activities.FinancialAccountsByProgramActivityObjectClass", object_class=obj_clas_1, program_activity=prg_atvy_1, treasury_account=tas_1, obligations_undelivered_orders_unpaid_total_cpe=8000, _fill_optional=True, submission=subm_2015_1, ) # ignored, superseded by the next submission in the FY mommy.make( "financial_activities.FinancialAccountsByProgramActivityObjectClass", object_class=obj_clas_2, program_activity=prg_atvy_2, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=1000, _fill_optional=True, submission=subm_2015_2, ) mommy.make( "financial_activities.FinancialAccountsByProgramActivityObjectClass", object_class=obj_clas_1, program_activity=prg_atvy_1, treasury_account=tas_1, obligations_undelivered_orders_unpaid_total_cpe=9000, _fill_optional=True, submission=subm_2016_1, ) # ignored, superseded by the next submission in the FY mommy.make( "financial_activities.FinancialAccountsByProgramActivityObjectClass", object_class=obj_clas_2, program_activity=prg_atvy_2, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=2000, _fill_optional=True, submission=subm_2016_2, ) FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy()
def test_get_quarterly_numbers(tas_oc_pa_data): """ Test the function that generates quarterly tas/obj class/pgm activity numbers. """ # retrieve all quarterly numbers and test results quarters = FinancialAccountsByProgramActivityObjectClass.get_quarterly_numbers( ) # number of quarterly adjusted records should = number of records # in FinancialAccountsByProgramActivityObjectClass assert len(list(quarters)) == 2 # submission 1: has no previous subission # submission 2: its previous submission is submission 1 sub1 = SubmissionAttributes.objects.get(previous_submission__isnull=True) sub2 = SubmissionAttributes.objects.get(previous_submission__isnull=False) for q in quarters: if q.submission == sub1: # qtrly values for year's first submission should remain unchanged assert q.gross_outlays_delivered_orders_paid_total_cpe == Decimal( '10.10') assert q.obligations_incurred_by_program_object_class_cpe == Decimal( '10.00') assert q.obligations_undelivered_orders_unpaid_total_fyb == Decimal( '99.99') assert q.ussgl490800_authority_outlayed_not_yet_disbursed_fyb == Decimal( '-30.00') else: # qtrly values for year's 2nd submission should be equal to 2nd # submission values - first submission values assert q.gross_outlays_delivered_orders_paid_total_cpe == Decimal( '10.50') assert q.obligations_incurred_by_program_object_class_cpe == Decimal( '-5.00') assert q.obligations_undelivered_orders_unpaid_total_fyb == Decimal( '0.00') # TODO: check on correct logic where previous submission is negative assert q.ussgl490800_authority_outlayed_not_yet_disbursed_fyb == Decimal( '60.00') # test getting quarterly results for a specific submission quarters = FinancialAccountsByProgramActivityObjectClass.get_quarterly_numbers( sub2.submission_id) # number of quarterly adjusted records should = number of records # in FinancialAccountsByProgramActivityObjectClass assert len(list(quarters)) == 1 q = quarters[0] # qtrly values for year's 2nd submission should return the same # values as the test above assert q.gross_outlays_delivered_orders_paid_total_cpe == Decimal('10.50') assert q.obligations_incurred_by_program_object_class_cpe == Decimal( '-5.00') assert q.obligations_undelivered_orders_unpaid_total_fyb == Decimal('0.00') assert q.ussgl490800_authority_outlayed_not_yet_disbursed_fyb == Decimal( '60.00') # requesting data for non-existent submission returns zero records quarters = FinancialAccountsByProgramActivityObjectClass.get_quarterly_numbers( -888) assert len(list(quarters)) == 0
def load_file_b(submission_attributes, prg_act_obj_cls_data, db_cursor): """ Process and load file B broker data (aka TAS balances by program activity and object class). """ reverse = re.compile(r"(_(cpe|fyb)$)|^transaction_obligated_amount$") skipped_tas = defaultdict( int) # tracks count of rows skipped due to "missing" TAS bulk_treasury_appropriation_account_tas_lookup(prg_act_obj_cls_data, db_cursor) save_manager = BulkCreateManager( FinancialAccountsByProgramActivityObjectClass) for row in prg_act_obj_cls_data: # Check and see if there is an entry for this TAS treasury_account, tas_rendering_label = get_treasury_appropriation_account_tas_lookup( row.get("tas_id")) if treasury_account is None: skipped_tas[tas_rendering_label] += 1 continue # get the corresponding account balances row (aka "File A" record) account_balances = AppropriationAccountBalances.objects.get( treasury_account_identifier=treasury_account, submission_id=submission_attributes.submission_id) financial_by_prg_act_obj_cls = FinancialAccountsByProgramActivityObjectClass( ) value_map = { "submission": submission_attributes, "reporting_period_start": submission_attributes.reporting_period_start, "reporting_period_end": submission_attributes.reporting_period_end, "treasury_account": treasury_account, "appropriation_account_balances": account_balances, "object_class": get_object_class(row["object_class"], row["by_direct_reimbursable_fun"]), "program_activity": get_program_activity(row, submission_attributes), "disaster_emergency_fund": get_disaster_emergency_fund(row), } save_manager.append( load_data_into_model(financial_by_prg_act_obj_cls, row, value_map=value_map, save=False, reverse=reverse)) save_manager.save_stragglers() for tas, count in skipped_tas.items(): logger.info(f"Skipped {count:,} rows due to {tas}") total_tas_skipped = sum([count for count in skipped_tas.values()]) if total_tas_skipped > 0: logger.info( f"SKIPPED {total_tas_skipped:,} ROWS of File B (missing TAS)") else: logger.info("All File B records in Broker loaded into USAspending")