def load_file_a(submission_attributes, appropriation_data, db_cursor): """ Process and load file A broker data (aka TAS balances, aka appropriation account balances). """ reverse = re.compile("gross_outlay_amount_by_tas_cpe") # dictionary to capture TAS that were skipped and some metadata # tas = top-level key # count = number of rows skipped # rows = row numbers skipped, corresponding to the original row numbers in the file that was submitted skipped_tas = {} # Create account objects for row in appropriation_data: # Check and see if there is an entry for this TAS treasury_account, tas_rendering_label = get_treasury_appropriation_account_tas_lookup( row.get("tas_id"), db_cursor) if treasury_account is None: update_skipped_tas(row, tas_rendering_label, skipped_tas) continue # Now that we have the account, we can load the appropriation balances # TODO: Figure out how we want to determine what row is overriden by what row # If we want to correlate, the following attributes are available in the data broker data that might be useful: # appropriation_id, row_number appropriation_balances = somethingsomething get appropriation balances... appropriation_balances = AppropriationAccountBalances() value_map = { "treasury_account_identifier": treasury_account, "submission": submission_attributes, "reporting_period_start": submission_attributes.reporting_period_start, "reporting_period_end": submission_attributes.reporting_period_end, } field_map = {} load_data_into_model(appropriation_balances, row, field_map=field_map, value_map=value_map, save=True, reverse=reverse) AppropriationAccountBalances.populate_final_of_fy() for key in skipped_tas: logger.info( f"Skipped {skipped_tas[key]['count']:,} rows due to missing TAS: {key}" ) total_tas_skipped = 0 for key in skipped_tas: total_tas_skipped += skipped_tas[key]["count"] logger.info( f"Skipped a total of {total_tas_skipped:,} TAS rows for File A")
def load_file_a(submission_attributes, appropriation_data, db_cursor): """ Process and load file A broker data (aka TAS balances, aka appropriation account balances). """ reverse = re.compile('gross_outlay_amount_by_tas_cpe') # dictionary to capture TAS that were skipped and some metadata # tas = top-level key # count = number of rows skipped # rows = row numbers skipped, corresponding to the original row numbers in the file that was submitted skipped_tas = {} # Create account objects for row in appropriation_data: # Check and see if there is an entry for this TAS treasury_account, tas_rendering_label = get_treasury_appropriation_account_tas_lookup( row.get('tas_id'), db_cursor) if treasury_account is None: update_skipped_tas(row, tas_rendering_label, skipped_tas) continue # Now that we have the account, we can load the appropriation balances # TODO: Figure out how we want to determine what row is overriden by what row # If we want to correlate, the following attributes are available in the data broker data that might be useful: # appropriation_id, row_number appropriation_balances = somethingsomething get appropriation balances... appropriation_balances = AppropriationAccountBalances() value_map = { 'treasury_account_identifier': treasury_account, 'submission': submission_attributes, 'reporting_period_start': submission_attributes.reporting_period_start, 'reporting_period_end': submission_attributes.reporting_period_end } field_map = {} load_data_into_model(appropriation_balances, row, field_map=field_map, value_map=value_map, save=True, reverse=reverse) AppropriationAccountBalances.populate_final_of_fy() # Insert File A quarterly numbers for this submission AppropriationAccountBalancesQuarterly.insert_quarterly_numbers(submission_attributes.submission_id) for key in skipped_tas: logger.info('Skipped %d rows due to missing TAS: %s', skipped_tas[key]['count'], key) total_tas_skipped = 0 for key in skipped_tas: total_tas_skipped += skipped_tas[key]['count'] logger.info('Skipped a total of {} TAS rows for File A'.format(total_tas_skipped))
def app_acc_bal_models(): sub_16_1 = mommy.make("submissions.SubmissionAttributes", reporting_period_start=date(2016, 1, 1)) sub_16_2 = mommy.make("submissions.SubmissionAttributes", reporting_period_start=date(2016, 4, 1)) sub_17_1 = mommy.make("submissions.SubmissionAttributes", reporting_period_start=date(2016, 10, 1)) sub_17_2 = mommy.make("submissions.SubmissionAttributes", reporting_period_start=date(2017, 2, 1)) tas_1 = mommy.make("accounts.TreasuryAppropriationAccount", tas_rendering_label="ABC", _fill_optional=True) tas_2 = mommy.make("accounts.TreasuryAppropriationAccount", tas_rendering_label="XYZ", _fill_optional=True) mommy.make( "accounts.AppropriationAccountBalances", treasury_account_identifier=tas_1, submission=sub_16_1, _fill_optional=True, ) mommy.make( "accounts.AppropriationAccountBalances", treasury_account_identifier=tas_1, submission=sub_16_2, _fill_optional=True, ) mommy.make( "accounts.AppropriationAccountBalances", treasury_account_identifier=tas_1, submission=sub_17_1, _fill_optional=True, ) mommy.make( "accounts.AppropriationAccountBalances", treasury_account_identifier=tas_1, submission=sub_17_2, _fill_optional=True, ) mommy.make( "accounts.AppropriationAccountBalances", treasury_account_identifier=tas_2, submission=sub_16_1, _fill_optional=True, ) mommy.make( "accounts.AppropriationAccountBalances", treasury_account_identifier=tas_2, submission=sub_16_2, _fill_optional=True, ) AppropriationAccountBalances.populate_final_of_fy()
def account_models(): # Add submission data subm_2015_1 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2014, 10, 1)) subm_2015_2 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2015, 8, 1)) subm_2016_1 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2016, 1, 1)) subm_2016_2 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2016, 6, 1)) # add object classes obj_clas_1 = mommy.make('references.ObjectClass', object_class=1) obj_clas_2 = mommy.make('references.ObjectClass', object_class=2) # add program activity prg_atvy_1 = mommy.make('references.RefProgramActivity', id=1) prg_atvy_2 = mommy.make('references.RefProgramActivity', id=2) # add tas data tas_1 = mommy.make('accounts.TreasuryAppropriationAccount', tas_rendering_label="ABC", _fill_optional=True) tas_2 = mommy.make('accounts.TreasuryAppropriationAccount', tas_rendering_label="XYZ", _fill_optional=True) # add file A data mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_1, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True, submission=subm_2015_1) mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_2, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True, submission=subm_2015_2) mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_1, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True, submission=subm_2016_1) mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_2, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True, submission=subm_2016_2) AppropriationAccountBalances.populate_final_of_fy() # add file B data mommy.make('financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_1, program_activity=prg_atvy_1, treasury_account=tas_1, obligations_undelivered_orders_unpaid_total_cpe=8000, _fill_optional=True, submission=subm_2015_1) # ignored, superseded by the next submission in the FY mommy.make('financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_2, program_activity=prg_atvy_2, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=1000, _fill_optional=True, submission=subm_2015_2) mommy.make('financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_1, program_activity=prg_atvy_1, treasury_account=tas_1, obligations_undelivered_orders_unpaid_total_cpe=9000, _fill_optional=True, submission=subm_2016_1) # ignored, superseded by the next submission in the FY mommy.make('financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_2, program_activity=prg_atvy_2, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=2000, _fill_optional=True, submission=subm_2016_2) FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy()
def load_file_a(submission_attributes, appropriation_data, db_cursor): """ Process and load file A broker data (aka TAS balances, aka appropriation account balances). """ reverse = re.compile('gross_outlay_amount_by_tas_cpe') # Create account objects for row in appropriation_data: # Check and see if there is an entry for this TAS treasury_account = get_treasury_appropriation_account_tas_lookup( row.get('tas_id'), db_cursor) if treasury_account is None: raise Exception('Could not find appropriation account for TAS: ' + row['tas']) # Now that we have the account, we can load the appropriation balances # TODO: Figure out how we want to determine what row is overriden by what row # If we want to correlate, the following attributes are available in the # data broker data that might be useful: appropriation_id, row_number # appropriation_balances = somethingsomething get appropriation balances... appropriation_balances = AppropriationAccountBalances() value_map = { 'treasury_account_identifier': treasury_account, 'submission': submission_attributes, 'reporting_period_start': submission_attributes.reporting_period_start, 'reporting_period_end': submission_attributes.reporting_period_end } field_map = {} load_data_into_model(appropriation_balances, row, field_map=field_map, value_map=value_map, save=True, reverse=reverse) AppropriationAccountBalances.populate_final_of_fy() # Insert File A quarterly numbers for this submission AppropriationAccountBalancesQuarterly.insert_quarterly_numbers( submission_attributes.submission_id)
def app_acc_bal_models(): sub_16_1 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2016, 1, 1)) sub_16_2 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2016, 4, 1)) sub_17_1 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2016, 10, 1)) sub_17_2 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2017, 2, 1)) tas_1 = mommy.make('accounts.TreasuryAppropriationAccount', tas_rendering_label="ABC", _fill_optional=True) tas_2 = mommy.make('accounts.TreasuryAppropriationAccount', tas_rendering_label="XYZ", _fill_optional=True) mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_1, submission=sub_16_1, _fill_optional=True) mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_1, submission=sub_16_2, _fill_optional=True) mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_1, submission=sub_17_1, _fill_optional=True) mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_1, submission=sub_17_2, _fill_optional=True) mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_2, submission=sub_16_1, _fill_optional=True) mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_2, submission=sub_16_2, _fill_optional=True) AppropriationAccountBalances.populate_final_of_fy()
def populate_final_of_fy(): AppropriationAccountBalances.populate_final_of_fy() FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy()
def account_models(): subm_2015_1 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2014, 10, 1)) subm_2015_2 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2015, 8, 1)) subm_2016_1 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2016, 1, 1)) subm_2016_2 = mommy.make('submissions.SubmissionAttributes', reporting_period_start=date(2016, 6, 1)) obj_clas_1 = mommy.make('references.ObjectClass', object_class=1) obj_clas_2 = mommy.make('references.ObjectClass', object_class=2) prg_atvy_1 = mommy.make('references.RefProgramActivity', id=1) prg_atvy_2 = mommy.make('references.RefProgramActivity', id=2) tas_1 = mommy.make('accounts.TreasuryAppropriationAccount', tas_rendering_label="ABC", _fill_optional=True) tas_2 = mommy.make('accounts.TreasuryAppropriationAccount', tas_rendering_label="XYZ", _fill_optional=True) mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_1, budget_authority_unobligated_balance_brought_forward_fyb=10, _quantity=2, _fill_optional=True) mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_2, budget_authority_unobligated_balance_brought_forward_fyb=10, _quantity=3, _fill_optional=True) AppropriationAccountBalances.populate_final_of_fy() mommy.make('accounts.AppropriationAccountBalancesQuarterly', treasury_account_identifier=tas_1, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True) mommy.make('accounts.AppropriationAccountBalancesQuarterly', treasury_account_identifier=tas_2, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True) mommy.make( 'financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_1, program_activity=prg_atvy_1, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=8000, _quantity=2, _fill_optional=True, submission=subm_2015_1 ) # ignored, superseded by the next submission in the FY mommy.make( 'financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_1, program_activity=prg_atvy_1, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=1000, _quantity=2, _fill_optional=True, submission=subm_2015_2) mommy.make( 'financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_2, program_activity=prg_atvy_2, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=9000, _quantity=2, _fill_optional=True, submission=subm_2016_1 ) # ignored, superseded by the next submission in the FY mommy.make( 'financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_2, program_activity=prg_atvy_2, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=2000, _quantity=2, _fill_optional=True, submission=subm_2016_2) mommy.make( 'financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_2, program_activity=prg_atvy_1, treasury_account=tas_1, obligations_undelivered_orders_unpaid_total_cpe=50, _fill_optional=True, submission=subm_2016_2) FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy() mommy.make('financial_activities.TasProgramActivityObjectClassQuarterly', object_class=obj_clas_1, program_activity=prg_atvy_1, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=5000, _quantity=2, _fill_optional=True) mommy.make('financial_activities.TasProgramActivityObjectClassQuarterly', object_class=obj_clas_2, program_activity=prg_atvy_2, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=3000, _quantity=2, _fill_optional=True) mommy.make('financial_activities.TasProgramActivityObjectClassQuarterly', object_class=obj_clas_2, program_activity=prg_atvy_1, treasury_account=tas_1, obligations_undelivered_orders_unpaid_total_cpe=100, _fill_optional=True)
def account_models(): # Add submission data subm_2015_1 = mommy.make("submissions.SubmissionAttributes", reporting_period_start=date(2014, 10, 1)) subm_2015_2 = mommy.make("submissions.SubmissionAttributes", reporting_period_start=date(2015, 8, 1)) subm_2016_1 = mommy.make("submissions.SubmissionAttributes", reporting_period_start=date(2016, 1, 1)) subm_2016_2 = mommy.make("submissions.SubmissionAttributes", reporting_period_start=date(2016, 6, 1)) # add object classes obj_clas_1 = mommy.make("references.ObjectClass", object_class=1) obj_clas_2 = mommy.make("references.ObjectClass", object_class=2) # add program activity prg_atvy_1 = mommy.make("references.RefProgramActivity", id=1) prg_atvy_2 = mommy.make("references.RefProgramActivity", id=2) # add tas data tas_1 = mommy.make("accounts.TreasuryAppropriationAccount", tas_rendering_label="ABC", _fill_optional=True) tas_2 = mommy.make("accounts.TreasuryAppropriationAccount", tas_rendering_label="XYZ", _fill_optional=True) # add file A data mommy.make( "accounts.AppropriationAccountBalances", treasury_account_identifier=tas_1, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True, submission=subm_2015_1, ) mommy.make( "accounts.AppropriationAccountBalances", treasury_account_identifier=tas_2, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True, submission=subm_2015_2, ) mommy.make( "accounts.AppropriationAccountBalances", treasury_account_identifier=tas_1, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True, submission=subm_2016_1, ) mommy.make( "accounts.AppropriationAccountBalances", treasury_account_identifier=tas_2, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True, submission=subm_2016_2, ) AppropriationAccountBalances.populate_final_of_fy() # add file B data mommy.make( "financial_activities.FinancialAccountsByProgramActivityObjectClass", object_class=obj_clas_1, program_activity=prg_atvy_1, treasury_account=tas_1, obligations_undelivered_orders_unpaid_total_cpe=8000, _fill_optional=True, submission=subm_2015_1, ) # ignored, superseded by the next submission in the FY mommy.make( "financial_activities.FinancialAccountsByProgramActivityObjectClass", object_class=obj_clas_2, program_activity=prg_atvy_2, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=1000, _fill_optional=True, submission=subm_2015_2, ) mommy.make( "financial_activities.FinancialAccountsByProgramActivityObjectClass", object_class=obj_clas_1, program_activity=prg_atvy_1, treasury_account=tas_1, obligations_undelivered_orders_unpaid_total_cpe=9000, _fill_optional=True, submission=subm_2016_1, ) # ignored, superseded by the next submission in the FY mommy.make( "financial_activities.FinancialAccountsByProgramActivityObjectClass", object_class=obj_clas_2, program_activity=prg_atvy_2, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=2000, _fill_optional=True, submission=subm_2016_2, ) FinancialAccountsByProgramActivityObjectClass.populate_final_of_fy()
def load_file_a(submission_attributes, appropriation_data, db_cursor): """ Process and load file A broker data (aka TAS balances, aka appropriation account balances). """ reverse = re.compile('gross_outlay_amount_by_tas_cpe') # dictionary to capture TAS that were skipped and some metadata # tas = top-level key # count = number of rows skipped # rows = row numbers skipped, corresponding to the original row numbers in the file that was submitted skipped_tas = {} # Create account objects for row in appropriation_data: # Check and see if there is an entry for this TAS treasury_account = get_treasury_appropriation_account_tas_lookup( row.get('tas_id'), db_cursor) if treasury_account is None: if row['tas'] not in skipped_tas: skipped_tas[row['tas']] = {} skipped_tas[row['tas']]['count'] = 1 skipped_tas[row['tas']]['rows'] = [row['row_number']] else: skipped_tas[row['tas']]['count'] += 1 skipped_tas[row['tas']]['rows'] += row['row_number'] continue # Now that we have the account, we can load the appropriation balances # TODO: Figure out how we want to determine what row is overriden by what row # If we want to correlate, the following attributes are available in the data broker data that might be useful: # appropriation_id, row_number appropriation_balances = somethingsomething get appropriation balances... appropriation_balances = AppropriationAccountBalances() value_map = { 'treasury_account_identifier': treasury_account, 'submission': submission_attributes, 'reporting_period_start': submission_attributes.reporting_period_start, 'reporting_period_end': submission_attributes.reporting_period_end } field_map = {} load_data_into_model(appropriation_balances, row, field_map=field_map, value_map=value_map, save=True, reverse=reverse) AppropriationAccountBalances.populate_final_of_fy() # Insert File A quarterly numbers for this submission AppropriationAccountBalancesQuarterly.insert_quarterly_numbers( submission_attributes.submission_id) for key in skipped_tas: logger.info('Skipped %d rows due to missing TAS: %s', skipped_tas[key]['count'], key) total_tas_skipped = 0 for key in skipped_tas: total_tas_skipped += skipped_tas[key]['count'] logger.info( 'Skipped a total of {} TAS rows for File A'.format(total_tas_skipped))
def account_models(): obj_clas_1 = mommy.make('references.ObjectClass', object_class=1) obj_clas_2 = mommy.make('references.ObjectClass', object_class=2) prg_atvy_1 = mommy.make('references.RefProgramActivity', id=1) prg_atvy_2 = mommy.make('references.RefProgramActivity', id=2) tas_1 = mommy.make('accounts.TreasuryAppropriationAccount', tas_rendering_label="ABC", _fill_optional=True) tas_2 = mommy.make('accounts.TreasuryAppropriationAccount', tas_rendering_label="XYZ", _fill_optional=True) mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_1, budget_authority_unobligated_balance_brought_forward_fyb=10, _quantity=2, _fill_optional=True) mommy.make('accounts.AppropriationAccountBalances', treasury_account_identifier=tas_2, budget_authority_unobligated_balance_brought_forward_fyb=10, _quantity=3, _fill_optional=True) AppropriationAccountBalances.populate_final_of_fy() mommy.make('accounts.AppropriationAccountBalancesQuarterly', treasury_account_identifier=tas_1, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True) mommy.make('accounts.AppropriationAccountBalancesQuarterly', treasury_account_identifier=tas_2, budget_authority_unobligated_balance_brought_forward_fyb=10, _fill_optional=True) mommy.make( 'financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_1, program_activity=prg_atvy_1, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=1000, _quantity=2, _fill_optional=True) mommy.make( 'financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_2, program_activity=prg_atvy_2, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=2000, _quantity=2, _fill_optional=True) mommy.make( 'financial_activities.FinancialAccountsByProgramActivityObjectClass', object_class=obj_clas_2, program_activity=prg_atvy_1, treasury_account=tas_1, obligations_undelivered_orders_unpaid_total_cpe=50, _fill_optional=True), mommy.make('financial_activities.TasProgramActivityObjectClassQuarterly', object_class=obj_clas_1, program_activity=prg_atvy_1, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=5000, _quantity=2, _fill_optional=True) mommy.make('financial_activities.TasProgramActivityObjectClassQuarterly', object_class=obj_clas_2, program_activity=prg_atvy_2, treasury_account=tas_2, obligations_undelivered_orders_unpaid_total_cpe=3000, _quantity=2, _fill_optional=True) mommy.make('financial_activities.TasProgramActivityObjectClassQuarterly', object_class=obj_clas_2, program_activity=prg_atvy_1, treasury_account=tas_1, obligations_undelivered_orders_unpaid_total_cpe=100, _fill_optional=True)