def test_load_program_activity_data_empty_file(mocked_get_pa_file, mocked_get_current_date, mocked_get_stored_date, mocked_set_stored_date, monkeypatch): """ Test actually loading the program activity data """ monkeypatch.setattr(load_program_activity, 'CONFIG_BROKER', { 'use_aws': False, 'local': False }) mocked_get_pa_file.return_value = StringIO("") mocked_get_current_date.return_value = datetime.datetime( 2017, 12, 31, 0, 0, 0) mocked_get_stored_date.return_value = datetime.datetime( 2016, 12, 31, 0, 0, 0) mocked_set_stored_date.return_value = None with pytest.raises(SystemExit) as se: load_program_activity.load_program_activity_data('some_path') assert se.value.code == 4 remove_metrics_file()
def test_load_program_activity_data(mocked_get_pa_file, mocked_get_current_date, mocked_get_stored_date, mocked_set_stored_date, database, monkeypatch): """ Test actually loading the program activity data """ monkeypatch.setattr(load_program_activity, 'CONFIG_BROKER', {'use_aws': False}) mocked_get_pa_file.return_value = StringIO( """AGENCY_CODE,ALLOCATION_ID,ACCOUNT_CODE,PA_CODE,PA_TITLE,FYQ\n2000,000,111,0000,1111,Test Name,FY2015Q1""" ) mocked_get_current_date.return_value = datetime.datetime( 2017, 12, 31, 0, 0, 0) mocked_get_stored_date.return_value = datetime.datetime( 2016, 12, 31, 0, 0, 0) mocked_set_stored_date.return_value = None sess = database.session load_program_activity.load_program_activity_data('some_path') pa = sess.query(ProgramActivity).one_or_none() assert pa.fiscal_year_quarter == 'FY2015Q1' assert pa.agency_id == '000' assert pa.allocation_transfer_id == '111' assert pa.account_number == '0000' assert pa.program_activity_code == '1111' assert pa.program_activity_name == 'test name'
def test_load_program_activity_data(mocked_get_pa_file, mocked_get_current_date, mocked_get_stored_date, mocked_set_stored_date, database, monkeypatch): """ Test actually loading the program activity data """ monkeypatch.setattr(load_program_activity, 'CONFIG_BROKER', {'use_aws': False}) mocked_get_pa_file.return_value = StringIO( """AGENCY_CODE,ALLOCATION_ID,ACCOUNT_CODE,PA_CODE,PA_TITLE,FYQ\n2000,000,111,0000,1111,Test Name,FY2015Q1""" ) mocked_get_current_date.return_value = datetime.datetime(2017, 12, 31, 0, 0, 0) mocked_get_stored_date.return_value = datetime.datetime(2016, 12, 31, 0, 0, 0) mocked_set_stored_date.return_value = None sess = database.session load_program_activity.load_program_activity_data('some_path') pa = sess.query(ProgramActivity).one_or_none() assert pa.fiscal_year_quarter == 'FY2015Q1' assert pa.agency_id == '000' assert pa.allocation_transfer_id == '111' assert pa.account_number == '0000' assert pa.program_activity_code == '1111' assert pa.program_activity_name == 'test name'
def load_domain_value_files(base_path): """Load domain values (Country codes, Program Activity, Object Class, CFDA).""" logger.info('Loading Object Class') load_object_class(base_path) logger.info('Loading CFDA Program') load_cfda_program(base_path) logger.info('Loading Program Activity') load_program_activity_data(base_path) logger.info('Loading Country codes') load_country_codes(base_path)
def load_domain_value_files(base_path): """Load domain values (Country codes, Program Activity, Object Class, CFDA).""" logger.info('Loading Object Class') load_object_class(base_path) logger.info('Loading CFDA Program') load_cfda_program(base_path) logger.info('Loading Program Activity') load_program_activity_data(base_path) logger.info('Loading Country codes') load_country_codes(base_path)
def test_load_program_activity_data_empty_file(mocked_get_pa_file, mocked_get_current_date, mocked_get_stored_date, mocked_set_stored_date, monkeypatch): """ Test actually loading the program activity data """ monkeypatch.setattr(load_program_activity, 'CONFIG_BROKER', {'use_aws': False, 'local': False}) mocked_get_pa_file.return_value = StringIO("") mocked_get_current_date.return_value = datetime.datetime(2017, 12, 31, 0, 0, 0) mocked_get_stored_date.return_value = datetime.datetime(2016, 12, 31, 0, 0, 0) mocked_set_stored_date.return_value = None with pytest.raises(SystemExit) as se: load_program_activity.load_program_activity_data('some_path') assert se.value.code == 4
def test_load_program_activity_data_only_header(mocked_get_pa_file, mocked_get_current_date, mocked_get_stored_date, mocked_set_stored_date, monkeypatch): """ Test actually loading the program activity data """ monkeypatch.setattr(load_program_activity, 'CONFIG_BROKER', { 'use_aws': False, 'local': False }) mocked_get_pa_file.return_value = StringIO( """AGENCY_CODE,ALLOCATION_ID,ACCOUNT_CODE,PA_CODE,PA_TITLE,FYQ""") mocked_get_current_date.return_value = datetime.datetime( 2017, 12, 31, 0, 0, 0) mocked_get_stored_date.return_value = datetime.datetime( 2016, 12, 31, 0, 0, 0) mocked_set_stored_date.return_value = None with pytest.raises(SystemExit) as se: load_program_activity.load_program_activity_data('some_path') assert se.value.code == 4
def main(): parser = argparse.ArgumentParser( description='Initialize the DATA Act Broker.') parser.add_argument('-i', '--initialize', help='Run all broker initialization tasks', action='store_true') parser.add_argument('-db', '--setup_db', help='Create broker database and helper tables', action='store_true') parser.add_argument('-a', '--create_admin', help='Create an admin user', action='store_true') parser.add_argument('-r', '--load_rules', help='Load SQL-based validation rules', action='store_true') parser.add_argument( '-d', '--update_domain', help='load slowly changing domain values such as object class', action='store_true') parser.add_argument('-cc', '--update_country_codes', help='update country codes', action='store_true') parser.add_argument('-oc', '--update_object_class', help='load object class to database', action='store_true') parser.add_argument('-cfda', '--cfda_load', help='Load CFDA to database', action='store_true') parser.add_argument('-pa', '--program_activity', help='Load program activity to database', action='store_true') parser.add_argument( '-c', '--load_agencies', help='Update agency data (CGACs, FRECs, SubTierAgencies)', action='store_true') parser.add_argument('-t', '--update_tas', help='Update broker TAS list', action='store_true') parser.add_argument('-s', '--update_sf133', help='Update broker SF-133 reports', action='store_true') parser.add_argument('-v', '--update_validator', help='Update validator schema', action='store_true') parser.add_argument('-l', '--load_location', help='Load city and county codes', action='store_true') parser.add_argument('-z', '--load_zips', help='Load zip code data', action='store_true') parser.add_argument('-o', '--load_offices', help='Load FPDS Office Codes', action='store_true') parser.add_argument('-u', '--uncache_file_requests', help='Un-cache file generation requests', action='store_true') args = parser.parse_args() if args.initialize: setup_db() load_sql_rules() load_domain_value_files(validator_config_path) load_agency_data(validator_config_path) load_tas_lookup() load_sf133() load_validator_schema() load_location_codes() load_zip_codes() load_offices() return if args.setup_db: setup_db() if args.create_admin: create_admin() if args.load_rules: load_sql_rules() if args.update_domain: load_domain_value_files(validator_config_path) if args.update_country_codes: load_country_codes(validator_config_path) if args.update_object_class: load_object_class(validator_config_path) if args.cfda_load: load_cfda_program(validator_config_path) if args.program_activity: load_program_activity_data(validator_config_path) if args.load_agencies: load_agency_data(validator_config_path) if args.update_tas: load_tas_lookup() if args.update_sf133: load_sf133() if args.update_validator: load_validator_schema() if args.load_location: load_location_codes() if args.load_zips: load_zip_codes() load_location_codes() if args.load_offices: load_offices() if args.uncache_file_requests: uncache_file_requests()
def main(): parser = argparse.ArgumentParser(description='Initialize the DATA Act Broker.') parser.add_argument('-i', '--initialize', help='Run all broker initialization tasks', action='store_true') parser.add_argument('-db', '--setup_db', help='Create broker database and helper tables', action='store_true') parser.add_argument('-a', '--create_admin', help='Create an admin user', action='store_true') parser.add_argument('-r', '--load_rules', help='Load SQL-based validation rules', action='store_true') parser.add_argument('-d', '--update_domain', help='load slowly changing domain values such as object class', action='store_true') parser.add_argument('-cc', '--update_country_codes', help='update country codes', action='store_true') parser.add_argument('-oc', '--update_object_class', help='load object class to database', action='store_true') parser.add_argument('-cfda', '--cfda_load', help='Load CFDA to database', action='store_true') parser.add_argument('-pa', '--program_activity', help='Load program activity to database', action='store_true') parser.add_argument('-c', '--load_agencies', help='Update agency data (CGACs, FRECs, SubTierAgencies)', action='store_true') parser.add_argument('-t', '--update_tas', help='Update broker TAS list', action='store_true') parser.add_argument('-s', '--update_sf133', help='Update broker SF-133 reports', action='store_true') parser.add_argument('-v', '--update_validator', help='Update validator schema', action='store_true') parser.add_argument('-l', '--load_location', help='Load city and county codes', action='store_true') parser.add_argument('-z', '--load_zips', help='Load zip code data', action='store_true') parser.add_argument('-o', '--load_offices', help='Load FPDS Office Codes', action='store_true') parser.add_argument('-u', '--uncache_all_files', help='Un-cache file generation requests', action='store_true') args = parser.parse_args() if args.initialize: setup_db() load_sql_rules() load_domain_value_files(validator_config_path) load_agency_data(validator_config_path) load_tas_lookup() load_sf133() load_validator_schema() load_location_codes() load_zip_codes() load_offices() return if args.setup_db: setup_db() if args.create_admin: create_admin() if args.load_rules: load_sql_rules() if args.update_domain: load_domain_value_files(validator_config_path) if args.update_country_codes: load_country_codes(validator_config_path) if args.update_object_class: load_object_class(validator_config_path) if args.cfda_load: load_cfda_program(validator_config_path) if args.program_activity: load_program_activity_data(validator_config_path) if args.load_agencies: load_agency_data(validator_config_path) if args.update_tas: load_tas_lookup() if args.update_sf133: load_sf133() if args.update_validator: load_validator_schema() if args.load_location: load_location_codes() if args.load_zips: load_zip_codes() load_location_codes() if args.load_offices: load_offices() if args.uncache_all_files: uncache_all_files()