def test_load_program_activity_data_empty_file(mocked_get_pa_file,
mocked_get_current_date,
mocked_get_stored_date,
mocked_set_stored_date,
monkeypatch):
""" Test actually loading the program activity data """
monkeypatch.setattr(load_program_activity, 'CONFIG_BROKER', {
'use_aws': False,
'local': False
})
mocked_get_pa_file.return_value = StringIO("")
mocked_get_current_date.return_value = datetime.datetime(
2017, 12, 31, 0, 0, 0)
mocked_get_stored_date.return_value = datetime.datetime(
2016, 12, 31, 0, 0, 0)
mocked_set_stored_date.return_value = None
with pytest.raises(SystemExit) as se:
load_program_activity.load_program_activity_data('some_path')
assert se.value.code == 4
remove_metrics_file()
def test_load_program_activity_data(mocked_get_pa_file,
mocked_get_current_date,
mocked_get_stored_date,
mocked_set_stored_date, database,
monkeypatch):
""" Test actually loading the program activity data """
monkeypatch.setattr(load_program_activity, 'CONFIG_BROKER',
{'use_aws': False})
mocked_get_pa_file.return_value = StringIO(
"""AGENCY_CODE,ALLOCATION_ID,ACCOUNT_CODE,PA_CODE,PA_TITLE,FYQ\n2000,000,111,0000,1111,Test Name,FY2015Q1"""
)
mocked_get_current_date.return_value = datetime.datetime(
2017, 12, 31, 0, 0, 0)
mocked_get_stored_date.return_value = datetime.datetime(
2016, 12, 31, 0, 0, 0)
mocked_set_stored_date.return_value = None
sess = database.session
load_program_activity.load_program_activity_data('some_path')
pa = sess.query(ProgramActivity).one_or_none()
assert pa.fiscal_year_quarter == 'FY2015Q1'
assert pa.agency_id == '000'
assert pa.allocation_transfer_id == '111'
assert pa.account_number == '0000'
assert pa.program_activity_code == '1111'
assert pa.program_activity_name == 'test name'
def test_load_program_activity_data(mocked_get_pa_file, mocked_get_current_date, mocked_get_stored_date,
mocked_set_stored_date, database, monkeypatch):
""" Test actually loading the program activity data """
monkeypatch.setattr(load_program_activity, 'CONFIG_BROKER', {'use_aws': False})
mocked_get_pa_file.return_value = StringIO(
"""AGENCY_CODE,ALLOCATION_ID,ACCOUNT_CODE,PA_CODE,PA_TITLE,FYQ\n2000,000,111,0000,1111,Test Name,FY2015Q1"""
)
mocked_get_current_date.return_value = datetime.datetime(2017, 12, 31, 0, 0, 0)
mocked_get_stored_date.return_value = datetime.datetime(2016, 12, 31, 0, 0, 0)
mocked_set_stored_date.return_value = None
sess = database.session
load_program_activity.load_program_activity_data('some_path')
pa = sess.query(ProgramActivity).one_or_none()
assert pa.fiscal_year_quarter == 'FY2015Q1'
assert pa.agency_id == '000'
assert pa.allocation_transfer_id == '111'
assert pa.account_number == '0000'
assert pa.program_activity_code == '1111'
assert pa.program_activity_name == 'test name'
def load_domain_value_files(base_path):
"""Load domain values (Country codes, Program Activity, Object Class, CFDA)."""
logger.info('Loading Object Class')
load_object_class(base_path)
logger.info('Loading CFDA Program')
load_cfda_program(base_path)
logger.info('Loading Program Activity')
load_program_activity_data(base_path)
logger.info('Loading Country codes')
load_country_codes(base_path)
def load_domain_value_files(base_path):
"""Load domain values (Country codes, Program Activity, Object Class, CFDA)."""
logger.info('Loading Object Class')
load_object_class(base_path)
logger.info('Loading CFDA Program')
load_cfda_program(base_path)
logger.info('Loading Program Activity')
load_program_activity_data(base_path)
logger.info('Loading Country codes')
load_country_codes(base_path)
def test_load_program_activity_data_empty_file(mocked_get_pa_file, mocked_get_current_date, mocked_get_stored_date,
mocked_set_stored_date, monkeypatch):
""" Test actually loading the program activity data """
monkeypatch.setattr(load_program_activity, 'CONFIG_BROKER', {'use_aws': False, 'local': False})
mocked_get_pa_file.return_value = StringIO("")
mocked_get_current_date.return_value = datetime.datetime(2017, 12, 31, 0, 0, 0)
mocked_get_stored_date.return_value = datetime.datetime(2016, 12, 31, 0, 0, 0)
mocked_set_stored_date.return_value = None
with pytest.raises(SystemExit) as se:
load_program_activity.load_program_activity_data('some_path')
assert se.value.code == 4
def test_load_program_activity_data_only_header(mocked_get_pa_file,
mocked_get_current_date,
mocked_get_stored_date,
mocked_set_stored_date,
monkeypatch):
""" Test actually loading the program activity data """
monkeypatch.setattr(load_program_activity, 'CONFIG_BROKER', {
'use_aws': False,
'local': False
})
mocked_get_pa_file.return_value = StringIO(
"""AGENCY_CODE,ALLOCATION_ID,ACCOUNT_CODE,PA_CODE,PA_TITLE,FYQ""")
mocked_get_current_date.return_value = datetime.datetime(
2017, 12, 31, 0, 0, 0)
mocked_get_stored_date.return_value = datetime.datetime(
2016, 12, 31, 0, 0, 0)
mocked_set_stored_date.return_value = None
with pytest.raises(SystemExit) as se:
load_program_activity.load_program_activity_data('some_path')
assert se.value.code == 4
def main():
parser = argparse.ArgumentParser(
description='Initialize the DATA Act Broker.')
parser.add_argument('-i',
'--initialize',
help='Run all broker initialization tasks',
action='store_true')
parser.add_argument('-db',
'--setup_db',
help='Create broker database and helper tables',
action='store_true')
parser.add_argument('-a',
'--create_admin',
help='Create an admin user',
action='store_true')
parser.add_argument('-r',
'--load_rules',
help='Load SQL-based validation rules',
action='store_true')
parser.add_argument(
'-d',
'--update_domain',
help='load slowly changing domain values such as object class',
action='store_true')
parser.add_argument('-cc',
'--update_country_codes',
help='update country codes',
action='store_true')
parser.add_argument('-oc',
'--update_object_class',
help='load object class to database',
action='store_true')
parser.add_argument('-cfda',
'--cfda_load',
help='Load CFDA to database',
action='store_true')
parser.add_argument('-pa',
'--program_activity',
help='Load program activity to database',
action='store_true')
parser.add_argument(
'-c',
'--load_agencies',
help='Update agency data (CGACs, FRECs, SubTierAgencies)',
action='store_true')
parser.add_argument('-t',
'--update_tas',
help='Update broker TAS list',
action='store_true')
parser.add_argument('-s',
'--update_sf133',
help='Update broker SF-133 reports',
action='store_true')
parser.add_argument('-v',
'--update_validator',
help='Update validator schema',
action='store_true')
parser.add_argument('-l',
'--load_location',
help='Load city and county codes',
action='store_true')
parser.add_argument('-z',
'--load_zips',
help='Load zip code data',
action='store_true')
parser.add_argument('-o',
'--load_offices',
help='Load FPDS Office Codes',
action='store_true')
parser.add_argument('-u',
'--uncache_file_requests',
help='Un-cache file generation requests',
action='store_true')
args = parser.parse_args()
if args.initialize:
setup_db()
load_sql_rules()
load_domain_value_files(validator_config_path)
load_agency_data(validator_config_path)
load_tas_lookup()
load_sf133()
load_validator_schema()
load_location_codes()
load_zip_codes()
load_offices()
return
if args.setup_db:
setup_db()
if args.create_admin:
create_admin()
if args.load_rules:
load_sql_rules()
if args.update_domain:
load_domain_value_files(validator_config_path)
if args.update_country_codes:
load_country_codes(validator_config_path)
if args.update_object_class:
load_object_class(validator_config_path)
if args.cfda_load:
load_cfda_program(validator_config_path)
if args.program_activity:
load_program_activity_data(validator_config_path)
if args.load_agencies:
load_agency_data(validator_config_path)
if args.update_tas:
load_tas_lookup()
if args.update_sf133:
load_sf133()
if args.update_validator:
load_validator_schema()
if args.load_location:
load_location_codes()
if args.load_zips:
load_zip_codes()
load_location_codes()
if args.load_offices:
load_offices()
if args.uncache_file_requests:
uncache_file_requests()
def main():
parser = argparse.ArgumentParser(description='Initialize the DATA Act Broker.')
parser.add_argument('-i', '--initialize', help='Run all broker initialization tasks', action='store_true')
parser.add_argument('-db', '--setup_db', help='Create broker database and helper tables', action='store_true')
parser.add_argument('-a', '--create_admin', help='Create an admin user', action='store_true')
parser.add_argument('-r', '--load_rules', help='Load SQL-based validation rules', action='store_true')
parser.add_argument('-d', '--update_domain', help='load slowly changing domain values such as object class',
action='store_true')
parser.add_argument('-cc', '--update_country_codes', help='update country codes', action='store_true')
parser.add_argument('-oc', '--update_object_class', help='load object class to database', action='store_true')
parser.add_argument('-cfda', '--cfda_load', help='Load CFDA to database', action='store_true')
parser.add_argument('-pa', '--program_activity', help='Load program activity to database', action='store_true')
parser.add_argument('-c', '--load_agencies', help='Update agency data (CGACs, FRECs, SubTierAgencies)',
action='store_true')
parser.add_argument('-t', '--update_tas', help='Update broker TAS list', action='store_true')
parser.add_argument('-s', '--update_sf133', help='Update broker SF-133 reports', action='store_true')
parser.add_argument('-v', '--update_validator', help='Update validator schema', action='store_true')
parser.add_argument('-l', '--load_location', help='Load city and county codes', action='store_true')
parser.add_argument('-z', '--load_zips', help='Load zip code data', action='store_true')
parser.add_argument('-o', '--load_offices', help='Load FPDS Office Codes', action='store_true')
parser.add_argument('-u', '--uncache_all_files', help='Un-cache file generation requests', action='store_true')
args = parser.parse_args()
if args.initialize:
setup_db()
load_sql_rules()
load_domain_value_files(validator_config_path)
load_agency_data(validator_config_path)
load_tas_lookup()
load_sf133()
load_validator_schema()
load_location_codes()
load_zip_codes()
load_offices()
return
if args.setup_db:
setup_db()
if args.create_admin:
create_admin()
if args.load_rules:
load_sql_rules()
if args.update_domain:
load_domain_value_files(validator_config_path)
if args.update_country_codes:
load_country_codes(validator_config_path)
if args.update_object_class:
load_object_class(validator_config_path)
if args.cfda_load:
load_cfda_program(validator_config_path)
if args.program_activity:
load_program_activity_data(validator_config_path)
if args.load_agencies:
load_agency_data(validator_config_path)
if args.update_tas:
load_tas_lookup()
if args.update_sf133:
load_sf133()
if args.update_validator:
load_validator_schema()
if args.load_location:
load_location_codes()
if args.load_zips:
load_zip_codes()
load_location_codes()
if args.load_offices:
load_offices()
if args.uncache_all_files:
uncache_all_files()