def test_success(database): """ If both are submitted, AwardingSubTierAgencyCode and AwardingOfficeCode must belong to the same AwardingAgencyCode (per the Federal Hierarchy). Ignored if one or both are missing. """ cgac = CGAC(cgac_id=1, cgac_code='001', agency_name='test') frec = FREC(frec_id=1, cgac_id=1, frec_code='0001', agency_name='test2') # sub tier codes are different on these offices to prove that we don't care if the office is under that sub tier # as long as the top tier codes match office_1 = OfficeFactory(office_code='12345a', sub_tier_code='abcd', agency_code=cgac.cgac_code) office_2 = OfficeFactory(office_code='123457', sub_tier_code='efgh', agency_code=frec.frec_code) agency_1 = SubTierAgency(sub_tier_agency_code='0000', cgac_id=1, frec_id=1, is_frec=False) agency_2 = SubTierAgency(sub_tier_agency_code='0001', cgac_id=1, frec_id=1, is_frec=True) # Same agency for cgac det_award_1 = DetachedAwardFinancialAssistanceFactory(awarding_sub_tier_agency_c=agency_1.sub_tier_agency_code, awarding_office_code=office_1.office_code) # Same agency for cgac (uppercased) det_award_2 = DetachedAwardFinancialAssistanceFactory(awarding_sub_tier_agency_c=agency_1.sub_tier_agency_code, awarding_office_code=office_1.office_code.upper()) # Same agency for frec det_award_3 = DetachedAwardFinancialAssistanceFactory(awarding_sub_tier_agency_c=agency_2.sub_tier_agency_code, awarding_office_code=office_2.office_code) # Missing sub tier code det_award_4 = DetachedAwardFinancialAssistanceFactory(awarding_sub_tier_agency_c='', awarding_office_code=office_2.office_code) # Missing office code det_award_5 = DetachedAwardFinancialAssistanceFactory(awarding_sub_tier_agency_c=agency_1.sub_tier_agency_code, awarding_office_code=None) errors = number_of_errors(_FILE, database, models=[cgac, frec, office_1, office_2, agency_1, agency_2, det_award_1, det_award_2, det_award_3, det_award_4, det_award_5]) assert errors == 0
def insert_agency_user_submission_data(sess, submission_id): """Insert jobs for the submission, and create a CGAC, FREC, and SubTierAgency""" for job_type in ['file_upload', 'csv_record_validation', 'validation']: sess.add( Job(file_type_id=FILE_TYPE_DICT['fabs'], job_status_id=FILE_STATUS_DICT['complete'], job_type_id=JOB_TYPE_DICT[job_type], submission_id=submission_id, original_filename=None, file_size=None, number_of_rows=None)) sess.commit() cgac = CGAC(cgac_code="NOT") sess.add(cgac) sess.commit() frec = FREC(cgac_id=cgac.cgac_id, frec_code="BLAH") sess.add(frec) sess.commit() sub = SubTierAgency(sub_tier_agency_code="WRONG", cgac_id=cgac.cgac_id, frec_id=frec.frec_id, is_frec=False) sess.add(sub) sess.commit()
def test_success(database): """ Test FundingSubTierAgencyCode is an optional field, but when provided must be a valid 4-digit sub-tier agency code. """ subcode = SubTierAgency(sub_tier_agency_code='A000', cgac_id='1') cgac = CGAC(cgac_id='1', cgac_code='001', agency_name='test') det_award = DetachedAwardFinancialAssistanceFactory( funding_sub_tier_agency_co='A000', correction_delete_indicatr='') det_award_2 = DetachedAwardFinancialAssistanceFactory( funding_sub_tier_agency_co='a000', correction_delete_indicatr=None) det_award_3 = DetachedAwardFinancialAssistanceFactory( funding_sub_tier_agency_co=None, correction_delete_indicatr='c') det_award_4 = DetachedAwardFinancialAssistanceFactory( funding_sub_tier_agency_co='', correction_delete_indicatr='C') # Ignore correction delete indicator of D det_award_5 = DetachedAwardFinancialAssistanceFactory( funding_sub_tier_agency_co='bad', correction_delete_indicatr='d') errors = number_of_errors(_FILE, database, models=[ det_award, det_award_2, det_award_3, det_award_4, det_award_5, subcode, cgac ]) assert errors == 0
def update_sub_tier_agencies(models, new_data, cgac_dict, frec_dict): """Modify existing models or create new ones""" for _, row in new_data.iterrows(): if row['cgac_code'] not in cgac_dict: new_data.drop(_) continue row['cgac_id'] = cgac_dict[row['cgac_code']] row['frec_id'] = frec_dict[row['frec_code']] sub_tier_agency_code = row['sub_tier_agency_code'] if sub_tier_agency_code not in models: models[sub_tier_agency_code] = SubTierAgency() for field, value in row.items(): setattr(models[sub_tier_agency_code], field, value)
def test_success(database): """ AwardingSubTierAgencyCode must be a valid 4-digit sub-tier agency code. """ agency = SubTierAgency(sub_tier_agency_code='0000', cgac_id='1') cgac = CGAC(cgac_id='1', cgac_code='001', agency_name='test') det_award = DetachedAwardFinancialAssistanceFactory( awarding_sub_tier_agency_c=agency.sub_tier_agency_code) det_award_2 = DetachedAwardFinancialAssistanceFactory( awarding_sub_tier_agency_c='0000') errors = number_of_errors(_FILE, database, models=[det_award, det_award_2, agency, cgac]) assert errors == 0
def test_failure(database): """ Test failure if both are submitted, AwardingSubTierAgencyCode and AwardingOfficeCode must belong to the same AwardingAgencyCode (per the Federal Hierarchy). """ cgac = CGAC(cgac_id=1, cgac_code='001', agency_name='test') frec = FREC(frec_id=1, cgac_id=1, frec_code='0001', agency_name='test2') office = OfficeFactory(office_code='123456', sub_tier_code='abcd', agency_code=cgac.cgac_code) agency = SubTierAgency(sub_tier_agency_code='0000', frec_id=1, cgac_id=1, is_frec=True) # Sub tier is FREC, office is based on CGAC, the numbers are different det_award = DetachedAwardFinancialAssistanceFactory(awarding_sub_tier_agency_c=agency.sub_tier_agency_code, awarding_office_code=office.office_code) errors = number_of_errors(_FILE, database, models=[det_award, cgac, frec, office, agency]) assert errors == 1
def test_success(database): """ FundingSubTierAgencyCode is an optional field, but when provided must be a valid 4-digit sub-tier agency code. """ subcode = SubTierAgency(sub_tier_agency_code='0000', cgac_id='1') cgac = CGAC(cgac_id='1', cgac_code='001', agency_name='test') det_award = DetachedAwardFinancialAssistanceFactory( funding_sub_tier_agency_co='0000') det_award_2 = DetachedAwardFinancialAssistanceFactory( funding_sub_tier_agency_co=None) det_award_3 = DetachedAwardFinancialAssistanceFactory( funding_sub_tier_agency_co='') errors = number_of_errors( _FILE, database, models=[det_award, det_award_2, det_award_3, subcode, cgac]) assert errors == 0
def test_success(database): """ AwardingSubTierAgencyCode must be a valid 4-digit sub-tier agency code. Doesn't fail when code not provided. """ agency = SubTierAgency(sub_tier_agency_code='a000', cgac_id='1') cgac = CGAC(cgac_id='1', cgac_code='001', agency_name='test') fabs = FABSFactory( awarding_sub_tier_agency_c=agency.sub_tier_agency_code.upper(), correction_delete_indicatr='') fabs_2 = FABSFactory(awarding_sub_tier_agency_c=None, correction_delete_indicatr=None) fabs_3 = FABSFactory(awarding_sub_tier_agency_c='', correction_delete_indicatr='c') # Ignore correction delete indicator of D fabs_4 = FABSFactory(awarding_sub_tier_agency_c='bad', correction_delete_indicatr='d') errors = number_of_errors( _FILE, database, models=[fabs, fabs_2, fabs_3, fabs_4, agency, cgac]) assert errors == 0
def update_sub_tier_agencies(models, new_data, cgac_dict, frec_dict): """ Modify existing models or create new ones Args: models: the list of existing models new_data: the data that was read in from the file cgac_dict: a dictionary of all cgac codes in the database frec_dict: a dictionary of all frec codes in the database """ for _, row in new_data.iterrows(): if row['cgac_code'] not in cgac_dict: new_data.drop(_) continue row['cgac_id'] = cgac_dict[row['cgac_code']] row['frec_id'] = frec_dict[row['frec_code']] sub_tier_agency_code = row['sub_tier_agency_code'] if sub_tier_agency_code not in models: models[sub_tier_agency_code] = SubTierAgency() for field, value in row.items(): setattr(models[sub_tier_agency_code], field, value)
def test_success(database): """ AwardingSubTierAgencyCode must be a valid 4-digit sub-tier agency code. Doesn't fail when code not provided. """ agency = SubTierAgency(sub_tier_agency_code='a000', cgac_id='1') cgac = CGAC(cgac_id='1', cgac_code='001', agency_name='test') det_award = DetachedAwardFinancialAssistanceFactory( awarding_sub_tier_agency_c=agency.sub_tier_agency_code.upper(), correction_delete_indicatr='') det_award_2 = DetachedAwardFinancialAssistanceFactory( awarding_sub_tier_agency_c=None, correction_delete_indicatr=None) det_award_3 = DetachedAwardFinancialAssistanceFactory( awarding_sub_tier_agency_c='', correction_delete_indicatr='c') # Ignore correction delete indicator of D det_award_4 = DetachedAwardFinancialAssistanceFactory( awarding_sub_tier_agency_c='bad', correction_delete_indicatr='d') errors = number_of_errors(_FILE, database, models=[ det_award, det_award_2, det_award_3, det_award_4, agency, cgac ]) assert errors == 0
def test_failure(database): """ Test failure if both are submitted, FundingSubTierAgencyCode and FundingOfficeCode must belong to the same FundingAgencyCode (per the Federal Hierarchy). """ cgac = CGAC(cgac_id=1, cgac_code='001', agency_name='test') frec = FREC(frec_id=1, cgac_id=1, frec_code='0001', agency_name='test2') office = OfficeFactory(office_code='123456', sub_tier_code='abcd', agency_code=cgac.cgac_code) agency = SubTierAgency(sub_tier_agency_code='0000', frec_id=1, cgac_id=1, is_frec=True) # Sub tier is FREC, office is based on CGAC, the numbers are different fabs = FABSFactory(funding_sub_tier_agency_co=agency.sub_tier_agency_code, funding_office_code=office.office_code, correction_delete_indicatr='') errors = number_of_errors(_FILE, database, models=[fabs, cgac, frec, office, agency]) assert errors == 1
def test_calculate_remaining_fields(database): """ Test that calculate_remaining_fields calculates fields based on content in the DB and inserts 999 for the code if the sub tier agency doesn't exist """ cgac = CGAC(cgac_id=1, cgac_code='1700', agency_name='test name') sub_tier = SubTierAgency(sub_tier_agency_code='0000', cgac_id=1) database.session.add(cgac) database.session.add(sub_tier) database.session.commit() tmp_obj = pullFPDSData.calculate_remaining_fields( { 'awarding_sub_tier_agency_c': "0000", 'funding_sub_tier_agency_co': None }, {sub_tier.sub_tier_agency_code: sub_tier}) tmp_obj_2 = pullFPDSData.calculate_remaining_fields( { 'awarding_sub_tier_agency_c': None, 'funding_sub_tier_agency_co': "0001", 'funding_sub_tier_agency_na': "Not Real" }, {sub_tier.sub_tier_agency_code: sub_tier}) assert tmp_obj['awarding_agency_code'] == '1700' assert tmp_obj['awarding_agency_name'] == 'test name' assert tmp_obj_2['funding_agency_code'] == '999' assert tmp_obj_2['funding_agency_name'] is None
def test_success(database): """ Test FundingSubTierAgencyCode is an optional field, but when provided must be a valid 4-digit sub-tier agency code. """ subcode = SubTierAgency(sub_tier_agency_code='A000', cgac_id='1') cgac = CGAC(cgac_id='1', cgac_code='001', agency_name='test') fabs = FABSFactory(funding_sub_tier_agency_co='A000', correction_delete_indicatr='') fabs_2 = FABSFactory(funding_sub_tier_agency_co='a000', correction_delete_indicatr=None) fabs_3 = FABSFactory(funding_sub_tier_agency_co=None, correction_delete_indicatr='c') fabs_4 = FABSFactory(funding_sub_tier_agency_co='', correction_delete_indicatr='C') # Ignore correction delete indicator of D fabs_5 = FABSFactory(funding_sub_tier_agency_co='bad', correction_delete_indicatr='d') errors = number_of_errors( _FILE, database, models=[fabs, fabs_2, fabs_3, fabs_4, fabs_5, subcode, cgac]) assert errors == 0
def test_success(database): """ If both are submitted, AwardingSubTierAgencyCode and AwardingOfficeCode must belong to the same AwardingAgencyCode (per the Federal Hierarchy). Ignored if one or both are missing. """ cgac = CGAC(cgac_id=1, cgac_code='001', agency_name='test') frec = FREC(frec_id=1, cgac_id=1, frec_code='0001', agency_name='test2') # sub tier codes are different on these offices to prove that we don't care if the office is under that sub tier # as long as the top tier codes match office_1 = OfficeFactory(office_code='12345a', sub_tier_code='abcd', agency_code=cgac.cgac_code) office_2 = OfficeFactory(office_code='123457', sub_tier_code='efgh', agency_code=frec.frec_code) agency_1 = SubTierAgency(sub_tier_agency_code='a000', cgac_id=1, frec_id=1, is_frec=False) agency_2 = SubTierAgency(sub_tier_agency_code='0001', cgac_id=1, frec_id=1, is_frec=True) # Same agency for cgac fabs_1 = FABSFactory( awarding_sub_tier_agency_c=agency_1.sub_tier_agency_code, awarding_office_code=office_1.office_code, correction_delete_indicatr='') # Same agency for cgac (uppercased) fabs_2 = FABSFactory( awarding_sub_tier_agency_c=agency_1.sub_tier_agency_code.upper(), awarding_office_code=office_1.office_code.upper(), correction_delete_indicatr=None) # Same agency for frec fabs_3 = FABSFactory( awarding_sub_tier_agency_c=agency_2.sub_tier_agency_code, awarding_office_code=office_2.office_code, correction_delete_indicatr='c') # Missing sub tier code fabs_4 = FABSFactory(awarding_sub_tier_agency_c='', awarding_office_code=office_2.office_code, correction_delete_indicatr='C') # Missing office code fabs_5 = FABSFactory( awarding_sub_tier_agency_c=agency_1.sub_tier_agency_code, awarding_office_code=None, correction_delete_indicatr='') # Ignore correction delete indicator of D fabs_6 = FABSFactory( awarding_sub_tier_agency_c=agency_1.sub_tier_agency_code, awarding_office_code=office_2.office_code, correction_delete_indicatr='d') errors = number_of_errors(_FILE, database, models=[ cgac, frec, office_1, office_2, agency_1, agency_2, fabs_1, fabs_2, fabs_3, fabs_4, fabs_5, fabs_6 ]) assert errors == 0
def test_calculate_remaining_fields(database): """ Test that calculate_remaining_fields calculates fields based on content in the DB and inserts 999 for the code if the sub tier agency doesn't exist """ sess = database.session cgac = CGAC(cgac_id=1, cgac_code='1700', agency_name='test name') zip_code = Zips(zip5='12345', zip_last4='6789', county_number='123') sub_tier = SubTierAgency(sub_tier_agency_code='0000', cgac_id=1) sess.add(cgac) sess.add(zip_code) sess.add(sub_tier) sess.commit() county_by_name = { 'GA': { 'COUNTY ONE': '123', 'GA COUNTY TWO': '321' }, 'MD': { 'JUST ONE MD': '024' } } county_by_code = { 'MD': { '024': 'JUST ONE MD' }, 'GA': { '123': 'COUNTY ONE' }, 'GU': { '123': 'GU COUNTY' } } state_codes = { 'GA': 'GEORGIA', 'MD': 'MARYLAND', 'PR': 'PUERTO RICO', 'GU': 'GUAM' } country_list = {'USA': 'UNITED STATES'} # build business category values business_category_dict = {} for field in BUSINESS_CATEGORY_FIELDS: business_category_dict[field] = None tmp_obj_data = { 'awarding_sub_tier_agency_c': "0000", 'funding_sub_tier_agency_co': None, 'place_of_perform_county_na': 'JUST ONE MD', 'place_of_performance_state': 'MD', 'place_of_perfor_state_desc': None, 'place_of_perform_country_c': 'USA', 'place_of_perf_country_desc': 'UNITED STATES', 'place_of_performance_zip4a': None, 'legal_entity_zip4': '987654321', 'legal_entity_country_code': 'USA', 'legal_entity_country_name': 'UNITED STATES', 'legal_entity_state_code': 'GA', 'legal_entity_state_descrip': 'GEORGIA' } tmp_obj_data.update(business_category_dict.copy()) tmp_obj_data['emerging_small_business'] = 'Y' tmp_obj = pullFPDSData.calculate_remaining_fields( tmp_obj_data, sess, {sub_tier.sub_tier_agency_code: sub_tier}, county_by_name, county_by_code, state_codes, country_list) tmp_obj_2_data = { 'awarding_sub_tier_agency_c': None, 'funding_sub_tier_agency_co': "0001", 'funding_sub_tier_agency_na': "Not Real", 'place_of_perform_county_na': 'JUST ONE MD', 'place_of_performance_state': 'GA', 'place_of_perfor_state_desc': 'GEORGIA', 'place_of_perform_country_c': 'USA', 'place_of_perf_country_desc': 'UNITED STATES', 'place_of_performance_zip4a': None, 'legal_entity_zip4': '123456789', 'legal_entity_country_code': 'USA', 'legal_entity_country_name': 'UNITED STATES', 'legal_entity_state_code': 'GA', 'legal_entity_state_descrip': 'GEORGIA' } tmp_obj_2_data.update(business_category_dict.copy()) tmp_obj_2_data['contracting_officers_deter'] = 'O' tmp_obj_2 = pullFPDSData.calculate_remaining_fields( tmp_obj_2_data, sess, {sub_tier.sub_tier_agency_code: sub_tier}, county_by_name, county_by_code, state_codes, country_list) tmp_obj_3_data = { 'awarding_sub_tier_agency_c': None, 'funding_sub_tier_agency_co': None, 'funding_sub_tier_agency_na': None, 'place_of_perform_county_na': None, 'place_of_performance_state': None, 'place_of_perfor_state_desc': None, 'place_of_perform_country_c': 'PRI', 'place_of_perf_country_desc': 'PUERTO RICO', 'place_of_performance_zip4a': '123456789', 'legal_entity_zip4': '12345', 'legal_entity_country_code': 'GUM', 'legal_entity_country_name': 'GUAM', 'legal_entity_state_code': 'GA', 'legal_entity_state_descrip': 'GEORGIA' } tmp_obj_3_data.update(business_category_dict.copy()) tmp_obj_3_data['alaskan_native_owned_corpo'] = 'True' tmp_obj_3 = pullFPDSData.calculate_remaining_fields( tmp_obj_3_data, sess, {sub_tier.sub_tier_agency_code: sub_tier}, county_by_name, county_by_code, state_codes, country_list) assert tmp_obj['awarding_agency_code'] == '1700' assert tmp_obj['awarding_agency_name'] == 'test name' assert tmp_obj['place_of_perform_county_co'] == '024' assert tmp_obj['place_of_perfor_state_desc'] == 'MARYLAND' assert tmp_obj['legal_entity_county_code'] is None assert tmp_obj['legal_entity_county_name'] is None assert sorted(tmp_obj['business_categories']) == [ 'category_business', 'emerging_small_business', 'small_business', 'special_designations' ] assert tmp_obj_2['funding_agency_code'] == '999' assert tmp_obj_2['funding_agency_name'] is None assert tmp_obj_2['place_of_perform_county_co'] is None assert tmp_obj_2['legal_entity_zip5'] == '12345' assert tmp_obj_2['legal_entity_zip_last4'] == '6789' assert tmp_obj_2['legal_entity_county_code'] == '123' assert tmp_obj_2['legal_entity_county_name'] == 'COUNTY ONE' assert sorted(tmp_obj_2['business_categories']) == [ 'category_business', 'other_than_small_business' ] assert tmp_obj_3['place_of_perform_country_c'] == 'USA' assert tmp_obj_3['place_of_perf_country_desc'] == 'UNITED STATES' assert tmp_obj_3['place_of_performance_state'] == 'PR' assert tmp_obj_3['place_of_perfor_state_desc'] == 'PUERTO RICO' assert tmp_obj_3['place_of_perform_county_co'] == '123' assert tmp_obj_3['place_of_performance_zip5'] == '12345' assert tmp_obj_3['place_of_perform_zip_last4'] == '6789' assert tmp_obj_3['legal_entity_country_code'] == 'USA' assert tmp_obj_3['legal_entity_country_name'] == 'UNITED STATES' assert tmp_obj_3['legal_entity_state_code'] == 'GU' assert tmp_obj_3['legal_entity_state_descrip'] == 'GUAM' assert tmp_obj_3['legal_entity_county_code'] == '123' assert tmp_obj_3['legal_entity_county_name'] == 'GU COUNTY' assert sorted(tmp_obj_3['business_categories']) == [ 'alaskan_native_owned_business', 'minority_owned_business' ]
def setUpClass(cls): """Set up resources to be shared within a test class""" cls.session_id = "" with create_validator_app().app_context(): # update application's db config options so unittests # run against test databases suite = cls.__name__.lower() config = dataactcore.config.CONFIG_DB cls.num = randint(1, 9999) config['db_name'] = 'unittest{}_{}_data_broker'.format( cls.num, suite) dataactcore.config.CONFIG_DB = config create_database(CONFIG_DB['db_name']) run_migrations() # drop and re-create test user db/tables setup_user_db() # drop and re-create test job db/tables setup_job_tracker_db() # drop and re-create test error db/tables setup_error_db() # drop and re-create test validation db/tables setup_validation_db() # load e-mail templates setup_emails() # set up default e-mails for tests test_users = { 'admin_user': '******', 'agency_user': '******', 'agency_user_2': '*****@*****.**', 'no_permissions_user': '******', 'editfabs_user': '******' } user_password = '******' admin_password = '******' # get user info and save as class variables for use by tests sess = GlobalDB.db().session admin_cgac = CGAC(cgac_code='SYS', agency_name='Admin Agency') cls.admin_cgac_code = admin_cgac.cgac_code sess.add(admin_cgac) sess.commit() cgac = CGAC(cgac_code='000', agency_name='Example Agency') sess.add(cgac) sess.commit() frec = FREC(frec_code='0001', cgac_id=cgac.cgac_id, agency_name='Example FREC') sess.add(frec) sess.commit() sub_tier = SubTierAgency(cgac_id=cgac.cgac_id, frec_id=frec.frec_id, sub_tier_agency_code='0000', sub_tier_agency_name='Example Sub Tier') sess.add(sub_tier) # set up users for status tests def add_user(email, name, username, permission_type=ALL_PERMISSION_TYPES_DICT['writer'], website_admin=False): user = UserFactory(email=email, website_admin=website_admin, name=name, username=username, affiliations=[ UserAffiliation( cgac=cgac, permission_type_id=permission_type) ]) user.salt, user.password_hash = get_password_hash( user_password, Bcrypt()) sess.add(user) add_user(test_users['agency_user'], "Test User", "testUser") add_user(test_users['agency_user_2'], "Test User 2", "testUser2") add_user(test_users['editfabs_user'], "Fabs Writer", "fabsWriter", permission_type=ALL_PERMISSION_TYPES_DICT['editfabs']) # add new users create_user_with_password(test_users["admin_user"], admin_password, Bcrypt(), website_admin=True) create_user_with_password(test_users["no_permissions_user"], user_password, Bcrypt()) agency_user = sess.query(User).filter( User.email == test_users['agency_user']).one() cls.agency_user_id = agency_user.user_id sess.commit() # set up info needed by the individual test classes cls.test_users = test_users cls.user_password = user_password cls.admin_password = admin_password cls.local = CONFIG_BROKER['local']