def validate_historic_dashboard_filters(filters, graphs=False):
""" Validate historic dashboard filters
Args:
filters: dictionary representing the filters provided to the historic dashboard endpoints
graphs: whether or not to validate the files and rules as well
Exceptions:
ResponseException if filter is invalid
"""
required_filters = ['quarters', 'fys', 'agencies']
if graphs:
required_filters.extend(['files', 'rules'])
missing_filters = [required_filter for required_filter in required_filters if required_filter not in filters]
if missing_filters:
raise ResponseException('The following filters were not provided: {}'.format(', '.join(missing_filters)),
status=StatusCode.CLIENT_ERROR)
wrong_filter_types = [key for key, value in filters.items() if not isinstance(value, list)]
if wrong_filter_types:
raise ResponseException('The following filters were not lists: {}'.format(', '.join(wrong_filter_types)),
status=StatusCode.CLIENT_ERROR)
for quarter in filters['quarters']:
if quarter not in range(1, 5):
raise ResponseException('Quarters must be a list of integers, each ranging 1-4, or an empty list.',
status=StatusCode.CLIENT_ERROR)
current_fy = fy(datetime.now())
for fiscal_year in filters['fys']:
if fiscal_year not in range(2017, current_fy + 1):
raise ResponseException('Fiscal Years must be a list of integers, each ranging from 2017 through the'
' current fiscal year, or an empty list.', status=StatusCode.CLIENT_ERROR)
for agency in filters['agencies']:
if not isinstance(agency, str):
raise ResponseException('Agencies must be a list of strings, or an empty list.',
status=StatusCode.CLIENT_ERROR)
if graphs:
for file_type in filters['files']:
if file_type not in FILE_TYPES:
raise ResponseException('Files must be a list of one or more of the following, or an empty list: {}'.
format(', '.join(FILE_TYPES)), status=StatusCode.CLIENT_ERROR)
for rule in filters['rules']:
if not isinstance(rule, str):
raise ResponseException('Rules must be a list of strings, or an empty list.',
status=StatusCode.CLIENT_ERROR)
def compare_contract_results(sub,
d1,
contract,
sub_contract,
parent_duns,
duns,
dom_country,
int_country,
created_at,
updated_at,
debug=False):
""" Helper function for contract results """
attr = {
'created_at': created_at,
'updated_at': updated_at,
'id': sub.id,
'unique_award_key': d1.unique_award_key,
'award_id': contract.contract_number,
'parent_award_id': contract.idv_reference_number,
'award_amount': contract.dollar_obligated,
'action_date': str(contract.date_signed),
'fy': 'FY{}'.format(fy(contract.date_signed)),
'awarding_agency_code': d1.awarding_agency_code,
'awarding_agency_name': d1.awarding_agency_name,
'awarding_sub_tier_agency_c': contract.contracting_office_aid,
'awarding_sub_tier_agency_n': contract.contracting_office_aname,
'awarding_office_code': contract.contracting_office_id,
'awarding_office_name': contract.contracting_office_name,
'funding_agency_code': d1.funding_agency_code,
'funding_agency_name': d1.funding_agency_name,
'funding_sub_tier_agency_co': contract.funding_agency_id,
'funding_sub_tier_agency_na': contract.funding_agency_name,
'funding_office_code': contract.funding_office_id,
'funding_office_name': contract.funding_office_name,
'awardee_or_recipient_uniqu': contract.duns,
'awardee_or_recipient_legal': contract.company_name,
'dba_name': contract.dba_name,
'ultimate_parent_unique_ide': contract.parent_duns,
'ultimate_parent_legal_enti': contract.parent_company_name,
'legal_entity_country_code': contract.company_address_country,
'legal_entity_country_name': dom_country.country_name,
'legal_entity_address_line1': contract.company_address_street,
'legal_entity_city_name': contract.company_address_city,
'legal_entity_state_code': contract.company_address_state,
'legal_entity_state_name': contract.company_address_state_name,
'legal_entity_zip': contract.company_address_zip,
'legal_entity_congressional': contract.company_address_district,
'legal_entity_foreign_posta': None,
'business_types': contract.bus_types,
'place_of_perform_city_name': contract.principle_place_city,
'place_of_perform_state_code': contract.principle_place_state,
'place_of_perform_state_name': contract.principle_place_state_name,
'place_of_performance_zip': contract.principle_place_zip,
'place_of_perform_congressio': contract.principle_place_district,
'place_of_perform_country_co': contract.principle_place_country,
'place_of_perform_country_na': int_country.country_name,
'award_description': d1.award_description,
'naics': contract.naics,
'naics_description': d1.naics_description,
'cfda_numbers': None,
'cfda_titles': None,
'subaward_type': 'sub-contract',
'subaward_report_year': contract.report_period_year,
'subaward_report_month': contract.report_period_mon,
'subaward_number': sub_contract.subcontract_num,
'subaward_amount': sub_contract.subcontract_amount,
'sub_action_date': str(sub_contract.subcontract_date),
'sub_awardee_or_recipient_uniqu': sub_contract.duns,
'sub_awardee_or_recipient_legal': sub_contract.company_name,
'sub_dba_name': sub_contract.dba_name,
'sub_ultimate_parent_unique_ide': sub_contract.parent_duns,
'sub_ultimate_parent_legal_enti': sub_contract.parent_company_name,
'sub_legal_entity_country_code': sub_contract.company_address_country,
'sub_legal_entity_country_name': int_country.country_name,
'sub_legal_entity_address_line1': sub_contract.company_address_street,
'sub_legal_entity_city_name': sub_contract.company_address_city,
'sub_legal_entity_state_code': sub_contract.company_address_state,
'sub_legal_entity_state_name': sub_contract.company_address_state_name,
'sub_legal_entity_zip': None,
'sub_legal_entity_congressional':
sub_contract.company_address_district,
'sub_legal_entity_foreign_posta': sub_contract.company_address_zip,
'sub_business_types': sub_contract.bus_types,
'sub_place_of_perform_city_name': sub_contract.principle_place_city,
'sub_place_of_perform_state_code': sub_contract.principle_place_state,
'sub_place_of_perform_state_name':
sub_contract.principle_place_state_name,
'sub_place_of_performance_zip': sub_contract.principle_place_zip,
'sub_place_of_perform_congressio':
sub_contract.principle_place_district,
'sub_place_of_perform_country_co':
sub_contract.principle_place_country,
'sub_place_of_perform_country_na': dom_country.country_name,
'subaward_description': sub_contract.overall_description,
'sub_high_comp_officer1_full_na': sub_contract.top_paid_fullname_1,
'sub_high_comp_officer1_amount': sub_contract.top_paid_amount_1,
'sub_high_comp_officer2_full_na': sub_contract.top_paid_fullname_2,
'sub_high_comp_officer2_amount': sub_contract.top_paid_amount_2,
'sub_high_comp_officer3_full_na': sub_contract.top_paid_fullname_3,
'sub_high_comp_officer3_amount': sub_contract.top_paid_amount_3,
'sub_high_comp_officer4_full_na': sub_contract.top_paid_fullname_4,
'sub_high_comp_officer4_amount': sub_contract.top_paid_amount_4,
'sub_high_comp_officer5_full_na': sub_contract.top_paid_fullname_5,
'sub_high_comp_officer5_amount': sub_contract.top_paid_amount_5,
'prime_id': contract.id,
'internal_id': contract.internal_id,
'date_submitted':
contract.date_submitted.strftime('%Y-%m-%d %H:%M:%S.%f'),
'report_type': contract.report_type,
'transaction_type': contract.transaction_type,
'program_title': contract.program_title,
'contract_agency_code': contract.contract_agency_code,
'contract_idv_agency_code': contract.contract_idv_agency_code,
'grant_funding_agency_id': None,
'grant_funding_agency_name': None,
'federal_agency_name': None,
'treasury_symbol': contract.treasury_symbol,
'dunsplus4': None,
'recovery_model_q1': str(contract.recovery_model_q1).lower(),
'recovery_model_q2': str(contract.recovery_model_q2).lower(),
'compensation_q1': None,
'compensation_q2': None,
'high_comp_officer1_full_na': contract.top_paid_fullname_1,
'high_comp_officer1_amount': contract.top_paid_amount_1,
'high_comp_officer2_full_na': contract.top_paid_fullname_2,
'high_comp_officer2_amount': contract.top_paid_amount_2,
'high_comp_officer3_full_na': contract.top_paid_fullname_3,
'high_comp_officer3_amount': contract.top_paid_amount_3,
'high_comp_officer4_full_na': contract.top_paid_fullname_4,
'high_comp_officer4_amount': contract.top_paid_amount_4,
'high_comp_officer5_full_na': contract.top_paid_fullname_5,
'high_comp_officer5_amount': contract.top_paid_amount_5,
'sub_id': sub_contract.id,
'sub_parent_id': sub_contract.parent_id,
'sub_federal_agency_id': None,
'sub_federal_agency_name': None,
'sub_funding_agency_id': sub_contract.funding_agency_id,
'sub_funding_agency_name': sub_contract.funding_agency_name,
'sub_funding_office_id': sub_contract.funding_office_id,
'sub_funding_office_name': sub_contract.funding_office_name,
'sub_naics': sub_contract.naics,
'sub_cfda_numbers': None,
'sub_dunsplus4': None,
'sub_recovery_subcontract_amt': sub_contract.recovery_subcontract_amt,
'sub_recovery_model_q1': str(sub_contract.recovery_model_q1).lower(),
'sub_recovery_model_q2': str(sub_contract.recovery_model_q2).lower(),
'sub_compensation_q1': None,
'sub_compensation_q2': None,
}
if debug and not (attr.items() <= sub.__dict__.items()):
print(sorted(attr.items()))
print(sorted(sub.__dict__.items()))
return attr.items() <= sub.__dict__.items()
def test_fy_type_exceptions(not_date):
assert fy(None) is None
with pytest.raises(TypeError):
fy(not_date)
def test_fy_returns_correct(raw_date, expected_fy):
assert fy(raw_date) == expected_fy
def test_fy_returns_integer(raw_date, expected_fy):
assert isinstance(fy(raw_date), int)
def compare_grant_results(sub, fabs, grant, sub_grant, parent_recipient, recipient, dom_country, int_country,
created_at, updated_at, debug=False):
""" Helper function for grant results """
attr = {
'created_at': created_at,
'updated_at': updated_at,
'id': sub.id,
'unique_award_key': fabs.unique_award_key if fabs else None,
'parent_award_id': None,
'award_amount': grant.total_fed_funding_amount,
'action_date': str(grant.obligation_date),
'fy': 'FY{}'.format(fy(grant.obligation_date)),
'awarding_agency_code': fabs.awarding_agency_code if fabs else None,
'awarding_agency_name': fabs.awarding_agency_name if fabs else None,
'awarding_sub_tier_agency_c': grant.federal_agency_id,
'awarding_sub_tier_agency_n': fabs.awarding_sub_tier_agency_n if fabs else None,
'awarding_office_code': fabs.awarding_office_code if fabs else None,
'awarding_office_name': fabs.awarding_office_name if fabs else None,
'funding_agency_code': fabs.funding_agency_code if fabs else None,
'funding_agency_name': fabs.funding_agency_name if fabs else None,
'funding_sub_tier_agency_co': fabs.funding_sub_tier_agency_co if fabs else None,
'funding_sub_tier_agency_na': fabs.funding_sub_tier_agency_na if fabs else None,
'funding_office_code': fabs.funding_office_code if fabs else None,
'funding_office_name': fabs.funding_office_name if fabs else None,
'awardee_or_recipient_uniqu': grant.duns,
'awardee_or_recipient_uei': grant.uei_number,
'awardee_or_recipient_legal': grant.awardee_name,
'dba_name': grant.dba_name,
'ultimate_parent_unique_ide': grant.parent_duns,
'ultimate_parent_legal_enti': parent_recipient.legal_business_name,
'legal_entity_country_code': int_country.country_code,
'legal_entity_country_name': int_country.country_name,
'legal_entity_address_line1': grant.awardee_address_street,
'legal_entity_city_name': grant.awardee_address_city,
'legal_entity_state_code': grant.awardee_address_state,
'legal_entity_state_name': grant.awardee_address_state_name,
'legal_entity_zip': None,
'legal_entity_congressional': grant.awardee_address_district,
'legal_entity_foreign_posta': grant.awardee_address_zip,
'business_types': fabs.business_types_desc if fabs else None,
'place_of_perform_city_name': grant.principle_place_city,
'place_of_perform_state_code': grant.principle_place_state,
'place_of_perform_state_name': grant.principle_place_state_name,
'place_of_performance_zip': grant.principle_place_zip,
'place_of_perform_congressio': grant.principle_place_district,
'place_of_perform_country_co': dom_country.country_code,
'place_of_perform_country_na': dom_country.country_name,
'award_description': grant.project_description,
'naics': None,
'naics_description': None,
'cfda_numbers': extract_cfda(grant.cfda_numbers, 'numbers'),
'cfda_titles': extract_cfda(grant.cfda_numbers, 'titles'),
'subaward_type': 'sub-grant',
'subaward_report_year': grant.report_period_year,
'subaward_report_month': grant.report_period_mon,
'subaward_number': sub_grant.subaward_num,
'subaward_amount': sub_grant.subaward_amount,
'sub_action_date': str(sub_grant.subaward_date),
'sub_awardee_or_recipient_uniqu': sub_grant.duns,
'sub_awardee_or_recipient_uei': sub_grant.uei_number,
'sub_awardee_or_recipient_legal': sub_grant.awardee_name,
'sub_dba_name': sub_grant.dba_name,
'sub_ultimate_parent_unique_ide': sub_grant.parent_duns,
'sub_ultimate_parent_legal_enti': parent_recipient.legal_business_name,
'sub_legal_entity_country_code': dom_country.country_code,
'sub_legal_entity_country_name': dom_country.country_name,
'sub_legal_entity_address_line1': sub_grant.awardee_address_street,
'sub_legal_entity_city_name': sub_grant.awardee_address_city,
'sub_legal_entity_state_code': sub_grant.awardee_address_state,
'sub_legal_entity_state_name': sub_grant.awardee_address_state_name,
'sub_legal_entity_zip': sub_grant.awardee_address_zip,
'sub_legal_entity_congressional': sub_grant.awardee_address_district,
'sub_legal_entity_foreign_posta': None,
'sub_business_types': ','.join(recipient.business_types) if recipient.business_types else None,
'sub_place_of_perform_city_name': sub_grant.principle_place_city,
'sub_place_of_perform_state_code': sub_grant.principle_place_state,
'sub_place_of_perform_state_name': sub_grant.principle_place_state_name,
'sub_place_of_performance_zip': sub_grant.principle_place_zip,
'sub_place_of_perform_congressio': sub_grant.principle_place_district,
'sub_place_of_perform_country_co': int_country.country_code,
'sub_place_of_perform_country_na': int_country.country_name,
'subaward_description': sub_grant.project_description,
'sub_high_comp_officer1_full_na': sub_grant.top_paid_fullname_1,
'sub_high_comp_officer1_amount': sub_grant.top_paid_amount_1,
'sub_high_comp_officer2_full_na': sub_grant.top_paid_fullname_2,
'sub_high_comp_officer2_amount': sub_grant.top_paid_amount_2,
'sub_high_comp_officer3_full_na': sub_grant.top_paid_fullname_3,
'sub_high_comp_officer3_amount': sub_grant.top_paid_amount_3,
'sub_high_comp_officer4_full_na': sub_grant.top_paid_fullname_4,
'sub_high_comp_officer4_amount': sub_grant.top_paid_amount_4,
'sub_high_comp_officer5_full_na': sub_grant.top_paid_fullname_5,
'sub_high_comp_officer5_amount': sub_grant.top_paid_amount_5,
'prime_id': grant.id,
'internal_id': grant.internal_id,
'date_submitted': grant.date_submitted.strftime('%Y-%m-%d %H:%M:%S.%f'),
'report_type': None,
'transaction_type': None,
'program_title': None,
'contract_agency_code': None,
'contract_idv_agency_code': None,
'grant_funding_agency_id': grant.funding_agency_id,
'grant_funding_agency_name': grant.funding_agency_name,
'federal_agency_name': grant.federal_agency_name,
'treasury_symbol': None,
'dunsplus4': grant.dunsplus4,
'recovery_model_q1': None,
'recovery_model_q2': None,
'compensation_q1': str(grant.compensation_q1).lower(),
'compensation_q2': str(grant.compensation_q2).lower(),
'high_comp_officer1_full_na': grant.top_paid_fullname_1,
'high_comp_officer1_amount': grant.top_paid_amount_1,
'high_comp_officer2_full_na': grant.top_paid_fullname_2,
'high_comp_officer2_amount': grant.top_paid_amount_2,
'high_comp_officer3_full_na': grant.top_paid_fullname_3,
'high_comp_officer3_amount': grant.top_paid_amount_3,
'high_comp_officer4_full_na': grant.top_paid_fullname_4,
'high_comp_officer4_amount': grant.top_paid_amount_4,
'high_comp_officer5_full_na': grant.top_paid_fullname_5,
'high_comp_officer5_amount': grant.top_paid_amount_5,
'place_of_perform_street': grant.principle_place_street,
'sub_id': sub_grant.id,
'sub_parent_id': sub_grant.parent_id,
'sub_federal_agency_id': sub_grant.federal_agency_id,
'sub_federal_agency_name': sub_grant.federal_agency_name,
'sub_funding_agency_id': sub_grant.funding_agency_id,
'sub_funding_agency_name': sub_grant.funding_agency_name,
'sub_funding_office_id': None,
'sub_funding_office_name': None,
'sub_naics': None,
'sub_cfda_numbers': sub_grant.cfda_numbers,
'sub_dunsplus4': sub_grant.dunsplus4,
'sub_recovery_subcontract_amt': None,
'sub_recovery_model_q1': None,
'sub_recovery_model_q2': None,
'sub_compensation_q1': str(sub_grant.compensation_q1).lower(),
'sub_compensation_q2': str(sub_grant.compensation_q2).lower(),
'sub_place_of_perform_street': sub_grant.principle_place_street
}
normal_compare = (attr.items() <= sub.__dict__.items())
dash_attrs = {
'award_id': grant.fain,
}
dash_compare = True
for da_name, da_value in dash_attrs.items():
dash_compare &= (da_value.replace('-', '') == sub.__dict__[da_name].replace('-', ''))
compare = normal_compare & dash_compare
if debug and not compare:
print(sorted(attr.items()))
print(sorted(sub.__dict__.items()))
return compare
def replicate_grant_results(sub, d2, grant, sub_grant, parent_duns, duns, dom_country, int_country):
""" Helper function for grant results """
return (
generate_unique_key(grant, d2),
grant.fain,
None,
grant.total_fed_funding_amount,
grant.obligation_date,
'FY{}'.format(fy(grant.obligation_date)),
d2.awarding_agency_code,
d2.awarding_agency_name,
grant.federal_agency_id,
d2.awarding_sub_tier_agency_n,
d2.awarding_office_code,
d2.awarding_office_name,
d2.funding_agency_code,
d2.funding_agency_name,
d2.funding_sub_tier_agency_co,
d2.funding_sub_tier_agency_na,
d2.funding_office_code,
d2.funding_office_name,
grant.duns,
grant.awardee_name,
grant.dba_name,
grant.parent_duns,
parent_duns.legal_business_name,
grant.awardee_address_country,
int_country.country_name,
grant.awardee_address_street,
grant.awardee_address_city,
grant.awardee_address_state,
grant.awardee_address_state_name,
None,
grant.awardee_address_district,
grant.awardee_address_zip,
d2.business_types_desc,
grant.principle_place_city,
grant.principle_place_state,
grant.principle_place_state_name,
grant.principle_place_zip,
grant.principle_place_district,
grant.principle_place_country,
dom_country.country_name,
grant.project_description,
None,
None,
extract_cfda(grant.cfda_numbers, 'numbers'),
extract_cfda(grant.cfda_numbers, 'titles'),
'sub-grant',
grant.report_period_year,
grant.report_period_mon,
sub_grant.subaward_num,
sub_grant.subaward_amount,
sub_grant.subaward_date,
sub_grant.duns,
sub_grant.awardee_name,
sub_grant.dba_name,
sub_grant.parent_duns,
parent_duns.legal_business_name,
sub_grant.awardee_address_country,
dom_country.country_name,
sub_grant.awardee_address_street,
sub_grant.awardee_address_city,
sub_grant.awardee_address_state,
sub_grant.awardee_address_state_name,
sub_grant.awardee_address_zip,
sub_grant.awardee_address_district,
None,
', '.join(parent_duns.business_types_codes),
sub_grant.principle_place_city,
sub_grant.principle_place_state,
sub_grant.principle_place_state_name,
sub_grant.principle_place_zip,
sub_grant.principle_place_district,
sub_grant.principle_place_country,
int_country.country_name,
sub_grant.project_description,
sub_grant.top_paid_fullname_1,
sub_grant.top_paid_amount_1,
sub_grant.top_paid_fullname_2,
sub_grant.top_paid_amount_2,
sub_grant.top_paid_fullname_3,
sub_grant.top_paid_amount_3,
sub_grant.top_paid_fullname_4,
sub_grant.top_paid_amount_4,
sub_grant.top_paid_fullname_5,
sub_grant.top_paid_amount_5
)
def replicate_contract_results(sub, d1, contract, sub_contract, parent_duns, duns, dom_country, int_country):
""" Helper function for contract results """
return (
generate_unique_key(contract, d1),
contract.contract_number,
contract.idv_reference_number,
contract.dollar_obligated,
contract.date_signed,
'FY{}'.format(fy(contract.date_signed)),
d1.awarding_agency_code,
d1.awarding_agency_name,
contract.contracting_office_aid,
contract.contracting_office_aname,
contract.contracting_office_id,
contract.contracting_office_name,
d1.funding_agency_code,
d1.funding_agency_name,
contract.funding_agency_id,
contract.funding_agency_name,
contract.funding_office_id,
contract.funding_office_name,
contract.duns,
contract.company_name,
contract.dba_name,
contract.parent_duns,
contract.parent_company_name,
contract.company_address_country,
dom_country.country_name,
contract.company_address_street,
contract.company_address_city,
contract.company_address_state,
contract.company_address_state_name,
contract.company_address_zip,
contract.company_address_district,
None,
contract.bus_types,
contract.principle_place_city,
contract.principle_place_state,
contract.principle_place_state_name,
contract.principle_place_zip,
contract.principle_place_district,
contract.principle_place_country,
int_country.country_name,
d1.award_description,
contract.naics,
d1.naics_description,
None,
None,
'sub-contract',
contract.report_period_year,
contract.report_period_mon,
sub_contract.subcontract_num,
sub_contract.subcontract_amount,
sub_contract.subcontract_date,
sub_contract.duns,
sub_contract.company_name,
sub_contract.dba_name,
sub_contract.parent_duns,
sub_contract.parent_company_name,
sub_contract.company_address_country,
int_country.country_name,
sub_contract.company_address_street,
sub_contract.company_address_city,
sub_contract.company_address_state,
sub_contract.company_address_state_name,
None,
sub_contract.company_address_district,
sub_contract.company_address_zip,
sub_contract.bus_types,
sub_contract.principle_place_city,
sub_contract.principle_place_state,
sub_contract.principle_place_state_name,
sub_contract.principle_place_zip,
sub_contract.principle_place_district,
sub_contract.principle_place_country,
dom_country.country_name,
sub_contract.overall_description,
sub_contract.top_paid_fullname_1,
sub_contract.top_paid_amount_1,
sub_contract.top_paid_fullname_2,
sub_contract.top_paid_amount_2,
sub_contract.top_paid_fullname_3,
sub_contract.top_paid_amount_3,
sub_contract.top_paid_fullname_4,
sub_contract.top_paid_amount_4,
sub_contract.top_paid_fullname_5,
sub_contract.top_paid_amount_5
)