def test_update_business_type_categories(recipients_data):
le = LegalEntity.objects.filter(legal_entity_id=1111).first()
le.business_types = 'P'
le.business_categories = get_business_categories({'business_types': le.business_types}, 'fabs')
assert le.business_categories == ['individuals']
le.business_types = 'L'
le.business_categories = get_business_categories({'business_types': le.business_types}, 'fabs')
assert 'authorities_and_commissions' in le.business_categories
assert 'government' in le.business_categories
le.business_types = 'M'
le.business_categories = get_business_categories({'business_types': le.business_types}, 'fabs')
assert 'nonprofit' in le.business_categories
def test_update_business_type_categories():
business_types = "P"
business_categories = get_business_categories(
{"business_types": business_types}, "fabs")
assert business_categories == ["individuals"]
business_types = "L"
business_categories = get_business_categories(
{"business_types": business_types}, "fabs")
assert "authorities_and_commissions" in business_categories
assert "government" in business_categories
business_types = "M"
business_categories = get_business_categories(
{"business_types": business_types}, "fabs")
assert "nonprofit" in business_categories
def extract_business_categories(recipient_name, recipient_duns):
""" Extract the business categories via the recipient hash
Args:
recipient_name: name of the recipient
recipient_duns: duns of the recipient
Returns:
list of business categories
"""
business_categories = set()
if recipient_name in SPECIAL_CASES:
return list(business_categories)
# Go through DUNS first
d_business_cat = DUNS.objects.filter(legal_business_name=recipient_name, awardee_or_recipient_uniqu=recipient_duns)\
.order_by('-update_date').values('business_types_codes', 'entity_structure').first()
if d_business_cat:
duns_types_mapping = get_duns_business_types_mapping()
business_types_codes = d_business_cat['business_types_codes']
if d_business_cat['entity_structure']:
business_types_codes.append(d_business_cat['entity_structure'])
business_types = {duns_types_mapping[type]: 'true' for type in d_business_cat['business_types_codes']
if type in duns_types_mapping}
business_categories |= set(get_business_categories(business_types, data_type='fpds'))
# combine with latest transaction's business categories
le_business_cat = LegalEntity.objects.filter(recipient_name=recipient_name, recipient_unique_id=recipient_duns) \
.order_by('-update_date').values('business_categories').first()
if le_business_cat and le_business_cat['business_categories']:
business_categories |= set(le_business_cat['business_categories'])
return list(business_categories)
def test_update_business_type_categories(recipients_data):
le = LegalEntity.objects.filter(legal_entity_id=1111).first()
le.business_types = "P"
le.business_categories = get_business_categories(
{"business_types": le.business_types}, "fabs")
assert le.business_categories == ["individuals"]
le.business_types = "L"
le.business_categories = get_business_categories(
{"business_types": le.business_types}, "fabs")
assert "authorities_and_commissions" in le.business_categories
assert "government" in le.business_categories
le.business_types = "M"
le.business_categories = get_business_categories(
{"business_types": le.business_types}, "fabs")
assert "nonprofit" in le.business_categories
def test_update_business_type_categories_faads_format(recipients_data):
le = LegalEntity.objects.filter(legal_entity_id=1111).first()
le.business_types = '01' # B equivalent
le.business_categories = get_business_categories({'business_types': le.business_types}, 'fabs')
assert 'government' in le.business_categories
assert 'local_government' in le.business_categories
le.business_types = '12' # M equivalent
le.business_categories = get_business_categories({'business_types': le.business_types}, 'fabs')
assert 'nonprofit' in le.business_categories
le.business_types = '21' # P equivalent
le.business_categories = get_business_categories({'business_types': le.business_types}, 'fabs')
assert 'individuals' in le.business_categories
le.business_types = '23' # R equivalent
le.business_categories = get_business_categories({'business_types': le.business_types}, 'fabs')
assert 'small_business' in le.business_categories
assert 'category_business' in le.business_categories
def test_update_business_type_categories_faads_format():
business_types = "01" # B equivalent
business_categories = get_business_categories(
{"business_types": business_types}, "fabs")
assert "government" in business_categories
assert "local_government" in business_categories
business_types = "12" # M equivalent
business_categories = get_business_categories(
{"business_types": business_types}, "fabs")
assert "nonprofit" in business_categories
business_types = "21" # P equivalent
business_categories = get_business_categories(
{"business_types": business_types}, "fabs")
assert "individuals" in business_categories
business_types = "23" # R equivalent
business_categories = get_business_categories(
{"business_types": business_types}, "fabs")
assert "small_business" in business_categories
assert "category_business" in business_categories
def test_update_business_type_categories_faads_format(recipients_data):
le = LegalEntity.objects.filter(legal_entity_id=1111).first()
le.business_types = "01" # B equivalent
le.business_categories = get_business_categories(
{"business_types": le.business_types}, "fabs")
assert "government" in le.business_categories
assert "local_government" in le.business_categories
le.business_types = "12" # M equivalent
le.business_categories = get_business_categories(
{"business_types": le.business_types}, "fabs")
assert "nonprofit" in le.business_categories
le.business_types = "21" # P equivalent
le.business_categories = get_business_categories(
{"business_types": le.business_types}, "fabs")
assert "individuals" in le.business_categories
le.business_types = "23" # R equivalent
le.business_categories = get_business_categories(
{"business_types": le.business_types}, "fabs")
assert "small_business" in le.business_categories
assert "category_business" in le.business_categories
def extract_business_categories(recipient_name, recipient_duns,
recipient_hash):
""" Extract the business categories via the recipient hash
Args:
recipient_name: name of the recipient
recipient_duns: duns of the recipient
recipient_hash: hash of name and duns
Returns:
list of business categories
"""
business_categories = set()
if recipient_name in SPECIAL_CASES:
return list(business_categories)
# Go through DUNS first
d_business_cat = (DUNS.objects.filter(
legal_business_name=recipient_name,
awardee_or_recipient_uniqu=recipient_duns).order_by(
"-update_date").values("business_types_codes",
"entity_structure").first())
if d_business_cat:
duns_types_mapping = get_duns_business_types_mapping()
business_types_codes = d_business_cat["business_types_codes"]
if d_business_cat["entity_structure"]:
business_types_codes.append(d_business_cat["entity_structure"])
business_types = {
duns_types_mapping[type]: "true"
for type in d_business_cat["business_types_codes"]
if type in duns_types_mapping
}
business_categories |= set(
get_business_categories(business_types, data_type="fpds"))
# combine with latest transaction's business categories
# from usaspending_api.references.models import LegalEntity
latest_transaction = (UniversalTransactionView.objects.filter(
recipient_hash=recipient_hash,
action_date__gte=settings.API_SEARCH_MIN_DATE).order_by(
"-action_date",
"-transaction_id").values("business_categories").first())
if latest_transaction and latest_transaction["business_categories"]:
business_categories |= set(latest_transaction["business_categories"])
return sorted(business_categories)
def extract_business_categories(recipient_name, recipient_duns):
""" Extract the business categories via the recipient hash
Args:
recipient_name: name of the recipient
recipient_duns: duns of the recipient
Returns:
list of business categories
"""
business_categories = set()
if recipient_name in SPECIAL_CASES:
return list(business_categories)
# Go through DUNS first
d_business_cat = (DUNS.objects.filter(
legal_business_name=recipient_name,
awardee_or_recipient_uniqu=recipient_duns).order_by(
"-update_date").values("business_types_codes",
"entity_structure").first())
if d_business_cat:
duns_types_mapping = get_duns_business_types_mapping()
business_types_codes = d_business_cat["business_types_codes"]
if d_business_cat["entity_structure"]:
business_types_codes.append(d_business_cat["entity_structure"])
business_types = {
duns_types_mapping[type]: "true"
for type in d_business_cat["business_types_codes"]
if type in duns_types_mapping
}
business_categories |= set(
get_business_categories(business_types, data_type="fpds"))
# combine with latest transaction's business categories
le_business_cat = (LegalEntity.objects.filter(
recipient_name=recipient_name,
recipient_unique_id=recipient_duns).order_by("-update_date").values(
"business_categories").first())
if le_business_cat and le_business_cat["business_categories"]:
business_categories |= set(le_business_cat["business_categories"])
return list(business_categories)
def insert_new_fpds(self, to_insert, total_rows):
place_of_performance_field_map = {
"location_country_code": "place_of_perform_country_c",
"country_name": "place_of_perf_country_desc",
"state_code": "place_of_performance_state",
"state_name": "place_of_perfor_state_desc",
"city_name": "place_of_perform_city_name",
"county_name": "place_of_perform_county_na",
"county_code": "place_of_perform_county_co",
"zip_4a": "place_of_performance_zip4a",
"congressional_code": "place_of_performance_congr",
"zip_last4": "place_of_perform_zip_last4",
"zip5": "place_of_performance_zip5",
}
legal_entity_location_field_map = {
"location_country_code": "legal_entity_country_code",
"country_name": "legal_entity_country_name",
"state_code": "legal_entity_state_code",
"state_name": "legal_entity_state_descrip",
"city_name": "legal_entity_city_name",
"county_name": "legal_entity_county_name",
"county_code": "legal_entity_county_code",
"address_line1": "legal_entity_address_line1",
"address_line2": "legal_entity_address_line2",
"address_line3": "legal_entity_address_line3",
"zip4": "legal_entity_zip4",
"congressional_code": "legal_entity_congressional",
"zip_last4": "legal_entity_zip_last4",
"zip5": "legal_entity_zip5",
}
for index, row in enumerate(to_insert, 1):
upper_case_dict_values(row)
# Create new LegalEntityLocation and LegalEntity from the row data
legal_entity_location = create_location(
legal_entity_location_field_map, row, {"recipient_flag": True, "is_fpds": True}
)
recipient_name = row["awardee_or_recipient_legal"]
legal_entity = LegalEntity.objects.create(
recipient_unique_id=row["awardee_or_recipient_uniqu"],
recipient_name=recipient_name if recipient_name is not None else "",
)
legal_entity_value_map = {
"location": legal_entity_location,
"business_categories": get_business_categories(row=row, data_type="fpds"),
"is_fpds": True,
}
set_legal_entity_boolean_fields(row)
legal_entity = load_data_into_model(legal_entity, row, value_map=legal_entity_value_map, save=True)
# Create the place of performance location
pop_location = create_location(place_of_performance_field_map, row, {"place_of_performance_flag": True})
# Find the toptier awards from the subtier awards
awarding_agency = Agency.get_by_subtier_only(row["awarding_sub_tier_agency_c"])
funding_agency = Agency.get_by_subtier_only(row["funding_sub_tier_agency_co"])
# Generate the unique Award ID
# "CONT_AW_" + agency_id + referenced_idv_agency_iden + piid + parent_award_id
generated_unique_id = (
"CONT_AW_"
+ (row["agency_id"] if row["agency_id"] else "-NONE-")
+ "_"
+ (row["referenced_idv_agency_iden"] if row["referenced_idv_agency_iden"] else "-NONE-")
+ "_"
+ (row["piid"] if row["piid"] else "-NONE-")
+ "_"
+ (row["parent_award_id"] if row["parent_award_id"] else "-NONE-")
)
# Create the summary Award
(created, award) = Award.get_or_create_summary_award(
generated_unique_award_id=generated_unique_id, piid=row["piid"]
)
award.parent_award_piid = row.get("parent_award_id")
award.save()
# Append row to list of Awards updated
AWARD_UPDATE_ID_LIST.append(award.id)
if row["last_modified"] and len(str(row["last_modified"])) == len("YYYY-MM-DD HH:MM:SS"): # 19 characters
dt_fmt = "%Y-%m-%d %H:%M:%S"
else:
dt_fmt = "%Y-%m-%d %H:%M:%S.%f" # try using this even if last_modified isn't a valid string
try:
last_mod_date = datetime.strptime(str(row["last_modified"]), dt_fmt).date()
except ValueError: # handle odd-string formats and NULLs from the upstream FPDS-NG system
info_message = "Invalid value '{}' does not match: '{}'".format(row["last_modified"], dt_fmt)
logger.info(info_message)
last_mod_date = None
award_type, award_type_desc = award_types(row)
parent_txn_value_map = {
"award": award,
"awarding_agency": awarding_agency,
"funding_agency": funding_agency,
"recipient": legal_entity,
"place_of_performance": pop_location,
"period_of_performance_start_date": format_date(row["period_of_performance_star"]),
"period_of_performance_current_end_date": format_date(row["period_of_performance_curr"]),
"action_date": format_date(row["action_date"]),
"last_modified_date": last_mod_date,
"transaction_unique_id": row["detached_award_proc_unique"],
"generated_unique_award_id": generated_unique_id,
"is_fpds": True,
"type": award_type,
"type_description": award_type_desc,
}
contract_field_map = {"description": "award_description"}
transaction_normalized_dict = load_data_into_model(
TransactionNormalized(), # thrown away
row,
field_map=contract_field_map,
value_map=parent_txn_value_map,
as_dict=True,
)
contract_instance = load_data_into_model(TransactionFPDS(), row, as_dict=True) # thrown away
detached_award_proc_unique = contract_instance["detached_award_proc_unique"]
unique_fpds = TransactionFPDS.objects.filter(detached_award_proc_unique=detached_award_proc_unique)
if unique_fpds.first():
transaction_normalized_dict["update_date"] = datetime.now(timezone.utc)
transaction_normalized_dict["fiscal_year"] = fy(transaction_normalized_dict["action_date"])
# update TransactionNormalized
TransactionNormalized.objects.filter(id=unique_fpds.first().transaction.id).update(
**transaction_normalized_dict
)
# update TransactionFPDS
unique_fpds.update(**contract_instance)
else:
# create TransactionNormalized
transaction = TransactionNormalized(**transaction_normalized_dict)
transaction.save()
# create TransactionFPDS
transaction_fpds = TransactionFPDS(transaction=transaction, **contract_instance)
transaction_fpds.save()
# Update legal entity to map back to transaction
legal_entity.transaction_unique_id = detached_award_proc_unique
legal_entity.save()
def business_categories(broker_input):
return get_business_categories(broker_input, "fpds")
def insert_new_fabs(to_insert):
fabs_normalized_field_map = {
"type": "assistance_type",
"description": "award_description",
"funding_amount": "total_funding_amount",
}
fabs_field_map = {
"officer_1_name": "high_comp_officer1_full_na",
"officer_1_amount": "high_comp_officer1_amount",
"officer_2_name": "high_comp_officer2_full_na",
"officer_2_amount": "high_comp_officer2_amount",
"officer_3_name": "high_comp_officer3_full_na",
"officer_3_amount": "high_comp_officer3_amount",
"officer_4_name": "high_comp_officer4_full_na",
"officer_4_amount": "high_comp_officer4_amount",
"officer_5_name": "high_comp_officer5_full_na",
"officer_5_amount": "high_comp_officer5_amount",
}
update_award_ids = []
for row in to_insert:
upper_case_dict_values(row)
# Find the toptier awards from the subtier awards
awarding_agency = Agency.get_by_subtier_only(
row["awarding_sub_tier_agency_c"])
funding_agency = Agency.get_by_subtier_only(
row["funding_sub_tier_agency_co"])
# Create the summary Award
(created, award) = Award.get_or_create_summary_award(
generated_unique_award_id=row["unique_award_key"],
fain=row["fain"],
uri=row["uri"],
record_type=row["record_type"],
)
award.save()
# Append row to list of Awards updated
update_award_ids.append(award.id)
try:
last_mod_date = datetime.strptime(str(row["modified_at"]),
"%Y-%m-%d %H:%M:%S.%f").date()
except ValueError:
last_mod_date = datetime.strptime(str(row["modified_at"]),
"%Y-%m-%d %H:%M:%S").date()
parent_txn_value_map = {
"award":
award,
"awarding_agency":
awarding_agency,
"funding_agency":
funding_agency,
"period_of_performance_start_date":
format_date(row["period_of_performance_star"]),
"period_of_performance_current_end_date":
format_date(row["period_of_performance_curr"]),
"action_date":
format_date(row["action_date"]),
"last_modified_date":
last_mod_date,
"type_description":
row["assistance_type_desc"],
"transaction_unique_id":
row["afa_generated_unique"],
"business_categories":
get_business_categories(row=row, data_type="fabs"),
}
transaction_normalized_dict = load_data_into_model(
TransactionNormalized(), # thrown away
row,
field_map=fabs_normalized_field_map,
value_map=parent_txn_value_map,
as_dict=True,
)
financial_assistance_data = load_data_into_model(
TransactionFABS(),
row,
field_map=fabs_field_map,
as_dict=True # thrown away
)
# Hack to cut back on the number of warnings dumped to the log.
financial_assistance_data["updated_at"] = cast_datetime_to_utc(
financial_assistance_data["updated_at"])
financial_assistance_data["created_at"] = cast_datetime_to_utc(
financial_assistance_data["created_at"])
financial_assistance_data["modified_at"] = cast_datetime_to_utc(
financial_assistance_data["modified_at"])
afa_generated_unique = financial_assistance_data[
"afa_generated_unique"]
unique_fabs = TransactionFABS.objects.filter(
afa_generated_unique=afa_generated_unique)
if unique_fabs.first():
transaction_normalized_dict["update_date"] = datetime.now(
timezone.utc)
transaction_normalized_dict["fiscal_year"] = fy(
transaction_normalized_dict["action_date"])
# Update TransactionNormalized
TransactionNormalized.objects.filter(
id=unique_fabs.first().transaction.id).update(
**transaction_normalized_dict)
# Update TransactionFABS
unique_fabs.update(**financial_assistance_data)
else:
# Create TransactionNormalized
transaction_normalized = TransactionNormalized(
**transaction_normalized_dict)
transaction_normalized.save()
# Create TransactionFABS
transaction_fabs = TransactionFABS(
transaction=transaction_normalized,
**financial_assistance_data)
transaction_fabs.save()
return update_award_ids
def insert_new_fabs(to_insert):
place_of_performance_field_map = {
"location_country_code": "place_of_perform_country_c",
"country_name": "place_of_perform_country_n",
"state_code": "place_of_perfor_state_code",
"state_name": "place_of_perform_state_nam",
"city_name": "place_of_performance_city",
"county_name": "place_of_perform_county_na",
"county_code": "place_of_perform_county_co",
"foreign_location_description": "place_of_performance_forei",
"zip_4a": "place_of_performance_zip4a",
"congressional_code": "place_of_performance_congr",
"performance_code": "place_of_performance_code",
"zip_last4": "place_of_perform_zip_last4",
"zip5": "place_of_performance_zip5",
}
legal_entity_location_field_map = {
"location_country_code": "legal_entity_country_code",
"country_name": "legal_entity_country_name",
"state_code": "legal_entity_state_code",
"state_name": "legal_entity_state_name",
"city_name": "legal_entity_city_name",
"city_code": "legal_entity_city_code",
"county_name": "legal_entity_county_name",
"county_code": "legal_entity_county_code",
"address_line1": "legal_entity_address_line1",
"address_line2": "legal_entity_address_line2",
"address_line3": "legal_entity_address_line3",
"foreign_location_description": "legal_entity_foreign_descr",
"congressional_code": "legal_entity_congressional",
"zip_last4": "legal_entity_zip_last4",
"zip5": "legal_entity_zip5",
"foreign_postal_code": "legal_entity_foreign_posta",
"foreign_province": "legal_entity_foreign_provi",
"foreign_city_name": "legal_entity_foreign_city",
}
fabs_normalized_field_map = {
"type": "assistance_type",
"description": "award_description",
"funding_amount": "total_funding_amount",
}
fabs_field_map = {
"officer_1_name": "high_comp_officer1_full_na",
"officer_1_amount": "high_comp_officer1_amount",
"officer_2_name": "high_comp_officer2_full_na",
"officer_2_amount": "high_comp_officer2_amount",
"officer_3_name": "high_comp_officer3_full_na",
"officer_3_amount": "high_comp_officer3_amount",
"officer_4_name": "high_comp_officer4_full_na",
"officer_4_amount": "high_comp_officer4_amount",
"officer_5_name": "high_comp_officer5_full_na",
"officer_5_amount": "high_comp_officer5_amount",
}
update_award_ids = []
for row in to_insert:
upper_case_dict_values(row)
# Create new LegalEntityLocation and LegalEntity from the row data
legal_entity_location = create_location(
legal_entity_location_field_map, row, {"recipient_flag": True})
recipient_name = row['awardee_or_recipient_legal']
legal_entity = LegalEntity.objects.create(
recipient_unique_id=row['awardee_or_recipient_uniqu'],
recipient_name=recipient_name
if recipient_name is not None else "",
parent_recipient_unique_id=row['ultimate_parent_unique_ide'],
)
legal_entity_value_map = {
"location":
legal_entity_location,
"business_categories":
get_business_categories(row=row, data_type='fabs'),
"business_types_description":
row['business_types_desc'],
}
legal_entity = load_data_into_model(legal_entity,
row,
value_map=legal_entity_value_map,
save=True)
# Create the place of performance location
pop_location = create_location(place_of_performance_field_map, row,
{"place_of_performance_flag": True})
# Find the toptier awards from the subtier awards
awarding_agency = Agency.get_by_subtier_only(
row["awarding_sub_tier_agency_c"])
funding_agency = Agency.get_by_subtier_only(
row["funding_sub_tier_agency_co"])
# Create the summary Award
(created, award) = Award.get_or_create_summary_award(
generated_unique_award_id=row['unique_award_key'],
fain=row['fain'],
uri=row['uri'],
record_type=row['record_type'],
)
award.save()
# Append row to list of Awards updated
update_award_ids.append(award.id)
try:
last_mod_date = datetime.strptime(str(row['modified_at']),
"%Y-%m-%d %H:%M:%S.%f").date()
except ValueError:
last_mod_date = datetime.strptime(str(row['modified_at']),
"%Y-%m-%d %H:%M:%S").date()
parent_txn_value_map = {
"award":
award,
"awarding_agency":
awarding_agency,
"funding_agency":
funding_agency,
"recipient":
legal_entity,
"place_of_performance":
pop_location,
"period_of_performance_start_date":
format_date(row['period_of_performance_star']),
"period_of_performance_current_end_date":
format_date(row['period_of_performance_curr']),
"action_date":
format_date(row['action_date']),
"last_modified_date":
last_mod_date,
"type_description":
row['assistance_type_desc'],
"transaction_unique_id":
row['afa_generated_unique'],
}
transaction_normalized_dict = load_data_into_model(
TransactionNormalized(), # thrown away
row,
field_map=fabs_normalized_field_map,
value_map=parent_txn_value_map,
as_dict=True,
)
financial_assistance_data = load_data_into_model(
TransactionFABS(), # thrown away
row,
field_map=fabs_field_map,
as_dict=True)
# Hack to cut back on the number of warnings dumped to the log.
financial_assistance_data['updated_at'] = cast_datetime_to_utc(
financial_assistance_data['updated_at'])
financial_assistance_data['created_at'] = cast_datetime_to_utc(
financial_assistance_data['created_at'])
financial_assistance_data['modified_at'] = cast_datetime_to_utc(
financial_assistance_data['modified_at'])
afa_generated_unique = financial_assistance_data[
'afa_generated_unique']
unique_fabs = TransactionFABS.objects.filter(
afa_generated_unique=afa_generated_unique)
if unique_fabs.first():
transaction_normalized_dict["update_date"] = datetime.now(
timezone.utc)
transaction_normalized_dict["fiscal_year"] = fy(
transaction_normalized_dict["action_date"])
# Update TransactionNormalized
TransactionNormalized.objects.filter(
id=unique_fabs.first().transaction.id).update(
**transaction_normalized_dict)
# Update TransactionFABS
unique_fabs.update(**financial_assistance_data)
else:
# Create TransactionNormalized
transaction_normalized = TransactionNormalized(
**transaction_normalized_dict)
transaction_normalized.save()
# Create TransactionFABS
transaction_fabs = TransactionFABS(
transaction=transaction_normalized,
**financial_assistance_data)
transaction_fabs.save()
# Update legal entity to map back to transaction
legal_entity.transaction_unique_id = afa_generated_unique
legal_entity.save()
return update_award_ids
def insert_new_fpds(self, to_insert, total_rows):
place_of_performance_field_map = {
"location_country_code": "place_of_perform_country_c",
"country_name": "place_of_perf_country_desc",
"state_code": "place_of_performance_state",
"state_name": "place_of_perfor_state_desc",
"city_name": "place_of_perform_city_name",
"county_name": "place_of_perform_county_na",
"county_code": "place_of_perform_county_co",
"zip_4a": "place_of_performance_zip4a",
"congressional_code": "place_of_performance_congr",
"zip_last4": "place_of_perform_zip_last4",
"zip5": "place_of_performance_zip5",
}
legal_entity_location_field_map = {
"location_country_code": "legal_entity_country_code",
"country_name": "legal_entity_country_name",
"state_code": "legal_entity_state_code",
"state_name": "legal_entity_state_descrip",
"city_name": "legal_entity_city_name",
"county_name": "legal_entity_county_name",
"county_code": "legal_entity_county_code",
"address_line1": "legal_entity_address_line1",
"address_line2": "legal_entity_address_line2",
"address_line3": "legal_entity_address_line3",
"zip4": "legal_entity_zip4",
"congressional_code": "legal_entity_congressional",
"zip_last4": "legal_entity_zip_last4",
"zip5": "legal_entity_zip5",
}
for index, row in enumerate(to_insert, 1):
upper_case_dict_values(row)
# Create new LegalEntityLocation and LegalEntity from the row data
legal_entity_location = create_location(
legal_entity_location_field_map, row, {
"recipient_flag": True,
"is_fpds": True
})
recipient_name = row["awardee_or_recipient_legal"]
legal_entity = LegalEntity.objects.create(
recipient_unique_id=row["awardee_or_recipient_uniqu"],
recipient_name=recipient_name
if recipient_name is not None else "",
)
legal_entity_value_map = {
"location":
legal_entity_location,
"business_categories":
get_business_categories(row=row, data_type="fpds"),
"is_fpds":
True,
}
set_legal_entity_boolean_fields(row)
legal_entity = load_data_into_model(
legal_entity, row, value_map=legal_entity_value_map, save=True)
# Create the place of performance location
pop_location = create_location(place_of_performance_field_map, row,
{"place_of_performance_flag": True})
# Find the toptier awards from the subtier awards
awarding_agency = Agency.get_by_subtier_only(
row["awarding_sub_tier_agency_c"])
funding_agency = Agency.get_by_subtier_only(
row["funding_sub_tier_agency_co"])
# Generate the unique Award ID
# "CONT_AW_" + agency_id + referenced_idv_agency_iden + piid + parent_award_id
generated_unique_id = (
"CONT_AW_" +
(row["agency_id"] if row["agency_id"] else "-NONE-") + "_" +
(row["referenced_idv_agency_iden"]
if row["referenced_idv_agency_iden"] else "-NONE-") + "_" +
(row["piid"] if row["piid"] else "-NONE-") + "_" +
(row["parent_award_id"]
if row["parent_award_id"] else "-NONE-"))
# Create the summary Award
(created, award) = Award.get_or_create_summary_award(
generated_unique_award_id=generated_unique_id,
piid=row["piid"])
award.parent_award_piid = row.get("parent_award_id")
award.save()
# Append row to list of Awards updated
AWARD_UPDATE_ID_LIST.append(award.id)
if row["last_modified"] and len(str(row["last_modified"])) == len(
"YYYY-MM-DD HH:MM:SS"): # 19 characters
dt_fmt = "%Y-%m-%d %H:%M:%S"
else:
dt_fmt = "%Y-%m-%d %H:%M:%S.%f" # try using this even if last_modified isn't a valid string
try:
last_mod_date = datetime.strptime(str(row["last_modified"]),
dt_fmt).date()
except ValueError: # handle odd-string formats and NULLs from the upstream FPDS-NG system
info_message = "Invalid value '{}' does not match: '{}'".format(
row["last_modified"], dt_fmt)
logger.info(info_message)
last_mod_date = None
award_type, award_type_desc = award_types(row)
parent_txn_value_map = {
"award":
award,
"awarding_agency":
awarding_agency,
"funding_agency":
funding_agency,
"recipient":
legal_entity,
"place_of_performance":
pop_location,
"period_of_performance_start_date":
format_date(row["period_of_performance_star"]),
"period_of_performance_current_end_date":
format_date(row["period_of_performance_curr"]),
"action_date":
format_date(row["action_date"]),
"last_modified_date":
last_mod_date,
"transaction_unique_id":
row["detached_award_proc_unique"],
"generated_unique_award_id":
generated_unique_id,
"is_fpds":
True,
"type":
award_type,
"type_description":
award_type_desc,
}
contract_field_map = {"description": "award_description"}
transaction_normalized_dict = load_data_into_model(
TransactionNormalized(), # thrown away
row,
field_map=contract_field_map,
value_map=parent_txn_value_map,
as_dict=True,
)
contract_instance = load_data_into_model(
TransactionFPDS(), row, as_dict=True) # thrown away
detached_award_proc_unique = contract_instance[
"detached_award_proc_unique"]
unique_fpds = TransactionFPDS.objects.filter(
detached_award_proc_unique=detached_award_proc_unique)
if unique_fpds.first():
transaction_normalized_dict["update_date"] = datetime.now(
timezone.utc)
transaction_normalized_dict["fiscal_year"] = fy(
transaction_normalized_dict["action_date"])
# update TransactionNormalized
TransactionNormalized.objects.filter(
id=unique_fpds.first().transaction.id).update(
**transaction_normalized_dict)
# update TransactionFPDS
unique_fpds.update(**contract_instance)
else:
# create TransactionNormalized
transaction = TransactionNormalized(
**transaction_normalized_dict)
transaction.save()
# create TransactionFPDS
transaction_fpds = TransactionFPDS(transaction=transaction,
**contract_instance)
transaction_fpds.save()
# Update legal entity to map back to transaction
legal_entity.transaction_unique_id = detached_award_proc_unique
legal_entity.save()
def insert_new_fabs(to_insert):
place_of_performance_field_map = {
"location_country_code": "place_of_perform_country_c",
"country_name": "place_of_perform_country_n",
"state_code": "place_of_perfor_state_code",
"state_name": "place_of_perform_state_nam",
"city_name": "place_of_performance_city",
"county_name": "place_of_perform_county_na",
"county_code": "place_of_perform_county_co",
"foreign_location_description": "place_of_performance_forei",
"zip_4a": "place_of_performance_zip4a",
"congressional_code": "place_of_performance_congr",
"performance_code": "place_of_performance_code",
"zip_last4": "place_of_perform_zip_last4",
"zip5": "place_of_performance_zip5",
}
legal_entity_location_field_map = {
"location_country_code": "legal_entity_country_code",
"country_name": "legal_entity_country_name",
"state_code": "legal_entity_state_code",
"state_name": "legal_entity_state_name",
"city_name": "legal_entity_city_name",
"city_code": "legal_entity_city_code",
"county_name": "legal_entity_county_name",
"county_code": "legal_entity_county_code",
"address_line1": "legal_entity_address_line1",
"address_line2": "legal_entity_address_line2",
"address_line3": "legal_entity_address_line3",
"foreign_location_description": "legal_entity_foreign_descr",
"congressional_code": "legal_entity_congressional",
"zip_last4": "legal_entity_zip_last4",
"zip5": "legal_entity_zip5",
"foreign_postal_code": "legal_entity_foreign_posta",
"foreign_province": "legal_entity_foreign_provi",
"foreign_city_name": "legal_entity_foreign_city",
}
update_award_ids = []
for row in to_insert:
upper_case_dict_values(row)
# Create new LegalEntityLocation and LegalEntity from the row data
legal_entity_location = create_location(legal_entity_location_field_map, row, {"recipient_flag": True})
recipient_name = row['awardee_or_recipient_legal']
legal_entity = LegalEntity.objects.create(
recipient_unique_id=row['awardee_or_recipient_uniqu'],
recipient_name=recipient_name if recipient_name is not None else "",
parent_recipient_unique_id=row['ultimate_parent_unique_ide'],
)
legal_entity_value_map = {
"location": legal_entity_location,
"business_categories": get_business_categories(row=row, data_type='fabs'),
"business_types_description": row['business_types_desc'],
}
legal_entity = load_data_into_model(legal_entity, row, value_map=legal_entity_value_map, save=True)
# Create the place of performance location
pop_location = create_location(place_of_performance_field_map, row, {"place_of_performance_flag": True})
# Find the toptier awards from the subtier awards
awarding_agency = Agency.get_by_subtier_only(row["awarding_sub_tier_agency_c"])
funding_agency = Agency.get_by_subtier_only(row["funding_sub_tier_agency_co"])
# Generate the unique Award ID
# "ASST_AW_" + awarding_sub_tier_agency_c + fain + uri
# this will raise an exception if the cast to an int fails, that's ok since we don't want to process
# non-numeric record type values
record_type_int = int(row['record_type'])
if record_type_int == 1:
uri = row['uri'] if row['uri'] else '-NONE-'
fain = '-NONE-'
elif record_type_int in (2, 3):
uri = '-NONE-'
fain = row['fain'] if row['fain'] else '-NONE-'
else:
msg = "Invalid record type encountered for the following afa_generated_unique record: {}"
raise Exception(msg.format(row['afa_generated_unique']))
astac = row["awarding_sub_tier_agency_c"] if row["awarding_sub_tier_agency_c"] else "-NONE-"
generated_unique_id = "ASST_AW_{}_{}_{}".format(astac, fain, uri)
# Create the summary Award
(created, award) = Award.get_or_create_summary_award(
generated_unique_award_id=generated_unique_id,
fain=row['fain'],
uri=row['uri'],
record_type=row['record_type'],
)
award.save()
# Append row to list of Awards updated
update_award_ids.append(award.id)
try:
last_mod_date = datetime.strptime(str(row['modified_at']), "%Y-%m-%d %H:%M:%S.%f").date()
except ValueError:
last_mod_date = datetime.strptime(str(row['modified_at']), "%Y-%m-%d %H:%M:%S").date()
parent_txn_value_map = {
"award": award,
"awarding_agency": awarding_agency,
"funding_agency": funding_agency,
"recipient": legal_entity,
"place_of_performance": pop_location,
"period_of_performance_start_date": format_date(row['period_of_performance_star']),
"period_of_performance_current_end_date": format_date(row['period_of_performance_curr']),
"action_date": format_date(row['action_date']),
"last_modified_date": last_mod_date,
"type_description": row['assistance_type_desc'],
"transaction_unique_id": row['afa_generated_unique'],
"generated_unique_award_id": generated_unique_id,
}
fad_field_map = {
"type": "assistance_type",
"description": "award_description",
"funding_amount": "total_funding_amount",
}
transaction_normalized_dict = load_data_into_model(
TransactionNormalized(), # thrown away
row,
field_map=fad_field_map,
value_map=parent_txn_value_map,
as_dict=True,
)
financial_assistance_data = load_data_into_model(TransactionFABS(), row, as_dict=True) # thrown away
# Hack to cut back on the number of warnings dumped to the log.
financial_assistance_data['updated_at'] = cast_datetime_to_utc(financial_assistance_data['updated_at'])
financial_assistance_data['created_at'] = cast_datetime_to_utc(financial_assistance_data['created_at'])
financial_assistance_data['modified_at'] = cast_datetime_to_utc(financial_assistance_data['modified_at'])
afa_generated_unique = financial_assistance_data['afa_generated_unique']
unique_fabs = TransactionFABS.objects.filter(afa_generated_unique=afa_generated_unique)
if unique_fabs.first():
transaction_normalized_dict["update_date"] = datetime.now(timezone.utc)
transaction_normalized_dict["fiscal_year"] = fy(transaction_normalized_dict["action_date"])
# Update TransactionNormalized
TransactionNormalized.objects.filter(id=unique_fabs.first().transaction.id).update(
**transaction_normalized_dict
)
# Update TransactionFABS
unique_fabs.update(**financial_assistance_data)
else:
# Create TransactionNormalized
transaction_normalized = TransactionNormalized(**transaction_normalized_dict)
transaction_normalized.save()
# Create TransactionFABS
transaction_fabs = TransactionFABS(transaction=transaction_normalized, **financial_assistance_data)
transaction_fabs.save()
# Update legal entity to map back to transaction
legal_entity.transaction_unique_id = afa_generated_unique
legal_entity.save()
return update_award_ids
def test_dev_2318_business_category_changes():
"""
The following changes were made to business categories as part of DEV-2318. Let's make sure they took.
- alaskan_native_owned_business => alaskan_native_corporation_owned_firm
- foreign_owned_and_located_business => foreign_owned
- native_hawaiian_owned_business => native_hawaiian_organization_owned_firm
- tribally_owned_business => tribally_owned_firm
"""
expected_business_categories = [
"alaskan_native_corporation_owned_firm",
"foreign_owned",
"minority_owned_business",
"native_hawaiian_organization_owned_firm",
"special_designations",
"tribally_owned_firm",
]
business_categories = get_business_categories(
{
"alaskan_native_owned_corpo": "true",
"native_hawaiian_owned_busi": "true",
"tribally_owned_business": "true",
"foreign_owned_and_located": "true",
},
"fpds",
)
assert business_categories == expected_business_categories
# Same test using the database function.
with connection.cursor() as cursor:
cursor.execute("""
select
compile_fpds_business_categories(
'', -- contracting_officers_deter
false, -- corporate_entity_tax_exemp
false, -- corporate_entity_not_tax_e
false, -- partnership_or_limited_lia
false, -- sole_proprietorship
false, -- manufacturer_of_goods
false, -- subchapter_s_corporation
false, -- limited_liability_corporat
false, -- for_profit_organization
TRUE, -- alaskan_native_owned_corpo
false, -- american_indian_owned_busi
false, -- asian_pacific_american_own
false, -- black_american_owned_busin
false, -- hispanic_american_owned_bu
false, -- native_american_owned_busi
TRUE, -- native_hawaiian_owned_busi
false, -- subcontinent_asian_asian_i
TRUE, -- tribally_owned_business
false, -- other_minority_owned_busin
false, -- minority_owned_business
false, -- women_owned_small_business
false, -- economically_disadvantaged
false, -- joint_venture_women_owned
false, -- joint_venture_economically
false, -- woman_owned_business
false, -- service_disabled_veteran_o
false, -- veteran_owned_business
false, -- c8a_program_participant
false, -- the_ability_one_program
false, -- dot_certified_disadvantage
false, -- emerging_small_business
false, -- federally_funded_research
false, -- historically_underutilized
false, -- labor_surplus_area_firm
false, -- sba_certified_8_a_joint_ve
false, -- self_certified_small_disad
false, -- small_agricultural_coopera
false, -- small_disadvantaged_busine
false, -- community_developed_corpor
'', -- domestic_or_foreign_entity
TRUE, -- foreign_owned_and_located
false, -- foreign_government
false, -- international_organization
false, -- domestic_shelter
false, -- hospital_flag
false, -- veterinary_hospital
false, -- foundation
false, -- community_development_corp
false, -- nonprofit_organization
false, -- educational_institution
false, -- other_not_for_profit_organ
false, -- state_controlled_instituti
false, -- c1862_land_grant_college
false, -- c1890_land_grant_college
false, -- c1994_land_grant_college
false, -- private_university_or_coll
false, -- minority_institution
false, -- historically_black_college
false, -- tribal_college
false, -- alaskan_native_servicing_i
false, -- native_hawaiian_servicing
false, -- hispanic_servicing_institu
false, -- school_of_forestry
false, -- veterinary_college
false, -- us_federal_government
false, -- federal_agency
false, -- us_government_entity
false, -- interstate_entity
false, -- us_state_government
false, -- council_of_governments
false, -- city_local_government
false, -- county_local_government
false, -- inter_municipal_local_gove
false, -- municipality_local_governm
false, -- township_local_government
false, -- us_local_government
false, -- local_government_owned
false, -- school_district_local_gove
false, -- us_tribal_government
false, -- indian_tribe_federally_rec
false, -- housing_authorities_public
false, -- airport_authority
false, -- port_authority
false, -- transit_authority
false -- planning_commission
)
""")
assert cursor.fetchall()[0][0] == expected_business_categories
display_names = get_business_category_display_names(
business_categories)
assert display_names == [
"Alaskan Native Corporation Owned Firm",
"Foreign Owned",
"Minority Owned Business",
"Native Hawaiian Organization Owned Firm",
"Special Designations",
"Tribally Owned Firm",
]
def insert_new_fabs(to_insert):
place_of_performance_field_map = {
"location_country_code": "place_of_perform_country_c",
"country_name": "place_of_perform_country_n",
"state_code": "place_of_perfor_state_code",
"state_name": "place_of_perform_state_nam",
"city_name": "place_of_performance_city",
"county_name": "place_of_perform_county_na",
"county_code": "place_of_perform_county_co",
"foreign_location_description": "place_of_performance_forei",
"zip_4a": "place_of_performance_zip4a",
"congressional_code": "place_of_performance_congr",
"performance_code": "place_of_performance_code",
"zip_last4": "place_of_perform_zip_last4",
"zip5": "place_of_performance_zip5",
}
legal_entity_location_field_map = {
"location_country_code": "legal_entity_country_code",
"country_name": "legal_entity_country_name",
"state_code": "legal_entity_state_code",
"state_name": "legal_entity_state_name",
"city_name": "legal_entity_city_name",
"city_code": "legal_entity_city_code",
"county_name": "legal_entity_county_name",
"county_code": "legal_entity_county_code",
"address_line1": "legal_entity_address_line1",
"address_line2": "legal_entity_address_line2",
"address_line3": "legal_entity_address_line3",
"foreign_location_description": "legal_entity_foreign_descr",
"congressional_code": "legal_entity_congressional",
"zip_last4": "legal_entity_zip_last4",
"zip5": "legal_entity_zip5",
"foreign_postal_code": "legal_entity_foreign_posta",
"foreign_province": "legal_entity_foreign_provi",
"foreign_city_name": "legal_entity_foreign_city",
}
update_award_ids = []
for row in to_insert:
upper_case_dict_values(row)
# Create new LegalEntityLocation and LegalEntity from the row data
legal_entity_location = create_location(
legal_entity_location_field_map, row, {"recipient_flag": True})
recipient_name = row['awardee_or_recipient_legal']
legal_entity = LegalEntity.objects.create(
recipient_unique_id=row['awardee_or_recipient_uniqu'],
recipient_name=recipient_name
if recipient_name is not None else "",
parent_recipient_unique_id=row['ultimate_parent_unique_ide'],
)
legal_entity_value_map = {
"location":
legal_entity_location,
"business_categories":
get_business_categories(row=row, data_type='fabs'),
"business_types_description":
row['business_types_desc'],
}
legal_entity = load_data_into_model(legal_entity,
row,
value_map=legal_entity_value_map,
save=True)
# Create the place of performance location
pop_location = create_location(place_of_performance_field_map, row,
{"place_of_performance_flag": True})
# Find the toptier awards from the subtier awards
awarding_agency = Agency.get_by_subtier_only(
row["awarding_sub_tier_agency_c"])
funding_agency = Agency.get_by_subtier_only(
row["funding_sub_tier_agency_co"])
# Generate the unique Award ID
# "ASST_AW_" + awarding_sub_tier_agency_c + fain + uri
# this will raise an exception if the cast to an int fails, that's ok since we don't want to process
# non-numeric record type values
record_type_int = int(row['record_type'])
if record_type_int == 1:
uri = row['uri'] if row['uri'] else '-NONE-'
fain = '-NONE-'
elif record_type_int in (2, 3):
uri = '-NONE-'
fain = row['fain'] if row['fain'] else '-NONE-'
else:
msg = "Invalid record type encountered for the following afa_generated_unique record: {}"
raise Exception(msg.format(row['afa_generated_unique']))
astac = row["awarding_sub_tier_agency_c"] if row[
"awarding_sub_tier_agency_c"] else "-NONE-"
generated_unique_id = "ASST_AW_{}_{}_{}".format(astac, fain, uri)
# Create the summary Award
(created, award) = Award.get_or_create_summary_award(
generated_unique_award_id=generated_unique_id,
fain=row['fain'],
uri=row['uri'],
record_type=row['record_type'],
)
award.save()
# Append row to list of Awards updated
update_award_ids.append(award.id)
try:
last_mod_date = datetime.strptime(str(row['modified_at']),
"%Y-%m-%d %H:%M:%S.%f").date()
except ValueError:
last_mod_date = datetime.strptime(str(row['modified_at']),
"%Y-%m-%d %H:%M:%S").date()
parent_txn_value_map = {
"award":
award,
"awarding_agency":
awarding_agency,
"funding_agency":
funding_agency,
"recipient":
legal_entity,
"place_of_performance":
pop_location,
"period_of_performance_start_date":
format_date(row['period_of_performance_star']),
"period_of_performance_current_end_date":
format_date(row['period_of_performance_curr']),
"action_date":
format_date(row['action_date']),
"last_modified_date":
last_mod_date,
"type_description":
row['assistance_type_desc'],
"transaction_unique_id":
row['afa_generated_unique'],
"generated_unique_award_id":
generated_unique_id,
}
fad_field_map = {
"type": "assistance_type",
"description": "award_description",
"funding_amount": "total_funding_amount",
}
transaction_normalized_dict = load_data_into_model(
TransactionNormalized(), # thrown away
row,
field_map=fad_field_map,
value_map=parent_txn_value_map,
as_dict=True,
)
financial_assistance_data = load_data_into_model(
TransactionFABS(), row, as_dict=True) # thrown away
# Hack to cut back on the number of warnings dumped to the log.
financial_assistance_data['updated_at'] = cast_datetime_to_utc(
financial_assistance_data['updated_at'])
financial_assistance_data['created_at'] = cast_datetime_to_utc(
financial_assistance_data['created_at'])
financial_assistance_data['modified_at'] = cast_datetime_to_utc(
financial_assistance_data['modified_at'])
afa_generated_unique = financial_assistance_data[
'afa_generated_unique']
unique_fabs = TransactionFABS.objects.filter(
afa_generated_unique=afa_generated_unique)
if unique_fabs.first():
transaction_normalized_dict["update_date"] = datetime.now(
timezone.utc)
transaction_normalized_dict["fiscal_year"] = fy(
transaction_normalized_dict["action_date"])
# Update TransactionNormalized
TransactionNormalized.objects.filter(
id=unique_fabs.first().transaction.id).update(
**transaction_normalized_dict)
# Update TransactionFABS
unique_fabs.update(**financial_assistance_data)
else:
# Create TransactionNormalized
transaction_normalized = TransactionNormalized(
**transaction_normalized_dict)
transaction_normalized.save()
# Create TransactionFABS
transaction_fabs = TransactionFABS(
transaction=transaction_normalized,
**financial_assistance_data)
transaction_fabs.save()
# Update legal entity to map back to transaction
legal_entity.transaction_unique_id = afa_generated_unique
legal_entity.save()
return update_award_ids
