def test_get_by_toptier():
"""Test Agency lookup by toptier CGAC code."""
toptier = mommy.make('references.ToptierAgency',
cgac_code='xyz',
name='yo')
subtier = mommy.make('references.SubtierAgency',
subtier_code='abc',
name='yo')
mommy.make('references.Agency',
toptier_agency=toptier,
subtier_agency=subtier)
mommy.make('references.Agency',
toptier_agency=toptier,
subtier_agency=mommy.make('references.SubtierAgency',
subtier_code='abc',
name='no'),
update_date=datetime.date(2017, 10, 10))
agency1 = mommy.make('references.Agency',
toptier_agency=toptier,
subtier_agency=subtier)
# lookup should return agency w/ most recent update_date that
# matches the cgac code
assert Agency.get_by_toptier('xyz') == agency1
# If there's no match, we should get none
assert Agency.get_by_toptier('nope') is None
def test_get_by_toptier():
"""Test Agency lookup by toptier CGAC code."""
toptier = mommy.make("references.ToptierAgency",
toptier_code="xyz",
name="yo")
subtier = mommy.make("references.SubtierAgency",
subtier_code="abc",
name="yo")
mommy.make(
"references.Agency",
toptier_agency=toptier,
subtier_agency=mommy.make("references.SubtierAgency",
subtier_code="bbb",
name="no"),
update_date=datetime.date(2017, 10, 10),
)
agency1 = mommy.make("references.Agency",
toptier_agency=toptier,
subtier_agency=subtier)
# lookup should return agency w/ most recent update_date that
# matches the cgac code
assert Agency.get_by_toptier("xyz") == agency1
# If there's no match, we should get none
assert Agency.get_by_toptier("nope") is None
def get_awarding_agency(row):
if row.txn:
# We found a matching transaction, so grab its awarding agency info and pass it get_or_create_summary_award
return Agency.objects.get(id=int(row.txn))
else:
# No matching transaction found, so find/create Award by using toptier agency only, since CGAC code is the only
# piece of awarding agency info that we have.
return Agency.get_by_toptier(row.agency_identifier)
def get_valid_awarding_agency(row):
agency_subtier_code = row['awarding_sub_tier_agency_c']
agency_toptier_code = row['awarding_agency_code']
valid_subtier_code = (agency_subtier_code and len(agency_subtier_code) > 0)
valid_toptier_code = (agency_toptier_code and len(agency_toptier_code) > 0)
if not valid_toptier_code and not valid_subtier_code:
return None
agency = None
# Get the awarding agency
if valid_subtier_code and valid_toptier_code:
agency = Agency.get_by_toptier_subtier(row['awarding_agency_code'], row['awarding_sub_tier_agency_c'])
if not agency and valid_subtier_code:
agency = Agency.get_by_subtier(row['awarding_sub_tier_agency_c'])
if not agency and valid_toptier_code:
agency = Agency.get_by_toptier(row['awarding_agency_code'])
return agency
def get_valid_awarding_agency(row):
agency_subtier_code = row['awarding_sub_tier_agency_c']
agency_toptier_code = row['awarding_agency_code']
valid_subtier_code = (agency_subtier_code and len(agency_subtier_code) > 0)
valid_toptier_code = (agency_toptier_code and len(agency_toptier_code) > 0)
if not valid_toptier_code and not valid_subtier_code:
return None
agency = None
# Get the awarding agency
if valid_subtier_code and valid_toptier_code:
agency = Agency.get_by_toptier_subtier(
row['awarding_agency_code'], row['awarding_sub_tier_agency_c'])
if not agency and valid_subtier_code:
agency = Agency.get_by_subtier(row['awarding_sub_tier_agency_c'])
if not agency and valid_toptier_code:
agency = Agency.get_by_toptier(row['awarding_agency_code'])
return agency
def get_or_create_summary_award(awarding_agency=None,
piid=None,
fain=None,
uri=None,
parent_award_id=None,
use_cache=False):
"""
Given a set of award identifiers and awarding agency information,
find a corresponding Award record. If we can't find one, create it.
Returns:
created: a list of new awards created (or that need to be created
if using cache), used to enable bulk insert
summary_award: the summary award that the calling process can map to
"""
# If an award transaction's ID is a piid, it's contract data
# If the ID is fain or a uri, it's financial assistance. If the award transaction
# has both a fain and a uri, fain takes precedence.
q_kwargs = {}
for i in [(piid, "piid"), (fain, "fain"), (uri, "uri")]:
if i[0]:
q_kwargs[i[1]] = i[0]
if parent_award_id:
q_kwargs["parent_award__" + i[1]] = parent_award_id
# parent_award__piid, parent_award__fain, parent_award__uri
else:
q_kwargs["parent_award"] = None
# Now search for it
# Do we want to log something if the the query below turns up
# more than one award record?
if use_cache:
q_kwargs_fixed = list(q_kwargs.items()) + [
('awarding_agency', awarding_agency),
]
q_kwargs_fixed.sort()
summary_award = awards_cache.get(q_kwargs_fixed)
if summary_award:
return [], summary_award
# Look for an existing award record
summary_award = Award.objects \
.filter(Q(**q_kwargs)) \
.filter(awarding_agency=awarding_agency) \
.first()
if (summary_award is None and awarding_agency is not None
and awarding_agency.toptier_agency.name !=
awarding_agency.subtier_agency.name):
# No award match found when searching by award id info +
# awarding subtier agency. Relax the awarding agency
# critera to just the toptier agency instead of the subtier
# agency and try the search again.
awarding_agency_toptier = Agency.get_by_toptier(
awarding_agency.toptier_agency.cgac_code)
summary_award = Award.objects \
.filter(Q(**q_kwargs)) \
.filter(awarding_agency=awarding_agency_toptier) \
.first()
if summary_award:
if use_cache:
awards_cache.set(q_kwargs_fixed, summary_award)
return [], summary_award
# We weren't able to match, so create a new award record.
if parent_award_id:
# If parent award id was supplied, recursively get/create
# an award record for it
parent_created, parent_award = Award.get_or_create_summary_award(
use_cache=use_cache,
**{
i[1]: parent_award_id,
'awarding_agency': awarding_agency
})
else:
parent_created, parent_award = [], None
# Now create the award record for this award transaction
summary_award = Award(
**{
i[1]: i[0],
"parent_award": parent_award,
"awarding_agency": awarding_agency
})
created = [
summary_award,
]
created.extend(parent_created)
if use_cache:
awards_cache.set(q_kwargs_fixed, summary_award)
else:
summary_award.save()
return created, summary_award
raise ValueError(
'Unable to find or create an award with the provided information: '
'piid={}, fain={}, uri={}, parent_id={}, awarding_agency={}'.
format(piid, fain, uri, parent_award_id, awarding_agency))
def load_file_c(submission_attributes, award_financial_data, db_cursor):
"""
Process and load file C broker data.
Note: this should run AFTER the D1 and D2 files are loaded because we try
to join to those records to retrieve some additional information
about the awarding sub-tier agency.
"""
# this matches the file b reverse directive, but am repeating it here
# to ensure that we don't overwrite it as we change up the order of
# file loading
reverse = re.compile(r'(_(cpe|fyb)$)|^transaction_obligated_amount$')
for row in award_financial_data:
# Check and see if there is an entry for this TAS
treasury_account = get_treasury_appropriation_account_tas_lookup(
row.get('tas_id'), db_cursor)
if treasury_account is None:
raise Exception('Could not find appropriation account for TAS: ' +
row['tas'])
# Find a matching transaction record, so we can use its
# subtier agency information to match to (or create) an Award record
awarding_cgac = row.get('agency_identifier') # cgac from record's TAS
txn = get_award_financial_transaction(
awarding_cgac,
piid=row.get('piid'),
parent_award_id=row.get('parent_award_id'),
fain=row.get('fain'),
uri=row.get('uri'))
if txn is not None:
# We found a matching transaction, so grab its awarding agency
# info and pass it get_or_create_summary_award
awarding_agency = txn.awarding_agency
else:
# No matching transaction found, so find/create Award by using
# topiter agency only, since CGAC code is the only piece of
# awarding agency info that we have.
awarding_agency = Agency.get_by_toptier(awarding_cgac)
# Find the award that this award transaction belongs to. If it doesn't exist, create it.
created, award = Award.get_or_create_summary_award(
awarding_agency=awarding_agency,
piid=row.get('piid'),
fain=row.get('fain'),
uri=row.get('uri'),
parent_award_id=row.get('parent_award_id'),
use_cache=False)
award_financial_data = FinancialAccountsByAwards()
value_map = {
'award':
award,
'submission':
submission_attributes,
'reporting_period_start':
submission_attributes.reporting_period_start,
'reporting_period_end':
submission_attributes.reporting_period_end,
'treasury_account':
treasury_account,
'object_class':
get_or_create_object_class(row['object_class'],
row['by_direct_reimbursable_fun'],
logger),
'program_activity':
get_or_create_program_activity(row, submission_attributes)
}
# Still using the cpe|fyb regex compiled above for reverse
afd = load_data_into_model(award_financial_data,
row,
value_map=value_map,
save=True,
reverse=reverse)
awards_cache.clear()
def get_or_create_summary_award(awarding_agency=None,
piid=None,
fain=None,
uri=None,
parent_award_id=None,
save=True):
"""
Given a set of award identifiers and awarding agency information,
find a corresponding Award record. If we can't find one, create it.
Returns:
created: a list of new awards created, used to enable bulk insert
summary_award: the summary award that the calling process can map to
"""
# If an award transaction's ID is a piid, it's contract data
# If the ID is fain or a uri, it's financial assistance. If the award transaction
# has both a fain and a uri, include both.
try:
# Look for an existing award record
# Check individual FILE C D linkage first
lookup_kwargs = {'recipient_id__isnull': False}
if piid is not None:
lookup_kwargs['piid'] = piid
if parent_award_id is not None:
lookup_kwargs['parent_award__piid'] = parent_award_id
if fain is not None:
lookup_kwargs['fain'] = fain
if uri is not None:
lookup_kwargs['uri'] = uri
award_queryset = Award.objects.filter(**lookup_kwargs)[:2]
award_count = len(award_queryset)
if award_count == 1:
summary_award = award_queryset[0]
return [], summary_award
# if nothing found revert to looking for an award that this record could've already created
lookup_kwargs = {"awarding_agency": awarding_agency}
for i in [(piid, "piid"), (fain, "fain"), (uri, "uri")]:
lookup_kwargs[i[1]] = i[0]
if parent_award_id:
# parent_award__piid
lookup_kwargs["parent_award_piid"] = parent_award_id
# Look for an existing award record
summary_award = Award.objects \
.filter(Q(**lookup_kwargs)) \
.filter(awarding_agency=awarding_agency) \
.first()
if (summary_award is None and awarding_agency is not None
and awarding_agency.toptier_agency.name !=
awarding_agency.subtier_agency.name):
# No award match found when searching by award id info +
# awarding subtier agency. Relax the awarding agency
# critera to just the toptier agency instead of the subtier
# agency and try the search again.
awarding_agency_toptier = Agency.get_by_toptier(
awarding_agency.toptier_agency.cgac_code)
summary_award = Award.objects \
.filter(Q(**lookup_kwargs)) \
.filter(awarding_agency=awarding_agency_toptier) \
.first()
if summary_award:
return [], summary_award
parent_created, parent_award = [], None
# Now create the award record for this award transaction
create_kwargs = {
"awarding_agency": awarding_agency,
"parent_award": parent_award,
"parent_award_piid": parent_award_id
}
for i in [(piid, "piid"), (fain, "fain"), (uri, "uri")]:
create_kwargs[i[1]] = i[0]
summary_award = Award(**create_kwargs)
created = [
summary_award,
]
created.extend(parent_created)
if save:
summary_award.save()
return created, summary_award
# Do not use bare except
except ValueError:
raise ValueError(
'Unable to find or create an award with the provided information: '
'piid={}, fain={}, uri={}, parent_id={}, awarding_agency={}'.
format(piid, fain, uri, parent_award_id, awarding_agency))