def test_get_previous_submission():
"""Test the process for determining the most recent submission in the current FY."""
# set up some related submissions
sub1 = mommy.make(SubmissionAttributes,
cgac_code='073',
reporting_fiscal_year=2017,
reporting_fiscal_period=9,
quarter_format_flag=True)
sub2 = mommy.make(SubmissionAttributes,
cgac_code='073',
reporting_fiscal_year=2017,
reporting_fiscal_period=6,
quarter_format_flag=True)
# Submission for same CGAC + a later period should return sub1 as previous submission
assert helpers.get_previous_submission('073', 2017, 12) == sub1
# Previous submission lookup should not find a match for an earlier submission
assert helpers.get_previous_submission('073', 2017, 3) is None
# Previous submission lookup should not match against a different fiscal year
assert helpers.get_previous_submission('073', 2018, 3) is None
# Previous submission lookup should not match against a different agency (CGAC)
assert helpers.get_previous_submission('ABC', 2017, 12) is None
sub3 = mommy.make(SubmissionAttributes,
cgac_code='020',
reporting_fiscal_year=2016,
reporting_fiscal_period=6,
quarter_format_flag=False)
# Previous submission lookup should only match a quarterly submission
assert helpers.get_previous_submission('020', 2016, 9) is None
def get_submission_attributes(broker_submission_id, submission_data):
"""
For a specified broker submission, return the existing corresponding usaspending submission record or create and
return a new one.
"""
# check if we already have an entry for this broker submission id; if not, create one
submission_attributes, created = SubmissionAttributes.\
objects.get_or_create(broker_submission_id=broker_submission_id)
if created:
# this is the first time we're loading this broker submission
logger.info('Creating broker submission id {}'.format(broker_submission_id))
else:
# we've already loaded this broker submission, so delete it before reloading if there's another submission that
# references this one as a "previous submission" do not proceed.
# TODO: now that we're chaining submisisons together, get clarification on what should happen when a submission
# in the middle of the chain is deleted
TasProgramActivityObjectClassQuarterly.refresh_downstream_quarterly_numbers(submission_attributes.submission_id)
logger.info('Broker submission id {} already exists. It will be deleted.'.format(broker_submission_id))
call_command('rm_submission', broker_submission_id)
logger.info("Merging CGAC and FREC columns")
submission_data["cgac_code"] = submission_data["cgac_code"]\
if submission_data["cgac_code"] else submission_data["frec_code"]
# Find the previous submission for this CGAC and fiscal year (if there is one)
previous_submission = get_previous_submission(
submission_data['cgac_code'],
submission_data['reporting_fiscal_year'],
submission_data['reporting_fiscal_period'])
# Update and save submission attributes
field_map = {
'reporting_period_start': 'reporting_start_date',
'reporting_period_end': 'reporting_end_date',
'quarter_format_flag': 'is_quarter_format',
}
# Create our value map - specific data to load
value_map = {
'broker_submission_id': broker_submission_id,
'reporting_fiscal_quarter': get_fiscal_quarter(submission_data['reporting_fiscal_period']),
'previous_submission': None if previous_submission is None else previous_submission,
# pull in broker's last update date to use as certified date
'certified_date': submission_data['updated_at'].date() if type(
submission_data['updated_at']) == datetime else None,
}
return load_data_into_model(
submission_attributes, submission_data,
field_map=field_map, value_map=value_map, save=True)
def test_get_previous_submission():
"""Test the process for determining the most recent submission in the current FY."""
# set up some related submissions
sub1 = mommy.make(
SubmissionAttributes,
cgac_code='073',
reporting_fiscal_year=2017,
reporting_fiscal_period=9,
quarter_format_flag=True
)
mommy.make(
SubmissionAttributes,
cgac_code='073',
reporting_fiscal_year=2017,
reporting_fiscal_period=6,
quarter_format_flag=True
)
# Submission for same CGAC + a later period should return sub1 as previous submission
assert helpers.get_previous_submission('073', 2017, 12) == sub1
# Previous submission lookup should not find a match for an earlier submission
assert helpers.get_previous_submission('073', 2017, 3) is None
# Previous submission lookup should not match against a different fiscal year
assert helpers.get_previous_submission('073', 2018, 3) is None
# Previous submission lookup should not match against a different agency (CGAC)
assert helpers.get_previous_submission('ABC', 2017, 12) is None
mommy.make(
SubmissionAttributes,
cgac_code='020',
reporting_fiscal_year=2016,
reporting_fiscal_period=6,
quarter_format_flag=False
)
# Previous submission lookup should only match a quarterly submission
assert helpers.get_previous_submission('020', 2016, 9) is None
def get_submission_attributes(broker_submission_id, submission_data):
"""
For a specified broker submission, return the existing corresponding usaspending submission record or create and
return a new one.
"""
# check if we already have an entry for this broker submission id; if not, create one
submission_attributes, created = SubmissionAttributes.objects.get_or_create(
broker_submission_id=broker_submission_id)
if created:
# this is the first time we're loading this broker submission
logger.info(
"Creating broker submission id {}".format(broker_submission_id))
else:
# we've already loaded this broker submission, so delete it before reloading if there's another submission that
# references this one as a "previous submission" do not proceed.
# TODO: now that we're chaining submissions together, get clarification on what should happen when a submission
# in the middle of the chain is deleted
TasProgramActivityObjectClassQuarterly.refresh_downstream_quarterly_numbers(
submission_attributes.submission_id)
logger.info(
"Broker submission id {} already exists. It will be deleted.".
format(broker_submission_id))
call_command("rm_submission", broker_submission_id)
logger.info("Merging CGAC and FREC columns")
submission_data["toptier_code"] = (submission_data["cgac_code"]
if submission_data["cgac_code"] else
submission_data["frec_code"])
# Find the previous submission for this CGAC and fiscal year (if there is one)
previous_submission = get_previous_submission(
submission_data["toptier_code"],
submission_data["reporting_fiscal_year"],
submission_data["reporting_fiscal_period"],
)
# if another submission lists the previous submission as its previous submission, set to null and update later
potential_conflicts = []
if previous_submission:
potential_conflicts = SubmissionAttributes.objects.filter(
previous_submission=previous_submission)
if potential_conflicts:
logger.info(
"==== ATTENTION! Previous Submission ID Conflict Detected ===="
)
for conflict in potential_conflicts:
logger.info(
"Temporarily setting {}'s Previous Submission ID from {} to null"
.format(conflict, previous_submission.submission_id))
conflict.previous_submission = None
conflict.save()
# Update and save submission attributes
field_map = {
"reporting_period_start": "reporting_start_date",
"reporting_period_end": "reporting_end_date",
"quarter_format_flag": "is_quarter_format",
}
# Create our value map - specific data to load
value_map = {
"broker_submission_id":
broker_submission_id,
"reporting_fiscal_quarter":
get_fiscal_quarter(submission_data["reporting_fiscal_period"]),
"previous_submission":
previous_submission,
# pull in broker's last update date to use as certified date
"certified_date":
submission_data["updated_at"].date()
if type(submission_data["updated_at"]) == datetime else None,
}
new_submission = load_data_into_model(submission_attributes,
submission_data,
field_map=field_map,
value_map=value_map,
save=True)
# If there were any submissions which were temporarily modified, reassign the submission
for conflict in potential_conflicts:
remapped_previous = get_previous_submission(
conflict.toptier_code, conflict.reporting_fiscal_year,
conflict.reporting_fiscal_period)
logger.info(
"New Previous Submission ID for Submission ID {} permanently mapped to {} "
.format(conflict.submission_id, remapped_previous))
conflict.previous_submission = remapped_previous
conflict.save()
return new_submission
def get_submission_attributes(broker_submission_id, submission_data):
"""
For a specified broker submission, return the existing corresponding usaspending
submission record or create and return a new one.
"""
# check if we already have an entry for this broker submission id; if not, create one
submission_attributes, created = SubmissionAttributes.objects.get_or_create(
broker_submission_id=broker_submission_id)
if created:
# this is the first time we're loading this broker submission
logger.info(
'Creating broker submission id {}'.format(broker_submission_id))
else:
# we've already loaded this broker submission, so delete it before reloading
# if there's another submission that references this one as a "previous submission"
# do not proceed.
# TODO: now that we're chaining submisisons together, get clarification on
# what should happen when a submission in the middle of the chain is deleted
downstream_submission = SubmissionAttributes.objects.filter(
previous_submission=submission_attributes).first()
if downstream_submission is not None:
message = (
'Broker submission {} (API submission id = {}) has a downstream submission (id={}) and '
'cannot be deleted'.format(
broker_submission_id, submission_attributes.submission_id,
downstream_submission.submission_id))
raise ValueError(message)
logger.info(
'Broker bubmission id {} already exists. It will be deleted.'.
format(broker_submission_id))
call_command('rm_submission', broker_submission_id)
# Find the previous submission for this CGAC and fiscal year (if there is one)
previous_submission = get_previous_submission(
submission_data['cgac_code'], submission_data['reporting_fiscal_year'],
submission_data['reporting_fiscal_period'])
# Update and save submission attributes
field_map = {
'reporting_period_start': 'reporting_start_date',
'reporting_period_end': 'reporting_end_date',
'quarter_format_flag': 'is_quarter_format',
}
# Create our value map - specific data to load
value_map = {
'broker_submission_id':
broker_submission_id,
'reporting_fiscal_quarter':
get_fiscal_quarter(submission_data['reporting_fiscal_period']),
'previous_submission':
None if previous_submission is None else previous_submission
}
return load_data_into_model(submission_attributes,
submission_data,
field_map=field_map,
value_map=value_map,
save=True)