def get_submission_attributes(submission_id, submission_data):
"""
For a specified broker submission, return the existing corresponding usaspending submission record or
create and return a new one.
"""
dabs_window = DABSSubmissionWindowSchedule.objects.filter(
submission_fiscal_year=submission_data["reporting_fiscal_year"],
submission_fiscal_month=submission_data["reporting_fiscal_period"],
is_quarter=submission_data["is_quarter_format"],
).first()
if not dabs_window:
raise RuntimeError(
f"Missing DABS Window record necessary for {submission_id}")
# check if we already have an entry for this submission id; if not, create one
submission_attributes, created = SubmissionAttributes.objects.get_or_create(
submission_id=submission_id,
defaults={"submission_window": dabs_window})
if created:
# this is the first time we're loading this submission
logger.info(f"Creating submission {submission_id}")
else:
# we've already loaded this submission, so delete it before reloading
logger.info(
f"Submission {submission_id} already exists. It will be deleted.")
call_command("rm_submission", submission_id)
submission_data["reporting_agency_name"] = retrive_agency_name_from_code(
submission_data["toptier_code"])
# Update and save submission attributes
field_map = {
"reporting_period_start": "reporting_start_date",
"reporting_period_end": "reporting_end_date",
"quarter_format_flag": "is_quarter_format",
}
# Create our value map - specific data to load
value_map = {
"reporting_fiscal_quarter":
get_fiscal_quarter(submission_data["reporting_fiscal_period"])
}
new_submission = load_data_into_model(submission_attributes,
submission_data,
field_map=field_map,
value_map=value_map,
save=True)
return new_submission
def get_submission_attributes(broker_submission_id, submission_data):
"""
For a specified broker submission, return the existing corresponding usaspending submission record or create and
return a new one.
"""
# check if we already have an entry for this broker submission id; if not, create one
submission_attributes, created = SubmissionAttributes.\
objects.get_or_create(broker_submission_id=broker_submission_id)
if created:
# this is the first time we're loading this broker submission
logger.info('Creating broker submission id {}'.format(broker_submission_id))
else:
# we've already loaded this broker submission, so delete it before reloading if there's another submission that
# references this one as a "previous submission" do not proceed.
# TODO: now that we're chaining submisisons together, get clarification on what should happen when a submission
# in the middle of the chain is deleted
TasProgramActivityObjectClassQuarterly.refresh_downstream_quarterly_numbers(submission_attributes.submission_id)
logger.info('Broker submission id {} already exists. It will be deleted.'.format(broker_submission_id))
call_command('rm_submission', broker_submission_id)
logger.info("Merging CGAC and FREC columns")
submission_data["cgac_code"] = submission_data["cgac_code"]\
if submission_data["cgac_code"] else submission_data["frec_code"]
# Find the previous submission for this CGAC and fiscal year (if there is one)
previous_submission = get_previous_submission(
submission_data['cgac_code'],
submission_data['reporting_fiscal_year'],
submission_data['reporting_fiscal_period'])
# Update and save submission attributes
field_map = {
'reporting_period_start': 'reporting_start_date',
'reporting_period_end': 'reporting_end_date',
'quarter_format_flag': 'is_quarter_format',
}
# Create our value map - specific data to load
value_map = {
'broker_submission_id': broker_submission_id,
'reporting_fiscal_quarter': get_fiscal_quarter(submission_data['reporting_fiscal_period']),
'previous_submission': None if previous_submission is None else previous_submission,
# pull in broker's last update date to use as certified date
'certified_date': submission_data['updated_at'].date() if type(
submission_data['updated_at']) == datetime else None,
}
return load_data_into_model(
submission_attributes, submission_data,
field_map=field_map, value_map=value_map, save=True)
def update_broker_submissions(apps, schema_editor):
"""Update new columns of existing broker submissions."""
subs = apps.get_model("submissions", "SubmissionAttributes")
# get all submissions that came from the broker
broker_subs = subs.objects.filter(
broker_submission_id__isnull=False)
# fill in new data for each submission, using the reporting period
# start date to calculate fiscal year, fiscal period, and fiscal quarter
for s in broker_subs:
fiscal_year, fiscal_period = get_fiscal_year_period(s.reporting_period_start)
s.reporting_fiscal_year = fiscal_year
s.reporting_fiscal_period = fiscal_period
s.reporting_fiscal_quarter = get_fiscal_quarter(fiscal_period)
s.quarter_format_flag = True # not used in USAspending code right now
s.save()
def get_submission_attributes(broker_submission_id, submission_data):
"""
For a specified broker submission, return the existing corresponding usaspending submission record or create and
return a new one.
"""
# check if we already have an entry for this broker submission id; if not, create one
submission_attributes, created = SubmissionAttributes.objects.get_or_create(
broker_submission_id=broker_submission_id)
if created:
# this is the first time we're loading this broker submission
logger.info(
"Creating broker submission id {}".format(broker_submission_id))
else:
# we've already loaded this broker submission, so delete it before reloading if there's another submission that
# references this one as a "previous submission" do not proceed.
# TODO: now that we're chaining submissions together, get clarification on what should happen when a submission
# in the middle of the chain is deleted
TasProgramActivityObjectClassQuarterly.refresh_downstream_quarterly_numbers(
submission_attributes.submission_id)
logger.info(
"Broker submission id {} already exists. It will be deleted.".
format(broker_submission_id))
call_command("rm_submission", broker_submission_id)
logger.info("Merging CGAC and FREC columns")
submission_data["toptier_code"] = (submission_data["cgac_code"]
if submission_data["cgac_code"] else
submission_data["frec_code"])
# Find the previous submission for this CGAC and fiscal year (if there is one)
previous_submission = get_previous_submission(
submission_data["toptier_code"],
submission_data["reporting_fiscal_year"],
submission_data["reporting_fiscal_period"],
)
# if another submission lists the previous submission as its previous submission, set to null and update later
potential_conflicts = []
if previous_submission:
potential_conflicts = SubmissionAttributes.objects.filter(
previous_submission=previous_submission)
if potential_conflicts:
logger.info(
"==== ATTENTION! Previous Submission ID Conflict Detected ===="
)
for conflict in potential_conflicts:
logger.info(
"Temporarily setting {}'s Previous Submission ID from {} to null"
.format(conflict, previous_submission.submission_id))
conflict.previous_submission = None
conflict.save()
# Update and save submission attributes
field_map = {
"reporting_period_start": "reporting_start_date",
"reporting_period_end": "reporting_end_date",
"quarter_format_flag": "is_quarter_format",
}
# Create our value map - specific data to load
value_map = {
"broker_submission_id":
broker_submission_id,
"reporting_fiscal_quarter":
get_fiscal_quarter(submission_data["reporting_fiscal_period"]),
"previous_submission":
previous_submission,
# pull in broker's last update date to use as certified date
"certified_date":
submission_data["updated_at"].date()
if type(submission_data["updated_at"]) == datetime else None,
}
new_submission = load_data_into_model(submission_attributes,
submission_data,
field_map=field_map,
value_map=value_map,
save=True)
# If there were any submissions which were temporarily modified, reassign the submission
for conflict in potential_conflicts:
remapped_previous = get_previous_submission(
conflict.toptier_code, conflict.reporting_fiscal_year,
conflict.reporting_fiscal_period)
logger.info(
"New Previous Submission ID for Submission ID {} permanently mapped to {} "
.format(conflict.submission_id, remapped_previous))
conflict.previous_submission = remapped_previous
conflict.save()
return new_submission
def get_submission_attributes(broker_submission_id, submission_data):
"""
For a specified broker submission, return the existing corresponding usaspending
submission record or create and return a new one.
"""
# check if we already have an entry for this broker submission id; if not, create one
submission_attributes, created = SubmissionAttributes.objects.get_or_create(
broker_submission_id=broker_submission_id)
if created:
# this is the first time we're loading this broker submission
logger.info(
'Creating broker submission id {}'.format(broker_submission_id))
else:
# we've already loaded this broker submission, so delete it before reloading
# if there's another submission that references this one as a "previous submission"
# do not proceed.
# TODO: now that we're chaining submisisons together, get clarification on
# what should happen when a submission in the middle of the chain is deleted
downstream_submission = SubmissionAttributes.objects.filter(
previous_submission=submission_attributes).first()
if downstream_submission is not None:
message = (
'Broker submission {} (API submission id = {}) has a downstream submission (id={}) and '
'cannot be deleted'.format(
broker_submission_id, submission_attributes.submission_id,
downstream_submission.submission_id))
raise ValueError(message)
logger.info(
'Broker bubmission id {} already exists. It will be deleted.'.
format(broker_submission_id))
call_command('rm_submission', broker_submission_id)
# Find the previous submission for this CGAC and fiscal year (if there is one)
previous_submission = get_previous_submission(
submission_data['cgac_code'], submission_data['reporting_fiscal_year'],
submission_data['reporting_fiscal_period'])
# Update and save submission attributes
field_map = {
'reporting_period_start': 'reporting_start_date',
'reporting_period_end': 'reporting_end_date',
'quarter_format_flag': 'is_quarter_format',
}
# Create our value map - specific data to load
value_map = {
'broker_submission_id':
broker_submission_id,
'reporting_fiscal_quarter':
get_fiscal_quarter(submission_data['reporting_fiscal_period']),
'previous_submission':
None if previous_submission is None else previous_submission
}
return load_data_into_model(submission_attributes,
submission_data,
field_map=field_map,
value_map=value_map,
save=True)