def test_term_id_lists(self, app):
all_term_ids = set(
berkeley.reverse_term_ids(include_future_terms=True,
include_legacy_terms=True))
canvas_integrated_term_ids = set(berkeley.reverse_term_ids())
future_term_ids = set(berkeley.future_term_ids())
legacy_term_ids = set(berkeley.legacy_term_ids())
assert canvas_integrated_term_ids < all_term_ids
assert berkeley.sis_term_id_for_name(
app.config['EARLIEST_LEGACY_TERM']) in all_term_ids
assert berkeley.sis_term_id_for_name(
app.config['EARLIEST_TERM']) in all_term_ids
assert berkeley.sis_term_id_for_name(
app.config['CURRENT_TERM']) in all_term_ids
assert berkeley.sis_term_id_for_name(
app.config['FUTURE_TERM']) in all_term_ids
assert berkeley.current_term_id() in canvas_integrated_term_ids
assert berkeley.earliest_term_id() in canvas_integrated_term_ids
assert future_term_ids.isdisjoint(canvas_integrated_term_ids)
assert future_term_ids < all_term_ids
assert berkeley.future_term_id() in future_term_ids
assert legacy_term_ids.isdisjoint(canvas_integrated_term_ids)
assert legacy_term_ids < all_term_ids
assert berkeley.earliest_legacy_term_id() in berkeley.legacy_term_ids()
def generate_feeds(self):
# Translation between canvas_user_id and UID/SID is needed to merge Canvas analytics data and SIS enrollment-based data.
advisees_by_canvas_id = {}
advisees_by_sid = {}
self.successes = []
self.failures = []
profile_tables = self.generate_student_profile_tables(
advisees_by_canvas_id, advisees_by_sid)
if not profile_tables:
raise BackgroundJobError(
'Failed to generate student profile tables.')
feed_path = app.config['LOCH_S3_BOAC_ANALYTICS_DATA_PATH'] + '/feeds/'
s3.upload_json(advisees_by_canvas_id,
feed_path + 'advisees_by_canvas_id.json')
upload_student_term_maps(advisees_by_sid)
# Avoid processing Canvas analytics data for future terms and pre-CS terms.
for term_id in (future_term_ids() + legacy_term_ids()):
enrollment_term_map = s3.get_object_json(
feed_path + f'enrollment_term_map_{term_id}.json')
if enrollment_term_map:
GenerateMergedEnrollmentTerm().refresh_student_enrollment_term(
term_id, enrollment_term_map)
canvas_integrated_term_ids = reverse_term_ids()
app.logger.info(
f'Will queue analytics generation for {len(canvas_integrated_term_ids)} terms on worker nodes.'
)
result = queue_merged_enrollment_term_jobs(self.job_id,
canvas_integrated_term_ids)
if not result:
raise BackgroundJobError('Failed to queue enrollment term jobs.')
refresh_all_from_staging(profile_tables)
self.update_redshift_academic_standing()
self.update_rds_profile_indexes()
app.logger.info(
'Profile generation complete; waiting for enrollment term generation to finish.'
)
while True:
sleep(1)
enrollment_results = get_merged_enrollment_term_job_status(
self.job_id)
if not enrollment_results:
raise BackgroundJobError('Failed to refresh RDS indexes.')
any_pending_job = next(
(row for row in enrollment_results
if row['status'] == 'created' or row['status'] == 'started'),
None)
if not any_pending_job:
break
app.logger.info('Exporting analytics data for archival purposes.')
unload_enrollment_terms([current_term_id(), future_term_id()])
app.logger.info('Refreshing enrollment terms in RDS.')
with rds.transaction() as transaction:
if self.refresh_rds_enrollment_terms(None, transaction):
transaction.commit()
app.logger.info('Refreshed RDS enrollment terms.')
else:
transaction.rollback()
raise BackgroundJobError(
'Failed to refresh RDS enrollment terms.')
status_string = f'Generated merged profiles ({len(self.successes)} successes, {len(self.failures)} failures).'
errored = False
for row in enrollment_results:
status_string += f" {row['details']}"
if row['status'] == 'error':
errored = True
truncate_staging_table('student_enrollment_terms')
if errored:
raise BackgroundJobError(status_string)
else:
return status_string