def update_all_for_term(cls, term_id):
app.logger.info('Starting alert update')
enrollments_for_term = data_loch.get_enrollments_for_term(str(term_id))
no_activity_alerts_enabled = cls.no_activity_alerts_enabled()
infrequent_activity_alerts_enabled = cls.infrequent_activity_alerts_enabled(
)
for row in enrollments_for_term:
enrollments = json.loads(row['enrollment_term']).get(
'enrollments', [])
for enrollment in enrollments:
cls.update_alerts_for_enrollment(
row['sid'], term_id, enrollment,
no_activity_alerts_enabled,
infrequent_activity_alerts_enabled)
if app.config['ALERT_HOLDS_ENABLED'] and str(
term_id) == current_term_id():
holds = data_loch.get_sis_holds()
for row in holds:
hold_feed = json.loads(row['feed'])
cls.update_hold_alerts(row['sid'], term_id,
hold_feed.get('type'),
hold_feed.get('reason'))
if app.config['ALERT_WITHDRAWAL_ENABLED'] and str(
term_id) == current_term_id():
profiles = data_loch.get_student_profiles()
for row in profiles:
profile_feed = json.loads(row['profile'])
if 'withdrawalCancel' in (profile_feed.get('sisProfile')
or {}):
cls.update_withdrawal_cancel_alerts(row['sid'], term_id)
app.logger.info('Alert update complete')
def get_summary_student_profiles(sids, term_id=None):
if not sids:
return []
benchmark = get_benchmarker('get_summary_student_profiles')
benchmark('begin')
# TODO It's probably more efficient to store summary profiles in the loch, rather than distilling them
# on the fly from full profiles.
profiles = get_full_student_profiles(sids)
# TODO Many views require no term enrollment information other than a units count. This datum too should be
# stored in the loch without BOAC having to crunch it.
if not term_id:
term_id = current_term_id()
benchmark('begin enrollments query')
enrollments_for_term = data_loch.get_enrollments_for_term(term_id, sids)
benchmark('end enrollments query')
enrollments_by_sid = {
row['sid']: json.loads(row['enrollment_term'])
for row in enrollments_for_term
}
benchmark('begin term GPA query')
term_gpas = get_term_gpas_by_sid(sids)
benchmark('end term GPA query')
benchmark('begin profile transformation')
for profile in profiles:
# Strip SIS details to lighten the API load.
sis_profile = profile.pop('sisProfile', None)
if sis_profile:
profile['cumulativeGPA'] = sis_profile.get('cumulativeGPA')
profile['cumulativeUnits'] = sis_profile.get('cumulativeUnits')
profile['currentTerm'] = sis_profile.get('currentTerm')
profile['expectedGraduationTerm'] = sis_profile.get(
'expectedGraduationTerm')
profile['level'] = _get_sis_level_description(sis_profile)
profile['majors'] = sorted(
plan.get('description')
for plan in sis_profile.get('plans', []))
profile['transfer'] = sis_profile.get('transfer')
if sis_profile.get('withdrawalCancel'):
profile['withdrawalCancel'] = sis_profile['withdrawalCancel']
# Add the singleton term.
term = enrollments_by_sid.get(profile['sid'])
profile['hasCurrentTermEnrollments'] = False
if term:
profile['analytics'] = term.pop('analytics', None)
profile['term'] = term
if term['termId'] == current_term_id() and len(
term['enrollments']) > 0:
profile['hasCurrentTermEnrollments'] = True
profile['termGpa'] = term_gpas.get(profile['sid'])
benchmark('end')
return profiles
def get_summary_student_profiles(sids, include_historical=False, term_id=None):
if not sids:
return []
benchmark = get_benchmarker('get_summary_student_profiles')
benchmark('begin')
# TODO It's probably more efficient to store summary profiles in the loch, rather than distilling them
# on the fly from full profiles.
profiles = get_full_student_profiles(sids)
# TODO Many views require no term enrollment information other than a units count. This datum too should be
# stored in the loch without BOAC having to crunch it.
if not term_id:
term_id = current_term_id()
benchmark('begin enrollments query')
enrollments_for_term = data_loch.get_enrollments_for_term(term_id, sids)
benchmark('end enrollments query')
enrollments_by_sid = {row['sid']: json.loads(row['enrollment_term']) for row in enrollments_for_term}
benchmark('begin term GPA query')
term_gpas = get_term_gpas_by_sid(sids)
benchmark('end term GPA query')
remaining_sids = list(set(sids) - set([p.get('sid') for p in profiles]))
if len(remaining_sids) and include_historical:
benchmark('begin historical profile supplement')
historical_profile_rows = data_loch.get_historical_student_profiles_for_sids(remaining_sids)
def _historicize_profile(row):
return {
**json.loads(row['profile']),
**{
'fullProfilePending': True,
},
}
historical_profiles = [_historicize_profile(row) for row in historical_profile_rows]
# We don't expect photo information to show for historical profiles, but we still need a placeholder element
# in the feed so the front end can show the proper fallback.
_merge_photo_urls(historical_profiles)
for historical_profile in historical_profiles:
ManuallyAddedAdvisee.find_or_create(historical_profile['sid'])
profiles += historical_profiles
historical_enrollments_for_term = data_loch.get_historical_enrollments_for_term(term_id, remaining_sids)
for row in historical_enrollments_for_term:
enrollments_by_sid[row['sid']] = json.loads(row['enrollment_term'])
benchmark('end historical profile supplement')
benchmark('begin profile transformation')
for profile in profiles:
summarize_profile(profile, enrollments=enrollments_by_sid, term_gpas=term_gpas)
benchmark('end')
return profiles
def get_student_profile_summaries(sids, term_id=None):
if not sids:
return []
benchmark = get_benchmarker('get_student_profile_summaries')
benchmark('begin')
profile_results = data_loch.get_student_profile_summaries(sids)
if not profile_results:
return []
profiles_by_sid = _get_profiles_by_sid(profile_results)
profiles = []
for sid in sids:
profile = profiles_by_sid.get(sid)
if profile:
profiles.append(profile)
benchmark('begin photo merge')
_merge_photo_urls(profiles)
benchmark('end photo merge')
scope = get_student_query_scope()
benchmark('begin ASC profile merge')
_merge_asc_student_profile_data(profiles_by_sid, scope)
benchmark('end ASC profile merge')
if 'COENG' in scope or 'ADMIN' in scope:
benchmark('begin COE profile merge')
_merge_coe_student_profile_data(profiles_by_sid)
benchmark('end COE profile merge')
# TODO Many views require no term enrollment information other than a units count. This datum too should be
# stored in the loch without BOAC having to crunch it.
if not term_id:
term_id = current_term_id()
benchmark('begin enrollments query')
enrollments_for_term = data_loch.get_enrollments_for_term(term_id, sids)
benchmark('end enrollments query')
enrollments_by_sid = {
row['sid']: json.loads(row['enrollment_term'])
for row in enrollments_for_term
}
for profile in profiles:
_merge_enrollments(profile, enrollments=enrollments_by_sid)
benchmark('end')
return profiles
def update_all_for_term(cls, term_id):
app.logger.info('Starting alert update')
enrollments_for_term = data_loch.get_enrollments_for_term(str(term_id))
no_activity_alerts_enabled = cls.no_activity_alerts_enabled()
infrequent_activity_alerts_enabled = cls.infrequent_activity_alerts_enabled(
)
for row in enrollments_for_term:
enrollments = json.loads(row['enrollment_term']).get(
'enrollments', [])
for enrollment in enrollments:
cls.update_alerts_for_enrollment(
sid=row['sid'],
term_id=term_id,
enrollment=enrollment,
no_activity_alerts_enabled=no_activity_alerts_enabled,
infrequent_activity_alerts_enabled=
infrequent_activity_alerts_enabled,
)
profiles = data_loch.get_student_profiles()
if app.config['ALERT_WITHDRAWAL_ENABLED'] and str(
term_id) == current_term_id():
for row in profiles:
sis_profile_feed = json.loads(
row['profile']).get('sisProfile') or {}
if sis_profile_feed.get('withdrawalCancel',
{}).get('termId') == str(term_id):
cls.update_withdrawal_cancel_alerts(row['sid'], term_id)
sids = [p['sid'] for p in profiles]
for sid, academic_standing_list in get_academic_standing_by_sid(
sids).items():
standing = next((s for s in academic_standing_list
if s['termId'] == str(term_id)), None)
if standing and standing['status'] in ('DIS', 'PRO', 'SUB'):
cls.update_academic_standing_alerts(
action_date=standing['actionDate'],
sid=standing['sid'],
status=standing['status'],
term_id=term_id,
)
app.logger.info('Alert update complete')
def get_summary_student_profiles(sids, include_historical=False, term_id=None):
if not sids:
return []
benchmark = get_benchmarker('get_summary_student_profiles')
benchmark('begin')
# TODO It's probably more efficient to store summary profiles in the loch, rather than distilling them
# on the fly from full profiles.
profiles = get_full_student_profiles(sids)
# TODO Many views require no term enrollment information other than a units count. This datum too should be
# stored in the loch without BOAC having to crunch it.
if not term_id:
term_id = current_term_id()
benchmark('begin enrollments query')
enrollments_for_term = data_loch.get_enrollments_for_term(term_id, sids)
benchmark('end enrollments query')
enrollments_by_sid = {row['sid']: json.loads(row['enrollment_term']) for row in enrollments_for_term}
benchmark('begin academic standing query')
academic_standing = get_academic_standing_by_sid(sids)
benchmark('end academic standing query')
benchmark('begin term GPA query')
term_gpas = get_term_gpas_by_sid(sids)
benchmark('end term GPA query')
remaining_sids = list(set(sids) - set([p.get('sid') for p in profiles]))
if len(remaining_sids) and include_historical:
benchmark('begin historical profile supplement')
historical_profiles = get_historical_student_profiles(remaining_sids)
profiles += historical_profiles
historical_enrollments_for_term = data_loch.get_historical_enrollments_for_term(str(term_id), remaining_sids)
for row in historical_enrollments_for_term:
enrollments_by_sid[row['sid']] = json.loads(row['enrollment_term'])
benchmark('end historical profile supplement')
benchmark('begin profile transformation')
for profile in profiles:
summarize_profile(profile, enrollments=enrollments_by_sid, academic_standing=academic_standing, term_gpas=term_gpas)
benchmark('end')
return profiles
def get_course_student_profiles(term_id, section_id, offset=None, limit=None, featured=None):
benchmark = get_benchmarker('get_course_student_profiles')
benchmark('begin')
enrollment_rows = data_loch.get_sis_section_enrollments(
term_id,
section_id,
offset=offset,
limit=limit,
)
sids = [str(r['sid']) for r in enrollment_rows]
if offset or len(sids) >= 50:
count_result = data_loch.get_sis_section_enrollments_count(term_id, section_id)
total_student_count = count_result[0]['count']
else:
total_student_count = len(sids)
# If we have a featured UID not already present in the result set, add the corresponding SID only if the
# student is enrolled.
if featured and not next((r for r in enrollment_rows if str(r['uid']) == featured), None):
featured_enrollment_rows = data_loch.get_sis_section_enrollment_for_uid(term_id, section_id, featured)
if featured_enrollment_rows:
sids = [str(featured_enrollment_rows[0]['sid'])] + sids
# TODO It's probably more efficient to store class profiles in the loch, rather than distilling them
# on the fly from full profiles.
students = get_full_student_profiles(sids)
benchmark('begin enrollments query')
enrollments_for_term = data_loch.get_enrollments_for_term(term_id, sids)
benchmark('end enrollments query')
enrollments_by_sid = {row['sid']: json.loads(row['enrollment_term']) for row in enrollments_for_term}
academic_standing = get_academic_standing_by_sid(sids, as_dicts=True)
term_gpas = get_term_gpas_by_sid(sids, as_dicts=True)
all_canvas_sites = {}
benchmark('begin profile transformation')
for student in students:
# Strip SIS details to lighten the API load.
sis_profile = student.pop('sisProfile', None)
if sis_profile:
student['academicCareerStatus'] = sis_profile.get('academicCareerStatus')
student['cumulativeGPA'] = sis_profile.get('cumulativeGPA')
student['cumulativeUnits'] = sis_profile.get('cumulativeUnits')
student['degree'] = sis_profile.get('degree')
student['level'] = _get_sis_level_description(sis_profile)
student['currentTerm'] = sis_profile.get('currentTerm')
student['majors'] = _get_active_plan_descriptions(sis_profile)
student['transfer'] = sis_profile.get('transfer')
term = enrollments_by_sid.get(student['sid'])
if term:
# Strip the enrollments list down to the section of interest.
enrollments = term.pop('enrollments', [])
for enrollment in enrollments:
_section = next((s for s in enrollment['sections'] if str(s['ccn']) == section_id), None)
if _section:
canvas_sites = enrollment.get('canvasSites', [])
student['enrollment'] = {
'canvasSites': canvas_sites,
'enrollmentStatus': _section.get('enrollmentStatus', None),
'grade': enrollment.get('grade', None),
'gradingBasis': enrollment.get('gradingBasis', None),
'midtermGrade': enrollment.get('midtermGrade', None),
}
student['analytics'] = analytics.mean_metrics_across_sites(canvas_sites, 'student')
# If more than one course site is associated with this section, derive mean metrics from as many sites as possible.
for site in canvas_sites:
if site['canvasCourseId'] not in all_canvas_sites:
all_canvas_sites[site['canvasCourseId']] = site
continue
student['academicStanding'] = academic_standing.get(student['sid'])
student['termGpa'] = term_gpas.get(student['sid'])
benchmark('end profile transformation')
mean_metrics = analytics.mean_metrics_across_sites(all_canvas_sites.values(), 'courseMean')
mean_metrics['gpa'] = {}
mean_gpas = data_loch.get_sis_section_mean_gpas(term_id, section_id)
for row in mean_gpas:
mean_metrics['gpa'][str(row['gpa_term_id'])] = row['avg_gpa']
benchmark('end')
return {
'students': students,
'totalStudentCount': total_student_count,
'meanMetrics': mean_metrics,
}
def low_assignment_scores(term_id=None):
if not term_id:
term_id = current_term_id()
examined_sids = set()
low_sids = set()
multiple_low_sids = set()
# Many students in a low percentile may have received a reasonably high score.
# Since instructors rarely grade on a curve, it may be fine to receive a score of 85
# even if all other students received 90 or above.
sids_with_low_raw_scores = set()
primary_sections = set()
primary_sections_with_scored_assignments = set()
primary_sections_with_plottable_assignments = set()
enrollments_for_term = data_loch.get_enrollments_for_term(term_id)
enrollments_by_sid = {
row['sid']: json.loads(row['enrollment_term'])
for row in enrollments_for_term
}
itr = iter(enrollments_by_sid.items())
for (sid, term) in itr:
examined_sids.add(sid)
for enr in term['enrollments']:
first_section = enr['sections'][0]
if not first_section.get('primary'):
continue
ccn = first_section['ccn']
primary_sections.add(ccn)
for site in enr['canvasSites']:
score_info = site['analytics']['currentScore']
if score_info['courseDeciles']:
primary_sections_with_scored_assignments.add(ccn)
if score_info['boxPlottable']:
primary_sections_with_plottable_assignments.add(ccn)
pct = score_info['student']['roundedUpPercentile']
if pct is not None:
if pct <= 25:
if sid in low_sids:
multiple_low_sids.add(sid)
low_sids.add(sid)
max_score = score_info['courseDeciles'][9]
if score_info['student']['raw'] < (max_score *
0.7):
sids_with_low_raw_scores.add(sid)
app.logger.warn(
f'Total of {len(examined_sids)} students in classes. {len(low_sids)} with low scores in a class.'
)
app.logger.warn(f'Low scorers: {sorted(low_sids)}')
app.logger.warn(
f' {len(multiple_low_sids)} Low scorers in multiple sites: {sorted(multiple_low_sids)}'
)
app.logger.warn(
f' {len(sids_with_low_raw_scores)} Low scorers with raw score < 70% of max: {sorted(sids_with_low_raw_scores)}'
)
app.logger.warn(
f'Total of {len(primary_sections)} primary sections. '
f'{len(primary_sections_with_scored_assignments)} have scores. '
f'{len(primary_sections_with_plottable_assignments)} have a reasonable range of scores.'
)
return {
'sids':
sorted(examined_sids),
'low_sids':
sorted(low_sids),
'multiple_low_sids':
sorted(multiple_low_sids),
'sids_with_low_raw_scores':
sorted(sids_with_low_raw_scores),
'primary_sections_count':
len(primary_sections),
'sections_scored_count':
len(primary_sections_with_scored_assignments),
'sections_with_range_of_scores_count':
len(primary_sections_with_plottable_assignments),
}
def get_course_student_profiles(term_id, section_id, offset=None, limit=None):
enrollment_rows = data_loch.get_sis_section_enrollments(
term_id,
section_id,
scope=get_student_query_scope(),
offset=offset,
limit=limit,
)
sids = [str(r['sid']) for r in enrollment_rows]
if offset or len(sids) >= 50:
count_result = data_loch.get_sis_section_enrollments_count(
term_id, section_id, scope=get_student_query_scope())
total_student_count = count_result[0]['count']
else:
total_student_count = len(sids)
# TODO It's probably more efficient to store class profiles in the loch, rather than distilling them
# on the fly from full profiles.
students = get_full_student_profiles(sids)
enrollments_for_term = data_loch.get_enrollments_for_term(term_id, sids)
enrollments_by_sid = {
row['sid']: json.loads(row['enrollment_term'])
for row in enrollments_for_term
}
all_canvas_sites = {}
for student in students:
# Strip SIS details to lighten the API load.
sis_profile = student.pop('sisProfile', None)
if sis_profile:
student['cumulativeGPA'] = sis_profile.get('cumulativeGPA')
student['cumulativeUnits'] = sis_profile.get('cumulativeUnits')
student['level'] = sis_profile.get('level', {}).get('description')
student['majors'] = sorted(
plan.get('description')
for plan in sis_profile.get('plans', []))
term = enrollments_by_sid.get(student['sid'])
student['hasCurrentTermEnrollments'] = False
if term:
# Strip the enrollments list down to the section of interest.
enrollments = term.pop('enrollments', [])
for enrollment in enrollments:
_section = next((s for s in enrollment['sections']
if str(s['ccn']) == section_id), None)
if _section:
canvas_sites = enrollment.get('canvasSites', [])
student['enrollment'] = {
'canvasSites': canvas_sites,
'enrollmentStatus':
_section.get('enrollmentStatus', None),
'grade': enrollment.get('grade', None),
'gradingBasis': enrollment.get('gradingBasis', None),
}
student['analytics'] = analytics.mean_metrics_across_sites(
canvas_sites, 'student')
# If more than one course site is associated with this section, derive mean metrics from as many sites as possible.
for site in canvas_sites:
if site['canvasCourseId'] not in all_canvas_sites:
all_canvas_sites[site['canvasCourseId']] = site
continue
mean_metrics = analytics.mean_metrics_across_sites(
all_canvas_sites.values(), 'courseMean')
return {
'students': students,
'totalStudentCount': total_student_count,
'meanMetrics': mean_metrics,
}
def get_term_units_by_sid(term, sids):
results = data_loch.get_enrollments_for_term(term, sids)
enrollments_by_sid = {row['sid']: json.loads(row['enrollment_term']) for row in results}
return {sid: str(enrollments_by_sid.get(sid).pop("enrolledUnits")) for sid in sids}
