def update_all_for_term(cls, term_id): app.logger.info('Starting alert update') enrollments_for_term = data_loch.get_enrollments_for_term(str(term_id)) no_activity_alerts_enabled = cls.no_activity_alerts_enabled() infrequent_activity_alerts_enabled = cls.infrequent_activity_alerts_enabled( ) for row in enrollments_for_term: enrollments = json.loads(row['enrollment_term']).get( 'enrollments', []) for enrollment in enrollments: cls.update_alerts_for_enrollment( row['sid'], term_id, enrollment, no_activity_alerts_enabled, infrequent_activity_alerts_enabled) if app.config['ALERT_HOLDS_ENABLED'] and str( term_id) == current_term_id(): holds = data_loch.get_sis_holds() for row in holds: hold_feed = json.loads(row['feed']) cls.update_hold_alerts(row['sid'], term_id, hold_feed.get('type'), hold_feed.get('reason')) if app.config['ALERT_WITHDRAWAL_ENABLED'] and str( term_id) == current_term_id(): profiles = data_loch.get_student_profiles() for row in profiles: profile_feed = json.loads(row['profile']) if 'withdrawalCancel' in (profile_feed.get('sisProfile') or {}): cls.update_withdrawal_cancel_alerts(row['sid'], term_id) app.logger.info('Alert update complete')
def get_summary_student_profiles(sids, term_id=None): if not sids: return [] benchmark = get_benchmarker('get_summary_student_profiles') benchmark('begin') # TODO It's probably more efficient to store summary profiles in the loch, rather than distilling them # on the fly from full profiles. profiles = get_full_student_profiles(sids) # TODO Many views require no term enrollment information other than a units count. This datum too should be # stored in the loch without BOAC having to crunch it. if not term_id: term_id = current_term_id() benchmark('begin enrollments query') enrollments_for_term = data_loch.get_enrollments_for_term(term_id, sids) benchmark('end enrollments query') enrollments_by_sid = { row['sid']: json.loads(row['enrollment_term']) for row in enrollments_for_term } benchmark('begin term GPA query') term_gpas = get_term_gpas_by_sid(sids) benchmark('end term GPA query') benchmark('begin profile transformation') for profile in profiles: # Strip SIS details to lighten the API load. sis_profile = profile.pop('sisProfile', None) if sis_profile: profile['cumulativeGPA'] = sis_profile.get('cumulativeGPA') profile['cumulativeUnits'] = sis_profile.get('cumulativeUnits') profile['currentTerm'] = sis_profile.get('currentTerm') profile['expectedGraduationTerm'] = sis_profile.get( 'expectedGraduationTerm') profile['level'] = _get_sis_level_description(sis_profile) profile['majors'] = sorted( plan.get('description') for plan in sis_profile.get('plans', [])) profile['transfer'] = sis_profile.get('transfer') if sis_profile.get('withdrawalCancel'): profile['withdrawalCancel'] = sis_profile['withdrawalCancel'] # Add the singleton term. term = enrollments_by_sid.get(profile['sid']) profile['hasCurrentTermEnrollments'] = False if term: profile['analytics'] = term.pop('analytics', None) profile['term'] = term if term['termId'] == current_term_id() and len( term['enrollments']) > 0: profile['hasCurrentTermEnrollments'] = True profile['termGpa'] = term_gpas.get(profile['sid']) benchmark('end') return profiles
def get_summary_student_profiles(sids, include_historical=False, term_id=None): if not sids: return [] benchmark = get_benchmarker('get_summary_student_profiles') benchmark('begin') # TODO It's probably more efficient to store summary profiles in the loch, rather than distilling them # on the fly from full profiles. profiles = get_full_student_profiles(sids) # TODO Many views require no term enrollment information other than a units count. This datum too should be # stored in the loch without BOAC having to crunch it. if not term_id: term_id = current_term_id() benchmark('begin enrollments query') enrollments_for_term = data_loch.get_enrollments_for_term(term_id, sids) benchmark('end enrollments query') enrollments_by_sid = {row['sid']: json.loads(row['enrollment_term']) for row in enrollments_for_term} benchmark('begin term GPA query') term_gpas = get_term_gpas_by_sid(sids) benchmark('end term GPA query') remaining_sids = list(set(sids) - set([p.get('sid') for p in profiles])) if len(remaining_sids) and include_historical: benchmark('begin historical profile supplement') historical_profile_rows = data_loch.get_historical_student_profiles_for_sids(remaining_sids) def _historicize_profile(row): return { **json.loads(row['profile']), **{ 'fullProfilePending': True, }, } historical_profiles = [_historicize_profile(row) for row in historical_profile_rows] # We don't expect photo information to show for historical profiles, but we still need a placeholder element # in the feed so the front end can show the proper fallback. _merge_photo_urls(historical_profiles) for historical_profile in historical_profiles: ManuallyAddedAdvisee.find_or_create(historical_profile['sid']) profiles += historical_profiles historical_enrollments_for_term = data_loch.get_historical_enrollments_for_term(term_id, remaining_sids) for row in historical_enrollments_for_term: enrollments_by_sid[row['sid']] = json.loads(row['enrollment_term']) benchmark('end historical profile supplement') benchmark('begin profile transformation') for profile in profiles: summarize_profile(profile, enrollments=enrollments_by_sid, term_gpas=term_gpas) benchmark('end') return profiles
def get_student_profile_summaries(sids, term_id=None): if not sids: return [] benchmark = get_benchmarker('get_student_profile_summaries') benchmark('begin') profile_results = data_loch.get_student_profile_summaries(sids) if not profile_results: return [] profiles_by_sid = _get_profiles_by_sid(profile_results) profiles = [] for sid in sids: profile = profiles_by_sid.get(sid) if profile: profiles.append(profile) benchmark('begin photo merge') _merge_photo_urls(profiles) benchmark('end photo merge') scope = get_student_query_scope() benchmark('begin ASC profile merge') _merge_asc_student_profile_data(profiles_by_sid, scope) benchmark('end ASC profile merge') if 'COENG' in scope or 'ADMIN' in scope: benchmark('begin COE profile merge') _merge_coe_student_profile_data(profiles_by_sid) benchmark('end COE profile merge') # TODO Many views require no term enrollment information other than a units count. This datum too should be # stored in the loch without BOAC having to crunch it. if not term_id: term_id = current_term_id() benchmark('begin enrollments query') enrollments_for_term = data_loch.get_enrollments_for_term(term_id, sids) benchmark('end enrollments query') enrollments_by_sid = { row['sid']: json.loads(row['enrollment_term']) for row in enrollments_for_term } for profile in profiles: _merge_enrollments(profile, enrollments=enrollments_by_sid) benchmark('end') return profiles
def update_all_for_term(cls, term_id): app.logger.info('Starting alert update') enrollments_for_term = data_loch.get_enrollments_for_term(str(term_id)) no_activity_alerts_enabled = cls.no_activity_alerts_enabled() infrequent_activity_alerts_enabled = cls.infrequent_activity_alerts_enabled( ) for row in enrollments_for_term: enrollments = json.loads(row['enrollment_term']).get( 'enrollments', []) for enrollment in enrollments: cls.update_alerts_for_enrollment( sid=row['sid'], term_id=term_id, enrollment=enrollment, no_activity_alerts_enabled=no_activity_alerts_enabled, infrequent_activity_alerts_enabled= infrequent_activity_alerts_enabled, ) profiles = data_loch.get_student_profiles() if app.config['ALERT_WITHDRAWAL_ENABLED'] and str( term_id) == current_term_id(): for row in profiles: sis_profile_feed = json.loads( row['profile']).get('sisProfile') or {} if sis_profile_feed.get('withdrawalCancel', {}).get('termId') == str(term_id): cls.update_withdrawal_cancel_alerts(row['sid'], term_id) sids = [p['sid'] for p in profiles] for sid, academic_standing_list in get_academic_standing_by_sid( sids).items(): standing = next((s for s in academic_standing_list if s['termId'] == str(term_id)), None) if standing and standing['status'] in ('DIS', 'PRO', 'SUB'): cls.update_academic_standing_alerts( action_date=standing['actionDate'], sid=standing['sid'], status=standing['status'], term_id=term_id, ) app.logger.info('Alert update complete')
def get_summary_student_profiles(sids, include_historical=False, term_id=None): if not sids: return [] benchmark = get_benchmarker('get_summary_student_profiles') benchmark('begin') # TODO It's probably more efficient to store summary profiles in the loch, rather than distilling them # on the fly from full profiles. profiles = get_full_student_profiles(sids) # TODO Many views require no term enrollment information other than a units count. This datum too should be # stored in the loch without BOAC having to crunch it. if not term_id: term_id = current_term_id() benchmark('begin enrollments query') enrollments_for_term = data_loch.get_enrollments_for_term(term_id, sids) benchmark('end enrollments query') enrollments_by_sid = {row['sid']: json.loads(row['enrollment_term']) for row in enrollments_for_term} benchmark('begin academic standing query') academic_standing = get_academic_standing_by_sid(sids) benchmark('end academic standing query') benchmark('begin term GPA query') term_gpas = get_term_gpas_by_sid(sids) benchmark('end term GPA query') remaining_sids = list(set(sids) - set([p.get('sid') for p in profiles])) if len(remaining_sids) and include_historical: benchmark('begin historical profile supplement') historical_profiles = get_historical_student_profiles(remaining_sids) profiles += historical_profiles historical_enrollments_for_term = data_loch.get_historical_enrollments_for_term(str(term_id), remaining_sids) for row in historical_enrollments_for_term: enrollments_by_sid[row['sid']] = json.loads(row['enrollment_term']) benchmark('end historical profile supplement') benchmark('begin profile transformation') for profile in profiles: summarize_profile(profile, enrollments=enrollments_by_sid, academic_standing=academic_standing, term_gpas=term_gpas) benchmark('end') return profiles
def get_course_student_profiles(term_id, section_id, offset=None, limit=None, featured=None): benchmark = get_benchmarker('get_course_student_profiles') benchmark('begin') enrollment_rows = data_loch.get_sis_section_enrollments( term_id, section_id, offset=offset, limit=limit, ) sids = [str(r['sid']) for r in enrollment_rows] if offset or len(sids) >= 50: count_result = data_loch.get_sis_section_enrollments_count(term_id, section_id) total_student_count = count_result[0]['count'] else: total_student_count = len(sids) # If we have a featured UID not already present in the result set, add the corresponding SID only if the # student is enrolled. if featured and not next((r for r in enrollment_rows if str(r['uid']) == featured), None): featured_enrollment_rows = data_loch.get_sis_section_enrollment_for_uid(term_id, section_id, featured) if featured_enrollment_rows: sids = [str(featured_enrollment_rows[0]['sid'])] + sids # TODO It's probably more efficient to store class profiles in the loch, rather than distilling them # on the fly from full profiles. students = get_full_student_profiles(sids) benchmark('begin enrollments query') enrollments_for_term = data_loch.get_enrollments_for_term(term_id, sids) benchmark('end enrollments query') enrollments_by_sid = {row['sid']: json.loads(row['enrollment_term']) for row in enrollments_for_term} academic_standing = get_academic_standing_by_sid(sids, as_dicts=True) term_gpas = get_term_gpas_by_sid(sids, as_dicts=True) all_canvas_sites = {} benchmark('begin profile transformation') for student in students: # Strip SIS details to lighten the API load. sis_profile = student.pop('sisProfile', None) if sis_profile: student['academicCareerStatus'] = sis_profile.get('academicCareerStatus') student['cumulativeGPA'] = sis_profile.get('cumulativeGPA') student['cumulativeUnits'] = sis_profile.get('cumulativeUnits') student['degree'] = sis_profile.get('degree') student['level'] = _get_sis_level_description(sis_profile) student['currentTerm'] = sis_profile.get('currentTerm') student['majors'] = _get_active_plan_descriptions(sis_profile) student['transfer'] = sis_profile.get('transfer') term = enrollments_by_sid.get(student['sid']) if term: # Strip the enrollments list down to the section of interest. enrollments = term.pop('enrollments', []) for enrollment in enrollments: _section = next((s for s in enrollment['sections'] if str(s['ccn']) == section_id), None) if _section: canvas_sites = enrollment.get('canvasSites', []) student['enrollment'] = { 'canvasSites': canvas_sites, 'enrollmentStatus': _section.get('enrollmentStatus', None), 'grade': enrollment.get('grade', None), 'gradingBasis': enrollment.get('gradingBasis', None), 'midtermGrade': enrollment.get('midtermGrade', None), } student['analytics'] = analytics.mean_metrics_across_sites(canvas_sites, 'student') # If more than one course site is associated with this section, derive mean metrics from as many sites as possible. for site in canvas_sites: if site['canvasCourseId'] not in all_canvas_sites: all_canvas_sites[site['canvasCourseId']] = site continue student['academicStanding'] = academic_standing.get(student['sid']) student['termGpa'] = term_gpas.get(student['sid']) benchmark('end profile transformation') mean_metrics = analytics.mean_metrics_across_sites(all_canvas_sites.values(), 'courseMean') mean_metrics['gpa'] = {} mean_gpas = data_loch.get_sis_section_mean_gpas(term_id, section_id) for row in mean_gpas: mean_metrics['gpa'][str(row['gpa_term_id'])] = row['avg_gpa'] benchmark('end') return { 'students': students, 'totalStudentCount': total_student_count, 'meanMetrics': mean_metrics, }
def low_assignment_scores(term_id=None): if not term_id: term_id = current_term_id() examined_sids = set() low_sids = set() multiple_low_sids = set() # Many students in a low percentile may have received a reasonably high score. # Since instructors rarely grade on a curve, it may be fine to receive a score of 85 # even if all other students received 90 or above. sids_with_low_raw_scores = set() primary_sections = set() primary_sections_with_scored_assignments = set() primary_sections_with_plottable_assignments = set() enrollments_for_term = data_loch.get_enrollments_for_term(term_id) enrollments_by_sid = { row['sid']: json.loads(row['enrollment_term']) for row in enrollments_for_term } itr = iter(enrollments_by_sid.items()) for (sid, term) in itr: examined_sids.add(sid) for enr in term['enrollments']: first_section = enr['sections'][0] if not first_section.get('primary'): continue ccn = first_section['ccn'] primary_sections.add(ccn) for site in enr['canvasSites']: score_info = site['analytics']['currentScore'] if score_info['courseDeciles']: primary_sections_with_scored_assignments.add(ccn) if score_info['boxPlottable']: primary_sections_with_plottable_assignments.add(ccn) pct = score_info['student']['roundedUpPercentile'] if pct is not None: if pct <= 25: if sid in low_sids: multiple_low_sids.add(sid) low_sids.add(sid) max_score = score_info['courseDeciles'][9] if score_info['student']['raw'] < (max_score * 0.7): sids_with_low_raw_scores.add(sid) app.logger.warn( f'Total of {len(examined_sids)} students in classes. {len(low_sids)} with low scores in a class.' ) app.logger.warn(f'Low scorers: {sorted(low_sids)}') app.logger.warn( f' {len(multiple_low_sids)} Low scorers in multiple sites: {sorted(multiple_low_sids)}' ) app.logger.warn( f' {len(sids_with_low_raw_scores)} Low scorers with raw score < 70% of max: {sorted(sids_with_low_raw_scores)}' ) app.logger.warn( f'Total of {len(primary_sections)} primary sections. ' f'{len(primary_sections_with_scored_assignments)} have scores. ' f'{len(primary_sections_with_plottable_assignments)} have a reasonable range of scores.' ) return { 'sids': sorted(examined_sids), 'low_sids': sorted(low_sids), 'multiple_low_sids': sorted(multiple_low_sids), 'sids_with_low_raw_scores': sorted(sids_with_low_raw_scores), 'primary_sections_count': len(primary_sections), 'sections_scored_count': len(primary_sections_with_scored_assignments), 'sections_with_range_of_scores_count': len(primary_sections_with_plottable_assignments), }
def get_course_student_profiles(term_id, section_id, offset=None, limit=None): enrollment_rows = data_loch.get_sis_section_enrollments( term_id, section_id, scope=get_student_query_scope(), offset=offset, limit=limit, ) sids = [str(r['sid']) for r in enrollment_rows] if offset or len(sids) >= 50: count_result = data_loch.get_sis_section_enrollments_count( term_id, section_id, scope=get_student_query_scope()) total_student_count = count_result[0]['count'] else: total_student_count = len(sids) # TODO It's probably more efficient to store class profiles in the loch, rather than distilling them # on the fly from full profiles. students = get_full_student_profiles(sids) enrollments_for_term = data_loch.get_enrollments_for_term(term_id, sids) enrollments_by_sid = { row['sid']: json.loads(row['enrollment_term']) for row in enrollments_for_term } all_canvas_sites = {} for student in students: # Strip SIS details to lighten the API load. sis_profile = student.pop('sisProfile', None) if sis_profile: student['cumulativeGPA'] = sis_profile.get('cumulativeGPA') student['cumulativeUnits'] = sis_profile.get('cumulativeUnits') student['level'] = sis_profile.get('level', {}).get('description') student['majors'] = sorted( plan.get('description') for plan in sis_profile.get('plans', [])) term = enrollments_by_sid.get(student['sid']) student['hasCurrentTermEnrollments'] = False if term: # Strip the enrollments list down to the section of interest. enrollments = term.pop('enrollments', []) for enrollment in enrollments: _section = next((s for s in enrollment['sections'] if str(s['ccn']) == section_id), None) if _section: canvas_sites = enrollment.get('canvasSites', []) student['enrollment'] = { 'canvasSites': canvas_sites, 'enrollmentStatus': _section.get('enrollmentStatus', None), 'grade': enrollment.get('grade', None), 'gradingBasis': enrollment.get('gradingBasis', None), } student['analytics'] = analytics.mean_metrics_across_sites( canvas_sites, 'student') # If more than one course site is associated with this section, derive mean metrics from as many sites as possible. for site in canvas_sites: if site['canvasCourseId'] not in all_canvas_sites: all_canvas_sites[site['canvasCourseId']] = site continue mean_metrics = analytics.mean_metrics_across_sites( all_canvas_sites.values(), 'courseMean') return { 'students': students, 'totalStudentCount': total_student_count, 'meanMetrics': mean_metrics, }
def get_term_units_by_sid(term, sids): results = data_loch.get_enrollments_for_term(term, sids) enrollments_by_sid = {row['sid']: json.loads(row['enrollment_term']) for row in results} return {sid: str(enrollments_by_sid.get(sid).pop("enrolledUnits")) for sid in sids}