def test_course_has_opted_out(self):
"""Do not send email to courses that have opted out."""
def _emails_sent():
return _get_emails_sent(email_template_type=email_template_type,
section_id=section_id,
term_id=term_id)
term_id = app.config['CURRENT_TERM_ID']
section_id = 50000
CoursePreference.update_opt_out(term_id=term_id,
section_id=section_id,
opt_out=True)
email_template_type = 'invitation'
recipient = {
'name': 'William Peter Blatty',
'uid': '10001',
}
QueuedEmail.create(section_id,
email_template_type,
term_id,
recipient=recipient)
std_commit(allow_test_environment=True)
emails_sent_before = _emails_sent()
# Run the job
QueuedEmailsJob(simply_yield).run()
std_commit(allow_test_environment=True)
# Expect no emails sent
emails_sent_after = _emails_sent()
assert len(emails_sent_after) == len(emails_sent_before)
assert list(map(lambda e: e.id, emails_sent_before)) == list(
map(lambda e: e.id, emails_sent_after))
def test_course_has_opted_out(self):
"""Do not send email to courses that have opted out."""
def _emails_sent():
return _get_emails_sent(email_template_type=email_template_type,
section_id=section_id,
term_id=term_id)
term_id = app.config['CURRENT_TERM_ID']
section_id = 28602
CoursePreference.update_opt_out(term_id=term_id,
section_id=section_id,
opt_out=True)
email_template_type = 'invitation'
QueuedEmail.create(section_id, email_template_type, term_id)
std_commit(allow_test_environment=True)
before = utc_now()
emails_sent_before = _emails_sent()
# Run the job
QueuedEmailsJob(app.app_context).run()
std_commit(allow_test_environment=True)
# Expect no emails sent
emails_sent_after = _emails_sent()
assert len(emails_sent_after) == len(emails_sent_before)
assert not next(
(e for e in emails_sent_after
if e.section_id == section_id and e.sent_at > before), None)
def unschedule():
params = request.get_json()
term_id = params.get('termId')
section_id = params.get('sectionId')
course = SisSection.get_course(term_id, section_id, include_deleted=True) if (term_id and section_id) else None
if not course:
raise BadRequestError('Required params missing or invalid')
if not (course['scheduled'] or course['hasNecessaryApprovals']):
raise BadRequestError(f'Section id {section_id}, term id {term_id} is not currently scheduled or queued for scheduling')
Approval.delete(term_id=term_id, section_id=section_id)
Scheduled.delete(term_id=term_id, section_id=section_id)
event_id = (course.get('scheduled') or {}).get('kalturaScheduleId')
if event_id:
try:
Kaltura().delete(event_id)
except (KalturaClientException, KalturaException) as e:
message = f'Failed to delete Kaltura schedule: {event_id}'
app.logger.error(message)
app.logger.exception(e)
send_system_error_email(
message=f'{message}\n\n<pre>{traceback.format_exc()}</pre>',
subject=message,
)
CoursePreference.update_opt_out(
term_id=term_id,
section_id=section_id,
opt_out=True,
)
return tolerant_jsonify(SisSection.get_course(term_id, section_id, include_deleted=True))
def test_opt_out_cross_listings(self, client, admin_session):
"""If a section opts out then its cross-listings are automatically opted out."""
with test_approvals_workflow(app):
# First, opt out
cross_listed_section_ids = [28475, 27950, 32827]
self._api_opt_out_update(
client,
term_id=self.term_id,
section_id=cross_listed_section_ids[-1],
opt_out=True,
)
section_ids_opted_out = CoursePreference.get_section_ids_opted_out(term_id=self.term_id)
for section_id in cross_listed_section_ids:
assert section_id in section_ids_opted_out
std_commit(allow_test_environment=True)
# Opt back in
self._api_opt_out_update(
client,
term_id=self.term_id,
section_id=cross_listed_section_ids[0],
opt_out=False,
)
section_ids_opted_out = CoursePreference.get_section_ids_opted_out(term_id=self.term_id)
for section_id in cross_listed_section_ids:
assert section_id not in section_ids_opted_out
def test_do_not_email_filter(self, client, db, admin_session):
"""Do Not Email filter: Courses in eligible room; "opt out" is true; all stages of approval; not scheduled."""
with test_approvals_workflow(app):
# Send invites them opt_out.
for section_id in (section_1_id, section_in_ineligible_room, section_3_id, section_4_id):
CoursePreference.update_opt_out(section_id=section_id, term_id=self.term_id, opt_out=True)
in_enabled_room = _is_course_in_enabled_room(section_id=section_id, term_id=self.term_id)
if section_id == section_in_ineligible_room:
# Courses in ineligible rooms will be excluded from the feed.
assert not in_enabled_room
else:
assert in_enabled_room
SentEmail.create(
section_id=section_id,
recipient_uids=_get_instructor_uids(section_id=section_id, term_id=self.term_id),
template_type='invitation',
term_id=self.term_id,
)
# If course has approvals but not scheduled then it will show up in the feed.
Approval.create(
approved_by_uid=_get_instructor_uids(section_id=section_1_id, term_id=self.term_id)[0],
approver_type_='instructor',
cross_listed_section_ids=[],
publish_type_='canvas',
recording_type_='presentation_audio',
room_id=Room.get_room_id(section_id=section_1_id, term_id=self.term_id),
section_id=section_1_id,
term_id=self.term_id,
)
# Feed will exclude scheduled.
_schedule_recordings(
section_id=section_3_id,
term_id=self.term_id,
)
std_commit(allow_test_environment=True)
api_json = self._api_courses(client, term_id=self.term_id, filter_='Do Not Email')
for section_id in (section_1_id, section_4_id):
# The 'Do Not Email' course is in the feed
assert _find_course(api_json=api_json, section_id=section_id)
for section_id in (section_3_id, section_in_ineligible_room):
# Excluded courses
assert not _find_course(api_json=api_json, section_id=section_id)
def test_authorized(self, client, admin_session):
"""Only admins can toggle the do-not-email preference of any given course."""
section_ids_opted_out = CoursePreference.get_section_ids_opted_out(term_id=self.term_id)
previously_opted_out = section_1_id not in section_ids_opted_out
opt_out = not previously_opted_out
self._api_opt_out_update(
client,
term_id=self.term_id,
section_id=section_1_id,
opt_out=opt_out,
)
std_commit(allow_test_environment=True)
section_ids_opted_out = CoursePreference.get_section_ids_opted_out(term_id=self.term_id)
if opt_out:
assert section_1_id in section_ids_opted_out
else:
assert section_1_id not in section_ids_opted_out
def update_opt_out():
params = request.get_json()
term_id = params.get('termId')
section_id = params.get('sectionId')
opt_out = params.get('optOut')
preferences = CoursePreference.update_opt_out(
term_id=term_id,
section_id=section_id,
opt_out=opt_out,
)
return tolerant_jsonify(preferences.to_api_json())
def test_course_opted_out(self, app):
"""Do not send email to courses that have opted out."""
term_id = app.config['CURRENT_TERM_ID']
with test_approvals_workflow(app):
section_id = 50006
CoursePreference.update_opt_out(term_id=term_id,
section_id=section_id,
opt_out=True)
std_commit(allow_test_environment=True)
timestamp = utc_now()
# Emails are queued but not sent.
InvitationJob(simply_yield).run()
assert len(_get_invitations_since(term_id, timestamp)) == 0
# Emails are sent.
QueuedEmailsJob(simply_yield).run()
invitations = _get_invitations_since(term_id, timestamp)
assert len(invitations) == 15
assert not next(
(e for e in invitations if e.section_id == section_id), None)
def _after_approval(course):
section_id = course['sectionId']
term_id = course['termId']
approvals = Approval.get_approvals(section_id=section_id, term_id=term_id)
if get_courses_ready_to_schedule(approvals=approvals, term_id=term_id):
# Queuing course for scheduling wipes any opt-out preference.
if course['hasOptedOut']:
CoursePreference.update_opt_out(
term_id=term_id,
section_id=section_id,
opt_out=False,
)
if app.config['FEATURE_FLAG_SCHEDULE_RECORDINGS_SYNCHRONOUSLY']:
# Feature flag intended for dev workstation ONLY. Do not enable in diablo-dev|qa|prod.
schedule_recordings(
all_approvals=approvals,
course=course,
)
return SisSection.get_course(section_id=course['sectionId'], term_id=course['termId'])
else:
return course
def test_do_not_email_filter(self, client, admin_session):
"""Do Not Email filter: Courses in eligible room; "opt out" is true; all stages of approval; not scheduled."""
with test_approvals_workflow(app):
# Send invites them opt_out.
for section_id in (section_1_id, section_in_ineligible_room, section_3_id, section_4_id):
CoursePreference.update_opt_out(section_id=section_id, term_id=self.term_id, opt_out=True)
in_enabled_room = _is_course_in_enabled_room(section_id=section_id, term_id=self.term_id)
if section_id == section_in_ineligible_room:
# Courses in ineligible rooms will be excluded from the feed.
assert not in_enabled_room
else:
assert in_enabled_room
self._send_invitation_email(section_id)
# If course has approvals but not scheduled then it will show up in the feed.
self._create_approval(section_4_id)
# Feed will exclude scheduled.
mock_scheduled(
section_id=section_3_id,
term_id=self.term_id,
)
std_commit(allow_test_environment=True)
api_json = self._api_courses(client, term_id=self.term_id, filter_='Do Not Email')
# Opted-out courses are in the feed, whether approved or not
course_1 = _find_course(api_json=api_json, section_id=section_1_id)
assert course_1['approvalStatus'] == 'Invited'
assert course_1['schedulingStatus'] == 'Not Scheduled'
course_4 = _find_course(api_json=api_json, section_id=section_4_id)
assert course_4['approvalStatus'] == 'Approved'
assert course_4['schedulingStatus'] == 'Queued for Scheduling'
for section_id in (section_3_id, section_in_ineligible_room):
# Excluded courses
assert not _find_course(api_json=api_json, section_id=section_id)
def update_opt_out():
params = request.get_json()
term_id = params.get('termId')
section_id = params.get('sectionId')
course = SisSection.get_course(term_id, section_id) if (term_id and section_id) else None
opt_out = params.get('optOut')
if not course or opt_out is None:
raise BadRequestError('Required params missing or invalid')
if course['scheduled']:
raise BadRequestError('Cannot update opt-out on scheduled course')
preferences = CoursePreference.update_opt_out(
term_id=term_id,
section_id=section_id,
opt_out=opt_out,
)
return tolerant_jsonify(preferences.to_api_json())
def schedule_recordings(all_approvals, course):
def _report_error(subject):
message = f'{subject}\n\n<pre>{course}</pre>'
app.logger.error(message)
send_system_error_email(message=message, subject=subject)
meetings = course.get('meetings', {}).get('eligible', [])
meeting = meetings[0] if len(meetings) == 1 else None
if not meeting:
_report_error(
subject=
f"{course['label']} not scheduled. Unique eligible meeting pattern not found."
)
return None
all_approvals.sort(key=lambda a: a.created_at.isoformat())
latest_approval = all_approvals[-1]
room = Room.get_room(latest_approval.room_id)
if room.location != meeting['location']:
_report_error(
subject=
f"{course['label']} not scheduled. Room change: {room.location} to {meeting['location']}"
)
return None
has_admin_approval = next(
(a for a in all_approvals if a.approver_type == 'admin'), None)
approved_by_uids = set(a.approved_by_uid for a in all_approvals)
instructor_uids = set([i['uid'] for i in course['instructors']])
if not has_admin_approval and not instructor_uids.issubset(
approved_by_uids):
_report_error(
subject=
f"{course['label']} not scheduled. We are missing instructor approval(s)."
)
return None
term_id = course['termId']
section_id = int(course['sectionId'])
scheduled = None
if room.kaltura_resource_id:
try:
kaltura_schedule_id = Kaltura().schedule_recording(
canvas_course_site_ids=[
c['courseSiteId'] for c in course['canvasCourseSites']
],
course_label=course['label'],
instructors=course['instructors'],
meeting=meeting,
publish_type=latest_approval.publish_type,
recording_type=latest_approval.recording_type,
room=room,
term_id=term_id,
)
scheduled = Scheduled.create(
course_display_name=course['label'],
instructor_uids=instructor_uids,
kaltura_schedule_id=kaltura_schedule_id,
meeting_days=meeting['days'],
meeting_end_date=get_recording_end_date(meeting),
meeting_end_time=meeting['endTime'],
meeting_start_date=get_recording_start_date(
meeting, return_today_if_past_start=True),
meeting_start_time=meeting['startTime'],
publish_type_=latest_approval.publish_type,
recording_type_=latest_approval.recording_type,
room_id=room.id,
section_id=section_id,
term_id=term_id,
)
# Turn off opt-out setting if present.
if section_id in CoursePreference.get_section_ids_opted_out(
term_id=term_id):
CoursePreference.update_opt_out(
term_id=term_id,
section_id=section_id,
opt_out=False,
)
notify_instructors_recordings_scheduled(course=course,
scheduled=scheduled)
uids = [approval.approved_by_uid for approval in all_approvals]
app.logger.info(
f'Recordings scheduled for course {section_id} per approvals: {", ".join(uids)}'
)
except (KalturaClientException, KalturaException) as e:
# Error codes: https://developer.kaltura.com/api-docs/Error_Codes
summary = f"Failed to schedule recordings {course['label']} (section_id: {course['sectionId']})"
app.logger.error(summary)
app.logger.exception(e)
send_system_error_email(
message=f'{summary}\n\n<pre>{traceback.format_exc()}</pre>',
subject=f'{summary[:50]}...' if len(summary) > 50 else summary,
)
else:
app.logger.warn(f"""
SKIP schedule recordings because room has no 'kaltura_resource_id'.
Course: {course['label']}
Room: {room.location}
Latest approved_by_uid: {latest_approval.approved_by_uid}
""")
return scheduled
def _to_api_json(term_id, rows, include_rooms=True):
rows = rows.fetchall()
section_ids = list(set(int(row['section_id']) for row in rows))
courses_per_id = {}
# Perform bulk queries and build data structures for feed generation.
section_ids_opted_out = CoursePreference.get_section_ids_opted_out(
term_id=term_id)
invited_uids_by_section_id = {section_id: [] for section_id in section_ids}
for invite in SentEmail.get_emails_of_type(section_ids=section_ids,
template_type='invitation',
term_id=term_id):
if invite.recipient_uid not in invited_uids_by_section_id[
invite.section_id]:
invited_uids_by_section_id[invite.section_id].append(
invite.recipient_uid)
approval_results = Approval.get_approvals_per_section_ids(
section_ids=section_ids, term_id=term_id)
scheduled_results = Scheduled.get_scheduled_per_section_ids(
section_ids=section_ids, term_id=term_id)
room_ids = set(row['room_id'] for row in rows)
room_ids.update(a.room_id for a in approval_results)
room_ids.update(s.room_id for s in scheduled_results)
rooms = Room.get_rooms(list(room_ids))
rooms_by_id = {room.id: room for room in rooms}
approvals_by_section_id = {section_id: [] for section_id in section_ids}
for approval in approval_results:
approvals_by_section_id[approval.section_id].append(
approval.to_api_json(rooms_by_id=rooms_by_id))
scheduled_by_section_id = {
s.section_id: s.to_api_json(rooms_by_id=rooms_by_id)
for s in scheduled_results
}
cross_listings_per_section_id, instructors_per_section_id, canvas_sites_by_section_id = _get_cross_listed_courses(
section_ids=section_ids,
term_id=term_id,
approvals=approvals_by_section_id,
invited_uids=invited_uids_by_section_id,
)
# Construct course objects.
# If course has multiple instructors or multiple rooms then the section_id will be represented across multiple rows.
# Multiple rooms are rare, but a course is sometimes associated with both an eligible and an ineligible room. We
# order rooms in SQL by capability, NULLS LAST, and use scheduling data from the first row available.
for row in rows:
section_id = int(row['section_id'])
if section_id in courses_per_id:
course = courses_per_id[section_id]
else:
# Approvals and scheduled (JSON)
approvals = approvals_by_section_id.get(section_id)
scheduled = scheduled_by_section_id.get(section_id)
# Instructors per cross-listings
cross_listed_courses = cross_listings_per_section_id.get(
section_id, [])
instructors = instructors_per_section_id.get(section_id, [])
# Construct course
course = {
'allowedUnits':
row['allowed_units'],
'approvals':
approvals,
'canvasCourseSites':
canvas_sites_by_section_id.get(section_id, []),
'courseName':
row['course_name'],
'courseTitle':
row['course_title'],
'crossListings':
cross_listed_courses,
'deletedAt':
safe_strftime(row['deleted_at'], '%Y-%m-%d'),
'hasOptedOut':
section_id in section_ids_opted_out,
'instructionFormat':
row['instruction_format'],
'instructors':
instructors,
'invitees':
invited_uids_by_section_id.get(section_id),
'isPrimary':
row['is_primary'],
'label':
_construct_course_label(
course_name=row['course_name'],
instruction_format=row['instruction_format'],
section_num=row['section_num'],
cross_listings=cross_listed_courses,
),
'meetings': {
'eligible': [],
'ineligible': [],
},
'nonstandardMeetingDates':
False,
'sectionId':
section_id,
'sectionNum':
row['section_num'],
'scheduled':
scheduled,
'termId':
row['term_id'],
}
courses_per_id[section_id] = course
# Note: Instructors associated with cross-listings are slurped up separately.
instructor_uid = row['instructor_uid']
if instructor_uid and instructor_uid not in [
i['uid'] for i in course['instructors']
]:
course['instructors'].append(
_to_instructor_json(
row=row,
approvals=course['approvals'],
invited_uids=course['invitees'],
), )
meeting = _to_meeting_json(row)
eligible_meetings = course['meetings']['eligible']
ineligible_meetings = course['meetings']['ineligible']
if not next((m for m in (eligible_meetings + ineligible_meetings)
if meeting.items() <= m.items()), None):
room = rooms_by_id.get(
row['room_id']) if 'room_id' in row.keys() else None
if room and room.capability:
meeting['eligible'] = True
meeting.update({
'recordingEndDate':
safe_strftime(get_recording_end_date(meeting), '%Y-%m-%d'),
'recordingStartDate':
safe_strftime(get_recording_start_date(meeting),
'%Y-%m-%d'),
})
eligible_meetings.append(meeting)
eligible_meetings.sort(
key=lambda m: f"{m['startDate']} {m['startTime']}")
if meeting['startDate'] != app.config[
'CURRENT_TERM_BEGIN'] or meeting[
'endDate'] != app.config['CURRENT_TERM_END']:
course['nonstandardMeetingDates'] = True
else:
meeting['eligible'] = False
ineligible_meetings.append(meeting)
ineligible_meetings.sort(
key=lambda m: f"{m['startDate']} {m['startTime']}")
if include_rooms:
meeting['room'] = room.to_api_json() if room else None
# Next, construct the feed
api_json = []
for section_id, course in courses_per_id.items():
_decorate_course(course)
# Add course to the feed
api_json.append(course)
return api_json
def _to_api_json(term_id, rows, include_rooms=True):
courses_per_id = {}
instructors_per_section_id = {}
section_ids_opted_out = CoursePreference.get_section_ids_opted_out(
term_id=term_id)
# If course has multiple instructors then the section_id will be represented across multiple rows.
for row in rows:
approvals = []
section_id = int(row['section_id'])
if section_id not in courses_per_id:
# Construct new course
instructors_per_section_id[section_id] = []
has_opted_out = section_id in section_ids_opted_out
cross_listings = _get_cross_listed_courses(section_id=section_id,
term_id=term_id)
approvals, scheduled = _get_approvals_and_scheduled(
section_ids=[section_id] +
[c['sectionId'] for c in cross_listings],
term_id=term_id,
)
course_name = row['course_name']
instruction_format = row['instruction_format']
section_num = row['section_num']
course = {
'allowedUnits': row['allowed_units'],
'canvasCourseSites': _canvas_course_sites(term_id, section_id),
'courseName': course_name,
'courseTitle': row['course_title'],
'crossListings': cross_listings,
'hasOptedOut': has_opted_out,
'instructionFormat': instruction_format,
'instructors': [],
'isPrimary': row['is_primary'],
'label': f'{course_name}, {instruction_format} {section_num}',
'meetingDays': format_days(row['meeting_days']),
'meetingEndDate': row['meeting_end_date'],
'meetingEndTime': format_time(row['meeting_end_time']),
'meetingLocation': row['meeting_location'],
'meetingStartDate': row['meeting_start_date'],
'meetingStartTime': format_time(row['meeting_start_time']),
'sectionId': section_id,
'sectionNum': section_num,
'termId': row['term_id'],
'approvals': approvals,
'scheduled': scheduled,
}
invites = SentEmail.get_emails_of_type(
section_id=section_id,
template_type='invitation',
term_id=term_id,
)
course['invitees'] = []
for invite in invites:
course['invitees'].extend(invite.recipient_uids)
if scheduled:
course['status'] = 'Scheduled'
elif approvals:
course['status'] = 'Partially Approved'
else:
course['status'] = 'Invited' if invites else 'Not Invited'
if include_rooms:
room = Room.get_room(
row['room_id']).to_api_json() if 'room_id' in row else None
course['room'] = room
courses_per_id[section_id] = course
# Build upon course object with one instructor per row.
instructor_uid = row['instructor_uid']
if instructor_uid not in [
i['uid'] for i in instructors_per_section_id[section_id]
]:
instructors_per_section_id[section_id].append({
'approval':
next((a for a in approvals
if a['approvedBy']['uid'] == instructor_uid), False),
'deptCode':
row['instructor_dept_code'],
'email':
row['instructor_email'],
'name':
row['instructor_name'],
'roleCode':
row['instructor_role_code'],
'uid':
instructor_uid,
'wasSentInvite':
instructor_uid in courses_per_id[section_id]['invitees'],
})
api_json = []
for section_id, course in courses_per_id.items():
room_id = course.get('room', {}).get('id')
def _add_and_verify_room(approval_or_scheduled):
action_room_id = approval_or_scheduled.get('room', {}).get('id')
is_obsolete_room = not room_id or room_id != action_room_id
approval_or_scheduled['hasObsoleteRoom'] = is_obsolete_room
course['instructors'] = instructors_per_section_id[section_id]
course['hasNecessaryApprovals'] = _has_necessary_approvals(course)
scheduled = course['scheduled']
# Check for course changes w.r.t. room, meeting times, and instructors.
if scheduled:
def _meeting(obj):
return f'{obj["meetingDays"]}-{obj["meetingStartTime"]}-{obj["meetingEndTime"]}'
instructor_uids = set(
[instructor['uid'] for instructor in course['instructors']])
scheduled['hasObsoleteInstructors'] = instructor_uids != set(
scheduled['instructorUids'])
scheduled['hasObsoleteMeetingTimes'] = _meeting(
course) != _meeting(scheduled)
_add_and_verify_room(scheduled)
for approval in course['approvals']:
_add_and_verify_room(approval)
# Add course to the feed
api_json.append(course)
return api_json