def _create_advising_notes(data, advisor, unit): try: notes = data['notes'] if not isinstance(notes, (list, tuple)): raise ValidationError("Notes not in list format") if len(notes) is 0: raise ValidationError("No advising notes present") except KeyError: raise ValidationError("No advising notes present") for note in notes: advisornote = AdvisorNote() try: emplid = note['emplid'] text = note['text'] if not isinstance(emplid, int): raise ValidationError("Note emplid must be an integer") if not isinstance(text, basestring): raise ValidationError("Note text must be a string") except KeyError: raise ValidationError("Emplid or text not present in note") try: student = Person.objects.get(emplid=emplid) except Person.DoesNotExist: raise ValidationError("Emplid '%d' doesn't exist" % emplid) advisornote.student = student advisornote.advisor = advisor advisornote.unit = unit advisornote.text = text #Check for fileupload if 'filename' in note and 'mediatype' in note and 'data' in note: filename = note['filename'] mediatype = note['mediatype'] file_data = note['data'] if not isinstance(filename, basestring): raise ValidationError("Note filename must be a string") if not isinstance(mediatype, basestring): raise ValidationError("Note mediatype must be a string") if not isinstance(file_data, basestring): raise ValidationError("Note file data must be a string") try: file_data = base64.b64decode(file_data) except TypeError: raise ValidationError( "Invalid base64 data for note file attachment") advisornote.file_attachment.save(name=filename, content=ContentFile(file_data), save=False) advisornote.file_mediatype = mediatype advisornote.save()
def _create_advising_notes(data, advisor, unit): try: notes = data['notes'] if not isinstance(notes, (list, tuple)): raise ValidationError("Notes not in list format") if len(notes) is 0: raise ValidationError("No advising notes present") except KeyError: raise ValidationError("No advising notes present") for note in notes: advisornote = AdvisorNote() try: emplid = note['emplid'] text = note['text'] if not isinstance(emplid, int): raise ValidationError("Note emplid must be an integer") if not isinstance(text, str): raise ValidationError("Note text must be a string") except KeyError: raise ValidationError("Emplid or text not present in note") try: student = Person.objects.get(emplid=emplid) except Person.DoesNotExist: raise ValidationError("Emplid '%d' doesn't exist" % emplid) advisornote.student = student advisornote.advisor = advisor advisornote.unit = unit advisornote.text = text #Check for fileupload if 'filename' in note and 'mediatype' in note and 'data' in note: filename = note['filename'] mediatype = note['mediatype'] file_data = note['data'] if not isinstance(filename, str): raise ValidationError("Note filename must be a string") if not isinstance(mediatype, str): raise ValidationError("Note mediatype must be a string") if not isinstance(file_data, str): raise ValidationError("Note file data must be a string") try: file_data = base64.b64decode(file_data) except TypeError: raise ValidationError("Invalid base64 data for note file attachment") advisornote.file_attachment.save(name=filename, content=ContentFile(file_data), save=False) advisornote.file_mediatype = mediatype advisornote.save()
def get_advisornote(self, key, person, advisor, created, delete_old_file=False, offset=0): """ get_or_create for this usage """ created = created + datetime.timedelta(minutes=offset) # look for previously-imported version of this note, so we're roughly idempotent oldnotes = AdvisorNote.objects.filter(student=person, advisor=advisor, created_at=created, unit=self.unit) oldnotes = [n for n in oldnotes if 'import_key' in n.config and n.config['import_key'] == key] if oldnotes: note = oldnotes[0] if delete_old_file and note.file_attachment and os.path.isfile(note.file_attachment.path): # let file be recreated below os.remove(note.file_attachment.path) note.file_attachment = None note.file_mediatype = None else: note = AdvisorNote(student=person, advisor=advisor, created_at=created, unit=self.unit) note.config['import_key'] = key note.config['src'] = 'crim_import' return note, bool(oldnotes)
def test_pages(self): client = Client() client.login_user("dzhao") adv = Person.objects.get(userid='dzhao') # create some notes to work with unit = Unit.objects.get(slug='cmpt') ns = NonStudent(first_name="Non", last_name="Student", high_school="North South Burnaby-Surrey", start_year=2000) ns.save() p1 = Person.objects.get(userid='0aaa6') n1 = AdvisorNote(student=p1, text="He seems like a nice student.", unit=unit, advisor=adv) n1.save() p2 = Person.objects.get(userid='0aaa8') p2.userid = None p2.save() n2 = AdvisorNote(student=p2, text="This guy doesn't have an active computing account.", unit=unit, advisor=adv) n2.save() n3 = AdvisorNote(nonstudent=ns, text="What a horrible person.", unit=unit, advisor=adv) n3.save() # index page url = reverse('advising:advising', kwargs={}) response = basic_page_tests(self, client, url) self.assertEqual(response.status_code, 200) # new nonstudent form url = reverse('advising:new_nonstudent', kwargs={}) response = basic_page_tests(self, client, url) self.assertEqual(response.status_code, 200) # student with userid url = reverse('advising:advising', kwargs={}) response = client.post(url, {'search': p1.emplid}) self.assertEqual(response.status_code, 302) redir_url = response['location'] student_url = reverse('advising:student_notes', kwargs={'userid': p1.userid}) self.assertIn(student_url, redir_url) response = basic_page_tests(self, client, student_url, check_valid=False) self.assertEqual(response.status_code, 200) new_url = reverse('advising:new_note', kwargs={'userid': p1.userid}) response = basic_page_tests(self, client, new_url) self.assertEqual(response.status_code, 200) # student with no userid response = client.post(url, {'search': p2.emplid}) self.assertEqual(response.status_code, 302) redir_url = response['location'] student_url = reverse('advising:student_notes', kwargs={'userid': p2.emplid}) self.assertIn(student_url, redir_url) response = basic_page_tests(self, client, student_url, check_valid=False) self.assertEqual(response.status_code, 200) new_url = reverse('advising:new_note', kwargs={'userid': p2.emplid}) response = basic_page_tests(self, client, new_url) self.assertEqual(response.status_code, 200) # non student response = client.post(url, {'search': ns.slug}) self.assertEqual(response.status_code, 302) redir_url = response['location'] student_url = reverse('advising:student_notes', kwargs={'userid': ns.slug}) self.assertIn(student_url, redir_url) response = basic_page_tests(self, client, student_url, check_valid=False) self.assertEqual(response.status_code, 200) new_url = reverse('advising:new_note', kwargs={'userid': ns.slug}) response = basic_page_tests(self, client, new_url) self.assertEqual(response.status_code, 200) # note content search url = reverse('advising:note_search', kwargs={}) + "?text-search=nice" response = basic_page_tests(self, client, url) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.context['notes']), 1) url = reverse('advising:note_search', kwargs={}) + "?text-search=" response = basic_page_tests(self, client, url) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.context['notes']), 3)
def process_row(self, i, row): """ Actually process each individual row """ # It's 0 indexed, and we already consumed the header row. # This is used for error messages so the user can refer to the input file and know the correct line number. row_num = i + 2 note_id = row['note_id'] # Just in case someone had a bunch of trailing slashes, etc, make sure we get solely the file name for the key. file_basename = os.path.basename(os.path.normpath(self.file.name)) # In order for the keys to match (to check for duplicates), they have to have been imported from this importer, # with the same filename, and with the same note_id. key = "notes_import-%s-%s" % (file_basename, note_id) # Find the recipient of the note: student_emplid = row['emplid'] # See if we can actually cast the emplid to int, since the function we call does so without checking. try: int(student_emplid) except ValueError: if self.verbose: error_msg = "ERROR, emplid is not valid for recipient on row %i (emplid %s). Ignoring" % \ (row_num, student_emplid) self.errors.append(error_msg) print(error_msg) return p = add_person(student_emplid, commit=self.commit) if not p: if self.verbose: error_msg = "ERROR: Can't find recipient on row %i (emplid %s). Ignoring." % ( row_num, student_emplid) self.errors.append(error_msg) print(error_msg) return # Find the advisor who entered the note: advisor_emplid = row['creator_emplid'] # Same thing for the advisor try: int(advisor_emplid) except ValueError: if self.verbose: error_msg = "ERROR, emplid is not valid for advisor on row %i (emplid %s). Ignoring" % \ (row_num, advisor_emplid) self.errors.append(error_msg) print(error_msg) return u = add_person(advisor_emplid, commit=self.commit) if not u: if self.verbose: error_msg = "ERROR: Can't find advisor %s on row %i (emplid %s). Ignoring." % \ (advisor_emplid, row_num, student_emplid) self.errors.append(error_msg) print(error_msg) return advisor_userid = row['creator_computing_id'] if u.userid != advisor_userid: if self.verbose: error_msg = "ERROR: The advisor emplid and userid do not match the same person. Emplid %s, userid " \ "%s at row %i. Ignoring." % (advisor_emplid, advisor_userid, row_num) self.errors.append(error_msg) print(error_msg) return read_date = row['date_created'] # We expect a certain date format, try that first, as the function is slightly faster. try: date_created = datetime.strptime(read_date, "%Y-%m-%d %H:%M:%S") except ValueError: # Fine, try to make the dateutils parser figure it out, then. try: date_created = dateparser.parse(read_date) except ValueError: if self.verbose: error_msg = "ERROR: Cannot deduce the correct date %s at line %i. Ignoring." % ( read_date, row_num) self.errors.append(error_msg) print(error_msg) return if date_created > timezone_today(): if self.verbose: error_msg = "ERROR: Creation date %s of note for %s at row %i is in the future. Ignoring. " % \ (read_date, student_emplid, row_num) self.errors.append(error_msg) print(error_msg) return # Let's check if we've already imported this note (or another which matches): matching_notes = AdvisorNote.objects.filter(student=p, advisor=u, created_at=date_created, unit=self.unit) key_matching_notes = [ n for n in matching_notes if 'import_key' in n.config and n.config['import_key'] == key ] if key_matching_notes: if self.verbose: error_msg = "Already imported note from this file with note_id %s on row %i, ignoring." % \ (note_id, row_num) #self.errors.append(error_msg) Don't actually add these to the error log, since these are due to us # running the importer before, as they have the correct key. Only print for verbose output, but not # in the error recap afterwards. print(error_msg) return # What if we have notes from the exact same time, from the same advisor, for the same recipient, but without # a matching key? That's fishy too. At first I wrote this to be just a warning and continue processing, but, # really, there's no reason a note should match this way. It serves as a good way to spot files that were # already imported under another name. if matching_notes.count() != len(key_matching_notes): if self.verbose: error_msg = "Found matching note, but without matching key. This is fishy. Note_id %s on row %i. "\ "Are you sure this file hasn't been processed already using a different filename? " \ "Ignoring this note." % (note_id, row_num) self.errors.append(error_msg) print(error_msg) return # We checked every possible case, let's create the new note. original_text = row['notes'] # The file we were given actually has some NULLs for some text content. No sense importing a null note. if not original_text or original_text == 'NULL': if self.verbose: error_msg = "No actual note content (empty string or NULL) for %s at row %i. Ignoring." % \ (student_emplid, row_num) self.errors.append(error_msg) print(error_msg) return text = ensure_sanitary_markup(original_text, self.markup) n = AdvisorNote(student=p, advisor=u, created_at=date_created, unit=self.unit, text=text) n.config['import_key'] = key n.markup = self.markup if self.verbose: print("Creating note for %s from row %i..." % (student_emplid, row_num), end='') if self.commit: n.save() self.saved += 1 if self.verbose: print("Saved.") else: if self.verbose: print("Not saved, dry-run only.") return
def process_row(self, i, row): """ Actually process each individual row """ # It's 0 indexed, and we already consumed the header row. # This is used for error messages so the user can refer to the input file and know the correct line number. row_num = i + 2 note_id = row['note_id'] # Just in case someone had a bunch of trailing slashes, etc, make sure we get solely the file name for the key. file_basename = os.path.basename(os.path.normpath(self.file.name)) # In order for the keys to match (to check for duplicates), they have to have been imported from this importer, # with the same filename, and with the same note_id. key = "notes_import-%s-%s" % (file_basename, note_id) # Find the recipient of the note: student_emplid = row['emplid'] # See if we can actually cast the emplid to int, since the function we call does so without checking. try: int(student_emplid) except ValueError: if self.verbose: error_msg = "ERROR, emplid is not valid for recipient on row %i (emplid %s). Ignoring" % \ (row_num, student_emplid) self.errors.append(error_msg) print(error_msg) return p = add_person(student_emplid, commit=self.commit) if not p: if self.verbose: error_msg = "ERROR: Can't find recipient on row %i (emplid %s). Ignoring." % (row_num, student_emplid) self.errors.append(error_msg) print(error_msg) return # Find the advisor who entered the note: advisor_emplid = row['creator_emplid'] # Same thing for the advisor try: int(advisor_emplid) except ValueError: if self.verbose: error_msg = "ERROR, emplid is not valid for advisor on row %i (emplid %s). Ignoring" % \ (row_num, advisor_emplid) self.errors.append(error_msg) print(error_msg) return u = add_person(advisor_emplid, commit=self.commit) if not u: if self.verbose: error_msg = "ERROR: Can't find advisor %s on row %i (emplid %s). Ignoring." % \ (advisor_emplid, row_num, student_emplid) self.errors.append(error_msg) print(error_msg) return advisor_userid = row['creator_computing_id'] if u.userid != advisor_userid: if self.verbose: error_msg = "ERROR: The advisor emplid and userid do not match the same person. Emplid %s, userid " \ "%s at row %i. Ignoring." % (advisor_emplid, advisor_userid, row_num) self.errors.append(error_msg) print(error_msg) return read_date = row['date_created'] # We expect a certain date format, try that first, as the function is slightly faster. try: date_created = datetime.strptime(read_date, "%Y-%m-%d %H:%M:%S") except ValueError: # Fine, try to make the dateutils parser figure it out, then. try: date_created = dateparser.parse(read_date) except ValueError: if self.verbose: error_msg = "ERROR: Cannot deduce the correct date %s at line %i. Ignoring." % (read_date, row_num) self.errors.append(error_msg) print(error_msg) return if date_created > timezone_today(): if self.verbose: error_msg = "ERROR: Creation date %s of note for %s at row %i is in the future. Ignoring. " % \ (read_date, student_emplid, row_num) self.errors.append(error_msg) print(error_msg) return # Let's check if we've already imported this note (or another which matches): matching_notes = AdvisorNote.objects.filter(student=p, advisor=u, created_at=date_created, unit=self.unit) key_matching_notes = [n for n in matching_notes if 'import_key' in n.config and n.config['import_key'] == key] if key_matching_notes: if self.verbose: error_msg = "Already imported note from this file with note_id %s on row %i, ignoring." % \ (note_id, row_num) #self.errors.append(error_msg) Don't actually add these to the error log, since these are due to us # running the importer before, as they have the correct key. Only print for verbose output, but not # in the error recap afterwards. print(error_msg) return # What if we have notes from the exact same time, from the same advisor, for the same recipient, but without # a matching key? That's fishy too. At first I wrote this to be just a warning and continue processing, but, # really, there's no reason a note should match this way. It serves as a good way to spot files that were # already imported under another name. if matching_notes.count() != len(key_matching_notes): if self.verbose: error_msg = "Found matching note, but without matching key. This is fishy. Note_id %s on row %i. "\ "Are you sure this file hasn't been processed already using a different filename? " \ "Ignoring this note." % (note_id, row_num) self.errors.append(error_msg) print(error_msg) return # We checked every possible case, let's create the new note. original_text = row['notes'] # The file we were given actually has some NULLs for some text content. No sense importing a null note. if not original_text or original_text == 'NULL': if self.verbose: error_msg = "No actual note content (empty string or NULL) for %s at row %i. Ignoring." % \ (student_emplid, row_num) self.errors.append(error_msg) print(error_msg) return text = ensure_sanitary_markup(original_text, self.markup) n = AdvisorNote(student=p, advisor=u, created_at=date_created, unit=self.unit, text=text) n.config['import_key'] = key n.markup = self.markup if self.verbose: print("Creating note for %s from row %i..." % (student_emplid, row_num), end='') if self.commit: n.save() self.saved += 1 if self.verbose: print("Saved.") else: if self.verbose: print("Not saved, dry-run only.") return
def test_pages(self): client = Client() client.login_user("dzhao") adv = Person.objects.get(userid='dzhao') # create some notes to work with unit = Unit.objects.get(slug='cmpt') ns = NonStudent(first_name="Non", last_name="Student", high_school="North South Burnaby-Surrey", start_year=2000) ns.save() p1 = Person.objects.get(userid='0aaa6') n1 = AdvisorNote(student=p1, text="He seems like a nice student.", unit=unit, advisor=adv) n1.save() p2 = Person.objects.get(userid='0aaa8') p2.userid = None p2.save() n2 = AdvisorNote(student=p2, text="This guy doesn't have an active computing account.", unit=unit, advisor=adv) n2.save() n3 = AdvisorNote(nonstudent=ns, text="What a horrible person.", unit=unit, advisor=adv) n3.save() # index page url = reverse('advisornotes.views.advising', kwargs={}) response = basic_page_tests(self, client, url) self.assertEqual(response.status_code, 200) # new nonstudent form url = reverse('advisornotes.views.new_nonstudent', kwargs={}) response = basic_page_tests(self, client, url) self.assertEqual(response.status_code, 200) # student with userid url = reverse('advisornotes.views.advising', kwargs={}) response = client.post(url, {'search': p1.emplid}) self.assertEqual(response.status_code, 302) redir_url = response['location'] student_url = reverse('advisornotes.views.student_notes', kwargs={'userid': p1.userid}) self.assertIn(student_url, redir_url) response = basic_page_tests(self, client, student_url, check_valid=False) self.assertEqual(response.status_code, 200) new_url = reverse('advisornotes.views.new_note', kwargs={'userid': p1.userid}) response = basic_page_tests(self, client, new_url) self.assertEqual(response.status_code, 200) # student with no userid response = client.post(url, {'search': p2.emplid}) self.assertEqual(response.status_code, 302) redir_url = response['location'] student_url = reverse('advisornotes.views.student_notes', kwargs={'userid': p2.emplid}) self.assertIn(student_url, redir_url) response = basic_page_tests(self, client, student_url, check_valid=False) self.assertEqual(response.status_code, 200) new_url = reverse('advisornotes.views.new_note', kwargs={'userid': p2.emplid}) response = basic_page_tests(self, client, new_url) self.assertEqual(response.status_code, 200) # non student response = client.post(url, {'search': ns.slug}) self.assertEqual(response.status_code, 302) redir_url = response['location'] student_url = reverse('advisornotes.views.student_notes', kwargs={'userid': ns.slug}) self.assertIn(student_url, redir_url) response = basic_page_tests(self, client, student_url, check_valid=False) self.assertEqual(response.status_code, 200) new_url = reverse('advisornotes.views.new_note', kwargs={'userid': ns.slug}) response = basic_page_tests(self, client, new_url) self.assertEqual(response.status_code, 200) # note content search url = reverse('advisornotes.views.note_search', kwargs={}) + "?text-search=nice" response = basic_page_tests(self, client, url) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.context['notes']), 1) url = reverse('advisornotes.views.note_search', kwargs={}) + "?text-search=" response = basic_page_tests(self, client, url) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.context['notes']), 3)
def edit_visit_initial(request, visit_slug): # This is for the initial edit, when the visit is first created. At this point, we want to show all the SIMS # stuff, set categories, and also potentially create a note. The end date/time is set when the form is submitted. visit = get_object_or_404(AdvisorVisit, slug=visit_slug, hidden=False) already_got_sims = False if request.method == 'POST': form = AdvisorVisitFormInitial(request.POST, request.FILES, instance=visit) if form.is_valid(): visit = form.save(commit=False) visit.categories.clear() if 'categories' in form.cleaned_data: for c in form.cleaned_data['categories']: visit.categories.add(c) visit.end_time = datetime.datetime.now() if 'programs' in form.cleaned_data: visit.programs = form.cleaned_data['programs'] if 'cgpa' in form.cleaned_data: visit.cgpa = form.cleaned_data['cgpa'] if 'credits' in form.cleaned_data: visit.credits = form.cleaned_data['credits'] if 'gender' in form.cleaned_data: visit.gender = form.cleaned_data['gender'] if 'citizenship' in form.cleaned_data: visit.citizenship = form.cleaned_data['citizenship'] visit.save() if 'note' in form.cleaned_data and form.cleaned_data['note']: note = AdvisorNote(student=visit.student, nonstudent=visit.nonstudent, advisor=visit.advisor, unit=visit.unit, text=form.cleaned_data['note']) if 'file_attachment' in request.FILES: upfile = request.FILES['file_attachment'] note.file_attachment = upfile note.file_mediatype = upfile.content_type if form.cleaned_data['email_student']: _email_student_note(note) note.emailed = True note.save() l = LogEntry(userid=request.user.username, description=("new advisor note from visit for %s") % visit.get_userid(), related_object=note) l.save() l = LogEntry(userid=request.user.username, description=("Recorded visit for %s") % visit.get_userid(), related_object=visit) l.save() script = '<script nonce='+request.csp_nonce+'>window.close();window.opener.location.reload();</script>' return HttpResponse(script) else: form = AdvisorVisitFormInitial(instance=visit) # If we've already fetched info from SIMS for this person, set a flag so we don't automatically fetch it again, # this would mean we're editing an already populated visit, and we should leave the choice to the user. if visit.cgpa: form.initial['cgpa'] = visit.cgpa already_got_sims = True if visit.programs: form.initial['programs'] = visit.programs already_got_sims = True if visit.credits: form.initial['credits'] = visit.credits already_got_sims = True return render(request, 'advisornotes/record_visit.html', {'userid': visit.get_userid(), 'visit': visit, 'form': form, 'fetch_automatically': not already_got_sims})