def find_gradstudent(self, verbosity, dry_run): gss = GradStudent.objects.filter(person__emplid=self.emplid, program__unit=self.unit).select_related('start_semester', 'program__unit', 'person') gss = list(gss) if self.admit_term < RELEVANT_PROGRAM_START: return for method, multiple_okay in GradCareer.GS_SELECTORS: by_selector = [gs for gs in gss if getattr(self, method)(gs)] #print method, by_selector if len(by_selector) == 1: return by_selector[0] elif len(by_selector) > 1: if multiple_okay: return by_selector[-1] else: raise ValueError, "Multiple records found by %s for %s." % (method, self) if GradCareer.program_map[self.last_program].unit.slug == 'cmpt' and self.admit_term < CMPT_CUTOFF: # Don't try to probe the depths of history for CMPT. You'll hurt yourself. # We have nice clean adm_appl_nbrs for CMPT_CUTOFF onwards, so the reliable GS_SELECTORS will find the student return if verbosity: print "New grad student career found: %s/%s in %s starting %s." % (self.emplid, self.unit.slug, self.last_program, self.admit_term) # can't find anything in database: create new gs = GradStudent(person=add_person(self.emplid, commit=(not dry_run))) # everything else updated by gs.update_status_fields later gs.program = GradCareer.program_map[self.last_program] # ...but this is needed to save if not dry_run: gs.save() # get gs.id filled in for foreign keys elsewhere return gs
def find_gradstudent(self, verbosity, dry_run): gss = GradStudent.objects.filter(person__emplid=self.emplid, program__unit=self.unit).select_related('start_semester', 'program__unit', 'person') gss = list(gss) if self.admit_term < RELEVANT_PROGRAM_START: return for method, multiple_okay in GradCareer.GS_SELECTORS: by_selector = [gs for gs in gss if getattr(self, method)(gs)] #print method, by_selector if len(by_selector) == 1: return by_selector[0] elif len(by_selector) > 1: if multiple_okay: return by_selector[-1] else: raise ValueError, "Multiple records found by %s for %s." % (method, self) if GradCareer.program_map[self.last_program].unit.slug == 'cmpt' and self.admit_term < CMPT_CUTOFF: # Don't try to probe the depths of history for CMPT. You'll hurt yourself. # We have nice clean adm_appl_nbrs for CMPT_CUTOFF onwards, so the reliable GS_SELECTORS will find the student return if verbosity: print "New grad student career found: %s/%s in %s starting %s." % (self.emplid, self.unit.slug, self.last_program, self.admit_term) # can't find anything in database: create new gs = GradStudent(person=add_person(self.emplid, commit=(not dry_run))) # everything else updated by gs.update_status_fields later gs.program = GradCareer.program_map[self.last_program] # ...but this is needed to save if not dry_run: gs.save() # get gs.id filled in for foreign keys elsewhere return gs
def update_local_data(self, student_info, verbosity, dry_run): # if self.grad_program.unit.slug == 'cmpt': # return key = self.import_key() local_committee = student_info['committee'] sup_type = COMMITTEE_MEMBER_MAP[self.committee_role] # cache People objects, so we don't query for them too much. if self.sup_emplid in CommitteeMembership.found_people: p = CommitteeMembership.found_people[self.sup_emplid] else: p = add_person(self.sup_emplid, external_email=True, commit=(not dry_run)) CommitteeMembership.found_people[self.sup_emplid] = p matches = [ m for m in local_committee if m.supervisor == p and m.supervisor_type == sup_type ] if matches: member = matches[0] else: similar = [m for m in local_committee if m.supervisor == p] if len(similar) > 0: if verbosity > 2: print( "* Found similar (but imperfect) committee member for %s is a %s for %s/%s" % (p.name(), SUPERVISOR_TYPE[sup_type], self.emplid, self.unit.slug)) member = similar[0] else: if verbosity: print("Adding committee member: %s is a %s for %s/%s" % (p.name(), SUPERVISOR_TYPE[sup_type], self.emplid, self.unit.slug)) member = Supervisor(student=student_info['student'], supervisor=p, supervisor_type=sup_type) member.created_at = self.effdt local_committee.append(member) if SIMS_SOURCE not in member.config: # record (the first) place we found this fact member.config[SIMS_SOURCE] = key # if it wasn't the product of a previous import it was hand-entered: take the effdt from SIMS member.created_at = self.effdt # TODO: try to match up external members with new real ones? That sounds hard. # TODO: remove members if added by this import (in the past) and not found in the newest committee version if not dry_run: member.save_if_dirty()
def find_person(userid): """ Find a this person, creating if necessary. """ people = Person.objects.filter(userid=userid) if people: return people[0] else: emplid = find_emplid(userid) p = add_person(emplid, commit=False) p.userid = userid p.save() return p
def find_person(userid): """ Find a this person, creating if necessary. """ people = Person.objects.filter(userid=userid) if people: return people[0] else: emplid = find_emplid(userid) p = add_person(emplid, commit=False) p.userid = userid p.save() return p
def clean(self, value): if isinstance(value, Person): return self.__check_email(value) else: if not self.required and not value: return None try: return self.__check_email(Person.objects.get(emplid=value)) except (ValueError, Person.DoesNotExist): # try to find the emplid in SIMS if they are missing from our DB if not value: raise forms.ValidationError("Could not find this emplid.") if not value.isdigit( ): # doesn't look like an emplid: try it as a userid try: return self.__check_email( Person.objects.get(userid=value)) except Person.DoesNotExist: value = userid_to_emplid(value) if not value: raise forms.ValidationError( "Could not find this emplid.") try: persondata = find_person(value) except SIMSProblem as e: raise forms.ValidationError( "Problem locating person in SIMS: " + str(e)) if not persondata: raise forms.ValidationError("Could not find this emplid.") # we found this emplid in SIMS: raise validation error, but offer to add next time. confirm = self.fieldname + '_confirm' checkemplid = self.fieldname + '_emplid' if confirm in self.formdata and checkemplid in self.formdata and self.formdata[ checkemplid] == value: # new person was presented in the form last time, and they confirmed import p = add_person(value) return self.__check_email(p) else: self.widget.found_sims = True self.widget.sims_data = persondata raise forms.ValidationError( "Person is new to this system: please confirm their import." )
def sims_add_person(request): if request.method == 'POST': emplid = request.POST.get('emplid', None) if emplid: try: p = add_person(emplid.strip()) except SIMSProblem: p = None if isinstance(p, Person): #LOG EVENT# l = LogEntry(userid=request.user.username, description=(u"added %s (%s) from SIMS") % (p.name(), p.emplid), related_object=p) l.save() messages.add_message(request, messages.SUCCESS, u'Record for %s created.' % (p.name())) return _redirect_to_notes(p) return HttpResponseRedirect(reverse('advisornotes.views.advising', kwargs={}))
def update_local_data(self, student_info, verbosity, dry_run): if self.grad_program.unit.slug == 'cmpt': return key = self.import_key() local_committee = student_info['committee'] sup_type = COMMITTEE_MEMBER_MAP[self.committee_role] # cache People objects, so we don't query for them too much. if self.sup_emplid in CommitteeMembership.found_people: p = CommitteeMembership.found_people[self.sup_emplid] else: p = add_person(self.sup_emplid, external_email=True, commit=(not dry_run)) CommitteeMembership.found_people[self.sup_emplid] = p matches = [m for m in local_committee if m.supervisor == p and m.supervisor_type == sup_type] if matches: member = matches[0] else: similar = [m for m in local_committee if m.supervisor == p] if len(similar) > 0: if verbosity > 2: print "* Found similar (but imperfect) committee member for %s is a %s for %s/%s" % (p.name(), SUPERVISOR_TYPE[sup_type], self.emplid, self.unit.slug) member = similar[0] else: if verbosity: print "Adding committee member: %s is a %s for %s/%s" % (p.name(), SUPERVISOR_TYPE[sup_type], self.emplid, self.unit.slug) member = Supervisor(student=student_info['student'], supervisor=p, supervisor_type=sup_type) member.created_at = self.effdt local_committee.append(member) if SIMS_SOURCE not in member.config: # record (the first) place we found this fact member.config[SIMS_SOURCE] = key # if it wasn't the product of a previous import it was hand-entered: take the effdt from SIMS member.created_at = self.effdt # TODO: try to match up external members with new real ones? That sounds hard. # TODO: remove members if added by this import (in the past) and not found in the newest committee version if not dry_run: member.save_if_dirty()
def clean_people(self): text = self.cleaned_data['people'] emplids = text.strip().split() people = [] for e in emplids: # this is going to be slow if there's a big list try: person = Person.objects.get(find_userid_or_emplid(e)) except Person.DoesNotExist: try: person = add_person(int(e)) if person is None: raise forms.ValidationError( 'Cannot find a person emplid/userid %r.' % (e, )) except (ValueError, SIMSProblem): raise forms.ValidationError( 'Cannot find a person emplid/userid %r.' % (e, )) people.append(person) return people
def clean(self, value): if isinstance(value, Person): return value else: if not self.required and not value: return None try: return Person.objects.get(emplid=value) except (ValueError, Person.DoesNotExist): # try to find the emplid in SIMS if they are missing from our DB if not value: raise forms.ValidationError, "Could not find this emplid." if not value.isdigit(): # doesn't look like an emplid: try it as a userid try: return Person.objects.get(userid=value) except Person.DoesNotExist: value = userid_to_emplid(value) if not value: raise forms.ValidationError, "Could not find this emplid." try: persondata = find_person(value) except SIMSProblem, e: raise forms.ValidationError, "Problem locating person in SIMS: " + unicode(e) if not persondata: raise forms.ValidationError, "Could not find this emplid." # we found this emplid in SIMS: raise validation error, but offer to add next time. confirm = self.fieldname+'_confirm' checkemplid = self.fieldname+'_emplid' if confirm in self.formdata and checkemplid in self.formdata and self.formdata[checkemplid] == value: # new person was presented in the form last time, and they confirmed import p = add_person(value) return p else: self.widget.found_sims = True self.widget.sims_data = persondata raise forms.ValidationError, "Person is new to this system: please confirm their import."
def process_pcs_row(row, column, rownum, unit, semester, user): """ Process a single row from the PCS import """ appsemester = semester.previous_semester() warnings = [] ident = "in row %i" % (rownum) appid = row[column['appid']] emplid = row[column['emplid']] program = row[column['program']] # get Person, from SIMS if necessary try: p = Person.objects.get(emplid=int(emplid)) except ValueError: warnings.append("Bad emplid %s: not processing that row." % (ident)) return warnings except Person.DoesNotExist: try: p = add_person(emplid) except SIMSProblem as e: return str(e) ident = 'for "%s"' % (p.name()) # update information on the Person email = row[column['email']] if email: p.config['applic_email'] = email dob = row[column['dob']] if dob: try: dt = datetime.datetime.strptime(dob, "%Y-%m-%d") p.config['birthdate'] = dt.date().isoformat() except ValueError: warnings.append("Bad birthdate %s." % (ident)) # get extended SIMS data data = grad_student_info(emplid) p.config.update(data) p.save() #print "Importing %s" % (p) # get GradStudent, creating if necessary # a unique identifier for this application, so we can detect repeated imports (and handle gracefully) uid = "%s-%s-%s-%s" % (unit.slug, semester.name, appid, emplid) # TODO: wrong, wrong, wrong. Figure out how to select program from import data program = GradProgram.objects.filter(unit=unit)[0] # find the old GradStudent if possible gss = GradStudent.objects.filter(program__unit=unit, person=p) gs = None for g in gss: if 'app_id' in g.config and g.config['app_id'] == uid: gs = g break if not gs: gs = GradStudent(program=program, person=p) gs.config['app_id'] = uid resarea = row[column['resarea']] firstlang = row[column['firstlang']] gs.research_area = resarea gs.mother_tongue = firstlang gs.created_by = user.userid gs.updated_by = user.userid gs.config['start_semester'] = semester.name gs.save() complete = row[column['complete']].strip() decision = row[column['decision']].strip() notes = row[column['notes']].strip() gs.config['decisionnotes'] = notes old_st = GradStatus.objects.filter(student=gs, start__name__gte=semester.name) if not old_st: # if no old status for current semester, create one # application completion status if complete == 'AppConfirm': st = GradStatus(student=gs, status="COMP", start=appsemester, end=None, notes="PCS import") st.save() elif complete == '': st = GradStatus(student=gs, status="INCO", start=appsemester, end=None, notes="PCS import") st.save() else: warnings.append('Unknown "Confirmation of Completion of Application" value %s.' % (ident)) # decision status if decision == 'DECL': st = GradStatus(student=gs, status="DECL", start=appsemester, end=None, notes="PCS import") st.save() elif decision == '': st = GradStatus(student=gs, status="OFFO", start=appsemester, end=None, notes="PCS import") st.save() elif decision == 'R': st = GradStatus(student=gs, status="REJE", start=appsemester, end=None, notes="PCS import") st.save() elif decision == 'HOLD': st = GradStatus(student=gs, status="HOLD", start=appsemester, end=None, notes="PCS import") st.save() elif decision == 'AMScT': # TODO: bump program to MSc thesis st = GradStatus(student=gs, status="CONF", start=appsemester, end=None, notes="PCS import") st.save() elif decision == 'AMScC': # TODO: bump program to MSc course-based st = GradStatus(student=gs, status="CONF", start=appsemester, end=None, notes="PCS import") st.save() # potential supervisor potsuper = row[column['potsuper']] if potsuper: superv = None external = None try: ps_last, ps_first = potsuper.split(', ') except ValueError: warnings.append('Bad potential supervisor name %s: will store them as an "external" supervisor.' % (ident)) external = potsuper else: potentials = possible_supervisor_people([unit]) potential_ids = [p.id for p in potentials] query = Q(last_name=ps_last, first_name=ps_first) | Q(last_name=ps_last, pref_first_name=ps_first) people = Person.objects.filter(query, id__in=potential_ids) if people.count() == 1: superv = people[0] else: warnings.append('Coundn\'t find potential supervisor %s: will store them as an "external" supervisor.' % (ident)) external = potsuper old_s = Supervisor.objects.filter(student=gs, supervisor_type='POT') if old_s: s = old_s[0] else: s = Supervisor(student=gs, supervisor_type='POT') s.superv = superv s.external = external s.position = 0 s.created_by = user.userid s.modified_by = user.userid s.save() l = LogEntry(userid=user.userid, description="Imported grad record for %s (%s) from PCS" % (p.name(), p.emplid), related_object=gs) l.save() return warnings
def process_pcs_row(row, column, rownum, unit, semester, user): """ Process a single row from the PCS import """ appsemester = semester.previous_semester() warnings = [] ident = "in row %i" % (rownum) appid = row[column['appid']] emplid = row[column['emplid']] program = row[column['program']] # get Person, from SIMS if necessary try: p = Person.objects.get(emplid=int(emplid)) except ValueError: warnings.append("Bad emplid %s: not processing that row." % (ident)) return warnings except Person.DoesNotExist: try: p = add_person(emplid) except SIMSProblem as e: return e.message ident = 'for "%s"' % (p.name()) # update information on the Person email = row[column['email']] if email: p.config['applic_email'] = email dob = row[column['dob']] if dob: try: dt = datetime.datetime.strptime(dob, "%Y-%m-%d") p.config['birthdate'] = dt.date().isoformat() except ValueError: warnings.append("Bad birthdate %s." % (ident)) # get extended SIMS data data = grad_student_info(emplid) p.config.update(data) p.save() #print "Importing %s" % (p) # get GradStudent, creating if necessary # a unique identifier for this application, so we can detect repeated imports (and handle gracefully) uid = "%s-%s-%s-%s" % (unit.slug, semester.name, appid, emplid) # TODO: wrong, wrong, wrong. Figure out how to select program from import data program = GradProgram.objects.filter(unit=unit)[0] # find the old GradStudent if possible gss = GradStudent.objects.filter(program__unit=unit, person=p) gs = None for g in gss: if 'app_id' in g.config and g.config['app_id'] == uid: gs = g break if not gs: gs = GradStudent(program=program, person=p) gs.config['app_id'] = uid resarea = row[column['resarea']] firstlang = row[column['firstlang']] gs.research_area = resarea gs.mother_tongue = firstlang gs.created_by = user.userid gs.updated_by = user.userid gs.config['start_semester'] = semester.name gs.save() complete = row[column['complete']].strip() decision = row[column['decision']].strip() notes = row[column['notes']].strip() gs.config['decisionnotes'] = notes old_st = GradStatus.objects.filter(student=gs, start__name__gte=semester.name) if not old_st: # if no old status for current semester, create one # application completion status if complete == 'AppConfirm': st = GradStatus(student=gs, status="COMP", start=appsemester, end=None, notes="PCS import") st.save() elif complete == '': st = GradStatus(student=gs, status="INCO", start=appsemester, end=None, notes="PCS import") st.save() else: warnings.append( 'Unknown "Confirmation of Completion of Application" value %s.' % (ident)) # decision status if decision == 'DECL': st = GradStatus(student=gs, status="DECL", start=appsemester, end=None, notes="PCS import") st.save() elif decision == '': st = GradStatus(student=gs, status="OFFO", start=appsemester, end=None, notes="PCS import") st.save() elif decision == 'R': st = GradStatus(student=gs, status="REJE", start=appsemester, end=None, notes="PCS import") st.save() elif decision == 'HOLD': st = GradStatus(student=gs, status="HOLD", start=appsemester, end=None, notes="PCS import") st.save() elif decision == 'AMScT': # TODO: bump program to MSc thesis st = GradStatus(student=gs, status="CONF", start=appsemester, end=None, notes="PCS import") st.save() elif decision == 'AMScC': # TODO: bump program to MSc course-based st = GradStatus(student=gs, status="CONF", start=appsemester, end=None, notes="PCS import") st.save() # potential supervisor potsuper = row[column['potsuper']] if potsuper: superv = None external = None try: ps_last, ps_first = potsuper.split(', ') except ValueError: warnings.append( 'Bad potential supervisor name %s: will store them as an "external" supervisor.' % (ident)) external = potsuper else: potentials = possible_supervisor_people([unit]) potential_ids = [p.id for p in potentials] query = Q(last_name=ps_last, first_name=ps_first) | Q( last_name=ps_last, pref_first_name=ps_first) people = Person.objects.filter(query, id__in=potential_ids) if people.count() == 1: superv = people[0] else: warnings.append( 'Coundn\'t find potential supervisor %s: will store them as an "external" supervisor.' % (ident)) external = potsuper old_s = Supervisor.objects.filter(student=gs, supervisor_type='POT') if old_s: s = old_s[0] else: s = Supervisor(student=gs, supervisor_type='POT') s.superv = superv s.external = external s.position = 0 s.created_by = user.userid s.modified_by = user.userid s.save() l = LogEntry(userid=user.userid, description="Imported grad record for %s (%s) from PCS" % (p.name(), p.emplid), related_object=gs) l.save() return warnings
def import_note(self, advisor, fn, i, row): emplid = row['Student ID'] date_str = row['Date Modified'] notes = normalize_newlines(row['Notes']) files = [ row.get('Transcript', None), row.get('Files', None), row.get('Files2', None), row.get('Files3', None), ] files = map(self.get_filepath, files) files = filter(bool, files) # fix mis-typed emplids we found # Lindsay if emplid == '960022098': emplid = '963022098' elif emplid == '30108409': emplid = '301078409' elif emplid == '30115964': emplid = '301115964' elif emplid == '30117882': emplid = '301178882' # Michael Sean elif emplid == '30105659': emplid = '301040985' # ? # Dijana elif emplid == '30120965': emplid = '301202965' if not emplid or emplid == '0': if self.verbosity > 0: print 'No emplid on row %i' % (i + 2) return p = add_person(emplid, commit=self.commit) if not p: if self.verbosity > 0: print u"Can't find person on row %i (emplid %s)" % (i + 2, emplid) return if self.verbosity > 1: print u"Importing %s with %i file(s)." % (emplid, len(files)) try: date = datetime.datetime.strptime(date_str, '%m-%d-%Y').date() except ValueError: date = datetime.datetime.strptime(date_str, '%Y-%m-%d').date() created = datetime.datetime.combine(date, datetime.time(hour=12, minute=0)) key = '%s-%i' % (fn, i) note, _ = self.get_advisornote(key, p, advisor, created, delete_old_file=self.commit) if files: path = files[0] self.attach_file(note, path) for j, path in enumerate(files[1:]): # these get stashed in accompanying notes k = key + '-auxfile-' + str(i) n, _ = self.get_advisornote(k, p, advisor, created, delete_old_file=self.commit, offset=(j + 1)) n.text = '[Additional file for previous note.]' self.attach_file(n, path) if self.commit: n.save() note.text = notes if self.commit: note.save()
def process_row(self, i, row): """ Actually process each individual row """ # It's 0 indexed, and we already consumed the header row. # This is used for error messages so the user can refer to the input file and know the correct line number. row_num = i + 2 note_id = row['note_id'] # Just in case someone had a bunch of trailing slashes, etc, make sure we get solely the file name for the key. file_basename = os.path.basename(os.path.normpath(self.file.name)) # In order for the keys to match (to check for duplicates), they have to have been imported from this importer, # with the same filename, and with the same note_id. key = "notes_import-%s-%s" % (file_basename, note_id) # Find the recipient of the note: student_emplid = row['emplid'] # See if we can actually cast the emplid to int, since the function we call does so without checking. try: int(student_emplid) except ValueError: if self.verbose: error_msg = "ERROR, emplid is not valid for recipient on row %i (emplid %s). Ignoring" % \ (row_num, student_emplid) self.errors.append(error_msg) print(error_msg) return p = add_person(student_emplid, commit=self.commit) if not p: if self.verbose: error_msg = "ERROR: Can't find recipient on row %i (emplid %s). Ignoring." % ( row_num, student_emplid) self.errors.append(error_msg) print(error_msg) return # Find the advisor who entered the note: advisor_emplid = row['creator_emplid'] # Same thing for the advisor try: int(advisor_emplid) except ValueError: if self.verbose: error_msg = "ERROR, emplid is not valid for advisor on row %i (emplid %s). Ignoring" % \ (row_num, advisor_emplid) self.errors.append(error_msg) print(error_msg) return u = add_person(advisor_emplid, commit=self.commit) if not u: if self.verbose: error_msg = "ERROR: Can't find advisor %s on row %i (emplid %s). Ignoring." % \ (advisor_emplid, row_num, student_emplid) self.errors.append(error_msg) print(error_msg) return advisor_userid = row['creator_computing_id'] if u.userid != advisor_userid: if self.verbose: error_msg = "ERROR: The advisor emplid and userid do not match the same person. Emplid %s, userid " \ "%s at row %i. Ignoring." % (advisor_emplid, advisor_userid, row_num) self.errors.append(error_msg) print(error_msg) return read_date = row['date_created'] # We expect a certain date format, try that first, as the function is slightly faster. try: date_created = datetime.strptime(read_date, "%Y-%m-%d %H:%M:%S") except ValueError: # Fine, try to make the dateutils parser figure it out, then. try: date_created = dateparser.parse(read_date) except ValueError: if self.verbose: error_msg = "ERROR: Cannot deduce the correct date %s at line %i. Ignoring." % ( read_date, row_num) self.errors.append(error_msg) print(error_msg) return if date_created > timezone_today(): if self.verbose: error_msg = "ERROR: Creation date %s of note for %s at row %i is in the future. Ignoring. " % \ (read_date, student_emplid, row_num) self.errors.append(error_msg) print(error_msg) return # Let's check if we've already imported this note (or another which matches): matching_notes = AdvisorNote.objects.filter(student=p, advisor=u, created_at=date_created, unit=self.unit) key_matching_notes = [ n for n in matching_notes if 'import_key' in n.config and n.config['import_key'] == key ] if key_matching_notes: if self.verbose: error_msg = "Already imported note from this file with note_id %s on row %i, ignoring." % \ (note_id, row_num) #self.errors.append(error_msg) Don't actually add these to the error log, since these are due to us # running the importer before, as they have the correct key. Only print for verbose output, but not # in the error recap afterwards. print(error_msg) return # What if we have notes from the exact same time, from the same advisor, for the same recipient, but without # a matching key? That's fishy too. At first I wrote this to be just a warning and continue processing, but, # really, there's no reason a note should match this way. It serves as a good way to spot files that were # already imported under another name. if matching_notes.count() != len(key_matching_notes): if self.verbose: error_msg = "Found matching note, but without matching key. This is fishy. Note_id %s on row %i. "\ "Are you sure this file hasn't been processed already using a different filename? " \ "Ignoring this note." % (note_id, row_num) self.errors.append(error_msg) print(error_msg) return # We checked every possible case, let's create the new note. original_text = row['notes'] # The file we were given actually has some NULLs for some text content. No sense importing a null note. if not original_text or original_text == 'NULL': if self.verbose: error_msg = "No actual note content (empty string or NULL) for %s at row %i. Ignoring." % \ (student_emplid, row_num) self.errors.append(error_msg) print(error_msg) return text = ensure_sanitary_markup(original_text, self.markup) n = AdvisorNote(student=p, advisor=u, created_at=date_created, unit=self.unit, text=text) n.config['import_key'] = key n.markup = self.markup if self.verbose: print("Creating note for %s from row %i..." % (student_emplid, row_num), end='') if self.commit: n.save() self.saved += 1 if self.verbose: print("Saved.") else: if self.verbose: print("Not saved, dry-run only.") return
def process_row(self, i, row): """ Actually process each individual row """ # It's 0 indexed, and we already consumed the header row. # This is used for error messages so the user can refer to the input file and know the correct line number. row_num = i + 2 note_id = row['note_id'] # Just in case someone had a bunch of trailing slashes, etc, make sure we get solely the file name for the key. file_basename = os.path.basename(os.path.normpath(self.file.name)) # In order for the keys to match (to check for duplicates), they have to have been imported from this importer, # with the same filename, and with the same note_id. key = "notes_import-%s-%s" % (file_basename, note_id) # Find the recipient of the note: student_emplid = row['emplid'] # See if we can actually cast the emplid to int, since the function we call does so without checking. try: int(student_emplid) except ValueError: if self.verbose: error_msg = "ERROR, emplid is not valid for recipient on row %i (emplid %s). Ignoring" % \ (row_num, student_emplid) self.errors.append(error_msg) print(error_msg) return p = add_person(student_emplid, commit=self.commit) if not p: if self.verbose: error_msg = "ERROR: Can't find recipient on row %i (emplid %s). Ignoring." % (row_num, student_emplid) self.errors.append(error_msg) print(error_msg) return # Find the advisor who entered the note: advisor_emplid = row['creator_emplid'] # Same thing for the advisor try: int(advisor_emplid) except ValueError: if self.verbose: error_msg = "ERROR, emplid is not valid for advisor on row %i (emplid %s). Ignoring" % \ (row_num, advisor_emplid) self.errors.append(error_msg) print(error_msg) return u = add_person(advisor_emplid, commit=self.commit) if not u: if self.verbose: error_msg = "ERROR: Can't find advisor %s on row %i (emplid %s). Ignoring." % \ (advisor_emplid, row_num, student_emplid) self.errors.append(error_msg) print(error_msg) return advisor_userid = row['creator_computing_id'] if u.userid != advisor_userid: if self.verbose: error_msg = "ERROR: The advisor emplid and userid do not match the same person. Emplid %s, userid " \ "%s at row %i. Ignoring." % (advisor_emplid, advisor_userid, row_num) self.errors.append(error_msg) print(error_msg) return read_date = row['date_created'] # We expect a certain date format, try that first, as the function is slightly faster. try: date_created = datetime.strptime(read_date, "%Y-%m-%d %H:%M:%S") except ValueError: # Fine, try to make the dateutils parser figure it out, then. try: date_created = dateparser.parse(read_date) except ValueError: if self.verbose: error_msg = "ERROR: Cannot deduce the correct date %s at line %i. Ignoring." % (read_date, row_num) self.errors.append(error_msg) print(error_msg) return if date_created > timezone_today(): if self.verbose: error_msg = "ERROR: Creation date %s of note for %s at row %i is in the future. Ignoring. " % \ (read_date, student_emplid, row_num) self.errors.append(error_msg) print(error_msg) return # Let's check if we've already imported this note (or another which matches): matching_notes = AdvisorNote.objects.filter(student=p, advisor=u, created_at=date_created, unit=self.unit) key_matching_notes = [n for n in matching_notes if 'import_key' in n.config and n.config['import_key'] == key] if key_matching_notes: if self.verbose: error_msg = "Already imported note from this file with note_id %s on row %i, ignoring." % \ (note_id, row_num) #self.errors.append(error_msg) Don't actually add these to the error log, since these are due to us # running the importer before, as they have the correct key. Only print for verbose output, but not # in the error recap afterwards. print(error_msg) return # What if we have notes from the exact same time, from the same advisor, for the same recipient, but without # a matching key? That's fishy too. At first I wrote this to be just a warning and continue processing, but, # really, there's no reason a note should match this way. It serves as a good way to spot files that were # already imported under another name. if matching_notes.count() != len(key_matching_notes): if self.verbose: error_msg = "Found matching note, but without matching key. This is fishy. Note_id %s on row %i. "\ "Are you sure this file hasn't been processed already using a different filename? " \ "Ignoring this note." % (note_id, row_num) self.errors.append(error_msg) print(error_msg) return # We checked every possible case, let's create the new note. original_text = row['notes'] # The file we were given actually has some NULLs for some text content. No sense importing a null note. if not original_text or original_text == 'NULL': if self.verbose: error_msg = "No actual note content (empty string or NULL) for %s at row %i. Ignoring." % \ (student_emplid, row_num) self.errors.append(error_msg) print(error_msg) return text = ensure_sanitary_markup(original_text, self.markup) n = AdvisorNote(student=p, advisor=u, created_at=date_created, unit=self.unit, text=text) n.config['import_key'] = key n.markup = self.markup if self.verbose: print("Creating note for %s from row %i..." % (student_emplid, row_num), end='') if self.commit: n.save() self.saved += 1 if self.verbose: print("Saved.") else: if self.verbose: print("Not saved, dry-run only.") return