def find_gradstudent(self, verbosity, dry_run):
gss = GradStudent.objects.filter(person__emplid=self.emplid, program__unit=self.unit).select_related('start_semester', 'program__unit', 'person')
gss = list(gss)
if self.admit_term < RELEVANT_PROGRAM_START:
return
for method, multiple_okay in GradCareer.GS_SELECTORS:
by_selector = [gs for gs in gss if getattr(self, method)(gs)]
#print method, by_selector
if len(by_selector) == 1:
return by_selector[0]
elif len(by_selector) > 1:
if multiple_okay:
return by_selector[-1]
else:
raise ValueError, "Multiple records found by %s for %s." % (method, self)
if GradCareer.program_map[self.last_program].unit.slug == 'cmpt' and self.admit_term < CMPT_CUTOFF:
# Don't try to probe the depths of history for CMPT. You'll hurt yourself.
# We have nice clean adm_appl_nbrs for CMPT_CUTOFF onwards, so the reliable GS_SELECTORS will find the student
return
if verbosity:
print "New grad student career found: %s/%s in %s starting %s." % (self.emplid, self.unit.slug, self.last_program, self.admit_term)
# can't find anything in database: create new
gs = GradStudent(person=add_person(self.emplid, commit=(not dry_run)))
# everything else updated by gs.update_status_fields later
gs.program = GradCareer.program_map[self.last_program] # ...but this is needed to save
if not dry_run:
gs.save() # get gs.id filled in for foreign keys elsewhere
return gs
def find_gradstudent(self, verbosity, dry_run):
gss = GradStudent.objects.filter(person__emplid=self.emplid, program__unit=self.unit).select_related('start_semester', 'program__unit', 'person')
gss = list(gss)
if self.admit_term < RELEVANT_PROGRAM_START:
return
for method, multiple_okay in GradCareer.GS_SELECTORS:
by_selector = [gs for gs in gss if getattr(self, method)(gs)]
#print method, by_selector
if len(by_selector) == 1:
return by_selector[0]
elif len(by_selector) > 1:
if multiple_okay:
return by_selector[-1]
else:
raise ValueError, "Multiple records found by %s for %s." % (method, self)
if GradCareer.program_map[self.last_program].unit.slug == 'cmpt' and self.admit_term < CMPT_CUTOFF:
# Don't try to probe the depths of history for CMPT. You'll hurt yourself.
# We have nice clean adm_appl_nbrs for CMPT_CUTOFF onwards, so the reliable GS_SELECTORS will find the student
return
if verbosity:
print "New grad student career found: %s/%s in %s starting %s." % (self.emplid, self.unit.slug, self.last_program, self.admit_term)
# can't find anything in database: create new
gs = GradStudent(person=add_person(self.emplid, commit=(not dry_run)))
# everything else updated by gs.update_status_fields later
gs.program = GradCareer.program_map[self.last_program] # ...but this is needed to save
if not dry_run:
gs.save() # get gs.id filled in for foreign keys elsewhere
return gs
def update_local_data(self, student_info, verbosity, dry_run):
# if self.grad_program.unit.slug == 'cmpt':
# return
key = self.import_key()
local_committee = student_info['committee']
sup_type = COMMITTEE_MEMBER_MAP[self.committee_role]
# cache People objects, so we don't query for them too much.
if self.sup_emplid in CommitteeMembership.found_people:
p = CommitteeMembership.found_people[self.sup_emplid]
else:
p = add_person(self.sup_emplid,
external_email=True,
commit=(not dry_run))
CommitteeMembership.found_people[self.sup_emplid] = p
matches = [
m for m in local_committee
if m.supervisor == p and m.supervisor_type == sup_type
]
if matches:
member = matches[0]
else:
similar = [m for m in local_committee if m.supervisor == p]
if len(similar) > 0:
if verbosity > 2:
print(
"* Found similar (but imperfect) committee member for %s is a %s for %s/%s"
% (p.name(), SUPERVISOR_TYPE[sup_type], self.emplid,
self.unit.slug))
member = similar[0]
else:
if verbosity:
print("Adding committee member: %s is a %s for %s/%s" %
(p.name(), SUPERVISOR_TYPE[sup_type], self.emplid,
self.unit.slug))
member = Supervisor(student=student_info['student'],
supervisor=p,
supervisor_type=sup_type)
member.created_at = self.effdt
local_committee.append(member)
if SIMS_SOURCE not in member.config:
# record (the first) place we found this fact
member.config[SIMS_SOURCE] = key
# if it wasn't the product of a previous import it was hand-entered: take the effdt from SIMS
member.created_at = self.effdt
# TODO: try to match up external members with new real ones? That sounds hard.
# TODO: remove members if added by this import (in the past) and not found in the newest committee version
if not dry_run:
member.save_if_dirty()
def find_person(userid):
"""
Find a this person, creating if necessary.
"""
people = Person.objects.filter(userid=userid)
if people:
return people[0]
else:
emplid = find_emplid(userid)
p = add_person(emplid, commit=False)
p.userid = userid
p.save()
return p
def find_person(userid):
"""
Find a this person, creating if necessary.
"""
people = Person.objects.filter(userid=userid)
if people:
return people[0]
else:
emplid = find_emplid(userid)
p = add_person(emplid, commit=False)
p.userid = userid
p.save()
return p
def clean(self, value):
if isinstance(value, Person):
return self.__check_email(value)
else:
if not self.required and not value:
return None
try:
return self.__check_email(Person.objects.get(emplid=value))
except (ValueError, Person.DoesNotExist):
# try to find the emplid in SIMS if they are missing from our DB
if not value:
raise forms.ValidationError("Could not find this emplid.")
if not value.isdigit(
): # doesn't look like an emplid: try it as a userid
try:
return self.__check_email(
Person.objects.get(userid=value))
except Person.DoesNotExist:
value = userid_to_emplid(value)
if not value:
raise forms.ValidationError(
"Could not find this emplid.")
try:
persondata = find_person(value)
except SIMSProblem as e:
raise forms.ValidationError(
"Problem locating person in SIMS: " + str(e))
if not persondata:
raise forms.ValidationError("Could not find this emplid.")
# we found this emplid in SIMS: raise validation error, but offer to add next time.
confirm = self.fieldname + '_confirm'
checkemplid = self.fieldname + '_emplid'
if confirm in self.formdata and checkemplid in self.formdata and self.formdata[
checkemplid] == value:
# new person was presented in the form last time, and they confirmed import
p = add_person(value)
return self.__check_email(p)
else:
self.widget.found_sims = True
self.widget.sims_data = persondata
raise forms.ValidationError(
"Person is new to this system: please confirm their import."
)
def sims_add_person(request):
if request.method == 'POST':
emplid = request.POST.get('emplid', None)
if emplid:
try:
p = add_person(emplid.strip())
except SIMSProblem:
p = None
if isinstance(p, Person):
#LOG EVENT#
l = LogEntry(userid=request.user.username,
description=(u"added %s (%s) from SIMS") % (p.name(), p.emplid),
related_object=p)
l.save()
messages.add_message(request, messages.SUCCESS, u'Record for %s created.' % (p.name()))
return _redirect_to_notes(p)
return HttpResponseRedirect(reverse('advisornotes.views.advising', kwargs={}))
def update_local_data(self, student_info, verbosity, dry_run):
if self.grad_program.unit.slug == 'cmpt':
return
key = self.import_key()
local_committee = student_info['committee']
sup_type = COMMITTEE_MEMBER_MAP[self.committee_role]
# cache People objects, so we don't query for them too much.
if self.sup_emplid in CommitteeMembership.found_people:
p = CommitteeMembership.found_people[self.sup_emplid]
else:
p = add_person(self.sup_emplid, external_email=True, commit=(not dry_run))
CommitteeMembership.found_people[self.sup_emplid] = p
matches = [m for m in local_committee if m.supervisor == p and m.supervisor_type == sup_type]
if matches:
member = matches[0]
else:
similar = [m for m in local_committee if m.supervisor == p]
if len(similar) > 0:
if verbosity > 2:
print "* Found similar (but imperfect) committee member for %s is a %s for %s/%s" % (p.name(), SUPERVISOR_TYPE[sup_type], self.emplid, self.unit.slug)
member = similar[0]
else:
if verbosity:
print "Adding committee member: %s is a %s for %s/%s" % (p.name(), SUPERVISOR_TYPE[sup_type], self.emplid, self.unit.slug)
member = Supervisor(student=student_info['student'], supervisor=p, supervisor_type=sup_type)
member.created_at = self.effdt
local_committee.append(member)
if SIMS_SOURCE not in member.config:
# record (the first) place we found this fact
member.config[SIMS_SOURCE] = key
# if it wasn't the product of a previous import it was hand-entered: take the effdt from SIMS
member.created_at = self.effdt
# TODO: try to match up external members with new real ones? That sounds hard.
# TODO: remove members if added by this import (in the past) and not found in the newest committee version
if not dry_run:
member.save_if_dirty()
def clean_people(self):
text = self.cleaned_data['people']
emplids = text.strip().split()
people = []
for e in emplids:
# this is going to be slow if there's a big list
try:
person = Person.objects.get(find_userid_or_emplid(e))
except Person.DoesNotExist:
try:
person = add_person(int(e))
if person is None:
raise forms.ValidationError(
'Cannot find a person emplid/userid %r.' % (e, ))
except (ValueError, SIMSProblem):
raise forms.ValidationError(
'Cannot find a person emplid/userid %r.' % (e, ))
people.append(person)
return people
def clean(self, value):
if isinstance(value, Person):
return value
else:
if not self.required and not value:
return None
try:
return Person.objects.get(emplid=value)
except (ValueError, Person.DoesNotExist):
# try to find the emplid in SIMS if they are missing from our DB
if not value:
raise forms.ValidationError, "Could not find this emplid."
if not value.isdigit(): # doesn't look like an emplid: try it as a userid
try:
return Person.objects.get(userid=value)
except Person.DoesNotExist:
value = userid_to_emplid(value)
if not value:
raise forms.ValidationError, "Could not find this emplid."
try:
persondata = find_person(value)
except SIMSProblem, e:
raise forms.ValidationError, "Problem locating person in SIMS: " + unicode(e)
if not persondata:
raise forms.ValidationError, "Could not find this emplid."
# we found this emplid in SIMS: raise validation error, but offer to add next time.
confirm = self.fieldname+'_confirm'
checkemplid = self.fieldname+'_emplid'
if confirm in self.formdata and checkemplid in self.formdata and self.formdata[checkemplid] == value:
# new person was presented in the form last time, and they confirmed import
p = add_person(value)
return p
else:
self.widget.found_sims = True
self.widget.sims_data = persondata
raise forms.ValidationError, "Person is new to this system: please confirm their import."
def process_pcs_row(row, column, rownum, unit, semester, user):
"""
Process a single row from the PCS import
"""
appsemester = semester.previous_semester()
warnings = []
ident = "in row %i" % (rownum)
appid = row[column['appid']]
emplid = row[column['emplid']]
program = row[column['program']]
# get Person, from SIMS if necessary
try:
p = Person.objects.get(emplid=int(emplid))
except ValueError:
warnings.append("Bad emplid %s: not processing that row." % (ident))
return warnings
except Person.DoesNotExist:
try:
p = add_person(emplid)
except SIMSProblem as e:
return str(e)
ident = 'for "%s"' % (p.name())
# update information on the Person
email = row[column['email']]
if email: p.config['applic_email'] = email
dob = row[column['dob']]
if dob:
try:
dt = datetime.datetime.strptime(dob, "%Y-%m-%d")
p.config['birthdate'] = dt.date().isoformat()
except ValueError:
warnings.append("Bad birthdate %s." % (ident))
# get extended SIMS data
data = grad_student_info(emplid)
p.config.update(data)
p.save()
#print "Importing %s" % (p)
# get GradStudent, creating if necessary
# a unique identifier for this application, so we can detect repeated imports (and handle gracefully)
uid = "%s-%s-%s-%s" % (unit.slug, semester.name, appid, emplid)
# TODO: wrong, wrong, wrong. Figure out how to select program from import data
program = GradProgram.objects.filter(unit=unit)[0]
# find the old GradStudent if possible
gss = GradStudent.objects.filter(program__unit=unit, person=p)
gs = None
for g in gss:
if 'app_id' in g.config and g.config['app_id'] == uid:
gs = g
break
if not gs:
gs = GradStudent(program=program, person=p)
gs.config['app_id'] = uid
resarea = row[column['resarea']]
firstlang = row[column['firstlang']]
gs.research_area = resarea
gs.mother_tongue = firstlang
gs.created_by = user.userid
gs.updated_by = user.userid
gs.config['start_semester'] = semester.name
gs.save()
complete = row[column['complete']].strip()
decision = row[column['decision']].strip()
notes = row[column['notes']].strip()
gs.config['decisionnotes'] = notes
old_st = GradStatus.objects.filter(student=gs, start__name__gte=semester.name)
if not old_st:
# if no old status for current semester, create one
# application completion status
if complete == 'AppConfirm':
st = GradStatus(student=gs, status="COMP", start=appsemester, end=None, notes="PCS import")
st.save()
elif complete == '':
st = GradStatus(student=gs, status="INCO", start=appsemester, end=None, notes="PCS import")
st.save()
else:
warnings.append('Unknown "Confirmation of Completion of Application" value %s.' % (ident))
# decision status
if decision == 'DECL':
st = GradStatus(student=gs, status="DECL", start=appsemester, end=None, notes="PCS import")
st.save()
elif decision == '':
st = GradStatus(student=gs, status="OFFO", start=appsemester, end=None, notes="PCS import")
st.save()
elif decision == 'R':
st = GradStatus(student=gs, status="REJE", start=appsemester, end=None, notes="PCS import")
st.save()
elif decision == 'HOLD':
st = GradStatus(student=gs, status="HOLD", start=appsemester, end=None, notes="PCS import")
st.save()
elif decision == 'AMScT':
# TODO: bump program to MSc thesis
st = GradStatus(student=gs, status="CONF", start=appsemester, end=None, notes="PCS import")
st.save()
elif decision == 'AMScC':
# TODO: bump program to MSc course-based
st = GradStatus(student=gs, status="CONF", start=appsemester, end=None, notes="PCS import")
st.save()
# potential supervisor
potsuper = row[column['potsuper']]
if potsuper:
superv = None
external = None
try:
ps_last, ps_first = potsuper.split(', ')
except ValueError:
warnings.append('Bad potential supervisor name %s: will store them as an "external" supervisor.' % (ident))
external = potsuper
else:
potentials = possible_supervisor_people([unit])
potential_ids = [p.id for p in potentials]
query = Q(last_name=ps_last, first_name=ps_first) | Q(last_name=ps_last, pref_first_name=ps_first)
people = Person.objects.filter(query, id__in=potential_ids)
if people.count() == 1:
superv = people[0]
else:
warnings.append('Coundn\'t find potential supervisor %s: will store them as an "external" supervisor.' % (ident))
external = potsuper
old_s = Supervisor.objects.filter(student=gs, supervisor_type='POT')
if old_s:
s = old_s[0]
else:
s = Supervisor(student=gs, supervisor_type='POT')
s.superv = superv
s.external = external
s.position = 0
s.created_by = user.userid
s.modified_by = user.userid
s.save()
l = LogEntry(userid=user.userid, description="Imported grad record for %s (%s) from PCS" % (p.name(), p.emplid), related_object=gs)
l.save()
return warnings
def process_pcs_row(row, column, rownum, unit, semester, user):
"""
Process a single row from the PCS import
"""
appsemester = semester.previous_semester()
warnings = []
ident = "in row %i" % (rownum)
appid = row[column['appid']]
emplid = row[column['emplid']]
program = row[column['program']]
# get Person, from SIMS if necessary
try:
p = Person.objects.get(emplid=int(emplid))
except ValueError:
warnings.append("Bad emplid %s: not processing that row." % (ident))
return warnings
except Person.DoesNotExist:
try:
p = add_person(emplid)
except SIMSProblem as e:
return e.message
ident = 'for "%s"' % (p.name())
# update information on the Person
email = row[column['email']]
if email: p.config['applic_email'] = email
dob = row[column['dob']]
if dob:
try:
dt = datetime.datetime.strptime(dob, "%Y-%m-%d")
p.config['birthdate'] = dt.date().isoformat()
except ValueError:
warnings.append("Bad birthdate %s." % (ident))
# get extended SIMS data
data = grad_student_info(emplid)
p.config.update(data)
p.save()
#print "Importing %s" % (p)
# get GradStudent, creating if necessary
# a unique identifier for this application, so we can detect repeated imports (and handle gracefully)
uid = "%s-%s-%s-%s" % (unit.slug, semester.name, appid, emplid)
# TODO: wrong, wrong, wrong. Figure out how to select program from import data
program = GradProgram.objects.filter(unit=unit)[0]
# find the old GradStudent if possible
gss = GradStudent.objects.filter(program__unit=unit, person=p)
gs = None
for g in gss:
if 'app_id' in g.config and g.config['app_id'] == uid:
gs = g
break
if not gs:
gs = GradStudent(program=program, person=p)
gs.config['app_id'] = uid
resarea = row[column['resarea']]
firstlang = row[column['firstlang']]
gs.research_area = resarea
gs.mother_tongue = firstlang
gs.created_by = user.userid
gs.updated_by = user.userid
gs.config['start_semester'] = semester.name
gs.save()
complete = row[column['complete']].strip()
decision = row[column['decision']].strip()
notes = row[column['notes']].strip()
gs.config['decisionnotes'] = notes
old_st = GradStatus.objects.filter(student=gs,
start__name__gte=semester.name)
if not old_st:
# if no old status for current semester, create one
# application completion status
if complete == 'AppConfirm':
st = GradStatus(student=gs,
status="COMP",
start=appsemester,
end=None,
notes="PCS import")
st.save()
elif complete == '':
st = GradStatus(student=gs,
status="INCO",
start=appsemester,
end=None,
notes="PCS import")
st.save()
else:
warnings.append(
'Unknown "Confirmation of Completion of Application" value %s.'
% (ident))
# decision status
if decision == 'DECL':
st = GradStatus(student=gs,
status="DECL",
start=appsemester,
end=None,
notes="PCS import")
st.save()
elif decision == '':
st = GradStatus(student=gs,
status="OFFO",
start=appsemester,
end=None,
notes="PCS import")
st.save()
elif decision == 'R':
st = GradStatus(student=gs,
status="REJE",
start=appsemester,
end=None,
notes="PCS import")
st.save()
elif decision == 'HOLD':
st = GradStatus(student=gs,
status="HOLD",
start=appsemester,
end=None,
notes="PCS import")
st.save()
elif decision == 'AMScT':
# TODO: bump program to MSc thesis
st = GradStatus(student=gs,
status="CONF",
start=appsemester,
end=None,
notes="PCS import")
st.save()
elif decision == 'AMScC':
# TODO: bump program to MSc course-based
st = GradStatus(student=gs,
status="CONF",
start=appsemester,
end=None,
notes="PCS import")
st.save()
# potential supervisor
potsuper = row[column['potsuper']]
if potsuper:
superv = None
external = None
try:
ps_last, ps_first = potsuper.split(', ')
except ValueError:
warnings.append(
'Bad potential supervisor name %s: will store them as an "external" supervisor.'
% (ident))
external = potsuper
else:
potentials = possible_supervisor_people([unit])
potential_ids = [p.id for p in potentials]
query = Q(last_name=ps_last, first_name=ps_first) | Q(
last_name=ps_last, pref_first_name=ps_first)
people = Person.objects.filter(query, id__in=potential_ids)
if people.count() == 1:
superv = people[0]
else:
warnings.append(
'Coundn\'t find potential supervisor %s: will store them as an "external" supervisor.'
% (ident))
external = potsuper
old_s = Supervisor.objects.filter(student=gs, supervisor_type='POT')
if old_s:
s = old_s[0]
else:
s = Supervisor(student=gs, supervisor_type='POT')
s.superv = superv
s.external = external
s.position = 0
s.created_by = user.userid
s.modified_by = user.userid
s.save()
l = LogEntry(userid=user.userid,
description="Imported grad record for %s (%s) from PCS" %
(p.name(), p.emplid),
related_object=gs)
l.save()
return warnings
def import_note(self, advisor, fn, i, row):
emplid = row['Student ID']
date_str = row['Date Modified']
notes = normalize_newlines(row['Notes'])
files = [
row.get('Transcript', None),
row.get('Files', None),
row.get('Files2', None),
row.get('Files3', None),
]
files = map(self.get_filepath, files)
files = filter(bool, files)
# fix mis-typed emplids we found
# Lindsay
if emplid == '960022098':
emplid = '963022098'
elif emplid == '30108409':
emplid = '301078409'
elif emplid == '30115964':
emplid = '301115964'
elif emplid == '30117882':
emplid = '301178882'
# Michael Sean
elif emplid == '30105659':
emplid = '301040985' # ?
# Dijana
elif emplid == '30120965':
emplid = '301202965'
if not emplid or emplid == '0':
if self.verbosity > 0:
print 'No emplid on row %i' % (i + 2)
return
p = add_person(emplid, commit=self.commit)
if not p:
if self.verbosity > 0:
print u"Can't find person on row %i (emplid %s)" % (i + 2,
emplid)
return
if self.verbosity > 1:
print u"Importing %s with %i file(s)." % (emplid, len(files))
try:
date = datetime.datetime.strptime(date_str, '%m-%d-%Y').date()
except ValueError:
date = datetime.datetime.strptime(date_str, '%Y-%m-%d').date()
created = datetime.datetime.combine(date,
datetime.time(hour=12, minute=0))
key = '%s-%i' % (fn, i)
note, _ = self.get_advisornote(key,
p,
advisor,
created,
delete_old_file=self.commit)
if files:
path = files[0]
self.attach_file(note, path)
for j, path in enumerate(files[1:]):
# these get stashed in accompanying notes
k = key + '-auxfile-' + str(i)
n, _ = self.get_advisornote(k,
p,
advisor,
created,
delete_old_file=self.commit,
offset=(j + 1))
n.text = '[Additional file for previous note.]'
self.attach_file(n, path)
if self.commit:
n.save()
note.text = notes
if self.commit:
note.save()
def process_row(self, i, row):
"""
Actually process each individual row
"""
# It's 0 indexed, and we already consumed the header row.
# This is used for error messages so the user can refer to the input file and know the correct line number.
row_num = i + 2
note_id = row['note_id']
# Just in case someone had a bunch of trailing slashes, etc, make sure we get solely the file name for the key.
file_basename = os.path.basename(os.path.normpath(self.file.name))
# In order for the keys to match (to check for duplicates), they have to have been imported from this importer,
# with the same filename, and with the same note_id.
key = "notes_import-%s-%s" % (file_basename, note_id)
# Find the recipient of the note:
student_emplid = row['emplid']
# See if we can actually cast the emplid to int, since the function we call does so without checking.
try:
int(student_emplid)
except ValueError:
if self.verbose:
error_msg = "ERROR, emplid is not valid for recipient on row %i (emplid %s). Ignoring" % \
(row_num, student_emplid)
self.errors.append(error_msg)
print(error_msg)
return
p = add_person(student_emplid, commit=self.commit)
if not p:
if self.verbose:
error_msg = "ERROR: Can't find recipient on row %i (emplid %s). Ignoring." % (
row_num, student_emplid)
self.errors.append(error_msg)
print(error_msg)
return
# Find the advisor who entered the note:
advisor_emplid = row['creator_emplid']
# Same thing for the advisor
try:
int(advisor_emplid)
except ValueError:
if self.verbose:
error_msg = "ERROR, emplid is not valid for advisor on row %i (emplid %s). Ignoring" % \
(row_num, advisor_emplid)
self.errors.append(error_msg)
print(error_msg)
return
u = add_person(advisor_emplid, commit=self.commit)
if not u:
if self.verbose:
error_msg = "ERROR: Can't find advisor %s on row %i (emplid %s). Ignoring." % \
(advisor_emplid, row_num, student_emplid)
self.errors.append(error_msg)
print(error_msg)
return
advisor_userid = row['creator_computing_id']
if u.userid != advisor_userid:
if self.verbose:
error_msg = "ERROR: The advisor emplid and userid do not match the same person. Emplid %s, userid " \
"%s at row %i. Ignoring." % (advisor_emplid, advisor_userid, row_num)
self.errors.append(error_msg)
print(error_msg)
return
read_date = row['date_created']
# We expect a certain date format, try that first, as the function is slightly faster.
try:
date_created = datetime.strptime(read_date, "%Y-%m-%d %H:%M:%S")
except ValueError:
# Fine, try to make the dateutils parser figure it out, then.
try:
date_created = dateparser.parse(read_date)
except ValueError:
if self.verbose:
error_msg = "ERROR: Cannot deduce the correct date %s at line %i. Ignoring." % (
read_date, row_num)
self.errors.append(error_msg)
print(error_msg)
return
if date_created > timezone_today():
if self.verbose:
error_msg = "ERROR: Creation date %s of note for %s at row %i is in the future. Ignoring. " % \
(read_date, student_emplid, row_num)
self.errors.append(error_msg)
print(error_msg)
return
# Let's check if we've already imported this note (or another which matches):
matching_notes = AdvisorNote.objects.filter(student=p,
advisor=u,
created_at=date_created,
unit=self.unit)
key_matching_notes = [
n for n in matching_notes
if 'import_key' in n.config and n.config['import_key'] == key
]
if key_matching_notes:
if self.verbose:
error_msg = "Already imported note from this file with note_id %s on row %i, ignoring." % \
(note_id, row_num)
#self.errors.append(error_msg) Don't actually add these to the error log, since these are due to us
# running the importer before, as they have the correct key. Only print for verbose output, but not
# in the error recap afterwards.
print(error_msg)
return
# What if we have notes from the exact same time, from the same advisor, for the same recipient, but without
# a matching key? That's fishy too. At first I wrote this to be just a warning and continue processing, but,
# really, there's no reason a note should match this way. It serves as a good way to spot files that were
# already imported under another name.
if matching_notes.count() != len(key_matching_notes):
if self.verbose:
error_msg = "Found matching note, but without matching key. This is fishy. Note_id %s on row %i. "\
"Are you sure this file hasn't been processed already using a different filename? " \
"Ignoring this note." % (note_id, row_num)
self.errors.append(error_msg)
print(error_msg)
return
# We checked every possible case, let's create the new note.
original_text = row['notes']
# The file we were given actually has some NULLs for some text content. No sense importing a null note.
if not original_text or original_text == 'NULL':
if self.verbose:
error_msg = "No actual note content (empty string or NULL) for %s at row %i. Ignoring." % \
(student_emplid, row_num)
self.errors.append(error_msg)
print(error_msg)
return
text = ensure_sanitary_markup(original_text, self.markup)
n = AdvisorNote(student=p,
advisor=u,
created_at=date_created,
unit=self.unit,
text=text)
n.config['import_key'] = key
n.markup = self.markup
if self.verbose:
print("Creating note for %s from row %i..." %
(student_emplid, row_num),
end='')
if self.commit:
n.save()
self.saved += 1
if self.verbose:
print("Saved.")
else:
if self.verbose:
print("Not saved, dry-run only.")
return
def process_row(self, i, row):
"""
Actually process each individual row
"""
# It's 0 indexed, and we already consumed the header row.
# This is used for error messages so the user can refer to the input file and know the correct line number.
row_num = i + 2
note_id = row['note_id']
# Just in case someone had a bunch of trailing slashes, etc, make sure we get solely the file name for the key.
file_basename = os.path.basename(os.path.normpath(self.file.name))
# In order for the keys to match (to check for duplicates), they have to have been imported from this importer,
# with the same filename, and with the same note_id.
key = "notes_import-%s-%s" % (file_basename, note_id)
# Find the recipient of the note:
student_emplid = row['emplid']
# See if we can actually cast the emplid to int, since the function we call does so without checking.
try:
int(student_emplid)
except ValueError:
if self.verbose:
error_msg = "ERROR, emplid is not valid for recipient on row %i (emplid %s). Ignoring" % \
(row_num, student_emplid)
self.errors.append(error_msg)
print(error_msg)
return
p = add_person(student_emplid, commit=self.commit)
if not p:
if self.verbose:
error_msg = "ERROR: Can't find recipient on row %i (emplid %s). Ignoring." % (row_num, student_emplid)
self.errors.append(error_msg)
print(error_msg)
return
# Find the advisor who entered the note:
advisor_emplid = row['creator_emplid']
# Same thing for the advisor
try:
int(advisor_emplid)
except ValueError:
if self.verbose:
error_msg = "ERROR, emplid is not valid for advisor on row %i (emplid %s). Ignoring" % \
(row_num, advisor_emplid)
self.errors.append(error_msg)
print(error_msg)
return
u = add_person(advisor_emplid, commit=self.commit)
if not u:
if self.verbose:
error_msg = "ERROR: Can't find advisor %s on row %i (emplid %s). Ignoring." % \
(advisor_emplid, row_num, student_emplid)
self.errors.append(error_msg)
print(error_msg)
return
advisor_userid = row['creator_computing_id']
if u.userid != advisor_userid:
if self.verbose:
error_msg = "ERROR: The advisor emplid and userid do not match the same person. Emplid %s, userid " \
"%s at row %i. Ignoring." % (advisor_emplid, advisor_userid, row_num)
self.errors.append(error_msg)
print(error_msg)
return
read_date = row['date_created']
# We expect a certain date format, try that first, as the function is slightly faster.
try:
date_created = datetime.strptime(read_date, "%Y-%m-%d %H:%M:%S")
except ValueError:
# Fine, try to make the dateutils parser figure it out, then.
try:
date_created = dateparser.parse(read_date)
except ValueError:
if self.verbose:
error_msg = "ERROR: Cannot deduce the correct date %s at line %i. Ignoring." % (read_date, row_num)
self.errors.append(error_msg)
print(error_msg)
return
if date_created > timezone_today():
if self.verbose:
error_msg = "ERROR: Creation date %s of note for %s at row %i is in the future. Ignoring. " % \
(read_date, student_emplid, row_num)
self.errors.append(error_msg)
print(error_msg)
return
# Let's check if we've already imported this note (or another which matches):
matching_notes = AdvisorNote.objects.filter(student=p, advisor=u, created_at=date_created, unit=self.unit)
key_matching_notes = [n for n in matching_notes if 'import_key' in n.config and n.config['import_key'] == key]
if key_matching_notes:
if self.verbose:
error_msg = "Already imported note from this file with note_id %s on row %i, ignoring." % \
(note_id, row_num)
#self.errors.append(error_msg) Don't actually add these to the error log, since these are due to us
# running the importer before, as they have the correct key. Only print for verbose output, but not
# in the error recap afterwards.
print(error_msg)
return
# What if we have notes from the exact same time, from the same advisor, for the same recipient, but without
# a matching key? That's fishy too. At first I wrote this to be just a warning and continue processing, but,
# really, there's no reason a note should match this way. It serves as a good way to spot files that were
# already imported under another name.
if matching_notes.count() != len(key_matching_notes):
if self.verbose:
error_msg = "Found matching note, but without matching key. This is fishy. Note_id %s on row %i. "\
"Are you sure this file hasn't been processed already using a different filename? " \
"Ignoring this note." % (note_id, row_num)
self.errors.append(error_msg)
print(error_msg)
return
# We checked every possible case, let's create the new note.
original_text = row['notes']
# The file we were given actually has some NULLs for some text content. No sense importing a null note.
if not original_text or original_text == 'NULL':
if self.verbose:
error_msg = "No actual note content (empty string or NULL) for %s at row %i. Ignoring." % \
(student_emplid, row_num)
self.errors.append(error_msg)
print(error_msg)
return
text = ensure_sanitary_markup(original_text, self.markup)
n = AdvisorNote(student=p, advisor=u, created_at=date_created, unit=self.unit, text=text)
n.config['import_key'] = key
n.markup = self.markup
if self.verbose:
print("Creating note for %s from row %i..." % (student_emplid, row_num), end='')
if self.commit:
n.save()
self.saved += 1
if self.verbose:
print("Saved.")
else:
if self.verbose:
print("Not saved, dry-run only.")
return
