def process_line(infile):
"""
Scan all lines in INFILE and create corresponding account/e-mail entries
in Cerebrum.
"""
stream = open(infile, 'r')
# Iterate over all persons:
for line in stream:
logger.debug5("Processing line: |%s|", line)
fields = string.split(line.strip(), ";")
print len(fields)
if len(fields) != 5:
logger.error("Bad line: %s. Skipping" % line)
continue
# fi
fnr, pname, uname, email, foo = fields
if not fnr == "":
logger.debug("Processing person %s", fnr)
try:
fodselsnr.personnr_ok(fnr)
except fodselsnr.InvalidFnrError:
logger.warn("Bad no_ssn %s, skipping", fnr)
continue
person.clear()
gender = constants.gender_male
if fodselsnr.er_kvinne(fnr):
gender = constants.gender_female
y, m, d = fodselsnr.fodt_dato(fnr)
# Can't just populate like this, we need to search for persons
# first.
try:
person.find_by_external_id(constants.externalid_fodselsnr, fnr)
except Errors.NotFoundError:
pass
person.populate(db.Date(y, m, d), gender)
person.affect_external_id(constants.system_ekstens,
constants.externalid_fodselsnr)
person.populate_external_id(constants.system_ekstens,
constants.externalid_fodselsnr,
fnr)
person.write_db()
update_names(fnr, pname)
person.write_db()
logger.debug("Created new person with fnr %s", fnr)
p_id = person.entity_id
account_id = process_user(p_id, uname)
process_mail(account_id, 'defaultmail', email)
def populate_external_ids(tpl):
"""
Locate (or create) a person holding the IDs contained in FIELDS and
register these external IDs if necessary.
This function both alters the PERSON object and retuns a boolean value
(True means the ID update/lookup was successful, False means the
update/lookup failed and nothing can be ascertained about the PERSON's
state).
There are two external IDs in SAP -- the Norwegian social security number
(11-siffret personnummer, fnr) and the SAP employee id (sap_id). SAP IDs
are (allegedly) permanent and unique, fnr can change.
"""
error, person = locate_person(tpl.sap_ansattnr, tpl.sap_fnr)
if error:
logger.error("Lookup for (sap_id; fnr) == (%s; %s) failed",
tpl.sap_ansattnr, tpl.sap_fnr)
return None
if person is not None:
logger.debug("A person owning IDs (%s, %s) already exists",
tpl.sap_ansattnr, tpl.sap_fnr)
# Now, we *must* check that the IDs registered in Cerebrum match
# those in SAP dump. I.e. we select the external IDs from Cerebrum
# and compare them to SAP_ID and FNR. They must either match
# exactly or be absent.
if not match_external_ids(person, tpl.sap_ansattnr, tpl.sap_fnr):
return None
else:
person = Factory.get("Person")(database)
logger.debug("New person for IDs (%s, %s)", tpl.sap_ansattnr,
tpl.sap_fnr)
try:
fodselsnr.personnr_ok(tpl.sap_fnr)
except fodselsnr.InvalidFnrError:
# IVR 2007-02-15 It is *wrong* to simply ignore these, but since they
# do occur, and they may be difficult to get rid of, we'll downgrade
# the severity to avoid being spammed to death.
logger.info("No valid checksum for FNR (%s)!", tpl.sap_fnr)
return None
gender = const.gender_male
if fodselsnr.er_kvinne(tpl.sap_fnr):
gender = const.gender_female
# This would allow us to update birthdays and gender information for
# both new and existing people.
person.populate(tpl.sap_birth_date, gender)
person.affect_external_id(const.system_sap, const.externalid_fodselsnr,
const.externalid_sap_ansattnr)
person.populate_external_id(const.system_sap,
const.externalid_sap_ansattnr,
tpl.sap_ansattnr)
person.populate_external_id(const.system_sap, const.externalid_fodselsnr,
tpl.sap_fnr)
person.write_db()
return person
def process_person(fnr):
"""
Find (or create, if necessary) and return the person_id corresponding to
FNR.
"""
logger.debug("Processing person %s", fnr)
if not fodselsnr.personnr_ok(fnr):
logger.warn("Bad no_ssn |%s|", fnr)
return None
# fi
if fnr2person_id.has_key(fnr):
logger.debug("Person with fnr %s exists in Cerebrum", fnr)
return fnr2person_id[fnr]
# fi
# ... otherwise, create a new person
person.clear()
gender = constants.gender_male
if fodselsnr.er_kvinne(fnr):
gender = constants.gender_female
# fi
year, mon, day = fodselsnr.fodt_dato(fnr)
person.populate(db.Date(year, mon, day), gender)
person.affect_external_id(constants.system_migrate,
constants.externalid_fodselsnr)
person.populate_external_id(constants.system_migrate,
constants.externalid_fodselsnr, fnr)
person.write_db()
logger.debug("Created new person with fnr %s", fnr)
e_id = person.entity_id
fnr2person_id[fnr] = e_id
return e_id
def process_person(fnr):
"""
Find (or create, if necessary) and return the person_id corresponding to
FNR.
"""
# If person already exists, return entity_id
if fnr2person_id.has_key(fnr):
return fnr2person_id[fnr]
try:
person.clear()
person.find_by_external_id(constants.externalid_fodselsnr, fnr)
e_id = person.entity_id
fnr2person_id[fnr] = e_id
return e_id
except Errors.NotFoundError:
pass
# ... otherwise, create a new person
person.clear()
gender = constants.gender_male
if fodselsnr.er_kvinne(fnr):
gender = constants.gender_female
year, mon, day = fodselsnr.fodt_dato(fnr)
person.populate(db.Date(year, mon, day), gender)
person.affect_external_id(constants.system_migrate,
constants.externalid_fodselsnr)
person.populate_external_id(constants.system_migrate,
constants.externalid_fodselsnr,
fnr)
person.write_db()
logger.debug("Created new person with fnr %s", fnr)
e_id = person.entity_id
fnr2person_id[fnr] = e_id
return e_id
def process_person(fnr):
"""
Find (or create, if necessary) and return the person_id corresponding to
FNR.
"""
# If person already exists, return entity_id
if fnr2person_id.has_key(fnr):
return fnr2person_id[fnr]
try:
person.clear()
person.find_by_external_id(constants.externalid_fodselsnr, fnr)
e_id = person.entity_id
fnr2person_id[fnr] = e_id
return e_id
except Errors.NotFoundError:
pass
# ... otherwise, create a new person
person.clear()
gender = constants.gender_male
if fodselsnr.er_kvinne(fnr):
gender = constants.gender_female
year, mon, day = fodselsnr.fodt_dato(fnr)
person.populate(db.Date(year, mon, day), gender)
person.affect_external_id(constants.system_migrate,
constants.externalid_fodselsnr)
person.populate_external_id(constants.system_migrate,
constants.externalid_fodselsnr, fnr)
person.write_db()
logger.debug("Created new person with fnr %s", fnr)
e_id = person.entity_id
fnr2person_id[fnr] = e_id
return e_id
def process_line(infile):
"""
Scan all lines in INFILE and create corresponding account/e-mail entries
in Cerebrum.
"""
stream = open(infile, 'r')
# Iterate over all persons:
for line in stream:
logger.debug5("Processing line: |%s|", line)
fields = string.split(line.strip(), ";")
print len(fields)
if len(fields) != 5:
logger.error("Bad line: %s. Skipping" % line)
continue
# fi
fnr, pname, uname, email, foo = fields
if not fnr == "":
logger.debug("Processing person %s", fnr)
try:
fodselsnr.personnr_ok(fnr)
except fodselsnr.InvalidFnrError:
logger.warn("Bad no_ssn %s, skipping", fnr)
continue
person.clear()
gender = constants.gender_male
if fodselsnr.er_kvinne(fnr):
gender = constants.gender_female
y, m, d = fodselsnr.fodt_dato(fnr)
# Can't just populate like this, we need to search for persons
# first.
try:
person.find_by_external_id(constants.externalid_fodselsnr, fnr)
except Errors.NotFoundError:
pass
person.populate(db.Date(y, m, d), gender)
person.affect_external_id(constants.system_ekstens,
constants.externalid_fodselsnr)
person.populate_external_id(constants.system_ekstens,
constants.externalid_fodselsnr, fnr)
person.write_db()
update_names(fnr, pname)
person.write_db()
logger.debug("Created new person with fnr %s", fnr)
p_id = person.entity_id
account_id = process_user(p_id, uname)
process_mail(account_id, 'defaultmail', email)
def process_person(fnr, lname, fname, bewid, set_names):
"""
Find or create a person; return the person_id corresponding to
fnr. Set name for new persons if set_name is True.
"""
logger.debug("Processing person %s %s (%s)", fname, lname, fnr)
try:
fodselsnr.personnr_ok(fnr)
except fodselsnr.InvalidFnrError:
logger.warn("Bad no_ssn |%s|", fnr)
return None
if fnr2person_id.has_key(fnr):
logger.debug("Person with fnr %s exists in Cerebrum", fnr)
return fnr2person_id[fnr]
# ... otherwise, create a new person
person.clear()
gender = constants.gender_male
if fodselsnr.er_kvinne(fnr):
gender = constants.gender_female
year, mon, day = fodselsnr.fodt_dato(fnr)
person.populate(db.Date(year, mon, day), gender)
if bewid:
person.affect_external_id(constants.system_migrate,
constants.externalid_fodselsnr,
constants.externalid_bewatorid)
else:
person.affect_external_id(constants.system_migrate,
constants.externalid_fodselsnr)
person.populate_external_id(constants.system_migrate,
constants.externalid_fodselsnr,
fnr)
person.write_db()
e_id = person.entity_id
logger.info("Created new person with id %s and fnr %s", e_id, fnr)
if bewid:
person.populate_external_id(constants.system_migrate,
constants.externalid_bewatorid,
bewid)
person.write_db()
logger.info("Added BewatorID %s for %s", bewid, fnr)
if set_names:
if lname and fname:
person.affect_names(constants.system_migrate,
constants.name_first, constants.name_last)
person.populate_name(constants.name_first, fname)
person.populate_name(constants.name_last, lname)
logger.info("Name %s %s set for person %s", fname, lname, fnr)
person.write_db()
else:
logger.warn("Couldn't set name %s %s for person %s",
fname, lname, fnr)
fnr2person_id[fnr] = e_id
return e_id
def process_person(fnr, card_id):
"""
Find or create a person; return the person_id corresponding to
fnr.
"""
logger.debug("Processing person %s", fnr)
try:
fodselsnr.personnr_ok(fnr)
except fodselsnr.InvalidFnrError:
logger.warn("Bad no_ssn |%s|", fnr)
return None
if fnr2person_id.has_key(fnr):
logger.debug("Person with fnr %s exists in Cerebrum", fnr)
return fnr2person_id[fnr]
# ... otherwise, create a new person
person.clear()
gender = constants.gender_male
if fodselsnr.er_kvinne(fnr):
gender = constants.gender_female
year, mon, day = fodselsnr.fodt_dato(fnr)
person.populate(db.Date(year, mon, day), gender)
person.affect_external_id(constants.system_migrate,
constants.externalid_fodselsnr,
constants.externalid_bewatorid)
person.populate_external_id(constants.system_migrate,
constants.externalid_fodselsnr,
fnr)
if card_id is not None and card_id != '':
person.populate_external_id(constants.system_migrate,
constants.externalid_bewatorid,
card_id)
else:
logger.debug("No Bewator-ID found for person %s" % fnr)
person.write_db()
logger.debug("Created new person with fnr %s and card-ID %s" % (fnr, card_id))
e_id = person.entity_id
fnr2person_id[fnr] = e_id
return e_id
def process_person(fnr):
"""
Find (or create, if necessary) and return the person_id corresponding to
FNR.
"""
logger.debug("Processing person %s", fnr)
if not fodselsnr.personnr_ok(fnr):
logger.warn("Bad no_ssn |%s|", fnr)
return None
# fi
if fnr2person_id.has_key(fnr):
logger.debug("Person with fnr %s exists in Cerebrum", fnr)
return fnr2person_id[fnr]
# fi
# ... otherwise, create a new person
person.clear()
gender = constants.gender_male
if fodselsnr.er_kvinne(fnr):
gender = constants.gender_female
# fi
year, mon, day = fodselsnr.fodt_dato(fnr)
person.populate(db.Date(year, mon, day), gender)
person.affect_external_id(constants.system_migrate,
constants.externalid_fodselsnr)
person.populate_external_id(constants.system_migrate,
constants.externalid_fodselsnr,
fnr)
person.write_db()
logger.debug("Created new person with fnr %s", fnr)
e_id = person.entity_id
fnr2person_id[fnr] = e_id
return e_id
def process_person_callback(person_info):
"""Called when we have fetched all data on a person from the xml
file. Updates/inserts name, address and affiliation
information."""
global no_name
try:
fnr = fodselsnr.personnr_ok("%06d%05d" % (int(person_info['fodselsdato']),
int(person_info['personnr'])))
fnr = fodselsnr.personnr_ok(fnr)
logger.info("Process %s " % (fnr))
(year, mon, day) = fodselsnr.fodt_dato(fnr)
if (year < 1970
and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1):
# Seems to be a bug in time.mktime on some machines
year = 1970
except fodselsnr.InvalidFnrError:
logger.warn("Ugyldig fødselsnr: %s" % fnr)
return
gender = co.gender_male
if(fodselsnr.er_kvinne(fnr)):
gender = co.gender_female
etternavn = fornavn = None
studentnr = None
affiliations = []
address_info = None
aktiv_sted = []
# Iterate over all person_info entries and extract relevant data
if person_info.has_key('aktiv'):
for row in person_info['aktiv']:
if studieprog2sko[row['studieprogramkode']] is not None:
aktiv_sted.append(int(studieprog2sko[row['studieprogramkode']]))
for dta_type in person_info.keys():
x = person_info[dta_type]
p = x[0]
if isinstance(p, str):
continue
# Get name
if dta_type in ('fagperson', 'evu', 'aktiv'):
etternavn = p['etternavn']
fornavn = p['fornavn']
if p.has_key('studentnr_tildelt'):
studentnr = p['studentnr_tildelt']
# Get affiliations
if dta_type in ('fagperson',):
_process_affiliation(co.affiliation_tilknyttet,
co.affiliation_status_tilknyttet_fagperson,
affiliations, _get_sko(p, 'faknr',
'instituttnr', 'gruppenr', 'institusjonsnr'))
elif dta_type in ('aktiv', ):
for row in x:
# aktiv_sted is necessary in order to avoid different affiliation statuses
# to a same 'stedkode' to be overwritten
# e.i. if a person has both affiliations status 'evu' and
# aktive to a single stedkode we want to register the status 'aktive'
# in cerebrum
if studieprog2sko[row['studieprogramkode']] is not None:
aktiv_sted.append(int(studieprog2sko[row['studieprogramkode']]))
_process_affiliation(co.affiliation_student,
co.affiliation_status_student_aktiv, affiliations,
studieprog2sko[row['studieprogramkode']])
elif dta_type in ('evu',):
subtype = co.affiliation_status_student_evu
if studieprog2sko[row['studieprogramkode']] in aktiv_sted:
subtype = co.affiliation_status_student_aktiv
_process_affiliation(co.affiliation_student,
subtype, affiliations,
studieprog2sko[row['studieprogramkode']])
if etternavn is None:
logger.debug("Ikke noe navn på %s" % fnr)
no_name += 1
return
# TODO: If the person already exist and has conflicting data from
# another source-system, some mechanism is needed to determine the
# superior setting.
new_person = Factory.get('Person')(db)
if fnr2person_id.has_key(fnr):
new_person.find(fnr2person_id[fnr])
new_person.populate(mx.DateTime.Date(year, mon, day), gender)
new_person.affect_names(co.system_fs, co.name_first, co.name_last)
new_person.populate_name(co.name_first, fornavn)
new_person.populate_name(co.name_last, etternavn)
if studentnr is not None:
new_person.affect_external_id(co.system_fs,
co.externalid_fodselsnr,
co.externalid_studentnr)
new_person.populate_external_id(co.system_fs, co.externalid_studentnr,
studentnr)
else:
new_person.affect_external_id(co.system_fs,
co.externalid_fodselsnr)
new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr)
ad_post, ad_post_private, ad_street = _calc_address(person_info)
for address_info, ad_const in ((ad_post, co.address_post),
(ad_post_private, co.address_post_private),
(ad_street, co.address_street)):
# TBD: Skal vi slette evt. eksisterende adresse v/None?
if address_info is not None:
logger.debug("Populating address...")
new_person.populate_address(co.system_fs, ad_const, **address_info)
# if this is a new Person, there is no entity_id assigned to it
# until written to the database.
op = new_person.write_db()
for a in filter_affiliations(affiliations):
ou, aff, aff_status = a
new_person.populate_affiliation(co.system_fs, ou, aff, aff_status)
if include_delete:
key_a = "%s:%s:%s" % (new_person.entity_id,ou,int(aff))
if old_aff.has_key(key_a):
old_aff[key_a] = False
register_cellphone(new_person, person_info)
op2 = new_person.write_db()
if op is None and op2 is None:
logger.info("**** EQUAL ****")
elif op == True:
logger.info("**** NEW ****")
else:
logger.info("**** UPDATE ****")
register_fagomrade(new_person, person_info)
# Reservations
if gen_groups:
should_add = False
if person_info.has_key('nettpubl'):
for row in person_info['nettpubl']:
if row.get('akseptansetypekode', "") == "NETTPUBL" and row.get('status_svar', "") == "J":
should_add = True
if should_add:
# The student has explicitly given us permission to be
# published in the directory.
_add_res(new_person.entity_id)
else:
# The student either hasn't registered an answer to
# the "Can we publish info about you in the directory"
# question at all, or has given an explicit "I don't
# want to appear in the directory" answer.
_rem_res(new_person.entity_id)
db.commit()
def process_person_callback(person_info):
"""Called when we have fetched all data on a person from the xml
file. Updates/inserts name, address and affiliation
information."""
global no_name
try:
fnr = fodselsnr.personnr_ok("%06d%05d" % (int(person_info["fodselsdato"]), int(person_info["personnr"])))
fnr = fodselsnr.personnr_ok(fnr)
logger.info("Process %s " % (fnr))
(year, mon, day) = fodselsnr.fodt_dato(fnr)
if year < 1970 and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1:
# Seems to be a bug in time.mktime on some machines
year = 1970
except fodselsnr.InvalidFnrError:
logger.warn("Ugyldig fødselsnr: %s" % fnr)
return
gender = co.gender_male
if fodselsnr.er_kvinne(fnr):
gender = co.gender_female
etternavn = fornavn = None
studentnr = None
affiliations = []
address_info = None
aktiv_sted = []
# Iterate over all person_info entries and extract relevant data
if person_info.has_key("aktiv"):
for row in person_info["aktiv"]:
if studieprog2sko[row["studieprogramkode"]] is not None:
aktiv_sted.append(int(studieprog2sko[row["studieprogramkode"]]))
logger.debug("App2akrivts")
for dta_type in person_info.keys():
x = person_info[dta_type]
p = x[0]
if isinstance(p, str):
continue
# Get name
if dta_type in ("fagperson", "evu", "aktiv"):
etternavn = p["etternavn"]
fornavn = p["fornavn"]
if p.has_key("studentnr_tildelt"):
studentnr = "%06d" % int(p["studentnr_tildelt"])
# Get affiliations
if dta_type in ("fagperson",):
_process_affiliation(
co.affiliation_tilknyttet,
co.affiliation_status_tilknyttet_fagperson,
affiliations,
_get_sko(p, "faknr", "instituttnr", "gruppenr", "institusjonsnr"),
)
elif dta_type in ("aktiv",):
for row in x:
# aktiv_sted is necessary in order to avoid different affiliation statuses
# to a same 'stedkode' to be overwritten
# e.i. if a person has both affiliations status 'evu' and
# aktive to a single stedkode we want to register the status 'aktive'
# in cerebrum
if studieprog2sko[row["studieprogramkode"]] is not None:
aktiv_sted.append(int(studieprog2sko[row["studieprogramkode"]]))
_process_affiliation(
co.affiliation_student,
co.affiliation_status_student_aktiv,
affiliations,
studieprog2sko[row["studieprogramkode"]],
)
elif dta_type in ("evu",):
subtype = co.affiliation_status_student_evu
if studieprog2sko[row["studieprogramkode"]] in aktiv_sted:
subtype = co.affiliation_status_student_aktiv
_process_affiliation(
co.affiliation_student, subtype, affiliations, studieprog2sko[row["studieprogramkode"]]
)
if etternavn is None:
logger.debug("Ikke noe navn på %s" % fnr)
no_name += 1
return
new_person = Factory.get("Person")(db)
if fnr2person_id.has_key(fnr):
new_person.find(fnr2person_id[fnr])
new_person.populate(mx.DateTime.Date(year, mon, day), gender)
new_person.affect_names(co.system_fs, co.name_first, co.name_last)
new_person.populate_name(co.name_first, fornavn)
new_person.populate_name(co.name_last, etternavn)
if studentnr is not None:
new_person.affect_external_id(
co.system_fs, co.externalid_fodselsnr, co.externalid_studentnr, co.externalid_bewatorid
)
new_person.populate_external_id(co.system_fs, co.externalid_studentnr, studentnr)
logger.debug("Studentnr is %s", studentnr)
new_bew_id = "01221%06d0" % int(studentnr)
logger.debug("Adding bewator-ID %s for %s", new_bew_id, studentnr)
# we have to use system_fs here (for technical reasons) even
# though we should be using system_manual
new_person.populate_external_id(co.system_fs, co.externalid_bewatorid, new_bew_id)
else:
new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr)
new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr)
ad_post, ad_post_private, ad_street = _calc_address(person_info)
for address_info, ad_const in (
(ad_post, co.address_post),
(ad_post_private, co.address_post_private),
(ad_street, co.address_street),
):
# TBD: Skal vi slette evt. eksisterende adresse v/None?
if address_info is not None:
logger.debug("Populating address...")
new_person.populate_address(co.system_fs, ad_const, **address_info)
# if this is a new Person, there is no entity_id assigned to it
# until written to the database.
op = new_person.write_db()
for a in filter_affiliations(affiliations):
ou, aff, aff_status = a
new_person.populate_affiliation(co.system_fs, ou, aff, aff_status)
if include_delete:
key_a = "%s:%s:%s" % (new_person.entity_id, ou, int(aff))
if old_aff.has_key(key_a):
old_aff[key_a] = False
register_cellphone(new_person, person_info)
op2 = new_person.write_db()
if op is None and op2 is None:
logger.info("**** EQUAL ****")
elif op == True:
logger.info("**** NEW ****")
else:
logger.info("**** UPDATE ****")
# Reservations
if gen_groups:
should_add = False
for dta_type in person_info.keys():
p = person_info[dta_type][0]
if isinstance(p, str):
continue
# Presence of 'fagperson' elements for a person should not
# affect that person's reservation status.
if dta_type in ("fagperson",):
continue
# We only fetch the column in these queries
if dta_type not in ("evu"):
continue
# If 'status_reserv_nettpubl' == "N": add to group
if p.get("status_reserv_nettpubl", "") == "N":
should_add = True
else:
should_add = False
if should_add:
# The student has explicitly given us permission to be
# published in the directory.
_add_res(new_person.entity_id)
else:
# The student either hasn't registered an answer to
# the "Can we publish info about you in the directory"
# question at all, or has given an explicit "I don't
# want to appear in the directory" answer.
_rem_res(new_person.entity_id)
db.commit()
def process_person_callback(person_info):
"""Called when we have fetched all data on a person from the xml
file. Updates/inserts name, address and affiliation
information."""
global no_name
try:
fnr = fodselsnr.personnr_ok(
"%06d%05d" %
(int(person_info['fodselsdato']), int(person_info['personnr'])))
fnr = fodselsnr.personnr_ok(fnr)
logger.info("Process %s " % (fnr))
(year, mon, day) = fodselsnr.fodt_dato(fnr)
if (year < 1970
and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1):
# Seems to be a bug in time.mktime on some machines
year = 1970
except fodselsnr.InvalidFnrError:
logger.warn("Ugyldig fødselsnr: %s" % fnr)
return
gender = co.gender_male
if (fodselsnr.er_kvinne(fnr)):
gender = co.gender_female
etternavn = fornavn = None
studentnr = None
affiliations = []
address_info = None
aktiv_sted = []
aktivemne_sted = []
# Iterate over all person_info entries and extract relevant data
if 'aktiv' in person_info:
for row in person_info['aktiv']:
if studieprog2sko[row['studieprogramkode']] is not None:
aktiv_sted.append(int(
studieprog2sko[row['studieprogramkode']]))
logger.debug("App2akrivts")
if 'emnestud' in person_info:
for row in person_info['emnestud']:
if emne2sko[row['emnekode']] is not None:
aktivemne_sted.append(int(emne2sko[row['emnekode']]))
logger.debug('Add sko %s based on emne %s',
int(emne2sko[row['emnekode']]), row['emnekode'])
for dta_type in person_info.keys():
x = person_info[dta_type]
p = x[0]
if isinstance(p, str):
continue
# Get name
if dta_type in (
'fagperson',
'opptak',
'tilbud',
'evu',
'privatist_emne',
'privatist_studieprogram',
'alumni',
'emnestud',
):
etternavn = p['etternavn']
fornavn = p['fornavn']
if 'studentnr_tildelt' in p:
studentnr = p['studentnr_tildelt']
# Get affiliations
if dta_type in ('fagperson', ):
_process_affiliation(
co.affiliation_tilknyttet, co.affiliation_tilknyttet_fagperson,
affiliations,
_get_sko(p, 'faknr', 'instituttnr', 'gruppenr',
'institusjonsnr'))
elif dta_type in ('opptak', ):
for row in x:
subtype = co.affiliation_status_student_opptak
if studieprog2sko[row['studieprogramkode']] in aktiv_sted:
subtype = co.affiliation_status_student_aktiv
elif row['studierettstatkode'] == 'EVU':
subtype = co.affiliation_status_student_evu
elif row['studierettstatkode'] == 'FULLFØRT':
subtype = co.affiliation_status_student_alumni
elif int(row['studienivakode']) >= 900:
subtype = co.affiliation_status_student_drgrad
elif _is_new_admission(row.get('dato_studierett_tildelt')):
subtype = co.affiliation_status_student_ny
_process_affiliation(co.affiliation_student, subtype,
affiliations,
studieprog2sko[row['studieprogramkode']])
elif dta_type in ('emnestud', ):
for row in x:
subtype = co.affiliation_status_student_emnestud
# We may have some situations here where students get
# emnestud and aonther affiliation to the same sko,
# but this seems to work for now.
try:
sko = emne2sko[row['emnekode']]
except KeyError:
logger.warn("Fant ingen emner med koden %s", p['emnekode'])
continue
if sko in aktiv_sted:
subtype = co.affiliation_status_student_aktiv
_process_affiliation(co.affiliation_student, subtype,
affiliations, sko)
elif dta_type in ('privatist_studieprogram', ):
_process_affiliation(co.affiliation_student,
co.affiliation_status_student_privatist,
affiliations,
studieprog2sko[p['studieprogramkode']])
elif dta_type in ('privatist_emne', ):
try:
sko = emne2sko[p['emnekode']]
except KeyError:
logger.warn("Fant ingen emner med koden %s" % p['emnekode'])
continue
_process_affiliation(co.affiliation_student,
co.affiliation_status_student_privatist,
affiliations, sko)
elif dta_type in ('perm', ):
_process_affiliation(co.affiliation_student,
co.affiliation_status_student_aktiv,
affiliations,
studieprog2sko[p['studieprogramkode']])
elif dta_type in ('tilbud', ):
for row in x:
_process_affiliation(co.affiliation_student,
co.affiliation_status_student_tilbud,
affiliations,
studieprog2sko[row['studieprogramkode']])
elif dta_type in ('evu', ):
_process_affiliation(
co.affiliation_student, co.affiliation_status_student_evu,
affiliations,
_get_sko(p, 'faknr_adm_ansvar', 'instituttnr_adm_ansvar',
'gruppenr_adm_ansvar'))
else:
logger.debug2("No such affiliation type: %s, skipping", dta_type)
if etternavn is None:
logger.debug("Ikke noe navn på %s" % fnr)
no_name += 1
return
# TODO: If the person already exist and has conflicting data from
# another source-system, some mechanism is needed to determine the
# superior setting.
new_person = Factory.get('Person')(db)
if fnr in fnr2person_id:
new_person.find(fnr2person_id[fnr])
new_person.populate(mx.DateTime.Date(year, mon, day), gender)
new_person.affect_names(co.system_fs, co.name_first, co.name_last)
new_person.populate_name(co.name_first, fornavn)
new_person.populate_name(co.name_last, etternavn)
if studentnr is not None:
new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr,
co.externalid_studentnr)
new_person.populate_external_id(co.system_fs, co.externalid_studentnr,
studentnr)
else:
new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr)
new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr)
ad_post, ad_post_private, ad_street = _calc_address(person_info)
for address_info, ad_const in ((ad_post, co.address_post),
(ad_post_private, co.address_post_private),
(ad_street, co.address_street)):
# TBD: Skal vi slette evt. eksisterende adresse v/None?
if address_info is not None:
logger.debug("Populating address %s for %s", ad_const, studentnr)
new_person.populate_address(co.system_fs, ad_const, **address_info)
# if this is a new Person, there is no entity_id assigned to it
# until written to the database.
try:
op = new_person.write_db()
except Exception, e:
logger.exception("write_db failed for person %s: %s", fnr, e)
# Roll back in case of db exceptions:
db.rollback()
return
def create_sysx_person(db, sxp, update_affs, stats):
"""
Create or update person with sysx data.
:param db:
:param sxp: Person dict from ``SYSX``
:param update_affs: A set of current SYSX aff keys
:param stats: A defaultdict with counts
"""
co = Factory.get('Constants')(db)
add_sysx_id = fnr_found = False
sysx_id = sxp['id']
fnr = sxp['fnr']
fornavn = sxp['fornavn']
etternavn = sxp['etternavn']
fullt_navn = "%s %s" % (fornavn, etternavn)
if check_expired_sourcedata(sxp['expire_date']):
logger.info("Skipping sysx_id=%r (%s), expire_date=%r", sysx_id,
fullt_navn, sxp['expire_date'])
stats['skipped'] += 1
return
# check if approved. Ditch if not.
if not sxp['approved']:
logger.error("Skipping sysx_id=%r (%s), not approved", sysx_id,
fullt_navn)
stats['skipped'] += 1
return
my_stedkode = Factory.get('OU')(db)
pers_sysx = Factory.get('Person')(db)
pers_fnr = Factory.get('Person')(db)
try:
pers_sysx.find_by_external_id(co.externalid_sys_x_id, sysx_id)
except Errors.NotFoundError:
add_sysx_id = True
if fnr:
try:
fnr = fodselsnr.personnr_ok(fnr)
except fodselsnr.InvalidFnrError:
logger.error("Skipping sysx_id=%r (%s), invalid fnr", sysx_id,
fullt_navn)
stats['skipped'] += 1
return
birth_date = datetime.date(*fodselsnr.fodt_dato(fnr))
if fodselsnr.er_kvinne(fnr):
gender = co.gender_female
else:
gender = co.gender_male
try:
pers_fnr.find_by_external_id(co.externalid_fodselsnr, fnr)
except Errors.NotFoundError:
pass
except Errors.TooManyRowsError as e:
# This persons fnr has multiple rows in entity_external_id table
# This is an error, person should not have not more than one entry.
# Don't know which person object to use, return error message.
logger.error(
"Skipping sysx_id=%r (%s), matched multiple persons (%s)",
sysx_id, fullt_navn, e)
stats['skipped'] += 1
return
else:
if not add_sysx_id and (pers_fnr.entity_id != pers_sysx.entity_id):
logger.error(
"Skipping sysx_id=%r (%s), matched multiple persons (sysx "
"id matched person_id=%r, sysx fnr matched person_id=%r)",
sysx_id, fullt_navn, pers_sysx.entity_id,
pers_fnr.entity_id)
stats['skipped'] += 1
return
fnr_found = True
else:
# foreigner without norwegian ssn,
birth_date = sxp['birth_date']
if not birth_date:
logger.error("sysx_id=%r (%s) is missing birth date", sysx_id,
fullt_navn)
if sxp['gender'] == 'M':
gender = co.gender_male
elif sxp['gender'] == 'F':
gender = co.gender_female
else:
logger.error("Skipping sysx_id=%r (%s), invalid gender %r",
sysx_id, fullt_navn, sxp['gender'])
stats['skipped'] += 1
return
logger.info("Processing sysx_id=%r (%s)", sysx_id, fullt_navn)
if fnr_found:
person = pers_fnr
else:
person = pers_sysx
# person object located, populate...
try:
person.populate(mx.DateTime.DateFrom(birth_date), gender)
except Errors.CerebrumError:
logger.error("Skipping sysx_id=%r (%s), populate() failed",
sysx_id,
fullt_navn,
exc_info=True)
stats['skipped'] += 1
return
person.affect_names(co.system_x, co.name_first, co.name_last, co.name_full)
person.populate_name(co.name_first, fornavn)
person.populate_name(co.name_last, etternavn)
person.populate_name(co.name_full, fullt_navn)
# add external ids
if fnr:
person.affect_external_id(co.system_x, co.externalid_fodselsnr,
co.externalid_sys_x_id)
person.populate_external_id(co.system_x, co.externalid_fodselsnr, fnr)
logger.debug("Set NO_BIRTHNO for sysx_id=%r (%s)", sysx_id, fullt_navn)
else:
person.affect_external_id(co.system_x, co.externalid_sys_x_id)
person.populate_external_id(co.system_x, co.externalid_sys_x_id, sysx_id)
logger.debug("Set sysx_id for sysx_id=%r (%s)", sysx_id, fullt_navn)
# setting affiliation and affiliation_status
affiliation = co.PersonAffiliation(sxp['affiliation'])
affiliation_status = co.PersonAffStatus(affiliation,
sxp['affiliation_status'])
# Assert that the affs are real
int(affiliation), int(affiliation_status)
# get ou_id of stedkode used
fak = sxp['ou'][0:2]
ins = sxp['ou'][2:4]
avd = sxp['ou'][4:6]
# KEB
if fak == '0':
logger.warning("sysx_id=%r (%s) has invalid sko=%r", sysx_id,
fullt_navn, sxp['ou'])
ins = avd = fak
try:
my_stedkode.find_stedkode(fak, ins, avd,
cereconf.DEFAULT_INSTITUSJONSNR)
except EntityExpiredError:
logger.error("Skipping sysx_id=%r (%s), expired sko=%r", sysx_id,
fullt_navn, sxp['ou'])
stats['skipped'] += 1
return
ou_id = int(my_stedkode.entity_id)
# if this is a new Person, there is no entity_id assigned to it
# until written to the database.
op = person.write_db()
# populate the person affiliation table
person.populate_affiliation(co.system_x, ou_id, int(affiliation),
int(affiliation_status))
# make sure we don't delete this aff when processing deleted affs
aff_key = affiliation_key(person.entity_id, ou_id, affiliation)
update_affs.discard(aff_key)
op2 = person.write_db()
logger.debug("OP codes: op=%s,op2=%s" % (op, op2))
if op is None and op2 is None:
logger.info("**** EQUAL ****")
stats['unchanged'] += 1
elif op is None and op2 is False:
logger.info("**** AFF UPDATE ****")
stats['updated'] += 1
elif op is True:
logger.info("**** NEW ****")
stats['added'] += 1
elif op is False:
logger.info("**** UPDATE ****")
stats['updated'] += 1
return
def import_person(persons, all_nodes):
print("import person")
global dryrun
global cere_list
global include_del
logger.info(
"database:%s" % cereconf.CEREBRUM_DATABASE_CONNECT_DATA['host'])
for person in persons:
ssn_not_valid = False
person_to_be_processed = {}
person = person.rstrip()
person_list = person.split(";")
logger.debug("--- Processing new person ---")
try:
# print "fnr:'%s'" % person_list[1]
person_to_be_processed['noreduorgnin'] = person_list[3]
person_to_be_processed['birth_day'] = person_list[0][0:2]
person_to_be_processed['birth_month'] = person_list[0][2:4]
person_to_be_processed['birth_year'] = person_list[0][4:6]
# Make sure year contains 4 digits.
if (person_to_be_processed['birth_year'] < 20):
birth_year = "20%s" % (person_to_be_processed['birth_year'])
else:
birth_year = "19%s" % (person_to_be_processed['birth_year'])
person_to_be_processed['birth_year'] = birth_year
# print "birth_day:'%s'" % person_to_be_processed['birth_day']
# print "birth_month:'%s'" % person_to_be_processed['birth_month']
# print "birth_year:'%s'" % person_to_be_processed['birth_year']
except ValueError:
logger.warning(
"Empty Birthdate for person named:%s %s. Continue with " +
"next person", person_list[1], person_list[2])
continue
except IndexError:
logger.warning("empty person file?")
continue
# Check if SSN is valid
# print "current fnr:%s" % person_list[0]
try:
fodselsnr.personnr_ok(person_list[0])
person_to_be_processed['ssn'] = person_list[0]
except fodselsnr.InvalidFnrError:
logger.warning(
"Empty or non-valid ssn %s for person:%s %s from :%s. " +
"Continue with next person",
person_list[0],
person_list[1],
person_list[2],
person_list[4])
# ssn_not_valid = True
# person_to_be_processed['ssn'] = ''
continue
# set person gender
# gender = const.gender_male
gender = const.gender_male
if not ssn_not_valid:
if fodselsnr.er_kvinne(person_to_be_processed['ssn']):
gender = const.gender_female
else:
# Impossible to set gender. Return error message and set gender to
# unknown
logger.warning(
"Impossible to set gender for person:%s %s. Using Unknown",
person_list[1], person_list[2])
gender = const.gender_unknown
# set gender
person_to_be_processed['gender'] = gender
# get person firstname
person_to_be_processed['firstname'] = person_list[1]
# print "firstname:%s" % person_to_be_processed['firstname']
# get person lastname
person_to_be_processed['lastname'] = person_list[2]
# print "lastname:%s" % person_to_be_processed['lastname']
if (person_to_be_processed['firstname'].isspace() or
person_to_be_processed['lastname'].isspace()):
# Firstname and/or lastname is made of whitespace ONLY.
# generate error message and continue with NEXT person
logger.warn(
"missing first and/or lastname for person:%s. Person NOT " +
"imported", person)
continue
# set correct encoding
person_to_be_processed['firstname'] = decode_text(
person_to_be_processed['firstname'])
person_to_be_processed['lastname'] = decode_text(
person_to_be_processed['lastname'])
#
# Finished building person_to_be_processed dict.
# pp.pprint(person_to_be_processed)
#
# create person object
db_person.clear()
try:
db_person.find_by_external_id(const.externalid_fodselsnr,
person_to_be_processed['ssn'])
logger.info("Ssn already in database. update person object")
# existing_person = True
except Errors.NotFoundError:
logger.warning("Unknown ssn:%s, create new person object" %
person_to_be_processed['ssn'])
# Unable to find person with ssn in the database.
pass
#
# Populate person object
#
try:
db_person.populate(
mx.DateTime.Date(int(person_to_be_processed['birth_year']),
int(person_to_be_processed['birth_month']),
int(person_to_be_processed['birth_day'])),
int(person_to_be_processed['gender']))
except Errors.CerebrumError as m:
# unable to populate person object. Return error message and
# continue with next person
logger.error("Person:%s population failed",
person_to_be_processed['ssn'],
m)
continue
# affect name and external id
db_person.affect_names(const.system_flyt, const.name_first,
const.name_last)
# populate firstname, lastname and external id
db_person.populate_name(const.name_first,
person_to_be_processed['firstname'])
db_person.populate_name(const.name_last,
person_to_be_processed['lastname'])
db_person.affect_external_id(const.system_flyt,
const.externalid_fodselsnr)
db_person.populate_external_id(const.system_flyt,
const.externalid_fodselsnr,
person_to_be_processed['ssn'])
# In case this is a new person, we will need to write to DB before
# we can continue.
try:
op = db_person.write_db()
except db.IntegrityError as e:
db_person.clear()
db.rollback()
logger.info("Error:%s - person not imported to BAS", e)
continue
# op = db_person.write_db()
# Determine person affiliation and affiliation_status
det_ou, det_affiliation = determine_affiliation(person_list, all_nodes)
# logger.debug(" --- from determine affiliation, the following is
# calculated ---")
# pp.pprint(det_affiliation)
for single_ou in det_ou:
for single_aff in det_affiliation:
new_aff = getattr(const, single_aff)
new_aff_stat = getattr(const, det_affiliation[single_aff])
ou.clear()
ou.find_stedkode(single_ou[0:2], single_ou[2:4],
single_ou[4:6],
cereconf.DEFAULT_INSTITUSJONSNR)
logger.debug(
"setting:: ou_id:%s, aff:%s, aff_stat:%s for person:%s",
int(ou.entity_id), int(new_aff), int(new_aff_stat),
db_person.entity_id)
db_person.populate_affiliation(const.system_flyt,
ou.entity_id,
new_aff,
new_aff_stat)
k = "%s:%s:%s" % (db_person.entity_id, int(ou.entity_id),
int(new_aff))
if include_del:
if k in cere_list:
cere_list[k] = False
# store mobile for those that has it
# contact = determine_contact(db_person)
if len(person_list[13]) > 1:
person_list[13]
logger.debug("has mobile:%s" % person_list[13])
number = person_list[13]
c_prefs = {}
c_type = int(const.contact_mobile_phone)
pref = c_prefs.get(c_type, 0)
db_person.populate_contact_info(const.system_flyt, c_type, number,
pref)
pref = c_prefs[c_type] = pref = 1
else:
logger.debug("No mobile registered for this user")
op2 = db_person.write_db()
if op is None and op2 is None:
logger.info("**** EQUAL ****")
elif op:
logger.info("**** NEW ****")
else:
logger.info("**** UPDATE (%s:%s) ****" % (op, op2))
def process_person_callback(person_info):
"""Called when we have fetched all data on a person from the xml
file. Updates/inserts name, address and affiliation
information."""
global no_name
try:
fnr = fodselsnr.personnr_ok(
"%06d%05d" %
(int(person_info['fodselsdato']), int(person_info['personnr'])))
fnr = fodselsnr.personnr_ok(fnr)
logger.info("Process %s " % (fnr))
(year, mon, day) = fodselsnr.fodt_dato(fnr)
if (year < 1970
and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1):
# Seems to be a bug in time.mktime on some machines
year = 1970
except fodselsnr.InvalidFnrError:
logger.warn("Ugyldig fødselsnr: %s" % fnr)
return
gender = co.gender_male
if (fodselsnr.er_kvinne(fnr)):
gender = co.gender_female
etternavn = fornavn = None
studentnr = None
affiliations = []
address_info = None
aktiv_sted = []
for dta_type in person_info.keys():
x = person_info[dta_type]
p = x[0]
if isinstance(p, str):
continue
# Get name
if dta_type in ('fagperson', 'evu', 'aktiv'):
etternavn = p['etternavn']
fornavn = p['fornavn']
if p.has_key('studentnr_tildelt'):
studentnr = '%06d' % int(p['studentnr_tildelt'])
if etternavn is None:
logger.debug("Ikke noe navn på %s" % fnr)
no_name += 1
return
new_person = Factory.get('Person')(db)
if fnr2person_id.has_key(fnr):
new_person.find(fnr2person_id[fnr])
new_person.populate(mx.DateTime.Date(year, mon, day), gender)
new_person.affect_names(co.system_fs, co.name_first, co.name_last)
new_person.populate_name(co.name_first, fornavn)
new_person.populate_name(co.name_last, etternavn)
if studentnr is not None:
new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr,
co.externalid_studentnr)
new_person.populate_external_id(co.system_fs, co.externalid_studentnr,
studentnr)
logger.debug("Studentnr is %s", studentnr)
else:
new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr)
new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr)
ad_post, ad_post_private, ad_street = _calc_address(person_info)
for address_info, ad_const in ((ad_post, co.address_post),
(ad_post_private, co.address_post_private),
(ad_street, co.address_street)):
# TBD: Skal vi slette evt. eksisterende adresse v/None?
if address_info is not None:
logger.debug("Populating address...")
new_person.populate_address(co.system_fs, ad_const, **address_info)
# if this is a new Person, there is no entity_id assigned to it
# until written to the database.
op = new_person.write_db()
# Iterate over all person_info entries and extract relevant data
if person_info.has_key('aktiv'):
for row in person_info['aktiv']:
try:
if studieprog2sko[row['studieprogramkode']] is not None:
aktiv_sted.append(
int(studieprog2sko[row['studieprogramkode']]))
logger.debug("App2akrivts")
except KeyError:
logger.warn(
'App2akrivts: Person id %d har stud.prog.kode %s som ikke eksisterer.',
new_person.entity_id, row['studieprogramkode'])
for dta_type in person_info.keys():
# Get affiliations
if dta_type in ('fagperson', ):
_process_affiliation(
co.affiliation_tilknyttet,
co.affiliation_status_tilknyttet_fagperson, affiliations,
_get_sko(p, 'faknr', 'instituttnr', 'gruppenr',
'institusjonsnr'))
elif dta_type in ('aktiv', ):
for row in x:
# aktiv_sted is necessary in order to avoid different affiliation statuses
# to a same 'stedkode' to be overwritten
# e.i. if a person has both affiliations status 'evu' and
# aktive to a single stedkode we want to register the status 'aktive'
# in cerebrum
try:
if studieprog2sko[row['studieprogramkode']] is not None:
aktiv_sted.append(
int(studieprog2sko[row['studieprogramkode']]))
_process_affiliation(
co.affiliation_student,
co.affiliation_status_student_aktiv, affiliations,
studieprog2sko[row['studieprogramkode']])
except KeyError:
logger.warn(
'AKTIV: Person id %d har stud.prog.kode %s som ikke eksisterer.',
new_person.entity_id, row['studieprogramkode'])
elif dta_type in ('evu', ):
subtype = co.affiliation_status_student_evu
try:
if studieprog2sko[row['studieprogramkode']] in aktiv_sted:
subtype = co.affiliation_status_student_aktiv
_process_affiliation(co.affiliation_student, subtype,
affiliations,
studieprog2sko[row['studieprogramkode']])
except KeyError:
logger.warn(
'EVU: Person id %d har stud.prog.kode %s som ikke eksisterer.',
new_person.entity_id, row['studieprogramkode'])
for a in filter_affiliations(affiliations):
ou, aff, aff_status = a
new_person.populate_affiliation(co.system_fs, ou, aff, aff_status)
if include_delete:
key_a = "%s:%s:%s" % (new_person.entity_id, ou, int(aff))
if old_aff.has_key(key_a):
old_aff[key_a] = False
register_contact(new_person, person_info)
op2 = new_person.write_db()
if op is None and op2 is None:
logger.info("**** EQUAL ****")
elif op == True:
logger.info("**** NEW ****")
else:
logger.info("**** UPDATE ****")
# Reservations
if gen_groups:
should_add = False
for dta_type in person_info.keys():
p = person_info[dta_type][0]
if isinstance(p, str):
continue
# Presence of 'fagperson' elements for a person should not
# affect that person's reservation status.
if dta_type in ('fagperson', ):
continue
# We only fetch the column in these queries
if dta_type not in ('evu'):
continue
# If 'status_reserv_nettpubl' == "N": add to group
if p.get('status_reserv_nettpubl', "") == "N":
should_add = True
else:
should_add = False
if should_add:
# The student has explicitly given us permission to be
# published in the directory.
_add_res(new_person.entity_id)
else:
# The student either hasn't registered an answer to
# the "Can we publish info about you in the directory"
# question at all, or has given an explicit "I don't
# want to appear in the directory" answer.
_rem_res(new_person.entity_id)
db.commit()
def process_person_callback(person_info):
"""Called when we have fetched all data on a person from the xml
file. Updates/inserts name, address and affiliation
information."""
global no_name
try:
fnr = fodselsnr.personnr_ok("%06d%05d" % (int(person_info['fodselsdato']),
int(person_info['personnr'])))
fnr = fodselsnr.personnr_ok(fnr)
logger.info("Process %s " % (fnr))
(year, mon, day) = fodselsnr.fodt_dato(fnr)
if (year < 1970
and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1):
# Seems to be a bug in time.mktime on some machines
year = 1970
except fodselsnr.InvalidFnrError:
logger.warn("Ugyldig fødselsnr: %s" % fnr)
return
gender = co.gender_male
if(fodselsnr.er_kvinne(fnr)):
gender = co.gender_female
etternavn = fornavn = None
studentnr = None
affiliations = []
address_info = None
aktiv_sted = []
# Iterate over all person_info entries and extract relevant data
if person_info.has_key('aktiv'):
for row in person_info['aktiv']:
if studieprog2sko[row['studieprogramkode']] is not None:
aktiv_sted.append(int(studieprog2sko[row['studieprogramkode']]))
logger.debug("App2akrivts")
for dta_type in person_info.keys():
x = person_info[dta_type]
p = x[0]
if isinstance(p, str):
continue
# Get name
if dta_type in ('fagperson', 'evu', 'aktiv'):
etternavn = p['etternavn']
fornavn = p['fornavn']
if p.has_key('studentnr_tildelt'):
studentnr = p['studentnr_tildelt']
# Get affiliations
if dta_type in ('fagperson',):
_process_affiliation(co.affiliation_tilknyttet,
co.affiliation_status_tilknyttet_fagperson,
affiliations, _get_sko(p, 'faknr',
'instituttnr', 'gruppenr', 'institusjonsnr'))
elif dta_type in ('aktiv', ):
for row in x:
# aktiv_sted is necessary in order to avoid different affiliation statuses
# to a same 'stedkode' to be overwritten
# e.i. if a person has both affiliations status 'evu' and
# aktive to a single stedkode we want to register the status 'aktive'
# in cerebrum
if studieprog2sko[row['studieprogramkode']] is not None:
aktiv_sted.append(int(studieprog2sko[row['studieprogramkode']]))
_process_affiliation(co.affiliation_student,
co.affiliation_status_student_aktiv, affiliations,
studieprog2sko[row['studieprogramkode']])
elif dta_type in ('evu',):
subtype = co.affiliation_status_student_evu
if studieprog2sko[row['studieprogramkode']] in aktiv_sted:
subtype = co.affiliation_status_student_aktiv
_process_affiliation(co.affiliation_student,
subtype, affiliations,
studieprog2sko[row['studieprogramkode']])
if etternavn is None:
logger.debug("Ikke noe navn på %s" % fnr)
no_name += 1
return
# TODO: If the person already exist and has conflicting data from
# another source-system, some mechanism is needed to determine the
# superior setting.
new_person = Factory.get('Person')(db)
if fnr2person_id.has_key(fnr):
new_person.find(fnr2person_id[fnr])
new_person.populate(mx.DateTime.Date(year, mon, day), gender)
new_person.affect_names(co.system_fs, co.name_first, co.name_last)
new_person.populate_name(co.name_first, fornavn)
new_person.populate_name(co.name_last, etternavn)
if studentnr is not None:
new_person.affect_external_id(co.system_fs,
co.externalid_fodselsnr,
co.externalid_studentnr)
new_person.populate_external_id(co.system_fs, co.externalid_studentnr,
studentnr)
else:
new_person.affect_external_id(co.system_fs,
co.externalid_fodselsnr)
new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr)
ad_post, ad_post_private, ad_street = _calc_address(person_info)
for address_info, ad_const in ((ad_post, co.address_post),
(ad_post_private, co.address_post_private),
(ad_street, co.address_street)):
# TBD: Skal vi slette evt. eksisterende adresse v/None?
if address_info is not None:
logger.debug("Populating address...")
new_person.populate_address(co.system_fs, ad_const, **address_info)
# if this is a new Person, there is no entity_id assigned to it
# until written to the database.
try:
op = new_person.write_db()
except Exception, e:
logger.exception("write_db failed for person %s: %s", fnr, e)
# Roll back in case of db exceptions:
db.rollback()
return
def process_person_callback(person_info):
"""Called when we have fetched all data on a person from the xml
file. Updates/inserts name, address and affiliation
information."""
global no_name
try:
fnr = fodselsnr.personnr_ok(
"%06d%05d" %
(int(person_info['fodselsdato']), int(person_info['personnr'])))
fnr = fodselsnr.personnr_ok(fnr)
logger.info("Process %s " % (fnr))
(year, mon, day) = fodselsnr.fodt_dato(fnr)
if (year < 1970
and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1):
# Seems to be a bug in time.mktime on some machines
year = 1970
except fodselsnr.InvalidFnrError:
logger.warn("Ugyldig fødselsnr: %s" % fnr)
return
gender = co.gender_male
if (fodselsnr.er_kvinne(fnr)):
gender = co.gender_female
etternavn = fornavn = None
studentnr = None
affiliations = []
address_info = None
aktiv_sted = []
# Iterate over all person_info entries and extract relevant data
if person_info.has_key('aktiv'):
for row in person_info['aktiv']:
if studieprog2sko[row['studieprogramkode']] is not None:
aktiv_sted.append(int(
studieprog2sko[row['studieprogramkode']]))
logger.debug("App2akrivts")
for dta_type in person_info.keys():
x = person_info[dta_type]
p = x[0]
if isinstance(p, str):
continue
# Get name
if dta_type in ('fagperson', 'evu', 'aktiv'):
etternavn = p['etternavn']
fornavn = p['fornavn']
if p.has_key('studentnr_tildelt'):
studentnr = p['studentnr_tildelt']
# Get affiliations
if dta_type in ('fagperson', ):
_process_affiliation(
co.affiliation_tilknyttet,
co.affiliation_status_tilknyttet_fagperson, affiliations,
_get_sko(p, 'faknr', 'instituttnr', 'gruppenr',
'institusjonsnr'))
elif dta_type in ('aktiv', ):
for row in x:
# aktiv_sted is necessary in order to avoid different affiliation statuses
# to a same 'stedkode' to be overwritten
# e.i. if a person has both affiliations status 'evu' and
# aktive to a single stedkode we want to register the status 'aktive'
# in cerebrum
if studieprog2sko[row['studieprogramkode']] is not None:
aktiv_sted.append(
int(studieprog2sko[row['studieprogramkode']]))
_process_affiliation(
co.affiliation_student,
co.affiliation_status_student_aktiv, affiliations,
studieprog2sko[row['studieprogramkode']])
elif dta_type in ('evu', ):
subtype = co.affiliation_status_student_evu
if studieprog2sko[row['studieprogramkode']] in aktiv_sted:
subtype = co.affiliation_status_student_aktiv
_process_affiliation(co.affiliation_student, subtype, affiliations,
studieprog2sko[row['studieprogramkode']])
if etternavn is None:
logger.debug("Ikke noe navn på %s" % fnr)
no_name += 1
return
# TODO: If the person already exist and has conflicting data from
# another source-system, some mechanism is needed to determine the
# superior setting.
new_person = Factory.get('Person')(db)
if fnr2person_id.has_key(fnr):
new_person.find(fnr2person_id[fnr])
new_person.populate(mx.DateTime.Date(year, mon, day), gender)
new_person.affect_names(co.system_fs, co.name_first, co.name_last)
new_person.populate_name(co.name_first, fornavn)
new_person.populate_name(co.name_last, etternavn)
if studentnr is not None:
new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr,
co.externalid_studentnr)
new_person.populate_external_id(co.system_fs, co.externalid_studentnr,
studentnr)
else:
new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr)
new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr)
ad_post, ad_post_private, ad_street = _calc_address(person_info)
for address_info, ad_const in ((ad_post, co.address_post),
(ad_post_private, co.address_post_private),
(ad_street, co.address_street)):
# TBD: Skal vi slette evt. eksisterende adresse v/None?
if address_info is not None:
logger.debug("Populating address...")
new_person.populate_address(co.system_fs, ad_const, **address_info)
# if this is a new Person, there is no entity_id assigned to it
# until written to the database.
try:
op = new_person.write_db()
except Exception, e:
logger.exception("write_db failed for person %s: %s", fnr, e)
# Roll back in case of db exceptions:
db.rollback()
return
def process_person_callback(person_info):
"""Called when we have fetched all data on a person from the xml
file. Updates/inserts name, address and affiliation
information."""
global no_name
try:
fnr = fodselsnr.personnr_ok("%06d%05d" % (
int(person_info['fodselsdato']),
int(person_info['personnr'])))
fnr = fodselsnr.personnr_ok(fnr)
logger.info("Processing %s", fnr)
(year, mon, day) = fodselsnr.fodt_dato(fnr)
if (year < 1970
and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1):
# Seems to be a bug in time.mktime on some machines
year = 1970
except fodselsnr.InvalidFnrError:
logger.warn(u"Ugyldig f�dselsnr for: %s",
person_info['fodselsdato'])
return
gender = co.gender_male
if(fodselsnr.er_kvinne(fnr)):
gender = co.gender_female
etternavn = fornavn = None
studentnr = None
affiliations = []
address_info = None
aktiv_sted = []
# Iterate over all person_info entries and extract relevant data
for dta_type in person_info.keys():
x = person_info[dta_type]
p = x[0]
if isinstance(p, str):
continue
if dta_type not in ('tilbud', 'eksamen', 'evu'):
if 'studentnr_tildelt' in p:
studentnr = p['studentnr_tildelt']
else:
logger.info("\n%s mangler studentnr!", fnr)
# Get name
if dta_type in ('aktiv', 'tilbud', 'evu', 'privatist_studieprogram', ):
etternavn = p['etternavn']
fornavn = p['fornavn']
# Get address
if address_info is None:
if dta_type in ('aktiv', 'privatist_studieprogram', ):
address_info = _ext_address_info(
p,
'adrlin1_semadr', 'adrlin2_semadr',
'adrlin3_semadr', 'postnr_semadr',
'adresseland_semadr')
if address_info is None:
address_info = _ext_address_info(
p,
'adrlin1_hjemsted', 'adrlin2_hjemsted',
'adrlin3_hjemsted', 'postnr_hjemsted',
'adresseland_hjemsted')
elif dta_type in ('evu',):
address_info = _ext_address_info(
p, 'adrlin1_hjem',
'adrlin2_hjem', 'adrlin3_hjem', 'postnr_hjem',
'adresseland_hjem')
if address_info is None:
address_info = _ext_address_info(
p,
'adrlin1_hjemsted', 'adrlin2_hjemsted',
'adrlin3_hjemsted', 'postnr_hjemsted',
'adresseland_hjemsted')
elif dta_type in ('tilbud',):
address_info = _ext_address_info(
p,
'adrlin1_kontakt', 'adrlin2_kontakt',
'adrlin3_kontakt', 'postnr_kontakt',
'adresseland_kontakt')
# Get affiliations
# Lots of changes here compared to import_FS.py @ uio
# TODO: split import_FS into a common part and organization spesific
# parts
if dta_type in ('aktiv', ):
for row in x:
# aktiv_sted is necessary in order to avoid different
# affiliation statuses to a same 'stedkode' to be overwritten
# e.i. if a person has both affiliations status 'tilbud' and
# aktive to a single stedkode we want to register the status
# 'aktive' in cerebrum
if studieprog2sko[row['studieprogramkode']] is not None:
aktiv_sted.append(
int(studieprog2sko[row['studieprogramkode']]))
_process_affiliation(
co.affiliation_student,
co.affiliation_status_student_aktiv, affiliations,
studieprog2sko[row['studieprogramkode']])
elif dta_type in ('evu',):
for row in x:
_process_affiliation(
co.affiliation_student,
co.affiliation_status_student_evu, affiliations,
_get_sko(p, 'faknr_adm_ansvar',
'instituttnr_adm_ansvar',
'gruppenr_adm_ansvar'))
elif dta_type in ('privatist_studieprogram', ):
for row in x:
_process_affiliation(
co.affiliation_student,
co.affiliation_status_student_privatist,
affiliations, studieprog2sko[row['studieprogramkode']])
elif dta_type in ('tilbud', ):
for row in x:
subtype = co.affiliation_status_student_tilbud
if studieprog2sko[row['studieprogramkode']] in aktiv_sted:
subtype = co.affiliation_status_student_aktiv
_process_affiliation(co.affiliation_student,
subtype, affiliations,
studieprog2sko[row['studieprogramkode']])
if etternavn is None:
logger.info("Ikke noe navn p� %s", fnr)
no_name += 1
return
# TODO: If the person already exist and has conflicting data from
# another source-system, some mechanism is needed to determine the
# superior setting.
fsids = [(co.externalid_fodselsnr, fnr)]
if studentnr is not None:
fsids.append((co.externalid_studentnr, studentnr))
new_person = Factory.get('Person')(db)
try:
new_person.find_by_external_ids(*fsids)
except Errors.NotFoundError:
pass
except Errors.TooManyRowsError, e:
logger.error("Trying to find studentnr %s, getting several persons: %s",
studentnr, e)
return
def populate_external_ids(tpl):
"""
Locate (or create) a person holding the IDs contained in FIELDS and
register these external IDs if necessary.
This function both alters the PERSON object and retuns a boolean value
(True means the ID update/lookup was successful, False means the
update/lookup failed and nothing can be ascertained about the PERSON's
state).
There are two external IDs in SAP -- the Norwegian social security number
(11-siffret personnummer, fnr) and the SAP employee id (sap_id). SAP IDs
are (allegedly) permanent and unique, fnr can change.
"""
error, person = locate_person(tpl.sap_ansattnr, tpl.sap_fnr)
if error:
logger.error("Lookup for (sap_id; fnr) == (%s; %s) failed",
tpl.sap_ansattnr, tpl.sap_fnr)
return None
if person is not None:
logger.debug("A person owning IDs (%s, %s) already exists",
tpl.sap_ansattnr, tpl.sap_fnr)
# Now, we *must* check that the IDs registered in Cerebrum match
# those in SAP dump. I.e. we select the external IDs from Cerebrum
# and compare them to SAP_ID and FNR. They must either match
# exactly or be absent.
if not match_external_ids(person,
tpl.sap_ansattnr, tpl.sap_fnr):
return None
else:
person = Factory.get("Person")(database)
logger.debug("New person for IDs (%s, %s)",
tpl.sap_ansattnr, tpl.sap_fnr)
try:
fodselsnr.personnr_ok(tpl.sap_fnr)
except fodselsnr.InvalidFnrError:
# IVR 2007-02-15 It is *wrong* to simply ignore these, but since they
# do occur, and they may be difficult to get rid of, we'll downgrade
# the severity to avoid being spammed to death.
logger.info("No valid checksum for FNR (%s)!", tpl.sap_fnr)
return None
gender = const.gender_male
if fodselsnr.er_kvinne(tpl.sap_fnr):
gender = const.gender_female
# This would allow us to update birthdays and gender information for
# both new and existing people.
person.populate(tpl.sap_birth_date, gender)
person.affect_external_id(const.system_sap,
const.externalid_fodselsnr,
const.externalid_sap_ansattnr)
person.populate_external_id(const.system_sap,
const.externalid_sap_ansattnr,
tpl.sap_ansattnr)
person.populate_external_id(const.system_sap,
const.externalid_fodselsnr,
tpl.sap_fnr)
person.write_db()
return person
def import_person(db, person, update_affs=None):
"""
Import a single person.
:param person:
A dict from py:func:`parse_person`
:param update_affs:
A set of affiliations from py:func:`load_sito_affiliations`.
Affiliations imported by this function will be removed from the set.
"""
new_person = Factory.get('Person')(db)
const = Factory.get('Constants')(db)
if update_affs is None:
update_affs = set()
employee_id = person['employee_id']
logger.info("Processing employee_id=%r", employee_id)
#
# Validate person data
#
# Birthdate
try:
birthdate = parse_date(person['birthdate'])
valid_birthdate = True
except ValueError:
logger.warning('Invalid birth date for employee_id=%r (%r)',
person['employee_id'], person['birthdate'])
valid_birthdate = False
birthdate = None
# SSN
try:
fodselsnr.personnr_ok(person['ssn'])
valid_ssn = True
except fodselsnr.InvalidFnrError:
logger.warning("Empty SSN for employee_id=%r",
person['employee_id'])
valid_ssn = False
# set person gender (checking both SSN and gender from sito input file)
if valid_ssn:
if fodselsnr.er_kvinne(person['ssn']):
gender = const.gender_female
else:
gender = const.gender_male
elif person['gender'] == 'Female':
gender = const.gender_female
elif person['gender'] == 'Male':
gender = const.gender_male
else:
logger.warning('Unknown gender value for employee_id=%r (%r)',
employee_id, person['gender'])
gender = const.gender_unknown
# Validate Birthdate against SSN
if valid_ssn and valid_birthdate:
ssndate = datetime.date(*fodselsnr.fodt_dato(person['ssn']))
if birthdate != ssndate:
raise SkipPerson('inconsistent birth date and ssn (%r, %r)',
birthdate, ssndate)
elif valid_ssn and not valid_birthdate:
logger.warning('Missing birth date for employee_id=%r, using date'
' from ssn', person['employee_id'])
birthdate = datetime.date(*fodselsnr.fodt_dato(person['ssn']))
elif not valid_ssn and valid_birthdate:
# person have birthdate but NOT ssn. Nothing to do here
pass
elif not valid_ssn and not valid_birthdate:
# person does not have birthdate nor ssn. This person cannot be
# built. SSN or Birthdate required. Return error message and
# continue with NEXT person
raise SkipPerson('missing ssn and birth date')
# Check names
if not person['first_name']:
raise SkipPerson('Missing first name')
if not person['last_name']:
raise SkipPerson('Missing last name')
if person['middle_name']:
fname = person['first_name'] + ' ' + person['middle_name']
else:
fname = person['first_name']
lname = person['last_name']
#
# Get person object som DB if it exists
#
found = False
new_person.clear()
try:
new_person.find_by_external_id(const.externalid_sito_ansattnr,
employee_id)
found = True
except Errors.NotFoundError:
# could not find person in DB based on ansattnr.
if valid_ssn:
# try to find person using ssn if ssn is valid
try:
new_person.clear()
new_person.find_by_external_id(const.externalid_fodselsnr,
person['ssn'])
found = True
except Errors.NotFoundError:
pass
if found:
logger.info('Updating person object for employee_id=%r', employee_id)
else:
logger.info('Creating person object for employee_id=%r', employee_id)
#
# Populate the person object
#
new_person.populate(mx.DateTime.DateFrom(birthdate), gender)
new_person.affect_names(const.system_sito, const.name_first,
const.name_last, const.name_work_title)
new_person.affect_external_id(const.system_sito,
const.externalid_fodselsnr,
const.externalid_sito_ansattnr)
new_person.populate_name(const.name_first, fname)
new_person.populate_name(const.name_last, lname)
if person['title']:
new_person.populate_name(const.name_work_title,
person['title'])
if valid_ssn:
new_person.populate_external_id(const.system_sito,
const.externalid_fodselsnr,
person['ssn'])
new_person.populate_external_id(const.system_sito,
const.externalid_sito_ansattnr,
employee_id)
# intermediary write to get an entity_id if this is a new person.
new_person.write_db()
new_person.populate_affiliation(const.system_sito)
new_person.populate_contact_info(const.system_sito)
# set person affiliation
for key, ou_id, aff, status in determine_affiliations(
db, new_person.entity_id, person):
logger.info("affiliation for employee_id=%r to ou_id=%r",
employee_id, ou_id)
new_person.populate_affiliation(const.system_sito, ou_id,
int(aff), int(status))
# set this persons affiliation entry to False
# this ensures that this persons affiliations will not be removed
# when the clean_affiliation function is called after import person
update_affs.discard(key)
# get person work, cellular and home phone numbers
c_prefs = {}
for con, number in person['phone'].items():
c_type = int(const.human2constant(con, const.ContactInfo))
if c_type in c_prefs:
pref = c_prefs[c_type]
else:
pref = 0
c_prefs[c_type] = 1
new_person.populate_contact_info(const.system_sito,
c_type, number, pref)
logger.debug("contact for employee_id=%r, system=%s, "
"c_type=%s, number=%s, pref=%s",
employee_id, const.system_sito, c_type, number, pref)
new_person.write_db()
return not found
def process_person_callback(person_info):
"""Called when we have fetched all data on a person from the xml
file. Updates/inserts name, address and affiliation
information."""
global no_name
try:
fnr = fodselsnr.personnr_ok(
"%06d%05d" % (int(person_info['fodselsdato']),
int(person_info['personnr'])))
fnr = fodselsnr.personnr_ok(fnr)
logger.info("Process %s " % (fnr))
(year, mon, day) = fodselsnr.fodt_dato(fnr)
if (year < 1970
and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1):
# Seems to be a bug in time.mktime on some machines
year = 1970
except fodselsnr.InvalidFnrError:
logger.warn("Ugyldig fødselsnr: %s" % fnr)
return
gender = co.gender_male
if(fodselsnr.er_kvinne(fnr)):
gender = co.gender_female
etternavn = fornavn = None
studentnr = None
affiliations = []
address_info = None
aktiv_sted = []
aktivemne_sted = []
# Iterate over all person_info entries and extract relevant data
if 'aktiv' in person_info:
for row in person_info['aktiv']:
if studieprog2sko[row['studieprogramkode']] is not None:
aktiv_sted.append(
int(studieprog2sko[row['studieprogramkode']]))
logger.debug("App2akrivts")
if 'emnestud' in person_info:
for row in person_info['emnestud']:
if emne2sko[row['emnekode']] is not None:
aktivemne_sted.append(int(emne2sko[row['emnekode']]))
logger.debug('Add sko %s based on emne %s',
int(emne2sko[row['emnekode']]), row['emnekode'])
for dta_type in person_info.keys():
x = person_info[dta_type]
p = x[0]
if isinstance(p, str):
continue
# Get name
if dta_type in (
'fagperson',
'opptak',
'tilbud',
'evu',
'privatist_emne',
'privatist_studieprogram',
'alumni',
'emnestud', ):
etternavn = p['etternavn']
fornavn = p['fornavn']
if 'studentnr_tildelt' in p:
studentnr = p['studentnr_tildelt']
# Get affiliations
if dta_type in ('fagperson',):
_process_affiliation(co.affiliation_tilknyttet,
co.affiliation_tilknyttet_fagperson,
affiliations,
_get_sko(p,
'faknr',
'instituttnr',
'gruppenr',
'institusjonsnr'))
elif dta_type in ('opptak', ):
for row in x:
subtype = co.affiliation_status_student_opptak
if studieprog2sko[row['studieprogramkode']] in aktiv_sted:
subtype = co.affiliation_status_student_aktiv
elif row['studierettstatkode'] == 'EVU':
subtype = co.affiliation_status_student_evu
elif row['studierettstatkode'] == 'FULLFØRT':
subtype = co.affiliation_status_student_alumni
elif int(row['studienivakode']) >= 900:
subtype = co.affiliation_status_student_drgrad
elif _is_new_admission(row.get('dato_studierett_tildelt')):
subtype = co.affiliation_status_student_ny
_process_affiliation(co.affiliation_student,
subtype,
affiliations,
studieprog2sko[row['studieprogramkode']])
elif dta_type in ('emnestud',):
for row in x:
subtype = co.affiliation_status_student_emnestud
# We may have some situations here where students get
# emnestud and aonther affiliation to the same sko,
# but this seems to work for now.
try:
sko = emne2sko[row['emnekode']]
except KeyError:
logger.warn("Fant ingen emner med koden %s", p['emnekode'])
continue
if sko in aktiv_sted:
subtype = co.affiliation_status_student_aktiv
_process_affiliation(co.affiliation_student, subtype,
affiliations, sko)
elif dta_type in ('privatist_studieprogram',):
_process_affiliation(co.affiliation_student,
co.affiliation_status_student_privatist,
affiliations,
studieprog2sko[p['studieprogramkode']])
elif dta_type in ('privatist_emne',):
try:
sko = emne2sko[p['emnekode']]
except KeyError:
logger.warn("Fant ingen emner med koden %s" % p['emnekode'])
continue
_process_affiliation(co.affiliation_student,
co.affiliation_status_student_privatist,
affiliations,
sko)
elif dta_type in ('perm',):
_process_affiliation(co.affiliation_student,
co.affiliation_status_student_aktiv,
affiliations,
studieprog2sko[p['studieprogramkode']])
elif dta_type in ('tilbud',):
for row in x:
_process_affiliation(co.affiliation_student,
co.affiliation_status_student_tilbud,
affiliations,
studieprog2sko[row['studieprogramkode']])
elif dta_type in ('evu', ):
_process_affiliation(co.affiliation_student,
co.affiliation_status_student_evu,
affiliations,
_get_sko(p,
'faknr_adm_ansvar',
'instituttnr_adm_ansvar',
'gruppenr_adm_ansvar'))
else:
logger.debug2("No such affiliation type: %s, skipping", dta_type)
if etternavn is None:
logger.debug("Ikke noe navn på %s" % fnr)
no_name += 1
return
# TODO: If the person already exist and has conflicting data from
# another source-system, some mechanism is needed to determine the
# superior setting.
new_person = Factory.get('Person')(db)
if fnr in fnr2person_id:
new_person.find(fnr2person_id[fnr])
new_person.populate(mx.DateTime.Date(year, mon, day), gender)
new_person.affect_names(co.system_fs, co.name_first, co.name_last)
new_person.populate_name(co.name_first, fornavn)
new_person.populate_name(co.name_last, etternavn)
if studentnr is not None:
new_person.affect_external_id(co.system_fs,
co.externalid_fodselsnr,
co.externalid_studentnr)
new_person.populate_external_id(co.system_fs, co.externalid_studentnr,
studentnr)
else:
new_person.affect_external_id(co.system_fs,
co.externalid_fodselsnr)
new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr)
ad_post, ad_post_private, ad_street = _calc_address(person_info)
for address_info, ad_const in ((ad_post, co.address_post),
(ad_post_private, co.address_post_private),
(ad_street, co.address_street)):
# TBD: Skal vi slette evt. eksisterende adresse v/None?
if address_info is not None:
logger.debug("Populating address %s for %s", ad_const, studentnr)
new_person.populate_address(co.system_fs, ad_const, **address_info)
# if this is a new Person, there is no entity_id assigned to it
# until written to the database.
try:
op = new_person.write_db()
except Exception, e:
logger.exception("write_db failed for person %s: %s", fnr, e)
# Roll back in case of db exceptions:
db.rollback()
return
def _process_person(self, person):
db = self.db
const = self.const
paga_nr = int(person['ansattnr'])
logger.info("Processing paga_id=%r", paga_nr)
try:
birthdate = parse_date(person['fodselsdato'])
except ValueError as e:
raise SkipPerson("Invalid birth date (%s)" % (e, ))
if person['kjonn'] == 'M':
gender = const.gender_male
else:
gender = const.gender_female
fnr = person['fnr']
if fnr and fnr[6:11] != '00000':
try:
fodselsnr.personnr_ok(fnr)
except Exception as e:
raise SkipPerson("Invalid fnr (%s)" % (e, ))
gender_chk = const.gender_male
if fodselsnr.er_kvinne(fnr):
gender_chk = const.gender_female
if gender_chk != gender:
raise SkipPerson("Inconsistent gender (gender=%r, ssn=%r)" %
(gender, gender_chk))
fnr_date = datetime.date(*(fodselsnr.fodt_dato(fnr)))
if not cmp_birthdates(birthdate, fnr_date):
raise SkipPerson("Inconsistent birth date (date=%r, ssn=%r)" %
(birthdate, fnr_date))
# Passport data
national_id_type = person['edag_id_type']
origin_country = person['country']
if not origin_country:
origin_country = 'NO'
if person['edag_id_nr']:
# generate external id on the form:
# <2-char national id>-<national_id>
national_id_val = '%s-%s' % (origin_country, person['edag_id_nr'])
else:
national_id_val = None
# Abort early if there are no valid identifiers from the source system:
if not any((
fnr and fnr[6:11] != '00000',
national_id_type == 'passnummer' and national_id_val,
)):
# TODO: Wouldn't it be enough with ansattnr?
raise SkipPerson("No valid identifier (fnr, passnummer)")
new_person = Factory.get('Person')(db)
identifiers = [(const.externalid_paga_ansattnr, paga_nr),
(const.externalid_fodselsnr, fnr)]
if national_id_type == 'passnummer' and national_id_val:
identifiers.append((const.externalid_pass_number, national_id_val))
for id_type, id_value in identifiers:
if _populate_existing(new_person, id_type, id_value):
break
if not person.get('fornavn', '').strip():
raise SkipPerson("Missing first name")
if not person.get('etternavn', '').strip():
raise SkipPerson("Missing last name")
new_person.populate(mx.DateTime.DateFrom(birthdate), gender)
new_person.affect_names(const.system_paga, const.name_first,
const.name_last)
new_person.affect_external_id(const.system_paga,
const.externalid_fodselsnr,
const.externalid_paga_ansattnr,
const.externalid_pass_number)
new_person.populate_name(const.name_first, person['fornavn'])
new_person.populate_name(const.name_last, person['etternavn'])
if fnr and fnr[6:11] != '00000':
# do not import external id where external_id type is fnr and
# fnr[6-11] =='00000'
new_person.populate_external_id(const.system_paga,
const.externalid_fodselsnr, fnr)
if national_id_type == 'passnummer' and national_id_val:
new_person.populate_external_id(const.system_paga,
const.externalid_pass_number,
national_id_val)
new_person.populate_external_id(const.system_paga,
const.externalid_paga_ansattnr,
paga_nr)
# If it's a new person, we need to call write_db() to have an entity_id
# assigned to it.
op = new_person.write_db()
if person.get('tittel_personlig'):
new_person.add_name_with_language(
name_variant=const.personal_title,
name_language=const.language_nb,
name=person['tittel_personlig'])
# work_title is set by _determine_affiliations
affiliations = _determine_affiliations(self.db, self.const, new_person,
person)
new_person.populate_affiliation(const.system_paga)
if 'fakultetnr_for_lonnsslip' in person:
sted = get_sted(db, person['fakultetnr_for_lonnsslip'],
person['instituttnr_for_lonnsslip'],
person['gruppenr_for_lonnsslip'])
if sted is not None:
if sted['addr_street'] is not None:
new_person.populate_address(const.system_paga,
type=const.address_street,
**sted['addr_street'])
if sted['addr_post'] is not None:
new_person.populate_address(const.system_paga,
type=const.address_post,
**sted['addr_post'])
if 'lokasjon' in person:
logger.debug(
'Populating paga_id=%r location address with '
'source=%s, type=%s, text=%r', paga_nr, const.system_paga,
const.address_location, person['lokasjon'])
new_person.populate_address(source_system=const.system_paga,
type=const.address_location,
address_text=person['lokasjon'])
else:
logger.warning("No location address for paga_id=%r", paga_nr)
for k, v in affiliations.items():
ou_id, aff, aff_stat = v
new_person.populate_affiliation(const.system_paga, ou_id, int(aff),
int(aff_stat))
self.old_affs.discard(k)
c_prefs = {}
new_person.populate_contact_info(const.system_paga)
for c_type, value in determine_contact(const, person):
c_type = int(c_type)
pref = c_prefs.get(c_type, 0)
new_person.populate_contact_info(const.system_paga, c_type, value,
pref)
c_prefs[c_type] = pref + 1
#
# Also add personal/home street address if it exists in the import file
#
priv_addr = (person.get('adresse'), person.get('postnr'),
person.get('poststed'))
if any(priv_addr):
logger.debug("Setting additional home address: %s %s %s",
priv_addr[0], priv_addr[1], priv_addr[2])
# TODO: Address country? Should probably use const.human2constant()
# country_code = new_person.get_country_code(origin_country)
# TODO: Is there really a guaranteed connection between passport
# origin country and home address?
new_person.populate_address(const.system_paga,
const.address_post_private,
priv_addr[0], None, priv_addr[1],
priv_addr[2], None)
op2 = new_person.write_db()
set_person_spreads(self.db, self.const, new_person, person)
if op is None and op2 is None:
logger.info("EQUAL: No change to person with paga_id=%r", paga_nr)
elif op:
logger.info("NEW: Created new person with paga_id=%r", paga_nr)
else:
logger.info("UPDATE: Updated person with paga_id=%r (%s, %s)",
paga_nr, op, op2)
def _get_gender(self, fnr):
gender = self.co.gender_male
if fodselsnr.er_kvinne(fnr):
gender = self.co.gender_female
return gender
