def process_line(infile): """ Scan all lines in INFILE and create corresponding account/e-mail entries in Cerebrum. """ stream = open(infile, 'r') # Iterate over all persons: for line in stream: logger.debug5("Processing line: |%s|", line) fields = string.split(line.strip(), ";") print len(fields) if len(fields) != 5: logger.error("Bad line: %s. Skipping" % line) continue # fi fnr, pname, uname, email, foo = fields if not fnr == "": logger.debug("Processing person %s", fnr) try: fodselsnr.personnr_ok(fnr) except fodselsnr.InvalidFnrError: logger.warn("Bad no_ssn %s, skipping", fnr) continue person.clear() gender = constants.gender_male if fodselsnr.er_kvinne(fnr): gender = constants.gender_female y, m, d = fodselsnr.fodt_dato(fnr) # Can't just populate like this, we need to search for persons # first. try: person.find_by_external_id(constants.externalid_fodselsnr, fnr) except Errors.NotFoundError: pass person.populate(db.Date(y, m, d), gender) person.affect_external_id(constants.system_ekstens, constants.externalid_fodselsnr) person.populate_external_id(constants.system_ekstens, constants.externalid_fodselsnr, fnr) person.write_db() update_names(fnr, pname) person.write_db() logger.debug("Created new person with fnr %s", fnr) p_id = person.entity_id account_id = process_user(p_id, uname) process_mail(account_id, 'defaultmail', email)
def populate_external_ids(tpl): """ Locate (or create) a person holding the IDs contained in FIELDS and register these external IDs if necessary. This function both alters the PERSON object and retuns a boolean value (True means the ID update/lookup was successful, False means the update/lookup failed and nothing can be ascertained about the PERSON's state). There are two external IDs in SAP -- the Norwegian social security number (11-siffret personnummer, fnr) and the SAP employee id (sap_id). SAP IDs are (allegedly) permanent and unique, fnr can change. """ error, person = locate_person(tpl.sap_ansattnr, tpl.sap_fnr) if error: logger.error("Lookup for (sap_id; fnr) == (%s; %s) failed", tpl.sap_ansattnr, tpl.sap_fnr) return None if person is not None: logger.debug("A person owning IDs (%s, %s) already exists", tpl.sap_ansattnr, tpl.sap_fnr) # Now, we *must* check that the IDs registered in Cerebrum match # those in SAP dump. I.e. we select the external IDs from Cerebrum # and compare them to SAP_ID and FNR. They must either match # exactly or be absent. if not match_external_ids(person, tpl.sap_ansattnr, tpl.sap_fnr): return None else: person = Factory.get("Person")(database) logger.debug("New person for IDs (%s, %s)", tpl.sap_ansattnr, tpl.sap_fnr) try: fodselsnr.personnr_ok(tpl.sap_fnr) except fodselsnr.InvalidFnrError: # IVR 2007-02-15 It is *wrong* to simply ignore these, but since they # do occur, and they may be difficult to get rid of, we'll downgrade # the severity to avoid being spammed to death. logger.info("No valid checksum for FNR (%s)!", tpl.sap_fnr) return None gender = const.gender_male if fodselsnr.er_kvinne(tpl.sap_fnr): gender = const.gender_female # This would allow us to update birthdays and gender information for # both new and existing people. person.populate(tpl.sap_birth_date, gender) person.affect_external_id(const.system_sap, const.externalid_fodselsnr, const.externalid_sap_ansattnr) person.populate_external_id(const.system_sap, const.externalid_sap_ansattnr, tpl.sap_ansattnr) person.populate_external_id(const.system_sap, const.externalid_fodselsnr, tpl.sap_fnr) person.write_db() return person
def process_person(fnr): """ Find (or create, if necessary) and return the person_id corresponding to FNR. """ logger.debug("Processing person %s", fnr) if not fodselsnr.personnr_ok(fnr): logger.warn("Bad no_ssn |%s|", fnr) return None # fi if fnr2person_id.has_key(fnr): logger.debug("Person with fnr %s exists in Cerebrum", fnr) return fnr2person_id[fnr] # fi # ... otherwise, create a new person person.clear() gender = constants.gender_male if fodselsnr.er_kvinne(fnr): gender = constants.gender_female # fi year, mon, day = fodselsnr.fodt_dato(fnr) person.populate(db.Date(year, mon, day), gender) person.affect_external_id(constants.system_migrate, constants.externalid_fodselsnr) person.populate_external_id(constants.system_migrate, constants.externalid_fodselsnr, fnr) person.write_db() logger.debug("Created new person with fnr %s", fnr) e_id = person.entity_id fnr2person_id[fnr] = e_id return e_id
def process_person(fnr): """ Find (or create, if necessary) and return the person_id corresponding to FNR. """ # If person already exists, return entity_id if fnr2person_id.has_key(fnr): return fnr2person_id[fnr] try: person.clear() person.find_by_external_id(constants.externalid_fodselsnr, fnr) e_id = person.entity_id fnr2person_id[fnr] = e_id return e_id except Errors.NotFoundError: pass # ... otherwise, create a new person person.clear() gender = constants.gender_male if fodselsnr.er_kvinne(fnr): gender = constants.gender_female year, mon, day = fodselsnr.fodt_dato(fnr) person.populate(db.Date(year, mon, day), gender) person.affect_external_id(constants.system_migrate, constants.externalid_fodselsnr) person.populate_external_id(constants.system_migrate, constants.externalid_fodselsnr, fnr) person.write_db() logger.debug("Created new person with fnr %s", fnr) e_id = person.entity_id fnr2person_id[fnr] = e_id return e_id
def process_person(fnr, lname, fname, bewid, set_names): """ Find or create a person; return the person_id corresponding to fnr. Set name for new persons if set_name is True. """ logger.debug("Processing person %s %s (%s)", fname, lname, fnr) try: fodselsnr.personnr_ok(fnr) except fodselsnr.InvalidFnrError: logger.warn("Bad no_ssn |%s|", fnr) return None if fnr2person_id.has_key(fnr): logger.debug("Person with fnr %s exists in Cerebrum", fnr) return fnr2person_id[fnr] # ... otherwise, create a new person person.clear() gender = constants.gender_male if fodselsnr.er_kvinne(fnr): gender = constants.gender_female year, mon, day = fodselsnr.fodt_dato(fnr) person.populate(db.Date(year, mon, day), gender) if bewid: person.affect_external_id(constants.system_migrate, constants.externalid_fodselsnr, constants.externalid_bewatorid) else: person.affect_external_id(constants.system_migrate, constants.externalid_fodselsnr) person.populate_external_id(constants.system_migrate, constants.externalid_fodselsnr, fnr) person.write_db() e_id = person.entity_id logger.info("Created new person with id %s and fnr %s", e_id, fnr) if bewid: person.populate_external_id(constants.system_migrate, constants.externalid_bewatorid, bewid) person.write_db() logger.info("Added BewatorID %s for %s", bewid, fnr) if set_names: if lname and fname: person.affect_names(constants.system_migrate, constants.name_first, constants.name_last) person.populate_name(constants.name_first, fname) person.populate_name(constants.name_last, lname) logger.info("Name %s %s set for person %s", fname, lname, fnr) person.write_db() else: logger.warn("Couldn't set name %s %s for person %s", fname, lname, fnr) fnr2person_id[fnr] = e_id return e_id
def process_person(fnr, card_id): """ Find or create a person; return the person_id corresponding to fnr. """ logger.debug("Processing person %s", fnr) try: fodselsnr.personnr_ok(fnr) except fodselsnr.InvalidFnrError: logger.warn("Bad no_ssn |%s|", fnr) return None if fnr2person_id.has_key(fnr): logger.debug("Person with fnr %s exists in Cerebrum", fnr) return fnr2person_id[fnr] # ... otherwise, create a new person person.clear() gender = constants.gender_male if fodselsnr.er_kvinne(fnr): gender = constants.gender_female year, mon, day = fodselsnr.fodt_dato(fnr) person.populate(db.Date(year, mon, day), gender) person.affect_external_id(constants.system_migrate, constants.externalid_fodselsnr, constants.externalid_bewatorid) person.populate_external_id(constants.system_migrate, constants.externalid_fodselsnr, fnr) if card_id is not None and card_id != '': person.populate_external_id(constants.system_migrate, constants.externalid_bewatorid, card_id) else: logger.debug("No Bewator-ID found for person %s" % fnr) person.write_db() logger.debug("Created new person with fnr %s and card-ID %s" % (fnr, card_id)) e_id = person.entity_id fnr2person_id[fnr] = e_id return e_id
def process_person_callback(person_info): """Called when we have fetched all data on a person from the xml file. Updates/inserts name, address and affiliation information.""" global no_name try: fnr = fodselsnr.personnr_ok("%06d%05d" % (int(person_info['fodselsdato']), int(person_info['personnr']))) fnr = fodselsnr.personnr_ok(fnr) logger.info("Process %s " % (fnr)) (year, mon, day) = fodselsnr.fodt_dato(fnr) if (year < 1970 and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1): # Seems to be a bug in time.mktime on some machines year = 1970 except fodselsnr.InvalidFnrError: logger.warn("Ugyldig fødselsnr: %s" % fnr) return gender = co.gender_male if(fodselsnr.er_kvinne(fnr)): gender = co.gender_female etternavn = fornavn = None studentnr = None affiliations = [] address_info = None aktiv_sted = [] # Iterate over all person_info entries and extract relevant data if person_info.has_key('aktiv'): for row in person_info['aktiv']: if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append(int(studieprog2sko[row['studieprogramkode']])) for dta_type in person_info.keys(): x = person_info[dta_type] p = x[0] if isinstance(p, str): continue # Get name if dta_type in ('fagperson', 'evu', 'aktiv'): etternavn = p['etternavn'] fornavn = p['fornavn'] if p.has_key('studentnr_tildelt'): studentnr = p['studentnr_tildelt'] # Get affiliations if dta_type in ('fagperson',): _process_affiliation(co.affiliation_tilknyttet, co.affiliation_status_tilknyttet_fagperson, affiliations, _get_sko(p, 'faknr', 'instituttnr', 'gruppenr', 'institusjonsnr')) elif dta_type in ('aktiv', ): for row in x: # aktiv_sted is necessary in order to avoid different affiliation statuses # to a same 'stedkode' to be overwritten # e.i. if a person has both affiliations status 'evu' and # aktive to a single stedkode we want to register the status 'aktive' # in cerebrum if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append(int(studieprog2sko[row['studieprogramkode']])) _process_affiliation(co.affiliation_student, co.affiliation_status_student_aktiv, affiliations, studieprog2sko[row['studieprogramkode']]) elif dta_type in ('evu',): subtype = co.affiliation_status_student_evu if studieprog2sko[row['studieprogramkode']] in aktiv_sted: subtype = co.affiliation_status_student_aktiv _process_affiliation(co.affiliation_student, subtype, affiliations, studieprog2sko[row['studieprogramkode']]) if etternavn is None: logger.debug("Ikke noe navn på %s" % fnr) no_name += 1 return # TODO: If the person already exist and has conflicting data from # another source-system, some mechanism is needed to determine the # superior setting. new_person = Factory.get('Person')(db) if fnr2person_id.has_key(fnr): new_person.find(fnr2person_id[fnr]) new_person.populate(mx.DateTime.Date(year, mon, day), gender) new_person.affect_names(co.system_fs, co.name_first, co.name_last) new_person.populate_name(co.name_first, fornavn) new_person.populate_name(co.name_last, etternavn) if studentnr is not None: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr, co.externalid_studentnr) new_person.populate_external_id(co.system_fs, co.externalid_studentnr, studentnr) else: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr) new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr) ad_post, ad_post_private, ad_street = _calc_address(person_info) for address_info, ad_const in ((ad_post, co.address_post), (ad_post_private, co.address_post_private), (ad_street, co.address_street)): # TBD: Skal vi slette evt. eksisterende adresse v/None? if address_info is not None: logger.debug("Populating address...") new_person.populate_address(co.system_fs, ad_const, **address_info) # if this is a new Person, there is no entity_id assigned to it # until written to the database. op = new_person.write_db() for a in filter_affiliations(affiliations): ou, aff, aff_status = a new_person.populate_affiliation(co.system_fs, ou, aff, aff_status) if include_delete: key_a = "%s:%s:%s" % (new_person.entity_id,ou,int(aff)) if old_aff.has_key(key_a): old_aff[key_a] = False register_cellphone(new_person, person_info) op2 = new_person.write_db() if op is None and op2 is None: logger.info("**** EQUAL ****") elif op == True: logger.info("**** NEW ****") else: logger.info("**** UPDATE ****") register_fagomrade(new_person, person_info) # Reservations if gen_groups: should_add = False if person_info.has_key('nettpubl'): for row in person_info['nettpubl']: if row.get('akseptansetypekode', "") == "NETTPUBL" and row.get('status_svar', "") == "J": should_add = True if should_add: # The student has explicitly given us permission to be # published in the directory. _add_res(new_person.entity_id) else: # The student either hasn't registered an answer to # the "Can we publish info about you in the directory" # question at all, or has given an explicit "I don't # want to appear in the directory" answer. _rem_res(new_person.entity_id) db.commit()
def process_person_callback(person_info): """Called when we have fetched all data on a person from the xml file. Updates/inserts name, address and affiliation information.""" global no_name try: fnr = fodselsnr.personnr_ok("%06d%05d" % (int(person_info["fodselsdato"]), int(person_info["personnr"]))) fnr = fodselsnr.personnr_ok(fnr) logger.info("Process %s " % (fnr)) (year, mon, day) = fodselsnr.fodt_dato(fnr) if year < 1970 and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1: # Seems to be a bug in time.mktime on some machines year = 1970 except fodselsnr.InvalidFnrError: logger.warn("Ugyldig fødselsnr: %s" % fnr) return gender = co.gender_male if fodselsnr.er_kvinne(fnr): gender = co.gender_female etternavn = fornavn = None studentnr = None affiliations = [] address_info = None aktiv_sted = [] # Iterate over all person_info entries and extract relevant data if person_info.has_key("aktiv"): for row in person_info["aktiv"]: if studieprog2sko[row["studieprogramkode"]] is not None: aktiv_sted.append(int(studieprog2sko[row["studieprogramkode"]])) logger.debug("App2akrivts") for dta_type in person_info.keys(): x = person_info[dta_type] p = x[0] if isinstance(p, str): continue # Get name if dta_type in ("fagperson", "evu", "aktiv"): etternavn = p["etternavn"] fornavn = p["fornavn"] if p.has_key("studentnr_tildelt"): studentnr = "%06d" % int(p["studentnr_tildelt"]) # Get affiliations if dta_type in ("fagperson",): _process_affiliation( co.affiliation_tilknyttet, co.affiliation_status_tilknyttet_fagperson, affiliations, _get_sko(p, "faknr", "instituttnr", "gruppenr", "institusjonsnr"), ) elif dta_type in ("aktiv",): for row in x: # aktiv_sted is necessary in order to avoid different affiliation statuses # to a same 'stedkode' to be overwritten # e.i. if a person has both affiliations status 'evu' and # aktive to a single stedkode we want to register the status 'aktive' # in cerebrum if studieprog2sko[row["studieprogramkode"]] is not None: aktiv_sted.append(int(studieprog2sko[row["studieprogramkode"]])) _process_affiliation( co.affiliation_student, co.affiliation_status_student_aktiv, affiliations, studieprog2sko[row["studieprogramkode"]], ) elif dta_type in ("evu",): subtype = co.affiliation_status_student_evu if studieprog2sko[row["studieprogramkode"]] in aktiv_sted: subtype = co.affiliation_status_student_aktiv _process_affiliation( co.affiliation_student, subtype, affiliations, studieprog2sko[row["studieprogramkode"]] ) if etternavn is None: logger.debug("Ikke noe navn på %s" % fnr) no_name += 1 return new_person = Factory.get("Person")(db) if fnr2person_id.has_key(fnr): new_person.find(fnr2person_id[fnr]) new_person.populate(mx.DateTime.Date(year, mon, day), gender) new_person.affect_names(co.system_fs, co.name_first, co.name_last) new_person.populate_name(co.name_first, fornavn) new_person.populate_name(co.name_last, etternavn) if studentnr is not None: new_person.affect_external_id( co.system_fs, co.externalid_fodselsnr, co.externalid_studentnr, co.externalid_bewatorid ) new_person.populate_external_id(co.system_fs, co.externalid_studentnr, studentnr) logger.debug("Studentnr is %s", studentnr) new_bew_id = "01221%06d0" % int(studentnr) logger.debug("Adding bewator-ID %s for %s", new_bew_id, studentnr) # we have to use system_fs here (for technical reasons) even # though we should be using system_manual new_person.populate_external_id(co.system_fs, co.externalid_bewatorid, new_bew_id) else: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr) new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr) ad_post, ad_post_private, ad_street = _calc_address(person_info) for address_info, ad_const in ( (ad_post, co.address_post), (ad_post_private, co.address_post_private), (ad_street, co.address_street), ): # TBD: Skal vi slette evt. eksisterende adresse v/None? if address_info is not None: logger.debug("Populating address...") new_person.populate_address(co.system_fs, ad_const, **address_info) # if this is a new Person, there is no entity_id assigned to it # until written to the database. op = new_person.write_db() for a in filter_affiliations(affiliations): ou, aff, aff_status = a new_person.populate_affiliation(co.system_fs, ou, aff, aff_status) if include_delete: key_a = "%s:%s:%s" % (new_person.entity_id, ou, int(aff)) if old_aff.has_key(key_a): old_aff[key_a] = False register_cellphone(new_person, person_info) op2 = new_person.write_db() if op is None and op2 is None: logger.info("**** EQUAL ****") elif op == True: logger.info("**** NEW ****") else: logger.info("**** UPDATE ****") # Reservations if gen_groups: should_add = False for dta_type in person_info.keys(): p = person_info[dta_type][0] if isinstance(p, str): continue # Presence of 'fagperson' elements for a person should not # affect that person's reservation status. if dta_type in ("fagperson",): continue # We only fetch the column in these queries if dta_type not in ("evu"): continue # If 'status_reserv_nettpubl' == "N": add to group if p.get("status_reserv_nettpubl", "") == "N": should_add = True else: should_add = False if should_add: # The student has explicitly given us permission to be # published in the directory. _add_res(new_person.entity_id) else: # The student either hasn't registered an answer to # the "Can we publish info about you in the directory" # question at all, or has given an explicit "I don't # want to appear in the directory" answer. _rem_res(new_person.entity_id) db.commit()
def process_person_callback(person_info): """Called when we have fetched all data on a person from the xml file. Updates/inserts name, address and affiliation information.""" global no_name try: fnr = fodselsnr.personnr_ok( "%06d%05d" % (int(person_info['fodselsdato']), int(person_info['personnr']))) fnr = fodselsnr.personnr_ok(fnr) logger.info("Process %s " % (fnr)) (year, mon, day) = fodselsnr.fodt_dato(fnr) if (year < 1970 and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1): # Seems to be a bug in time.mktime on some machines year = 1970 except fodselsnr.InvalidFnrError: logger.warn("Ugyldig fødselsnr: %s" % fnr) return gender = co.gender_male if (fodselsnr.er_kvinne(fnr)): gender = co.gender_female etternavn = fornavn = None studentnr = None affiliations = [] address_info = None aktiv_sted = [] aktivemne_sted = [] # Iterate over all person_info entries and extract relevant data if 'aktiv' in person_info: for row in person_info['aktiv']: if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append(int( studieprog2sko[row['studieprogramkode']])) logger.debug("App2akrivts") if 'emnestud' in person_info: for row in person_info['emnestud']: if emne2sko[row['emnekode']] is not None: aktivemne_sted.append(int(emne2sko[row['emnekode']])) logger.debug('Add sko %s based on emne %s', int(emne2sko[row['emnekode']]), row['emnekode']) for dta_type in person_info.keys(): x = person_info[dta_type] p = x[0] if isinstance(p, str): continue # Get name if dta_type in ( 'fagperson', 'opptak', 'tilbud', 'evu', 'privatist_emne', 'privatist_studieprogram', 'alumni', 'emnestud', ): etternavn = p['etternavn'] fornavn = p['fornavn'] if 'studentnr_tildelt' in p: studentnr = p['studentnr_tildelt'] # Get affiliations if dta_type in ('fagperson', ): _process_affiliation( co.affiliation_tilknyttet, co.affiliation_tilknyttet_fagperson, affiliations, _get_sko(p, 'faknr', 'instituttnr', 'gruppenr', 'institusjonsnr')) elif dta_type in ('opptak', ): for row in x: subtype = co.affiliation_status_student_opptak if studieprog2sko[row['studieprogramkode']] in aktiv_sted: subtype = co.affiliation_status_student_aktiv elif row['studierettstatkode'] == 'EVU': subtype = co.affiliation_status_student_evu elif row['studierettstatkode'] == 'FULLFØRT': subtype = co.affiliation_status_student_alumni elif int(row['studienivakode']) >= 900: subtype = co.affiliation_status_student_drgrad elif _is_new_admission(row.get('dato_studierett_tildelt')): subtype = co.affiliation_status_student_ny _process_affiliation(co.affiliation_student, subtype, affiliations, studieprog2sko[row['studieprogramkode']]) elif dta_type in ('emnestud', ): for row in x: subtype = co.affiliation_status_student_emnestud # We may have some situations here where students get # emnestud and aonther affiliation to the same sko, # but this seems to work for now. try: sko = emne2sko[row['emnekode']] except KeyError: logger.warn("Fant ingen emner med koden %s", p['emnekode']) continue if sko in aktiv_sted: subtype = co.affiliation_status_student_aktiv _process_affiliation(co.affiliation_student, subtype, affiliations, sko) elif dta_type in ('privatist_studieprogram', ): _process_affiliation(co.affiliation_student, co.affiliation_status_student_privatist, affiliations, studieprog2sko[p['studieprogramkode']]) elif dta_type in ('privatist_emne', ): try: sko = emne2sko[p['emnekode']] except KeyError: logger.warn("Fant ingen emner med koden %s" % p['emnekode']) continue _process_affiliation(co.affiliation_student, co.affiliation_status_student_privatist, affiliations, sko) elif dta_type in ('perm', ): _process_affiliation(co.affiliation_student, co.affiliation_status_student_aktiv, affiliations, studieprog2sko[p['studieprogramkode']]) elif dta_type in ('tilbud', ): for row in x: _process_affiliation(co.affiliation_student, co.affiliation_status_student_tilbud, affiliations, studieprog2sko[row['studieprogramkode']]) elif dta_type in ('evu', ): _process_affiliation( co.affiliation_student, co.affiliation_status_student_evu, affiliations, _get_sko(p, 'faknr_adm_ansvar', 'instituttnr_adm_ansvar', 'gruppenr_adm_ansvar')) else: logger.debug2("No such affiliation type: %s, skipping", dta_type) if etternavn is None: logger.debug("Ikke noe navn på %s" % fnr) no_name += 1 return # TODO: If the person already exist and has conflicting data from # another source-system, some mechanism is needed to determine the # superior setting. new_person = Factory.get('Person')(db) if fnr in fnr2person_id: new_person.find(fnr2person_id[fnr]) new_person.populate(mx.DateTime.Date(year, mon, day), gender) new_person.affect_names(co.system_fs, co.name_first, co.name_last) new_person.populate_name(co.name_first, fornavn) new_person.populate_name(co.name_last, etternavn) if studentnr is not None: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr, co.externalid_studentnr) new_person.populate_external_id(co.system_fs, co.externalid_studentnr, studentnr) else: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr) new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr) ad_post, ad_post_private, ad_street = _calc_address(person_info) for address_info, ad_const in ((ad_post, co.address_post), (ad_post_private, co.address_post_private), (ad_street, co.address_street)): # TBD: Skal vi slette evt. eksisterende adresse v/None? if address_info is not None: logger.debug("Populating address %s for %s", ad_const, studentnr) new_person.populate_address(co.system_fs, ad_const, **address_info) # if this is a new Person, there is no entity_id assigned to it # until written to the database. try: op = new_person.write_db() except Exception, e: logger.exception("write_db failed for person %s: %s", fnr, e) # Roll back in case of db exceptions: db.rollback() return
def create_sysx_person(db, sxp, update_affs, stats): """ Create or update person with sysx data. :param db: :param sxp: Person dict from ``SYSX`` :param update_affs: A set of current SYSX aff keys :param stats: A defaultdict with counts """ co = Factory.get('Constants')(db) add_sysx_id = fnr_found = False sysx_id = sxp['id'] fnr = sxp['fnr'] fornavn = sxp['fornavn'] etternavn = sxp['etternavn'] fullt_navn = "%s %s" % (fornavn, etternavn) if check_expired_sourcedata(sxp['expire_date']): logger.info("Skipping sysx_id=%r (%s), expire_date=%r", sysx_id, fullt_navn, sxp['expire_date']) stats['skipped'] += 1 return # check if approved. Ditch if not. if not sxp['approved']: logger.error("Skipping sysx_id=%r (%s), not approved", sysx_id, fullt_navn) stats['skipped'] += 1 return my_stedkode = Factory.get('OU')(db) pers_sysx = Factory.get('Person')(db) pers_fnr = Factory.get('Person')(db) try: pers_sysx.find_by_external_id(co.externalid_sys_x_id, sysx_id) except Errors.NotFoundError: add_sysx_id = True if fnr: try: fnr = fodselsnr.personnr_ok(fnr) except fodselsnr.InvalidFnrError: logger.error("Skipping sysx_id=%r (%s), invalid fnr", sysx_id, fullt_navn) stats['skipped'] += 1 return birth_date = datetime.date(*fodselsnr.fodt_dato(fnr)) if fodselsnr.er_kvinne(fnr): gender = co.gender_female else: gender = co.gender_male try: pers_fnr.find_by_external_id(co.externalid_fodselsnr, fnr) except Errors.NotFoundError: pass except Errors.TooManyRowsError as e: # This persons fnr has multiple rows in entity_external_id table # This is an error, person should not have not more than one entry. # Don't know which person object to use, return error message. logger.error( "Skipping sysx_id=%r (%s), matched multiple persons (%s)", sysx_id, fullt_navn, e) stats['skipped'] += 1 return else: if not add_sysx_id and (pers_fnr.entity_id != pers_sysx.entity_id): logger.error( "Skipping sysx_id=%r (%s), matched multiple persons (sysx " "id matched person_id=%r, sysx fnr matched person_id=%r)", sysx_id, fullt_navn, pers_sysx.entity_id, pers_fnr.entity_id) stats['skipped'] += 1 return fnr_found = True else: # foreigner without norwegian ssn, birth_date = sxp['birth_date'] if not birth_date: logger.error("sysx_id=%r (%s) is missing birth date", sysx_id, fullt_navn) if sxp['gender'] == 'M': gender = co.gender_male elif sxp['gender'] == 'F': gender = co.gender_female else: logger.error("Skipping sysx_id=%r (%s), invalid gender %r", sysx_id, fullt_navn, sxp['gender']) stats['skipped'] += 1 return logger.info("Processing sysx_id=%r (%s)", sysx_id, fullt_navn) if fnr_found: person = pers_fnr else: person = pers_sysx # person object located, populate... try: person.populate(mx.DateTime.DateFrom(birth_date), gender) except Errors.CerebrumError: logger.error("Skipping sysx_id=%r (%s), populate() failed", sysx_id, fullt_navn, exc_info=True) stats['skipped'] += 1 return person.affect_names(co.system_x, co.name_first, co.name_last, co.name_full) person.populate_name(co.name_first, fornavn) person.populate_name(co.name_last, etternavn) person.populate_name(co.name_full, fullt_navn) # add external ids if fnr: person.affect_external_id(co.system_x, co.externalid_fodselsnr, co.externalid_sys_x_id) person.populate_external_id(co.system_x, co.externalid_fodselsnr, fnr) logger.debug("Set NO_BIRTHNO for sysx_id=%r (%s)", sysx_id, fullt_navn) else: person.affect_external_id(co.system_x, co.externalid_sys_x_id) person.populate_external_id(co.system_x, co.externalid_sys_x_id, sysx_id) logger.debug("Set sysx_id for sysx_id=%r (%s)", sysx_id, fullt_navn) # setting affiliation and affiliation_status affiliation = co.PersonAffiliation(sxp['affiliation']) affiliation_status = co.PersonAffStatus(affiliation, sxp['affiliation_status']) # Assert that the affs are real int(affiliation), int(affiliation_status) # get ou_id of stedkode used fak = sxp['ou'][0:2] ins = sxp['ou'][2:4] avd = sxp['ou'][4:6] # KEB if fak == '0': logger.warning("sysx_id=%r (%s) has invalid sko=%r", sysx_id, fullt_navn, sxp['ou']) ins = avd = fak try: my_stedkode.find_stedkode(fak, ins, avd, cereconf.DEFAULT_INSTITUSJONSNR) except EntityExpiredError: logger.error("Skipping sysx_id=%r (%s), expired sko=%r", sysx_id, fullt_navn, sxp['ou']) stats['skipped'] += 1 return ou_id = int(my_stedkode.entity_id) # if this is a new Person, there is no entity_id assigned to it # until written to the database. op = person.write_db() # populate the person affiliation table person.populate_affiliation(co.system_x, ou_id, int(affiliation), int(affiliation_status)) # make sure we don't delete this aff when processing deleted affs aff_key = affiliation_key(person.entity_id, ou_id, affiliation) update_affs.discard(aff_key) op2 = person.write_db() logger.debug("OP codes: op=%s,op2=%s" % (op, op2)) if op is None and op2 is None: logger.info("**** EQUAL ****") stats['unchanged'] += 1 elif op is None and op2 is False: logger.info("**** AFF UPDATE ****") stats['updated'] += 1 elif op is True: logger.info("**** NEW ****") stats['added'] += 1 elif op is False: logger.info("**** UPDATE ****") stats['updated'] += 1 return
def import_person(persons, all_nodes): print("import person") global dryrun global cere_list global include_del logger.info( "database:%s" % cereconf.CEREBRUM_DATABASE_CONNECT_DATA['host']) for person in persons: ssn_not_valid = False person_to_be_processed = {} person = person.rstrip() person_list = person.split(";") logger.debug("--- Processing new person ---") try: # print "fnr:'%s'" % person_list[1] person_to_be_processed['noreduorgnin'] = person_list[3] person_to_be_processed['birth_day'] = person_list[0][0:2] person_to_be_processed['birth_month'] = person_list[0][2:4] person_to_be_processed['birth_year'] = person_list[0][4:6] # Make sure year contains 4 digits. if (person_to_be_processed['birth_year'] < 20): birth_year = "20%s" % (person_to_be_processed['birth_year']) else: birth_year = "19%s" % (person_to_be_processed['birth_year']) person_to_be_processed['birth_year'] = birth_year # print "birth_day:'%s'" % person_to_be_processed['birth_day'] # print "birth_month:'%s'" % person_to_be_processed['birth_month'] # print "birth_year:'%s'" % person_to_be_processed['birth_year'] except ValueError: logger.warning( "Empty Birthdate for person named:%s %s. Continue with " + "next person", person_list[1], person_list[2]) continue except IndexError: logger.warning("empty person file?") continue # Check if SSN is valid # print "current fnr:%s" % person_list[0] try: fodselsnr.personnr_ok(person_list[0]) person_to_be_processed['ssn'] = person_list[0] except fodselsnr.InvalidFnrError: logger.warning( "Empty or non-valid ssn %s for person:%s %s from :%s. " + "Continue with next person", person_list[0], person_list[1], person_list[2], person_list[4]) # ssn_not_valid = True # person_to_be_processed['ssn'] = '' continue # set person gender # gender = const.gender_male gender = const.gender_male if not ssn_not_valid: if fodselsnr.er_kvinne(person_to_be_processed['ssn']): gender = const.gender_female else: # Impossible to set gender. Return error message and set gender to # unknown logger.warning( "Impossible to set gender for person:%s %s. Using Unknown", person_list[1], person_list[2]) gender = const.gender_unknown # set gender person_to_be_processed['gender'] = gender # get person firstname person_to_be_processed['firstname'] = person_list[1] # print "firstname:%s" % person_to_be_processed['firstname'] # get person lastname person_to_be_processed['lastname'] = person_list[2] # print "lastname:%s" % person_to_be_processed['lastname'] if (person_to_be_processed['firstname'].isspace() or person_to_be_processed['lastname'].isspace()): # Firstname and/or lastname is made of whitespace ONLY. # generate error message and continue with NEXT person logger.warn( "missing first and/or lastname for person:%s. Person NOT " + "imported", person) continue # set correct encoding person_to_be_processed['firstname'] = decode_text( person_to_be_processed['firstname']) person_to_be_processed['lastname'] = decode_text( person_to_be_processed['lastname']) # # Finished building person_to_be_processed dict. # pp.pprint(person_to_be_processed) # # create person object db_person.clear() try: db_person.find_by_external_id(const.externalid_fodselsnr, person_to_be_processed['ssn']) logger.info("Ssn already in database. update person object") # existing_person = True except Errors.NotFoundError: logger.warning("Unknown ssn:%s, create new person object" % person_to_be_processed['ssn']) # Unable to find person with ssn in the database. pass # # Populate person object # try: db_person.populate( mx.DateTime.Date(int(person_to_be_processed['birth_year']), int(person_to_be_processed['birth_month']), int(person_to_be_processed['birth_day'])), int(person_to_be_processed['gender'])) except Errors.CerebrumError as m: # unable to populate person object. Return error message and # continue with next person logger.error("Person:%s population failed", person_to_be_processed['ssn'], m) continue # affect name and external id db_person.affect_names(const.system_flyt, const.name_first, const.name_last) # populate firstname, lastname and external id db_person.populate_name(const.name_first, person_to_be_processed['firstname']) db_person.populate_name(const.name_last, person_to_be_processed['lastname']) db_person.affect_external_id(const.system_flyt, const.externalid_fodselsnr) db_person.populate_external_id(const.system_flyt, const.externalid_fodselsnr, person_to_be_processed['ssn']) # In case this is a new person, we will need to write to DB before # we can continue. try: op = db_person.write_db() except db.IntegrityError as e: db_person.clear() db.rollback() logger.info("Error:%s - person not imported to BAS", e) continue # op = db_person.write_db() # Determine person affiliation and affiliation_status det_ou, det_affiliation = determine_affiliation(person_list, all_nodes) # logger.debug(" --- from determine affiliation, the following is # calculated ---") # pp.pprint(det_affiliation) for single_ou in det_ou: for single_aff in det_affiliation: new_aff = getattr(const, single_aff) new_aff_stat = getattr(const, det_affiliation[single_aff]) ou.clear() ou.find_stedkode(single_ou[0:2], single_ou[2:4], single_ou[4:6], cereconf.DEFAULT_INSTITUSJONSNR) logger.debug( "setting:: ou_id:%s, aff:%s, aff_stat:%s for person:%s", int(ou.entity_id), int(new_aff), int(new_aff_stat), db_person.entity_id) db_person.populate_affiliation(const.system_flyt, ou.entity_id, new_aff, new_aff_stat) k = "%s:%s:%s" % (db_person.entity_id, int(ou.entity_id), int(new_aff)) if include_del: if k in cere_list: cere_list[k] = False # store mobile for those that has it # contact = determine_contact(db_person) if len(person_list[13]) > 1: person_list[13] logger.debug("has mobile:%s" % person_list[13]) number = person_list[13] c_prefs = {} c_type = int(const.contact_mobile_phone) pref = c_prefs.get(c_type, 0) db_person.populate_contact_info(const.system_flyt, c_type, number, pref) pref = c_prefs[c_type] = pref = 1 else: logger.debug("No mobile registered for this user") op2 = db_person.write_db() if op is None and op2 is None: logger.info("**** EQUAL ****") elif op: logger.info("**** NEW ****") else: logger.info("**** UPDATE (%s:%s) ****" % (op, op2))
def process_person_callback(person_info): """Called when we have fetched all data on a person from the xml file. Updates/inserts name, address and affiliation information.""" global no_name try: fnr = fodselsnr.personnr_ok( "%06d%05d" % (int(person_info['fodselsdato']), int(person_info['personnr']))) fnr = fodselsnr.personnr_ok(fnr) logger.info("Process %s " % (fnr)) (year, mon, day) = fodselsnr.fodt_dato(fnr) if (year < 1970 and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1): # Seems to be a bug in time.mktime on some machines year = 1970 except fodselsnr.InvalidFnrError: logger.warn("Ugyldig fødselsnr: %s" % fnr) return gender = co.gender_male if (fodselsnr.er_kvinne(fnr)): gender = co.gender_female etternavn = fornavn = None studentnr = None affiliations = [] address_info = None aktiv_sted = [] for dta_type in person_info.keys(): x = person_info[dta_type] p = x[0] if isinstance(p, str): continue # Get name if dta_type in ('fagperson', 'evu', 'aktiv'): etternavn = p['etternavn'] fornavn = p['fornavn'] if p.has_key('studentnr_tildelt'): studentnr = '%06d' % int(p['studentnr_tildelt']) if etternavn is None: logger.debug("Ikke noe navn på %s" % fnr) no_name += 1 return new_person = Factory.get('Person')(db) if fnr2person_id.has_key(fnr): new_person.find(fnr2person_id[fnr]) new_person.populate(mx.DateTime.Date(year, mon, day), gender) new_person.affect_names(co.system_fs, co.name_first, co.name_last) new_person.populate_name(co.name_first, fornavn) new_person.populate_name(co.name_last, etternavn) if studentnr is not None: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr, co.externalid_studentnr) new_person.populate_external_id(co.system_fs, co.externalid_studentnr, studentnr) logger.debug("Studentnr is %s", studentnr) else: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr) new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr) ad_post, ad_post_private, ad_street = _calc_address(person_info) for address_info, ad_const in ((ad_post, co.address_post), (ad_post_private, co.address_post_private), (ad_street, co.address_street)): # TBD: Skal vi slette evt. eksisterende adresse v/None? if address_info is not None: logger.debug("Populating address...") new_person.populate_address(co.system_fs, ad_const, **address_info) # if this is a new Person, there is no entity_id assigned to it # until written to the database. op = new_person.write_db() # Iterate over all person_info entries and extract relevant data if person_info.has_key('aktiv'): for row in person_info['aktiv']: try: if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append( int(studieprog2sko[row['studieprogramkode']])) logger.debug("App2akrivts") except KeyError: logger.warn( 'App2akrivts: Person id %d har stud.prog.kode %s som ikke eksisterer.', new_person.entity_id, row['studieprogramkode']) for dta_type in person_info.keys(): # Get affiliations if dta_type in ('fagperson', ): _process_affiliation( co.affiliation_tilknyttet, co.affiliation_status_tilknyttet_fagperson, affiliations, _get_sko(p, 'faknr', 'instituttnr', 'gruppenr', 'institusjonsnr')) elif dta_type in ('aktiv', ): for row in x: # aktiv_sted is necessary in order to avoid different affiliation statuses # to a same 'stedkode' to be overwritten # e.i. if a person has both affiliations status 'evu' and # aktive to a single stedkode we want to register the status 'aktive' # in cerebrum try: if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append( int(studieprog2sko[row['studieprogramkode']])) _process_affiliation( co.affiliation_student, co.affiliation_status_student_aktiv, affiliations, studieprog2sko[row['studieprogramkode']]) except KeyError: logger.warn( 'AKTIV: Person id %d har stud.prog.kode %s som ikke eksisterer.', new_person.entity_id, row['studieprogramkode']) elif dta_type in ('evu', ): subtype = co.affiliation_status_student_evu try: if studieprog2sko[row['studieprogramkode']] in aktiv_sted: subtype = co.affiliation_status_student_aktiv _process_affiliation(co.affiliation_student, subtype, affiliations, studieprog2sko[row['studieprogramkode']]) except KeyError: logger.warn( 'EVU: Person id %d har stud.prog.kode %s som ikke eksisterer.', new_person.entity_id, row['studieprogramkode']) for a in filter_affiliations(affiliations): ou, aff, aff_status = a new_person.populate_affiliation(co.system_fs, ou, aff, aff_status) if include_delete: key_a = "%s:%s:%s" % (new_person.entity_id, ou, int(aff)) if old_aff.has_key(key_a): old_aff[key_a] = False register_contact(new_person, person_info) op2 = new_person.write_db() if op is None and op2 is None: logger.info("**** EQUAL ****") elif op == True: logger.info("**** NEW ****") else: logger.info("**** UPDATE ****") # Reservations if gen_groups: should_add = False for dta_type in person_info.keys(): p = person_info[dta_type][0] if isinstance(p, str): continue # Presence of 'fagperson' elements for a person should not # affect that person's reservation status. if dta_type in ('fagperson', ): continue # We only fetch the column in these queries if dta_type not in ('evu'): continue # If 'status_reserv_nettpubl' == "N": add to group if p.get('status_reserv_nettpubl', "") == "N": should_add = True else: should_add = False if should_add: # The student has explicitly given us permission to be # published in the directory. _add_res(new_person.entity_id) else: # The student either hasn't registered an answer to # the "Can we publish info about you in the directory" # question at all, or has given an explicit "I don't # want to appear in the directory" answer. _rem_res(new_person.entity_id) db.commit()
def process_person_callback(person_info): """Called when we have fetched all data on a person from the xml file. Updates/inserts name, address and affiliation information.""" global no_name try: fnr = fodselsnr.personnr_ok("%06d%05d" % (int(person_info['fodselsdato']), int(person_info['personnr']))) fnr = fodselsnr.personnr_ok(fnr) logger.info("Process %s " % (fnr)) (year, mon, day) = fodselsnr.fodt_dato(fnr) if (year < 1970 and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1): # Seems to be a bug in time.mktime on some machines year = 1970 except fodselsnr.InvalidFnrError: logger.warn("Ugyldig fødselsnr: %s" % fnr) return gender = co.gender_male if(fodselsnr.er_kvinne(fnr)): gender = co.gender_female etternavn = fornavn = None studentnr = None affiliations = [] address_info = None aktiv_sted = [] # Iterate over all person_info entries and extract relevant data if person_info.has_key('aktiv'): for row in person_info['aktiv']: if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append(int(studieprog2sko[row['studieprogramkode']])) logger.debug("App2akrivts") for dta_type in person_info.keys(): x = person_info[dta_type] p = x[0] if isinstance(p, str): continue # Get name if dta_type in ('fagperson', 'evu', 'aktiv'): etternavn = p['etternavn'] fornavn = p['fornavn'] if p.has_key('studentnr_tildelt'): studentnr = p['studentnr_tildelt'] # Get affiliations if dta_type in ('fagperson',): _process_affiliation(co.affiliation_tilknyttet, co.affiliation_status_tilknyttet_fagperson, affiliations, _get_sko(p, 'faknr', 'instituttnr', 'gruppenr', 'institusjonsnr')) elif dta_type in ('aktiv', ): for row in x: # aktiv_sted is necessary in order to avoid different affiliation statuses # to a same 'stedkode' to be overwritten # e.i. if a person has both affiliations status 'evu' and # aktive to a single stedkode we want to register the status 'aktive' # in cerebrum if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append(int(studieprog2sko[row['studieprogramkode']])) _process_affiliation(co.affiliation_student, co.affiliation_status_student_aktiv, affiliations, studieprog2sko[row['studieprogramkode']]) elif dta_type in ('evu',): subtype = co.affiliation_status_student_evu if studieprog2sko[row['studieprogramkode']] in aktiv_sted: subtype = co.affiliation_status_student_aktiv _process_affiliation(co.affiliation_student, subtype, affiliations, studieprog2sko[row['studieprogramkode']]) if etternavn is None: logger.debug("Ikke noe navn på %s" % fnr) no_name += 1 return # TODO: If the person already exist and has conflicting data from # another source-system, some mechanism is needed to determine the # superior setting. new_person = Factory.get('Person')(db) if fnr2person_id.has_key(fnr): new_person.find(fnr2person_id[fnr]) new_person.populate(mx.DateTime.Date(year, mon, day), gender) new_person.affect_names(co.system_fs, co.name_first, co.name_last) new_person.populate_name(co.name_first, fornavn) new_person.populate_name(co.name_last, etternavn) if studentnr is not None: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr, co.externalid_studentnr) new_person.populate_external_id(co.system_fs, co.externalid_studentnr, studentnr) else: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr) new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr) ad_post, ad_post_private, ad_street = _calc_address(person_info) for address_info, ad_const in ((ad_post, co.address_post), (ad_post_private, co.address_post_private), (ad_street, co.address_street)): # TBD: Skal vi slette evt. eksisterende adresse v/None? if address_info is not None: logger.debug("Populating address...") new_person.populate_address(co.system_fs, ad_const, **address_info) # if this is a new Person, there is no entity_id assigned to it # until written to the database. try: op = new_person.write_db() except Exception, e: logger.exception("write_db failed for person %s: %s", fnr, e) # Roll back in case of db exceptions: db.rollback() return
def process_person_callback(person_info): """Called when we have fetched all data on a person from the xml file. Updates/inserts name, address and affiliation information.""" global no_name try: fnr = fodselsnr.personnr_ok( "%06d%05d" % (int(person_info['fodselsdato']), int(person_info['personnr']))) fnr = fodselsnr.personnr_ok(fnr) logger.info("Process %s " % (fnr)) (year, mon, day) = fodselsnr.fodt_dato(fnr) if (year < 1970 and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1): # Seems to be a bug in time.mktime on some machines year = 1970 except fodselsnr.InvalidFnrError: logger.warn("Ugyldig fødselsnr: %s" % fnr) return gender = co.gender_male if (fodselsnr.er_kvinne(fnr)): gender = co.gender_female etternavn = fornavn = None studentnr = None affiliations = [] address_info = None aktiv_sted = [] # Iterate over all person_info entries and extract relevant data if person_info.has_key('aktiv'): for row in person_info['aktiv']: if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append(int( studieprog2sko[row['studieprogramkode']])) logger.debug("App2akrivts") for dta_type in person_info.keys(): x = person_info[dta_type] p = x[0] if isinstance(p, str): continue # Get name if dta_type in ('fagperson', 'evu', 'aktiv'): etternavn = p['etternavn'] fornavn = p['fornavn'] if p.has_key('studentnr_tildelt'): studentnr = p['studentnr_tildelt'] # Get affiliations if dta_type in ('fagperson', ): _process_affiliation( co.affiliation_tilknyttet, co.affiliation_status_tilknyttet_fagperson, affiliations, _get_sko(p, 'faknr', 'instituttnr', 'gruppenr', 'institusjonsnr')) elif dta_type in ('aktiv', ): for row in x: # aktiv_sted is necessary in order to avoid different affiliation statuses # to a same 'stedkode' to be overwritten # e.i. if a person has both affiliations status 'evu' and # aktive to a single stedkode we want to register the status 'aktive' # in cerebrum if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append( int(studieprog2sko[row['studieprogramkode']])) _process_affiliation( co.affiliation_student, co.affiliation_status_student_aktiv, affiliations, studieprog2sko[row['studieprogramkode']]) elif dta_type in ('evu', ): subtype = co.affiliation_status_student_evu if studieprog2sko[row['studieprogramkode']] in aktiv_sted: subtype = co.affiliation_status_student_aktiv _process_affiliation(co.affiliation_student, subtype, affiliations, studieprog2sko[row['studieprogramkode']]) if etternavn is None: logger.debug("Ikke noe navn på %s" % fnr) no_name += 1 return # TODO: If the person already exist and has conflicting data from # another source-system, some mechanism is needed to determine the # superior setting. new_person = Factory.get('Person')(db) if fnr2person_id.has_key(fnr): new_person.find(fnr2person_id[fnr]) new_person.populate(mx.DateTime.Date(year, mon, day), gender) new_person.affect_names(co.system_fs, co.name_first, co.name_last) new_person.populate_name(co.name_first, fornavn) new_person.populate_name(co.name_last, etternavn) if studentnr is not None: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr, co.externalid_studentnr) new_person.populate_external_id(co.system_fs, co.externalid_studentnr, studentnr) else: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr) new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr) ad_post, ad_post_private, ad_street = _calc_address(person_info) for address_info, ad_const in ((ad_post, co.address_post), (ad_post_private, co.address_post_private), (ad_street, co.address_street)): # TBD: Skal vi slette evt. eksisterende adresse v/None? if address_info is not None: logger.debug("Populating address...") new_person.populate_address(co.system_fs, ad_const, **address_info) # if this is a new Person, there is no entity_id assigned to it # until written to the database. try: op = new_person.write_db() except Exception, e: logger.exception("write_db failed for person %s: %s", fnr, e) # Roll back in case of db exceptions: db.rollback() return
def process_person_callback(person_info): """Called when we have fetched all data on a person from the xml file. Updates/inserts name, address and affiliation information.""" global no_name try: fnr = fodselsnr.personnr_ok("%06d%05d" % ( int(person_info['fodselsdato']), int(person_info['personnr']))) fnr = fodselsnr.personnr_ok(fnr) logger.info("Processing %s", fnr) (year, mon, day) = fodselsnr.fodt_dato(fnr) if (year < 1970 and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1): # Seems to be a bug in time.mktime on some machines year = 1970 except fodselsnr.InvalidFnrError: logger.warn(u"Ugyldig f�dselsnr for: %s", person_info['fodselsdato']) return gender = co.gender_male if(fodselsnr.er_kvinne(fnr)): gender = co.gender_female etternavn = fornavn = None studentnr = None affiliations = [] address_info = None aktiv_sted = [] # Iterate over all person_info entries and extract relevant data for dta_type in person_info.keys(): x = person_info[dta_type] p = x[0] if isinstance(p, str): continue if dta_type not in ('tilbud', 'eksamen', 'evu'): if 'studentnr_tildelt' in p: studentnr = p['studentnr_tildelt'] else: logger.info("\n%s mangler studentnr!", fnr) # Get name if dta_type in ('aktiv', 'tilbud', 'evu', 'privatist_studieprogram', ): etternavn = p['etternavn'] fornavn = p['fornavn'] # Get address if address_info is None: if dta_type in ('aktiv', 'privatist_studieprogram', ): address_info = _ext_address_info( p, 'adrlin1_semadr', 'adrlin2_semadr', 'adrlin3_semadr', 'postnr_semadr', 'adresseland_semadr') if address_info is None: address_info = _ext_address_info( p, 'adrlin1_hjemsted', 'adrlin2_hjemsted', 'adrlin3_hjemsted', 'postnr_hjemsted', 'adresseland_hjemsted') elif dta_type in ('evu',): address_info = _ext_address_info( p, 'adrlin1_hjem', 'adrlin2_hjem', 'adrlin3_hjem', 'postnr_hjem', 'adresseland_hjem') if address_info is None: address_info = _ext_address_info( p, 'adrlin1_hjemsted', 'adrlin2_hjemsted', 'adrlin3_hjemsted', 'postnr_hjemsted', 'adresseland_hjemsted') elif dta_type in ('tilbud',): address_info = _ext_address_info( p, 'adrlin1_kontakt', 'adrlin2_kontakt', 'adrlin3_kontakt', 'postnr_kontakt', 'adresseland_kontakt') # Get affiliations # Lots of changes here compared to import_FS.py @ uio # TODO: split import_FS into a common part and organization spesific # parts if dta_type in ('aktiv', ): for row in x: # aktiv_sted is necessary in order to avoid different # affiliation statuses to a same 'stedkode' to be overwritten # e.i. if a person has both affiliations status 'tilbud' and # aktive to a single stedkode we want to register the status # 'aktive' in cerebrum if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append( int(studieprog2sko[row['studieprogramkode']])) _process_affiliation( co.affiliation_student, co.affiliation_status_student_aktiv, affiliations, studieprog2sko[row['studieprogramkode']]) elif dta_type in ('evu',): for row in x: _process_affiliation( co.affiliation_student, co.affiliation_status_student_evu, affiliations, _get_sko(p, 'faknr_adm_ansvar', 'instituttnr_adm_ansvar', 'gruppenr_adm_ansvar')) elif dta_type in ('privatist_studieprogram', ): for row in x: _process_affiliation( co.affiliation_student, co.affiliation_status_student_privatist, affiliations, studieprog2sko[row['studieprogramkode']]) elif dta_type in ('tilbud', ): for row in x: subtype = co.affiliation_status_student_tilbud if studieprog2sko[row['studieprogramkode']] in aktiv_sted: subtype = co.affiliation_status_student_aktiv _process_affiliation(co.affiliation_student, subtype, affiliations, studieprog2sko[row['studieprogramkode']]) if etternavn is None: logger.info("Ikke noe navn p� %s", fnr) no_name += 1 return # TODO: If the person already exist and has conflicting data from # another source-system, some mechanism is needed to determine the # superior setting. fsids = [(co.externalid_fodselsnr, fnr)] if studentnr is not None: fsids.append((co.externalid_studentnr, studentnr)) new_person = Factory.get('Person')(db) try: new_person.find_by_external_ids(*fsids) except Errors.NotFoundError: pass except Errors.TooManyRowsError, e: logger.error("Trying to find studentnr %s, getting several persons: %s", studentnr, e) return
def import_person(db, person, update_affs=None): """ Import a single person. :param person: A dict from py:func:`parse_person` :param update_affs: A set of affiliations from py:func:`load_sito_affiliations`. Affiliations imported by this function will be removed from the set. """ new_person = Factory.get('Person')(db) const = Factory.get('Constants')(db) if update_affs is None: update_affs = set() employee_id = person['employee_id'] logger.info("Processing employee_id=%r", employee_id) # # Validate person data # # Birthdate try: birthdate = parse_date(person['birthdate']) valid_birthdate = True except ValueError: logger.warning('Invalid birth date for employee_id=%r (%r)', person['employee_id'], person['birthdate']) valid_birthdate = False birthdate = None # SSN try: fodselsnr.personnr_ok(person['ssn']) valid_ssn = True except fodselsnr.InvalidFnrError: logger.warning("Empty SSN for employee_id=%r", person['employee_id']) valid_ssn = False # set person gender (checking both SSN and gender from sito input file) if valid_ssn: if fodselsnr.er_kvinne(person['ssn']): gender = const.gender_female else: gender = const.gender_male elif person['gender'] == 'Female': gender = const.gender_female elif person['gender'] == 'Male': gender = const.gender_male else: logger.warning('Unknown gender value for employee_id=%r (%r)', employee_id, person['gender']) gender = const.gender_unknown # Validate Birthdate against SSN if valid_ssn and valid_birthdate: ssndate = datetime.date(*fodselsnr.fodt_dato(person['ssn'])) if birthdate != ssndate: raise SkipPerson('inconsistent birth date and ssn (%r, %r)', birthdate, ssndate) elif valid_ssn and not valid_birthdate: logger.warning('Missing birth date for employee_id=%r, using date' ' from ssn', person['employee_id']) birthdate = datetime.date(*fodselsnr.fodt_dato(person['ssn'])) elif not valid_ssn and valid_birthdate: # person have birthdate but NOT ssn. Nothing to do here pass elif not valid_ssn and not valid_birthdate: # person does not have birthdate nor ssn. This person cannot be # built. SSN or Birthdate required. Return error message and # continue with NEXT person raise SkipPerson('missing ssn and birth date') # Check names if not person['first_name']: raise SkipPerson('Missing first name') if not person['last_name']: raise SkipPerson('Missing last name') if person['middle_name']: fname = person['first_name'] + ' ' + person['middle_name'] else: fname = person['first_name'] lname = person['last_name'] # # Get person object som DB if it exists # found = False new_person.clear() try: new_person.find_by_external_id(const.externalid_sito_ansattnr, employee_id) found = True except Errors.NotFoundError: # could not find person in DB based on ansattnr. if valid_ssn: # try to find person using ssn if ssn is valid try: new_person.clear() new_person.find_by_external_id(const.externalid_fodselsnr, person['ssn']) found = True except Errors.NotFoundError: pass if found: logger.info('Updating person object for employee_id=%r', employee_id) else: logger.info('Creating person object for employee_id=%r', employee_id) # # Populate the person object # new_person.populate(mx.DateTime.DateFrom(birthdate), gender) new_person.affect_names(const.system_sito, const.name_first, const.name_last, const.name_work_title) new_person.affect_external_id(const.system_sito, const.externalid_fodselsnr, const.externalid_sito_ansattnr) new_person.populate_name(const.name_first, fname) new_person.populate_name(const.name_last, lname) if person['title']: new_person.populate_name(const.name_work_title, person['title']) if valid_ssn: new_person.populate_external_id(const.system_sito, const.externalid_fodselsnr, person['ssn']) new_person.populate_external_id(const.system_sito, const.externalid_sito_ansattnr, employee_id) # intermediary write to get an entity_id if this is a new person. new_person.write_db() new_person.populate_affiliation(const.system_sito) new_person.populate_contact_info(const.system_sito) # set person affiliation for key, ou_id, aff, status in determine_affiliations( db, new_person.entity_id, person): logger.info("affiliation for employee_id=%r to ou_id=%r", employee_id, ou_id) new_person.populate_affiliation(const.system_sito, ou_id, int(aff), int(status)) # set this persons affiliation entry to False # this ensures that this persons affiliations will not be removed # when the clean_affiliation function is called after import person update_affs.discard(key) # get person work, cellular and home phone numbers c_prefs = {} for con, number in person['phone'].items(): c_type = int(const.human2constant(con, const.ContactInfo)) if c_type in c_prefs: pref = c_prefs[c_type] else: pref = 0 c_prefs[c_type] = 1 new_person.populate_contact_info(const.system_sito, c_type, number, pref) logger.debug("contact for employee_id=%r, system=%s, " "c_type=%s, number=%s, pref=%s", employee_id, const.system_sito, c_type, number, pref) new_person.write_db() return not found
def process_person_callback(person_info): """Called when we have fetched all data on a person from the xml file. Updates/inserts name, address and affiliation information.""" global no_name try: fnr = fodselsnr.personnr_ok( "%06d%05d" % (int(person_info['fodselsdato']), int(person_info['personnr']))) fnr = fodselsnr.personnr_ok(fnr) logger.info("Process %s " % (fnr)) (year, mon, day) = fodselsnr.fodt_dato(fnr) if (year < 1970 and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1): # Seems to be a bug in time.mktime on some machines year = 1970 except fodselsnr.InvalidFnrError: logger.warn("Ugyldig fødselsnr: %s" % fnr) return gender = co.gender_male if(fodselsnr.er_kvinne(fnr)): gender = co.gender_female etternavn = fornavn = None studentnr = None affiliations = [] address_info = None aktiv_sted = [] aktivemne_sted = [] # Iterate over all person_info entries and extract relevant data if 'aktiv' in person_info: for row in person_info['aktiv']: if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append( int(studieprog2sko[row['studieprogramkode']])) logger.debug("App2akrivts") if 'emnestud' in person_info: for row in person_info['emnestud']: if emne2sko[row['emnekode']] is not None: aktivemne_sted.append(int(emne2sko[row['emnekode']])) logger.debug('Add sko %s based on emne %s', int(emne2sko[row['emnekode']]), row['emnekode']) for dta_type in person_info.keys(): x = person_info[dta_type] p = x[0] if isinstance(p, str): continue # Get name if dta_type in ( 'fagperson', 'opptak', 'tilbud', 'evu', 'privatist_emne', 'privatist_studieprogram', 'alumni', 'emnestud', ): etternavn = p['etternavn'] fornavn = p['fornavn'] if 'studentnr_tildelt' in p: studentnr = p['studentnr_tildelt'] # Get affiliations if dta_type in ('fagperson',): _process_affiliation(co.affiliation_tilknyttet, co.affiliation_tilknyttet_fagperson, affiliations, _get_sko(p, 'faknr', 'instituttnr', 'gruppenr', 'institusjonsnr')) elif dta_type in ('opptak', ): for row in x: subtype = co.affiliation_status_student_opptak if studieprog2sko[row['studieprogramkode']] in aktiv_sted: subtype = co.affiliation_status_student_aktiv elif row['studierettstatkode'] == 'EVU': subtype = co.affiliation_status_student_evu elif row['studierettstatkode'] == 'FULLFØRT': subtype = co.affiliation_status_student_alumni elif int(row['studienivakode']) >= 900: subtype = co.affiliation_status_student_drgrad elif _is_new_admission(row.get('dato_studierett_tildelt')): subtype = co.affiliation_status_student_ny _process_affiliation(co.affiliation_student, subtype, affiliations, studieprog2sko[row['studieprogramkode']]) elif dta_type in ('emnestud',): for row in x: subtype = co.affiliation_status_student_emnestud # We may have some situations here where students get # emnestud and aonther affiliation to the same sko, # but this seems to work for now. try: sko = emne2sko[row['emnekode']] except KeyError: logger.warn("Fant ingen emner med koden %s", p['emnekode']) continue if sko in aktiv_sted: subtype = co.affiliation_status_student_aktiv _process_affiliation(co.affiliation_student, subtype, affiliations, sko) elif dta_type in ('privatist_studieprogram',): _process_affiliation(co.affiliation_student, co.affiliation_status_student_privatist, affiliations, studieprog2sko[p['studieprogramkode']]) elif dta_type in ('privatist_emne',): try: sko = emne2sko[p['emnekode']] except KeyError: logger.warn("Fant ingen emner med koden %s" % p['emnekode']) continue _process_affiliation(co.affiliation_student, co.affiliation_status_student_privatist, affiliations, sko) elif dta_type in ('perm',): _process_affiliation(co.affiliation_student, co.affiliation_status_student_aktiv, affiliations, studieprog2sko[p['studieprogramkode']]) elif dta_type in ('tilbud',): for row in x: _process_affiliation(co.affiliation_student, co.affiliation_status_student_tilbud, affiliations, studieprog2sko[row['studieprogramkode']]) elif dta_type in ('evu', ): _process_affiliation(co.affiliation_student, co.affiliation_status_student_evu, affiliations, _get_sko(p, 'faknr_adm_ansvar', 'instituttnr_adm_ansvar', 'gruppenr_adm_ansvar')) else: logger.debug2("No such affiliation type: %s, skipping", dta_type) if etternavn is None: logger.debug("Ikke noe navn på %s" % fnr) no_name += 1 return # TODO: If the person already exist and has conflicting data from # another source-system, some mechanism is needed to determine the # superior setting. new_person = Factory.get('Person')(db) if fnr in fnr2person_id: new_person.find(fnr2person_id[fnr]) new_person.populate(mx.DateTime.Date(year, mon, day), gender) new_person.affect_names(co.system_fs, co.name_first, co.name_last) new_person.populate_name(co.name_first, fornavn) new_person.populate_name(co.name_last, etternavn) if studentnr is not None: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr, co.externalid_studentnr) new_person.populate_external_id(co.system_fs, co.externalid_studentnr, studentnr) else: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr) new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr) ad_post, ad_post_private, ad_street = _calc_address(person_info) for address_info, ad_const in ((ad_post, co.address_post), (ad_post_private, co.address_post_private), (ad_street, co.address_street)): # TBD: Skal vi slette evt. eksisterende adresse v/None? if address_info is not None: logger.debug("Populating address %s for %s", ad_const, studentnr) new_person.populate_address(co.system_fs, ad_const, **address_info) # if this is a new Person, there is no entity_id assigned to it # until written to the database. try: op = new_person.write_db() except Exception, e: logger.exception("write_db failed for person %s: %s", fnr, e) # Roll back in case of db exceptions: db.rollback() return
def _process_person(self, person): db = self.db const = self.const paga_nr = int(person['ansattnr']) logger.info("Processing paga_id=%r", paga_nr) try: birthdate = parse_date(person['fodselsdato']) except ValueError as e: raise SkipPerson("Invalid birth date (%s)" % (e, )) if person['kjonn'] == 'M': gender = const.gender_male else: gender = const.gender_female fnr = person['fnr'] if fnr and fnr[6:11] != '00000': try: fodselsnr.personnr_ok(fnr) except Exception as e: raise SkipPerson("Invalid fnr (%s)" % (e, )) gender_chk = const.gender_male if fodselsnr.er_kvinne(fnr): gender_chk = const.gender_female if gender_chk != gender: raise SkipPerson("Inconsistent gender (gender=%r, ssn=%r)" % (gender, gender_chk)) fnr_date = datetime.date(*(fodselsnr.fodt_dato(fnr))) if not cmp_birthdates(birthdate, fnr_date): raise SkipPerson("Inconsistent birth date (date=%r, ssn=%r)" % (birthdate, fnr_date)) # Passport data national_id_type = person['edag_id_type'] origin_country = person['country'] if not origin_country: origin_country = 'NO' if person['edag_id_nr']: # generate external id on the form: # <2-char national id>-<national_id> national_id_val = '%s-%s' % (origin_country, person['edag_id_nr']) else: national_id_val = None # Abort early if there are no valid identifiers from the source system: if not any(( fnr and fnr[6:11] != '00000', national_id_type == 'passnummer' and national_id_val, )): # TODO: Wouldn't it be enough with ansattnr? raise SkipPerson("No valid identifier (fnr, passnummer)") new_person = Factory.get('Person')(db) identifiers = [(const.externalid_paga_ansattnr, paga_nr), (const.externalid_fodselsnr, fnr)] if national_id_type == 'passnummer' and national_id_val: identifiers.append((const.externalid_pass_number, national_id_val)) for id_type, id_value in identifiers: if _populate_existing(new_person, id_type, id_value): break if not person.get('fornavn', '').strip(): raise SkipPerson("Missing first name") if not person.get('etternavn', '').strip(): raise SkipPerson("Missing last name") new_person.populate(mx.DateTime.DateFrom(birthdate), gender) new_person.affect_names(const.system_paga, const.name_first, const.name_last) new_person.affect_external_id(const.system_paga, const.externalid_fodselsnr, const.externalid_paga_ansattnr, const.externalid_pass_number) new_person.populate_name(const.name_first, person['fornavn']) new_person.populate_name(const.name_last, person['etternavn']) if fnr and fnr[6:11] != '00000': # do not import external id where external_id type is fnr and # fnr[6-11] =='00000' new_person.populate_external_id(const.system_paga, const.externalid_fodselsnr, fnr) if national_id_type == 'passnummer' and national_id_val: new_person.populate_external_id(const.system_paga, const.externalid_pass_number, national_id_val) new_person.populate_external_id(const.system_paga, const.externalid_paga_ansattnr, paga_nr) # If it's a new person, we need to call write_db() to have an entity_id # assigned to it. op = new_person.write_db() if person.get('tittel_personlig'): new_person.add_name_with_language( name_variant=const.personal_title, name_language=const.language_nb, name=person['tittel_personlig']) # work_title is set by _determine_affiliations affiliations = _determine_affiliations(self.db, self.const, new_person, person) new_person.populate_affiliation(const.system_paga) if 'fakultetnr_for_lonnsslip' in person: sted = get_sted(db, person['fakultetnr_for_lonnsslip'], person['instituttnr_for_lonnsslip'], person['gruppenr_for_lonnsslip']) if sted is not None: if sted['addr_street'] is not None: new_person.populate_address(const.system_paga, type=const.address_street, **sted['addr_street']) if sted['addr_post'] is not None: new_person.populate_address(const.system_paga, type=const.address_post, **sted['addr_post']) if 'lokasjon' in person: logger.debug( 'Populating paga_id=%r location address with ' 'source=%s, type=%s, text=%r', paga_nr, const.system_paga, const.address_location, person['lokasjon']) new_person.populate_address(source_system=const.system_paga, type=const.address_location, address_text=person['lokasjon']) else: logger.warning("No location address for paga_id=%r", paga_nr) for k, v in affiliations.items(): ou_id, aff, aff_stat = v new_person.populate_affiliation(const.system_paga, ou_id, int(aff), int(aff_stat)) self.old_affs.discard(k) c_prefs = {} new_person.populate_contact_info(const.system_paga) for c_type, value in determine_contact(const, person): c_type = int(c_type) pref = c_prefs.get(c_type, 0) new_person.populate_contact_info(const.system_paga, c_type, value, pref) c_prefs[c_type] = pref + 1 # # Also add personal/home street address if it exists in the import file # priv_addr = (person.get('adresse'), person.get('postnr'), person.get('poststed')) if any(priv_addr): logger.debug("Setting additional home address: %s %s %s", priv_addr[0], priv_addr[1], priv_addr[2]) # TODO: Address country? Should probably use const.human2constant() # country_code = new_person.get_country_code(origin_country) # TODO: Is there really a guaranteed connection between passport # origin country and home address? new_person.populate_address(const.system_paga, const.address_post_private, priv_addr[0], None, priv_addr[1], priv_addr[2], None) op2 = new_person.write_db() set_person_spreads(self.db, self.const, new_person, person) if op is None and op2 is None: logger.info("EQUAL: No change to person with paga_id=%r", paga_nr) elif op: logger.info("NEW: Created new person with paga_id=%r", paga_nr) else: logger.info("UPDATE: Updated person with paga_id=%r (%s, %s)", paga_nr, op, op2)
def _get_gender(self, fnr): gender = self.co.gender_male if fodselsnr.er_kvinne(fnr): gender = self.co.gender_female return gender