def p_create(fnr, lname, fname): gender = constants.gender_female try: fodselsnr.personnr_ok(fnr) except fodselsnr.InvalidFnrError: logger.error("Cannot create person, bad no_ssn |%s|", fnr) return None if not (lname and fname): logger.error("Cannot create person %s, missing name") return None if fodselsnr.er_mann(fnr): gender = constants.gender_female # this is very wrong, but as we do not have any birth days # registered we need to try producing them. year, mon, day = fodselsnr.fodt_dato(fnr) # in order to be a guardian you need to be over 18 years old which # excludes anyone born in after 1990 so this will work for now :-) birth_date = db.Date(year, mon, day) person.clear() # Create person person.populate(birth_date, gender) person.affect_external_id(constants.system_manual, constants.externalid_fodselsnr) person.populate_external_id(constants.system_manual, constants.externalid_fodselsnr, fnr) person.write_db() logger.info("Created new person with fnr %s", fnr) # Add person_name person.affect_names(constants.system_manual, constants.name_first, constants.name_last) person.populate_name(constants.name_first, fname) person.populate_name(constants.name_last, lname) person.write_db() logger.debug("Added name %s %s to fnr %s", fname, lname, fnr) return person.entity_id
def process_line(infile): """ Scan all lines in INFILE and create corresponding account/e-mail entries in Cerebrum. """ stream = open(infile, 'r') # Iterate over all persons: for line in stream: logger.debug5("Processing line: |%s|", line) fields = string.split(line.strip(), ";") print len(fields) if len(fields) != 5: logger.error("Bad line: %s. Skipping" % line) continue # fi fnr, pname, uname, email, foo = fields if not fnr == "": logger.debug("Processing person %s", fnr) try: fodselsnr.personnr_ok(fnr) except fodselsnr.InvalidFnrError: logger.warn("Bad no_ssn %s, skipping", fnr) continue person.clear() gender = constants.gender_male if fodselsnr.er_kvinne(fnr): gender = constants.gender_female y, m, d = fodselsnr.fodt_dato(fnr) # Can't just populate like this, we need to search for persons # first. try: person.find_by_external_id(constants.externalid_fodselsnr, fnr) except Errors.NotFoundError: pass person.populate(db.Date(y, m, d), gender) person.affect_external_id(constants.system_ekstens, constants.externalid_fodselsnr) person.populate_external_id(constants.system_ekstens, constants.externalid_fodselsnr, fnr) person.write_db() update_names(fnr, pname) person.write_db() logger.debug("Created new person with fnr %s", fnr) p_id = person.entity_id account_id = process_user(p_id, uname) process_mail(account_id, 'defaultmail', email)
def process_person(fnr): """ Find (or create, if necessary) and return the person_id corresponding to FNR. """ logger.debug("Processing person %s", fnr) if not fodselsnr.personnr_ok(fnr): logger.warn("Bad no_ssn |%s|", fnr) return None # fi if fnr2person_id.has_key(fnr): logger.debug("Person with fnr %s exists in Cerebrum", fnr) return fnr2person_id[fnr] # fi # ... otherwise, create a new person person.clear() gender = constants.gender_male if fodselsnr.er_kvinne(fnr): gender = constants.gender_female # fi year, mon, day = fodselsnr.fodt_dato(fnr) person.populate(db.Date(year, mon, day), gender) person.affect_external_id(constants.system_migrate, constants.externalid_fodselsnr) person.populate_external_id(constants.system_migrate, constants.externalid_fodselsnr, fnr) person.write_db() logger.debug("Created new person with fnr %s", fnr) e_id = person.entity_id fnr2person_id[fnr] = e_id return e_id
def process_person(fnr): """ Find (or create, if necessary) and return the person_id corresponding to FNR. """ # If person already exists, return entity_id if fnr2person_id.has_key(fnr): return fnr2person_id[fnr] try: person.clear() person.find_by_external_id(constants.externalid_fodselsnr, fnr) e_id = person.entity_id fnr2person_id[fnr] = e_id return e_id except Errors.NotFoundError: pass # ... otherwise, create a new person person.clear() gender = constants.gender_male if fodselsnr.er_kvinne(fnr): gender = constants.gender_female year, mon, day = fodselsnr.fodt_dato(fnr) person.populate(db.Date(year, mon, day), gender) person.affect_external_id(constants.system_migrate, constants.externalid_fodselsnr) person.populate_external_id(constants.system_migrate, constants.externalid_fodselsnr, fnr) person.write_db() logger.debug("Created new person with fnr %s", fnr) e_id = person.entity_id fnr2person_id[fnr] = e_id return e_id
def process_person(fnr, lname, fname, bewid, set_names): """ Find or create a person; return the person_id corresponding to fnr. Set name for new persons if set_name is True. """ logger.debug("Processing person %s %s (%s)", fname, lname, fnr) try: fodselsnr.personnr_ok(fnr) except fodselsnr.InvalidFnrError: logger.warn("Bad no_ssn |%s|", fnr) return None if fnr2person_id.has_key(fnr): logger.debug("Person with fnr %s exists in Cerebrum", fnr) return fnr2person_id[fnr] # ... otherwise, create a new person person.clear() gender = constants.gender_male if fodselsnr.er_kvinne(fnr): gender = constants.gender_female year, mon, day = fodselsnr.fodt_dato(fnr) person.populate(db.Date(year, mon, day), gender) if bewid: person.affect_external_id(constants.system_migrate, constants.externalid_fodselsnr, constants.externalid_bewatorid) else: person.affect_external_id(constants.system_migrate, constants.externalid_fodselsnr) person.populate_external_id(constants.system_migrate, constants.externalid_fodselsnr, fnr) person.write_db() e_id = person.entity_id logger.info("Created new person with id %s and fnr %s", e_id, fnr) if bewid: person.populate_external_id(constants.system_migrate, constants.externalid_bewatorid, bewid) person.write_db() logger.info("Added BewatorID %s for %s", bewid, fnr) if set_names: if lname and fname: person.affect_names(constants.system_migrate, constants.name_first, constants.name_last) person.populate_name(constants.name_first, fname) person.populate_name(constants.name_last, lname) logger.info("Name %s %s set for person %s", fname, lname, fnr) person.write_db() else: logger.warn("Couldn't set name %s %s for person %s", fname, lname, fnr) fnr2person_id[fnr] = e_id return e_id
def process_person(fnr, card_id): """ Find or create a person; return the person_id corresponding to fnr. """ logger.debug("Processing person %s", fnr) try: fodselsnr.personnr_ok(fnr) except fodselsnr.InvalidFnrError: logger.warn("Bad no_ssn |%s|", fnr) return None if fnr2person_id.has_key(fnr): logger.debug("Person with fnr %s exists in Cerebrum", fnr) return fnr2person_id[fnr] # ... otherwise, create a new person person.clear() gender = constants.gender_male if fodselsnr.er_kvinne(fnr): gender = constants.gender_female year, mon, day = fodselsnr.fodt_dato(fnr) person.populate(db.Date(year, mon, day), gender) person.affect_external_id(constants.system_migrate, constants.externalid_fodselsnr, constants.externalid_bewatorid) person.populate_external_id(constants.system_migrate, constants.externalid_fodselsnr, fnr) if card_id is not None and card_id != '': person.populate_external_id(constants.system_migrate, constants.externalid_bewatorid, card_id) else: logger.debug("No Bewator-ID found for person %s" % fnr) person.write_db() logger.debug("Created new person with fnr %s and card-ID %s" % (fnr, card_id)) e_id = person.entity_id fnr2person_id[fnr] = e_id return e_id
def process_person_callback(person_info): """Called when we have fetched all data on a person from the xml file. Updates/inserts name, address and affiliation information.""" global no_name try: fnr = fodselsnr.personnr_ok("%06d%05d" % (int(person_info['fodselsdato']), int(person_info['personnr']))) fnr = fodselsnr.personnr_ok(fnr) logger.info("Process %s " % (fnr)) (year, mon, day) = fodselsnr.fodt_dato(fnr) if (year < 1970 and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1): # Seems to be a bug in time.mktime on some machines year = 1970 except fodselsnr.InvalidFnrError: logger.warn("Ugyldig fødselsnr: %s" % fnr) return gender = co.gender_male if(fodselsnr.er_kvinne(fnr)): gender = co.gender_female etternavn = fornavn = None studentnr = None affiliations = [] address_info = None aktiv_sted = [] # Iterate over all person_info entries and extract relevant data if person_info.has_key('aktiv'): for row in person_info['aktiv']: if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append(int(studieprog2sko[row['studieprogramkode']])) for dta_type in person_info.keys(): x = person_info[dta_type] p = x[0] if isinstance(p, str): continue # Get name if dta_type in ('fagperson', 'evu', 'aktiv'): etternavn = p['etternavn'] fornavn = p['fornavn'] if p.has_key('studentnr_tildelt'): studentnr = p['studentnr_tildelt'] # Get affiliations if dta_type in ('fagperson',): _process_affiliation(co.affiliation_tilknyttet, co.affiliation_status_tilknyttet_fagperson, affiliations, _get_sko(p, 'faknr', 'instituttnr', 'gruppenr', 'institusjonsnr')) elif dta_type in ('aktiv', ): for row in x: # aktiv_sted is necessary in order to avoid different affiliation statuses # to a same 'stedkode' to be overwritten # e.i. if a person has both affiliations status 'evu' and # aktive to a single stedkode we want to register the status 'aktive' # in cerebrum if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append(int(studieprog2sko[row['studieprogramkode']])) _process_affiliation(co.affiliation_student, co.affiliation_status_student_aktiv, affiliations, studieprog2sko[row['studieprogramkode']]) elif dta_type in ('evu',): subtype = co.affiliation_status_student_evu if studieprog2sko[row['studieprogramkode']] in aktiv_sted: subtype = co.affiliation_status_student_aktiv _process_affiliation(co.affiliation_student, subtype, affiliations, studieprog2sko[row['studieprogramkode']]) if etternavn is None: logger.debug("Ikke noe navn på %s" % fnr) no_name += 1 return # TODO: If the person already exist and has conflicting data from # another source-system, some mechanism is needed to determine the # superior setting. new_person = Factory.get('Person')(db) if fnr2person_id.has_key(fnr): new_person.find(fnr2person_id[fnr]) new_person.populate(mx.DateTime.Date(year, mon, day), gender) new_person.affect_names(co.system_fs, co.name_first, co.name_last) new_person.populate_name(co.name_first, fornavn) new_person.populate_name(co.name_last, etternavn) if studentnr is not None: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr, co.externalid_studentnr) new_person.populate_external_id(co.system_fs, co.externalid_studentnr, studentnr) else: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr) new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr) ad_post, ad_post_private, ad_street = _calc_address(person_info) for address_info, ad_const in ((ad_post, co.address_post), (ad_post_private, co.address_post_private), (ad_street, co.address_street)): # TBD: Skal vi slette evt. eksisterende adresse v/None? if address_info is not None: logger.debug("Populating address...") new_person.populate_address(co.system_fs, ad_const, **address_info) # if this is a new Person, there is no entity_id assigned to it # until written to the database. op = new_person.write_db() for a in filter_affiliations(affiliations): ou, aff, aff_status = a new_person.populate_affiliation(co.system_fs, ou, aff, aff_status) if include_delete: key_a = "%s:%s:%s" % (new_person.entity_id,ou,int(aff)) if old_aff.has_key(key_a): old_aff[key_a] = False register_cellphone(new_person, person_info) op2 = new_person.write_db() if op is None and op2 is None: logger.info("**** EQUAL ****") elif op == True: logger.info("**** NEW ****") else: logger.info("**** UPDATE ****") register_fagomrade(new_person, person_info) # Reservations if gen_groups: should_add = False if person_info.has_key('nettpubl'): for row in person_info['nettpubl']: if row.get('akseptansetypekode', "") == "NETTPUBL" and row.get('status_svar', "") == "J": should_add = True if should_add: # The student has explicitly given us permission to be # published in the directory. _add_res(new_person.entity_id) else: # The student either hasn't registered an answer to # the "Can we publish info about you in the directory" # question at all, or has given an explicit "I don't # want to appear in the directory" answer. _rem_res(new_person.entity_id) db.commit()
def process_person_callback(person_info): """Called when we have fetched all data on a person from the xml file. Updates/inserts name, address and affiliation information.""" global no_name try: fnr = fodselsnr.personnr_ok("%06d%05d" % (int(person_info["fodselsdato"]), int(person_info["personnr"]))) fnr = fodselsnr.personnr_ok(fnr) logger.info("Process %s " % (fnr)) (year, mon, day) = fodselsnr.fodt_dato(fnr) if year < 1970 and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1: # Seems to be a bug in time.mktime on some machines year = 1970 except fodselsnr.InvalidFnrError: logger.warn("Ugyldig fødselsnr: %s" % fnr) return gender = co.gender_male if fodselsnr.er_kvinne(fnr): gender = co.gender_female etternavn = fornavn = None studentnr = None affiliations = [] address_info = None aktiv_sted = [] # Iterate over all person_info entries and extract relevant data if person_info.has_key("aktiv"): for row in person_info["aktiv"]: if studieprog2sko[row["studieprogramkode"]] is not None: aktiv_sted.append(int(studieprog2sko[row["studieprogramkode"]])) logger.debug("App2akrivts") for dta_type in person_info.keys(): x = person_info[dta_type] p = x[0] if isinstance(p, str): continue # Get name if dta_type in ("fagperson", "evu", "aktiv"): etternavn = p["etternavn"] fornavn = p["fornavn"] if p.has_key("studentnr_tildelt"): studentnr = "%06d" % int(p["studentnr_tildelt"]) # Get affiliations if dta_type in ("fagperson",): _process_affiliation( co.affiliation_tilknyttet, co.affiliation_status_tilknyttet_fagperson, affiliations, _get_sko(p, "faknr", "instituttnr", "gruppenr", "institusjonsnr"), ) elif dta_type in ("aktiv",): for row in x: # aktiv_sted is necessary in order to avoid different affiliation statuses # to a same 'stedkode' to be overwritten # e.i. if a person has both affiliations status 'evu' and # aktive to a single stedkode we want to register the status 'aktive' # in cerebrum if studieprog2sko[row["studieprogramkode"]] is not None: aktiv_sted.append(int(studieprog2sko[row["studieprogramkode"]])) _process_affiliation( co.affiliation_student, co.affiliation_status_student_aktiv, affiliations, studieprog2sko[row["studieprogramkode"]], ) elif dta_type in ("evu",): subtype = co.affiliation_status_student_evu if studieprog2sko[row["studieprogramkode"]] in aktiv_sted: subtype = co.affiliation_status_student_aktiv _process_affiliation( co.affiliation_student, subtype, affiliations, studieprog2sko[row["studieprogramkode"]] ) if etternavn is None: logger.debug("Ikke noe navn på %s" % fnr) no_name += 1 return new_person = Factory.get("Person")(db) if fnr2person_id.has_key(fnr): new_person.find(fnr2person_id[fnr]) new_person.populate(mx.DateTime.Date(year, mon, day), gender) new_person.affect_names(co.system_fs, co.name_first, co.name_last) new_person.populate_name(co.name_first, fornavn) new_person.populate_name(co.name_last, etternavn) if studentnr is not None: new_person.affect_external_id( co.system_fs, co.externalid_fodselsnr, co.externalid_studentnr, co.externalid_bewatorid ) new_person.populate_external_id(co.system_fs, co.externalid_studentnr, studentnr) logger.debug("Studentnr is %s", studentnr) new_bew_id = "01221%06d0" % int(studentnr) logger.debug("Adding bewator-ID %s for %s", new_bew_id, studentnr) # we have to use system_fs here (for technical reasons) even # though we should be using system_manual new_person.populate_external_id(co.system_fs, co.externalid_bewatorid, new_bew_id) else: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr) new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr) ad_post, ad_post_private, ad_street = _calc_address(person_info) for address_info, ad_const in ( (ad_post, co.address_post), (ad_post_private, co.address_post_private), (ad_street, co.address_street), ): # TBD: Skal vi slette evt. eksisterende adresse v/None? if address_info is not None: logger.debug("Populating address...") new_person.populate_address(co.system_fs, ad_const, **address_info) # if this is a new Person, there is no entity_id assigned to it # until written to the database. op = new_person.write_db() for a in filter_affiliations(affiliations): ou, aff, aff_status = a new_person.populate_affiliation(co.system_fs, ou, aff, aff_status) if include_delete: key_a = "%s:%s:%s" % (new_person.entity_id, ou, int(aff)) if old_aff.has_key(key_a): old_aff[key_a] = False register_cellphone(new_person, person_info) op2 = new_person.write_db() if op is None and op2 is None: logger.info("**** EQUAL ****") elif op == True: logger.info("**** NEW ****") else: logger.info("**** UPDATE ****") # Reservations if gen_groups: should_add = False for dta_type in person_info.keys(): p = person_info[dta_type][0] if isinstance(p, str): continue # Presence of 'fagperson' elements for a person should not # affect that person's reservation status. if dta_type in ("fagperson",): continue # We only fetch the column in these queries if dta_type not in ("evu"): continue # If 'status_reserv_nettpubl' == "N": add to group if p.get("status_reserv_nettpubl", "") == "N": should_add = True else: should_add = False if should_add: # The student has explicitly given us permission to be # published in the directory. _add_res(new_person.entity_id) else: # The student either hasn't registered an answer to # the "Can we publish info about you in the directory" # question at all, or has given an explicit "I don't # want to appear in the directory" answer. _rem_res(new_person.entity_id) db.commit()
def process_person_callback(person_info): """Called when we have fetched all data on a person from the xml file. Updates/inserts name, address and affiliation information.""" global no_name try: fnr = fodselsnr.personnr_ok( "%06d%05d" % (int(person_info['fodselsdato']), int(person_info['personnr']))) fnr = fodselsnr.personnr_ok(fnr) logger.info("Process %s " % (fnr)) (year, mon, day) = fodselsnr.fodt_dato(fnr) if (year < 1970 and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1): # Seems to be a bug in time.mktime on some machines year = 1970 except fodselsnr.InvalidFnrError: logger.warn("Ugyldig fødselsnr: %s" % fnr) return gender = co.gender_male if (fodselsnr.er_kvinne(fnr)): gender = co.gender_female etternavn = fornavn = None studentnr = None affiliations = [] address_info = None aktiv_sted = [] aktivemne_sted = [] # Iterate over all person_info entries and extract relevant data if 'aktiv' in person_info: for row in person_info['aktiv']: if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append(int( studieprog2sko[row['studieprogramkode']])) logger.debug("App2akrivts") if 'emnestud' in person_info: for row in person_info['emnestud']: if emne2sko[row['emnekode']] is not None: aktivemne_sted.append(int(emne2sko[row['emnekode']])) logger.debug('Add sko %s based on emne %s', int(emne2sko[row['emnekode']]), row['emnekode']) for dta_type in person_info.keys(): x = person_info[dta_type] p = x[0] if isinstance(p, str): continue # Get name if dta_type in ( 'fagperson', 'opptak', 'tilbud', 'evu', 'privatist_emne', 'privatist_studieprogram', 'alumni', 'emnestud', ): etternavn = p['etternavn'] fornavn = p['fornavn'] if 'studentnr_tildelt' in p: studentnr = p['studentnr_tildelt'] # Get affiliations if dta_type in ('fagperson', ): _process_affiliation( co.affiliation_tilknyttet, co.affiliation_tilknyttet_fagperson, affiliations, _get_sko(p, 'faknr', 'instituttnr', 'gruppenr', 'institusjonsnr')) elif dta_type in ('opptak', ): for row in x: subtype = co.affiliation_status_student_opptak if studieprog2sko[row['studieprogramkode']] in aktiv_sted: subtype = co.affiliation_status_student_aktiv elif row['studierettstatkode'] == 'EVU': subtype = co.affiliation_status_student_evu elif row['studierettstatkode'] == 'FULLFØRT': subtype = co.affiliation_status_student_alumni elif int(row['studienivakode']) >= 900: subtype = co.affiliation_status_student_drgrad elif _is_new_admission(row.get('dato_studierett_tildelt')): subtype = co.affiliation_status_student_ny _process_affiliation(co.affiliation_student, subtype, affiliations, studieprog2sko[row['studieprogramkode']]) elif dta_type in ('emnestud', ): for row in x: subtype = co.affiliation_status_student_emnestud # We may have some situations here where students get # emnestud and aonther affiliation to the same sko, # but this seems to work for now. try: sko = emne2sko[row['emnekode']] except KeyError: logger.warn("Fant ingen emner med koden %s", p['emnekode']) continue if sko in aktiv_sted: subtype = co.affiliation_status_student_aktiv _process_affiliation(co.affiliation_student, subtype, affiliations, sko) elif dta_type in ('privatist_studieprogram', ): _process_affiliation(co.affiliation_student, co.affiliation_status_student_privatist, affiliations, studieprog2sko[p['studieprogramkode']]) elif dta_type in ('privatist_emne', ): try: sko = emne2sko[p['emnekode']] except KeyError: logger.warn("Fant ingen emner med koden %s" % p['emnekode']) continue _process_affiliation(co.affiliation_student, co.affiliation_status_student_privatist, affiliations, sko) elif dta_type in ('perm', ): _process_affiliation(co.affiliation_student, co.affiliation_status_student_aktiv, affiliations, studieprog2sko[p['studieprogramkode']]) elif dta_type in ('tilbud', ): for row in x: _process_affiliation(co.affiliation_student, co.affiliation_status_student_tilbud, affiliations, studieprog2sko[row['studieprogramkode']]) elif dta_type in ('evu', ): _process_affiliation( co.affiliation_student, co.affiliation_status_student_evu, affiliations, _get_sko(p, 'faknr_adm_ansvar', 'instituttnr_adm_ansvar', 'gruppenr_adm_ansvar')) else: logger.debug2("No such affiliation type: %s, skipping", dta_type) if etternavn is None: logger.debug("Ikke noe navn på %s" % fnr) no_name += 1 return # TODO: If the person already exist and has conflicting data from # another source-system, some mechanism is needed to determine the # superior setting. new_person = Factory.get('Person')(db) if fnr in fnr2person_id: new_person.find(fnr2person_id[fnr]) new_person.populate(mx.DateTime.Date(year, mon, day), gender) new_person.affect_names(co.system_fs, co.name_first, co.name_last) new_person.populate_name(co.name_first, fornavn) new_person.populate_name(co.name_last, etternavn) if studentnr is not None: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr, co.externalid_studentnr) new_person.populate_external_id(co.system_fs, co.externalid_studentnr, studentnr) else: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr) new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr) ad_post, ad_post_private, ad_street = _calc_address(person_info) for address_info, ad_const in ((ad_post, co.address_post), (ad_post_private, co.address_post_private), (ad_street, co.address_street)): # TBD: Skal vi slette evt. eksisterende adresse v/None? if address_info is not None: logger.debug("Populating address %s for %s", ad_const, studentnr) new_person.populate_address(co.system_fs, ad_const, **address_info) # if this is a new Person, there is no entity_id assigned to it # until written to the database. try: op = new_person.write_db() except Exception, e: logger.exception("write_db failed for person %s: %s", fnr, e) # Roll back in case of db exceptions: db.rollback() return
def create_sysx_person(db, sxp, update_affs, stats): """ Create or update person with sysx data. :param db: :param sxp: Person dict from ``SYSX`` :param update_affs: A set of current SYSX aff keys :param stats: A defaultdict with counts """ co = Factory.get('Constants')(db) add_sysx_id = fnr_found = False sysx_id = sxp['id'] fnr = sxp['fnr'] fornavn = sxp['fornavn'] etternavn = sxp['etternavn'] fullt_navn = "%s %s" % (fornavn, etternavn) if check_expired_sourcedata(sxp['expire_date']): logger.info("Skipping sysx_id=%r (%s), expire_date=%r", sysx_id, fullt_navn, sxp['expire_date']) stats['skipped'] += 1 return # check if approved. Ditch if not. if not sxp['approved']: logger.error("Skipping sysx_id=%r (%s), not approved", sysx_id, fullt_navn) stats['skipped'] += 1 return my_stedkode = Factory.get('OU')(db) pers_sysx = Factory.get('Person')(db) pers_fnr = Factory.get('Person')(db) try: pers_sysx.find_by_external_id(co.externalid_sys_x_id, sysx_id) except Errors.NotFoundError: add_sysx_id = True if fnr: try: fnr = fodselsnr.personnr_ok(fnr) except fodselsnr.InvalidFnrError: logger.error("Skipping sysx_id=%r (%s), invalid fnr", sysx_id, fullt_navn) stats['skipped'] += 1 return birth_date = datetime.date(*fodselsnr.fodt_dato(fnr)) if fodselsnr.er_kvinne(fnr): gender = co.gender_female else: gender = co.gender_male try: pers_fnr.find_by_external_id(co.externalid_fodselsnr, fnr) except Errors.NotFoundError: pass except Errors.TooManyRowsError as e: # This persons fnr has multiple rows in entity_external_id table # This is an error, person should not have not more than one entry. # Don't know which person object to use, return error message. logger.error( "Skipping sysx_id=%r (%s), matched multiple persons (%s)", sysx_id, fullt_navn, e) stats['skipped'] += 1 return else: if not add_sysx_id and (pers_fnr.entity_id != pers_sysx.entity_id): logger.error( "Skipping sysx_id=%r (%s), matched multiple persons (sysx " "id matched person_id=%r, sysx fnr matched person_id=%r)", sysx_id, fullt_navn, pers_sysx.entity_id, pers_fnr.entity_id) stats['skipped'] += 1 return fnr_found = True else: # foreigner without norwegian ssn, birth_date = sxp['birth_date'] if not birth_date: logger.error("sysx_id=%r (%s) is missing birth date", sysx_id, fullt_navn) if sxp['gender'] == 'M': gender = co.gender_male elif sxp['gender'] == 'F': gender = co.gender_female else: logger.error("Skipping sysx_id=%r (%s), invalid gender %r", sysx_id, fullt_navn, sxp['gender']) stats['skipped'] += 1 return logger.info("Processing sysx_id=%r (%s)", sysx_id, fullt_navn) if fnr_found: person = pers_fnr else: person = pers_sysx # person object located, populate... try: person.populate(mx.DateTime.DateFrom(birth_date), gender) except Errors.CerebrumError: logger.error("Skipping sysx_id=%r (%s), populate() failed", sysx_id, fullt_navn, exc_info=True) stats['skipped'] += 1 return person.affect_names(co.system_x, co.name_first, co.name_last, co.name_full) person.populate_name(co.name_first, fornavn) person.populate_name(co.name_last, etternavn) person.populate_name(co.name_full, fullt_navn) # add external ids if fnr: person.affect_external_id(co.system_x, co.externalid_fodselsnr, co.externalid_sys_x_id) person.populate_external_id(co.system_x, co.externalid_fodselsnr, fnr) logger.debug("Set NO_BIRTHNO for sysx_id=%r (%s)", sysx_id, fullt_navn) else: person.affect_external_id(co.system_x, co.externalid_sys_x_id) person.populate_external_id(co.system_x, co.externalid_sys_x_id, sysx_id) logger.debug("Set sysx_id for sysx_id=%r (%s)", sysx_id, fullt_navn) # setting affiliation and affiliation_status affiliation = co.PersonAffiliation(sxp['affiliation']) affiliation_status = co.PersonAffStatus(affiliation, sxp['affiliation_status']) # Assert that the affs are real int(affiliation), int(affiliation_status) # get ou_id of stedkode used fak = sxp['ou'][0:2] ins = sxp['ou'][2:4] avd = sxp['ou'][4:6] # KEB if fak == '0': logger.warning("sysx_id=%r (%s) has invalid sko=%r", sysx_id, fullt_navn, sxp['ou']) ins = avd = fak try: my_stedkode.find_stedkode(fak, ins, avd, cereconf.DEFAULT_INSTITUSJONSNR) except EntityExpiredError: logger.error("Skipping sysx_id=%r (%s), expired sko=%r", sysx_id, fullt_navn, sxp['ou']) stats['skipped'] += 1 return ou_id = int(my_stedkode.entity_id) # if this is a new Person, there is no entity_id assigned to it # until written to the database. op = person.write_db() # populate the person affiliation table person.populate_affiliation(co.system_x, ou_id, int(affiliation), int(affiliation_status)) # make sure we don't delete this aff when processing deleted affs aff_key = affiliation_key(person.entity_id, ou_id, affiliation) update_affs.discard(aff_key) op2 = person.write_db() logger.debug("OP codes: op=%s,op2=%s" % (op, op2)) if op is None and op2 is None: logger.info("**** EQUAL ****") stats['unchanged'] += 1 elif op is None and op2 is False: logger.info("**** AFF UPDATE ****") stats['updated'] += 1 elif op is True: logger.info("**** NEW ****") stats['added'] += 1 elif op is False: logger.info("**** UPDATE ****") stats['updated'] += 1 return
def import_person(db, person, update_affs=None): """ Import a single person. :param person: A dict from py:func:`parse_person` :param update_affs: A set of affiliations from py:func:`load_sito_affiliations`. Affiliations imported by this function will be removed from the set. """ new_person = Factory.get('Person')(db) const = Factory.get('Constants')(db) if update_affs is None: update_affs = set() employee_id = person['employee_id'] logger.info("Processing employee_id=%r", employee_id) # # Validate person data # # Birthdate try: birthdate = parse_date(person['birthdate']) valid_birthdate = True except ValueError: logger.warning('Invalid birth date for employee_id=%r (%r)', person['employee_id'], person['birthdate']) valid_birthdate = False birthdate = None # SSN try: fodselsnr.personnr_ok(person['ssn']) valid_ssn = True except fodselsnr.InvalidFnrError: logger.warning("Empty SSN for employee_id=%r", person['employee_id']) valid_ssn = False # set person gender (checking both SSN and gender from sito input file) if valid_ssn: if fodselsnr.er_kvinne(person['ssn']): gender = const.gender_female else: gender = const.gender_male elif person['gender'] == 'Female': gender = const.gender_female elif person['gender'] == 'Male': gender = const.gender_male else: logger.warning('Unknown gender value for employee_id=%r (%r)', employee_id, person['gender']) gender = const.gender_unknown # Validate Birthdate against SSN if valid_ssn and valid_birthdate: ssndate = datetime.date(*fodselsnr.fodt_dato(person['ssn'])) if birthdate != ssndate: raise SkipPerson('inconsistent birth date and ssn (%r, %r)', birthdate, ssndate) elif valid_ssn and not valid_birthdate: logger.warning('Missing birth date for employee_id=%r, using date' ' from ssn', person['employee_id']) birthdate = datetime.date(*fodselsnr.fodt_dato(person['ssn'])) elif not valid_ssn and valid_birthdate: # person have birthdate but NOT ssn. Nothing to do here pass elif not valid_ssn and not valid_birthdate: # person does not have birthdate nor ssn. This person cannot be # built. SSN or Birthdate required. Return error message and # continue with NEXT person raise SkipPerson('missing ssn and birth date') # Check names if not person['first_name']: raise SkipPerson('Missing first name') if not person['last_name']: raise SkipPerson('Missing last name') if person['middle_name']: fname = person['first_name'] + ' ' + person['middle_name'] else: fname = person['first_name'] lname = person['last_name'] # # Get person object som DB if it exists # found = False new_person.clear() try: new_person.find_by_external_id(const.externalid_sito_ansattnr, employee_id) found = True except Errors.NotFoundError: # could not find person in DB based on ansattnr. if valid_ssn: # try to find person using ssn if ssn is valid try: new_person.clear() new_person.find_by_external_id(const.externalid_fodselsnr, person['ssn']) found = True except Errors.NotFoundError: pass if found: logger.info('Updating person object for employee_id=%r', employee_id) else: logger.info('Creating person object for employee_id=%r', employee_id) # # Populate the person object # new_person.populate(mx.DateTime.DateFrom(birthdate), gender) new_person.affect_names(const.system_sito, const.name_first, const.name_last, const.name_work_title) new_person.affect_external_id(const.system_sito, const.externalid_fodselsnr, const.externalid_sito_ansattnr) new_person.populate_name(const.name_first, fname) new_person.populate_name(const.name_last, lname) if person['title']: new_person.populate_name(const.name_work_title, person['title']) if valid_ssn: new_person.populate_external_id(const.system_sito, const.externalid_fodselsnr, person['ssn']) new_person.populate_external_id(const.system_sito, const.externalid_sito_ansattnr, employee_id) # intermediary write to get an entity_id if this is a new person. new_person.write_db() new_person.populate_affiliation(const.system_sito) new_person.populate_contact_info(const.system_sito) # set person affiliation for key, ou_id, aff, status in determine_affiliations( db, new_person.entity_id, person): logger.info("affiliation for employee_id=%r to ou_id=%r", employee_id, ou_id) new_person.populate_affiliation(const.system_sito, ou_id, int(aff), int(status)) # set this persons affiliation entry to False # this ensures that this persons affiliations will not be removed # when the clean_affiliation function is called after import person update_affs.discard(key) # get person work, cellular and home phone numbers c_prefs = {} for con, number in person['phone'].items(): c_type = int(const.human2constant(con, const.ContactInfo)) if c_type in c_prefs: pref = c_prefs[c_type] else: pref = 0 c_prefs[c_type] = 1 new_person.populate_contact_info(const.system_sito, c_type, number, pref) logger.debug("contact for employee_id=%r, system=%s, " "c_type=%s, number=%s, pref=%s", employee_id, const.system_sito, c_type, number, pref) new_person.write_db() return not found
def process_person_callback(person_info): """Called when we have fetched all data on a person from the xml file. Updates/inserts name, address and affiliation information.""" global no_name try: fnr = fodselsnr.personnr_ok( "%06d%05d" % (int(person_info['fodselsdato']), int(person_info['personnr']))) fnr = fodselsnr.personnr_ok(fnr) logger.info("Process %s " % (fnr)) (year, mon, day) = fodselsnr.fodt_dato(fnr) if (year < 1970 and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1): # Seems to be a bug in time.mktime on some machines year = 1970 except fodselsnr.InvalidFnrError: logger.warn("Ugyldig fødselsnr: %s" % fnr) return gender = co.gender_male if (fodselsnr.er_kvinne(fnr)): gender = co.gender_female etternavn = fornavn = None studentnr = None affiliations = [] address_info = None aktiv_sted = [] for dta_type in person_info.keys(): x = person_info[dta_type] p = x[0] if isinstance(p, str): continue # Get name if dta_type in ('fagperson', 'evu', 'aktiv'): etternavn = p['etternavn'] fornavn = p['fornavn'] if p.has_key('studentnr_tildelt'): studentnr = '%06d' % int(p['studentnr_tildelt']) if etternavn is None: logger.debug("Ikke noe navn på %s" % fnr) no_name += 1 return new_person = Factory.get('Person')(db) if fnr2person_id.has_key(fnr): new_person.find(fnr2person_id[fnr]) new_person.populate(mx.DateTime.Date(year, mon, day), gender) new_person.affect_names(co.system_fs, co.name_first, co.name_last) new_person.populate_name(co.name_first, fornavn) new_person.populate_name(co.name_last, etternavn) if studentnr is not None: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr, co.externalid_studentnr) new_person.populate_external_id(co.system_fs, co.externalid_studentnr, studentnr) logger.debug("Studentnr is %s", studentnr) else: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr) new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr) ad_post, ad_post_private, ad_street = _calc_address(person_info) for address_info, ad_const in ((ad_post, co.address_post), (ad_post_private, co.address_post_private), (ad_street, co.address_street)): # TBD: Skal vi slette evt. eksisterende adresse v/None? if address_info is not None: logger.debug("Populating address...") new_person.populate_address(co.system_fs, ad_const, **address_info) # if this is a new Person, there is no entity_id assigned to it # until written to the database. op = new_person.write_db() # Iterate over all person_info entries and extract relevant data if person_info.has_key('aktiv'): for row in person_info['aktiv']: try: if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append( int(studieprog2sko[row['studieprogramkode']])) logger.debug("App2akrivts") except KeyError: logger.warn( 'App2akrivts: Person id %d har stud.prog.kode %s som ikke eksisterer.', new_person.entity_id, row['studieprogramkode']) for dta_type in person_info.keys(): # Get affiliations if dta_type in ('fagperson', ): _process_affiliation( co.affiliation_tilknyttet, co.affiliation_status_tilknyttet_fagperson, affiliations, _get_sko(p, 'faknr', 'instituttnr', 'gruppenr', 'institusjonsnr')) elif dta_type in ('aktiv', ): for row in x: # aktiv_sted is necessary in order to avoid different affiliation statuses # to a same 'stedkode' to be overwritten # e.i. if a person has both affiliations status 'evu' and # aktive to a single stedkode we want to register the status 'aktive' # in cerebrum try: if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append( int(studieprog2sko[row['studieprogramkode']])) _process_affiliation( co.affiliation_student, co.affiliation_status_student_aktiv, affiliations, studieprog2sko[row['studieprogramkode']]) except KeyError: logger.warn( 'AKTIV: Person id %d har stud.prog.kode %s som ikke eksisterer.', new_person.entity_id, row['studieprogramkode']) elif dta_type in ('evu', ): subtype = co.affiliation_status_student_evu try: if studieprog2sko[row['studieprogramkode']] in aktiv_sted: subtype = co.affiliation_status_student_aktiv _process_affiliation(co.affiliation_student, subtype, affiliations, studieprog2sko[row['studieprogramkode']]) except KeyError: logger.warn( 'EVU: Person id %d har stud.prog.kode %s som ikke eksisterer.', new_person.entity_id, row['studieprogramkode']) for a in filter_affiliations(affiliations): ou, aff, aff_status = a new_person.populate_affiliation(co.system_fs, ou, aff, aff_status) if include_delete: key_a = "%s:%s:%s" % (new_person.entity_id, ou, int(aff)) if old_aff.has_key(key_a): old_aff[key_a] = False register_contact(new_person, person_info) op2 = new_person.write_db() if op is None and op2 is None: logger.info("**** EQUAL ****") elif op == True: logger.info("**** NEW ****") else: logger.info("**** UPDATE ****") # Reservations if gen_groups: should_add = False for dta_type in person_info.keys(): p = person_info[dta_type][0] if isinstance(p, str): continue # Presence of 'fagperson' elements for a person should not # affect that person's reservation status. if dta_type in ('fagperson', ): continue # We only fetch the column in these queries if dta_type not in ('evu'): continue # If 'status_reserv_nettpubl' == "N": add to group if p.get('status_reserv_nettpubl', "") == "N": should_add = True else: should_add = False if should_add: # The student has explicitly given us permission to be # published in the directory. _add_res(new_person.entity_id) else: # The student either hasn't registered an answer to # the "Can we publish info about you in the directory" # question at all, or has given an explicit "I don't # want to appear in the directory" answer. _rem_res(new_person.entity_id) db.commit()
def process_person_callback(person_info): """Called when we have fetched all data on a person from the xml file. Updates/inserts name, address and affiliation information.""" global no_name try: fnr = fodselsnr.personnr_ok("%06d%05d" % (int(person_info['fodselsdato']), int(person_info['personnr']))) fnr = fodselsnr.personnr_ok(fnr) logger.info("Process %s " % (fnr)) (year, mon, day) = fodselsnr.fodt_dato(fnr) if (year < 1970 and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1): # Seems to be a bug in time.mktime on some machines year = 1970 except fodselsnr.InvalidFnrError: logger.warn("Ugyldig fødselsnr: %s" % fnr) return gender = co.gender_male if(fodselsnr.er_kvinne(fnr)): gender = co.gender_female etternavn = fornavn = None studentnr = None affiliations = [] address_info = None aktiv_sted = [] # Iterate over all person_info entries and extract relevant data if person_info.has_key('aktiv'): for row in person_info['aktiv']: if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append(int(studieprog2sko[row['studieprogramkode']])) logger.debug("App2akrivts") for dta_type in person_info.keys(): x = person_info[dta_type] p = x[0] if isinstance(p, str): continue # Get name if dta_type in ('fagperson', 'evu', 'aktiv'): etternavn = p['etternavn'] fornavn = p['fornavn'] if p.has_key('studentnr_tildelt'): studentnr = p['studentnr_tildelt'] # Get affiliations if dta_type in ('fagperson',): _process_affiliation(co.affiliation_tilknyttet, co.affiliation_status_tilknyttet_fagperson, affiliations, _get_sko(p, 'faknr', 'instituttnr', 'gruppenr', 'institusjonsnr')) elif dta_type in ('aktiv', ): for row in x: # aktiv_sted is necessary in order to avoid different affiliation statuses # to a same 'stedkode' to be overwritten # e.i. if a person has both affiliations status 'evu' and # aktive to a single stedkode we want to register the status 'aktive' # in cerebrum if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append(int(studieprog2sko[row['studieprogramkode']])) _process_affiliation(co.affiliation_student, co.affiliation_status_student_aktiv, affiliations, studieprog2sko[row['studieprogramkode']]) elif dta_type in ('evu',): subtype = co.affiliation_status_student_evu if studieprog2sko[row['studieprogramkode']] in aktiv_sted: subtype = co.affiliation_status_student_aktiv _process_affiliation(co.affiliation_student, subtype, affiliations, studieprog2sko[row['studieprogramkode']]) if etternavn is None: logger.debug("Ikke noe navn på %s" % fnr) no_name += 1 return # TODO: If the person already exist and has conflicting data from # another source-system, some mechanism is needed to determine the # superior setting. new_person = Factory.get('Person')(db) if fnr2person_id.has_key(fnr): new_person.find(fnr2person_id[fnr]) new_person.populate(mx.DateTime.Date(year, mon, day), gender) new_person.affect_names(co.system_fs, co.name_first, co.name_last) new_person.populate_name(co.name_first, fornavn) new_person.populate_name(co.name_last, etternavn) if studentnr is not None: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr, co.externalid_studentnr) new_person.populate_external_id(co.system_fs, co.externalid_studentnr, studentnr) else: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr) new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr) ad_post, ad_post_private, ad_street = _calc_address(person_info) for address_info, ad_const in ((ad_post, co.address_post), (ad_post_private, co.address_post_private), (ad_street, co.address_street)): # TBD: Skal vi slette evt. eksisterende adresse v/None? if address_info is not None: logger.debug("Populating address...") new_person.populate_address(co.system_fs, ad_const, **address_info) # if this is a new Person, there is no entity_id assigned to it # until written to the database. try: op = new_person.write_db() except Exception, e: logger.exception("write_db failed for person %s: %s", fnr, e) # Roll back in case of db exceptions: db.rollback() return
def process_person_callback(person_info): """Called when we have fetched all data on a person from the xml file. Updates/inserts name, address and affiliation information.""" global no_name try: fnr = fodselsnr.personnr_ok( "%06d%05d" % (int(person_info['fodselsdato']), int(person_info['personnr']))) fnr = fodselsnr.personnr_ok(fnr) logger.info("Process %s " % (fnr)) (year, mon, day) = fodselsnr.fodt_dato(fnr) if (year < 1970 and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1): # Seems to be a bug in time.mktime on some machines year = 1970 except fodselsnr.InvalidFnrError: logger.warn("Ugyldig fødselsnr: %s" % fnr) return gender = co.gender_male if (fodselsnr.er_kvinne(fnr)): gender = co.gender_female etternavn = fornavn = None studentnr = None affiliations = [] address_info = None aktiv_sted = [] # Iterate over all person_info entries and extract relevant data if person_info.has_key('aktiv'): for row in person_info['aktiv']: if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append(int( studieprog2sko[row['studieprogramkode']])) logger.debug("App2akrivts") for dta_type in person_info.keys(): x = person_info[dta_type] p = x[0] if isinstance(p, str): continue # Get name if dta_type in ('fagperson', 'evu', 'aktiv'): etternavn = p['etternavn'] fornavn = p['fornavn'] if p.has_key('studentnr_tildelt'): studentnr = p['studentnr_tildelt'] # Get affiliations if dta_type in ('fagperson', ): _process_affiliation( co.affiliation_tilknyttet, co.affiliation_status_tilknyttet_fagperson, affiliations, _get_sko(p, 'faknr', 'instituttnr', 'gruppenr', 'institusjonsnr')) elif dta_type in ('aktiv', ): for row in x: # aktiv_sted is necessary in order to avoid different affiliation statuses # to a same 'stedkode' to be overwritten # e.i. if a person has both affiliations status 'evu' and # aktive to a single stedkode we want to register the status 'aktive' # in cerebrum if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append( int(studieprog2sko[row['studieprogramkode']])) _process_affiliation( co.affiliation_student, co.affiliation_status_student_aktiv, affiliations, studieprog2sko[row['studieprogramkode']]) elif dta_type in ('evu', ): subtype = co.affiliation_status_student_evu if studieprog2sko[row['studieprogramkode']] in aktiv_sted: subtype = co.affiliation_status_student_aktiv _process_affiliation(co.affiliation_student, subtype, affiliations, studieprog2sko[row['studieprogramkode']]) if etternavn is None: logger.debug("Ikke noe navn på %s" % fnr) no_name += 1 return # TODO: If the person already exist and has conflicting data from # another source-system, some mechanism is needed to determine the # superior setting. new_person = Factory.get('Person')(db) if fnr2person_id.has_key(fnr): new_person.find(fnr2person_id[fnr]) new_person.populate(mx.DateTime.Date(year, mon, day), gender) new_person.affect_names(co.system_fs, co.name_first, co.name_last) new_person.populate_name(co.name_first, fornavn) new_person.populate_name(co.name_last, etternavn) if studentnr is not None: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr, co.externalid_studentnr) new_person.populate_external_id(co.system_fs, co.externalid_studentnr, studentnr) else: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr) new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr) ad_post, ad_post_private, ad_street = _calc_address(person_info) for address_info, ad_const in ((ad_post, co.address_post), (ad_post_private, co.address_post_private), (ad_street, co.address_street)): # TBD: Skal vi slette evt. eksisterende adresse v/None? if address_info is not None: logger.debug("Populating address...") new_person.populate_address(co.system_fs, ad_const, **address_info) # if this is a new Person, there is no entity_id assigned to it # until written to the database. try: op = new_person.write_db() except Exception, e: logger.exception("write_db failed for person %s: %s", fnr, e) # Roll back in case of db exceptions: db.rollback() return
def process_person_callback(person_info): """Called when we have fetched all data on a person from the xml file. Updates/inserts name, address and affiliation information.""" global no_name try: fnr = fodselsnr.personnr_ok("%06d%05d" % ( int(person_info['fodselsdato']), int(person_info['personnr']))) fnr = fodselsnr.personnr_ok(fnr) logger.info("Processing %s", fnr) (year, mon, day) = fodselsnr.fodt_dato(fnr) if (year < 1970 and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1): # Seems to be a bug in time.mktime on some machines year = 1970 except fodselsnr.InvalidFnrError: logger.warn(u"Ugyldig f�dselsnr for: %s", person_info['fodselsdato']) return gender = co.gender_male if(fodselsnr.er_kvinne(fnr)): gender = co.gender_female etternavn = fornavn = None studentnr = None affiliations = [] address_info = None aktiv_sted = [] # Iterate over all person_info entries and extract relevant data for dta_type in person_info.keys(): x = person_info[dta_type] p = x[0] if isinstance(p, str): continue if dta_type not in ('tilbud', 'eksamen', 'evu'): if 'studentnr_tildelt' in p: studentnr = p['studentnr_tildelt'] else: logger.info("\n%s mangler studentnr!", fnr) # Get name if dta_type in ('aktiv', 'tilbud', 'evu', 'privatist_studieprogram', ): etternavn = p['etternavn'] fornavn = p['fornavn'] # Get address if address_info is None: if dta_type in ('aktiv', 'privatist_studieprogram', ): address_info = _ext_address_info( p, 'adrlin1_semadr', 'adrlin2_semadr', 'adrlin3_semadr', 'postnr_semadr', 'adresseland_semadr') if address_info is None: address_info = _ext_address_info( p, 'adrlin1_hjemsted', 'adrlin2_hjemsted', 'adrlin3_hjemsted', 'postnr_hjemsted', 'adresseland_hjemsted') elif dta_type in ('evu',): address_info = _ext_address_info( p, 'adrlin1_hjem', 'adrlin2_hjem', 'adrlin3_hjem', 'postnr_hjem', 'adresseland_hjem') if address_info is None: address_info = _ext_address_info( p, 'adrlin1_hjemsted', 'adrlin2_hjemsted', 'adrlin3_hjemsted', 'postnr_hjemsted', 'adresseland_hjemsted') elif dta_type in ('tilbud',): address_info = _ext_address_info( p, 'adrlin1_kontakt', 'adrlin2_kontakt', 'adrlin3_kontakt', 'postnr_kontakt', 'adresseland_kontakt') # Get affiliations # Lots of changes here compared to import_FS.py @ uio # TODO: split import_FS into a common part and organization spesific # parts if dta_type in ('aktiv', ): for row in x: # aktiv_sted is necessary in order to avoid different # affiliation statuses to a same 'stedkode' to be overwritten # e.i. if a person has both affiliations status 'tilbud' and # aktive to a single stedkode we want to register the status # 'aktive' in cerebrum if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append( int(studieprog2sko[row['studieprogramkode']])) _process_affiliation( co.affiliation_student, co.affiliation_status_student_aktiv, affiliations, studieprog2sko[row['studieprogramkode']]) elif dta_type in ('evu',): for row in x: _process_affiliation( co.affiliation_student, co.affiliation_status_student_evu, affiliations, _get_sko(p, 'faknr_adm_ansvar', 'instituttnr_adm_ansvar', 'gruppenr_adm_ansvar')) elif dta_type in ('privatist_studieprogram', ): for row in x: _process_affiliation( co.affiliation_student, co.affiliation_status_student_privatist, affiliations, studieprog2sko[row['studieprogramkode']]) elif dta_type in ('tilbud', ): for row in x: subtype = co.affiliation_status_student_tilbud if studieprog2sko[row['studieprogramkode']] in aktiv_sted: subtype = co.affiliation_status_student_aktiv _process_affiliation(co.affiliation_student, subtype, affiliations, studieprog2sko[row['studieprogramkode']]) if etternavn is None: logger.info("Ikke noe navn p� %s", fnr) no_name += 1 return # TODO: If the person already exist and has conflicting data from # another source-system, some mechanism is needed to determine the # superior setting. fsids = [(co.externalid_fodselsnr, fnr)] if studentnr is not None: fsids.append((co.externalid_studentnr, studentnr)) new_person = Factory.get('Person')(db) try: new_person.find_by_external_ids(*fsids) except Errors.NotFoundError: pass except Errors.TooManyRowsError, e: logger.error("Trying to find studentnr %s, getting several persons: %s", studentnr, e) return
def next_object(self, element): def get_value(element_value): return ensure_unicode(element_value, self.encoding) def extract(element_attr): return get_value(element.get(element_attr, "")) result = HRDataPerson() # Pull out all names self._register_names(result, element) # Pull out fnr tmp = "%02d%02d%02d%05d" % tuple([int(element.get(x)) for x in ("fodtdag", "fodtmnd", "fodtar", "personnr")]) fnr = fodselsnr.personnr_ok(tmp) result.add_id(result.NO_SSN, fnr) # Since LT does not provide birth date directly, we extract it from fnr result.birth_date = Date(*fodselsnr.fodt_dato(fnr)) # ... and gender if fodselsnr.er_mann(fnr): result.gender = result.GENDER_MALE else: result.gender = result.GENDER_FEMALE # fi # Register address # extract = lambda y: ensure_unicode(element.get(y, ""), self.encoding) result.address = DataAddress( kind=DataAddress.ADDRESS_PRIVATE, street=(extract("adresselinje1_privatadresse"), extract("adresselinje2_privatadresse")), zip=extract("poststednr_privatadresse"), city=extract("poststednavn_privatadresse")) # Contact information and jobs # FIXME: We do not have anything more intelligent for priorities priorities = dict() for sub in element.getiterator(): if sub.tag in ("bilag", "gjest", "tils",): emp = self._make_employment(sub) result.add_employment(emp) elif sub.tag in ("komm", "arbtlf",): for contact in self._make_contact(sub, priorities): result.add_contact(contact) # od # od # Reservation rules. Roughly, all employees are not reserved, unless # they say otherwise. Everyone else *is* reserved, unless they # explicitly allow publication in catalogues. has_active = result.has_active_employments() if has_active: to_reserve = False for resv in element.findall("res"): if (resv.get("katalogkode") == "ELKAT" and resv.get("felttypekode") not in ("PRIVADR", "PRIVTLF") and resv.get("resnivakode") != "SAMTYKKE"): to_reserve = True else: to_reserve = True for resv in element.findall("res"): if (resv.get("katalogkode") == "ELKAT" and resv.get("felttypekode") not in ("PRIVADR", "PRIVTLF")): to_reserve = resv.get("resnivakode") != "SAMTYKKE" result.reserved = to_reserve if (element.get("fakultetnr_for_lonnsslip") and element.get("instituttnr_for_lonnsslip") and element.get("gruppenr_for_lonnsslip")): result.primary_ou = (cereconf.DEFAULT_INSTITUSJONSNR, extract("fakultetnr_for_lonnsslip"), extract("instituttnr_for_lonnsslip"), extract("gruppenr_for_lonnsslip")) if not (result.get_name(result.NAME_FIRST) and result.get_name(result.NAME_LAST)): raise AssertionError("Missing name for %s" % list(result.iterids())) return result
def process_person_callback(person_info): """Called when we have fetched all data on a person from the xml file. Updates/inserts name, address and affiliation information.""" global no_name try: fnr = fodselsnr.personnr_ok( "%06d%05d" % (int(person_info['fodselsdato']), int(person_info['personnr']))) fnr = fodselsnr.personnr_ok(fnr) logger.info("Process %s " % (fnr)) (year, mon, day) = fodselsnr.fodt_dato(fnr) if (year < 1970 and getattr(cereconf, "ENABLE_MKTIME_WORKAROUND", 0) == 1): # Seems to be a bug in time.mktime on some machines year = 1970 except fodselsnr.InvalidFnrError: logger.warn("Ugyldig fødselsnr: %s" % fnr) return gender = co.gender_male if(fodselsnr.er_kvinne(fnr)): gender = co.gender_female etternavn = fornavn = None studentnr = None affiliations = [] address_info = None aktiv_sted = [] aktivemne_sted = [] # Iterate over all person_info entries and extract relevant data if 'aktiv' in person_info: for row in person_info['aktiv']: if studieprog2sko[row['studieprogramkode']] is not None: aktiv_sted.append( int(studieprog2sko[row['studieprogramkode']])) logger.debug("App2akrivts") if 'emnestud' in person_info: for row in person_info['emnestud']: if emne2sko[row['emnekode']] is not None: aktivemne_sted.append(int(emne2sko[row['emnekode']])) logger.debug('Add sko %s based on emne %s', int(emne2sko[row['emnekode']]), row['emnekode']) for dta_type in person_info.keys(): x = person_info[dta_type] p = x[0] if isinstance(p, str): continue # Get name if dta_type in ( 'fagperson', 'opptak', 'tilbud', 'evu', 'privatist_emne', 'privatist_studieprogram', 'alumni', 'emnestud', ): etternavn = p['etternavn'] fornavn = p['fornavn'] if 'studentnr_tildelt' in p: studentnr = p['studentnr_tildelt'] # Get affiliations if dta_type in ('fagperson',): _process_affiliation(co.affiliation_tilknyttet, co.affiliation_tilknyttet_fagperson, affiliations, _get_sko(p, 'faknr', 'instituttnr', 'gruppenr', 'institusjonsnr')) elif dta_type in ('opptak', ): for row in x: subtype = co.affiliation_status_student_opptak if studieprog2sko[row['studieprogramkode']] in aktiv_sted: subtype = co.affiliation_status_student_aktiv elif row['studierettstatkode'] == 'EVU': subtype = co.affiliation_status_student_evu elif row['studierettstatkode'] == 'FULLFØRT': subtype = co.affiliation_status_student_alumni elif int(row['studienivakode']) >= 900: subtype = co.affiliation_status_student_drgrad elif _is_new_admission(row.get('dato_studierett_tildelt')): subtype = co.affiliation_status_student_ny _process_affiliation(co.affiliation_student, subtype, affiliations, studieprog2sko[row['studieprogramkode']]) elif dta_type in ('emnestud',): for row in x: subtype = co.affiliation_status_student_emnestud # We may have some situations here where students get # emnestud and aonther affiliation to the same sko, # but this seems to work for now. try: sko = emne2sko[row['emnekode']] except KeyError: logger.warn("Fant ingen emner med koden %s", p['emnekode']) continue if sko in aktiv_sted: subtype = co.affiliation_status_student_aktiv _process_affiliation(co.affiliation_student, subtype, affiliations, sko) elif dta_type in ('privatist_studieprogram',): _process_affiliation(co.affiliation_student, co.affiliation_status_student_privatist, affiliations, studieprog2sko[p['studieprogramkode']]) elif dta_type in ('privatist_emne',): try: sko = emne2sko[p['emnekode']] except KeyError: logger.warn("Fant ingen emner med koden %s" % p['emnekode']) continue _process_affiliation(co.affiliation_student, co.affiliation_status_student_privatist, affiliations, sko) elif dta_type in ('perm',): _process_affiliation(co.affiliation_student, co.affiliation_status_student_aktiv, affiliations, studieprog2sko[p['studieprogramkode']]) elif dta_type in ('tilbud',): for row in x: _process_affiliation(co.affiliation_student, co.affiliation_status_student_tilbud, affiliations, studieprog2sko[row['studieprogramkode']]) elif dta_type in ('evu', ): _process_affiliation(co.affiliation_student, co.affiliation_status_student_evu, affiliations, _get_sko(p, 'faknr_adm_ansvar', 'instituttnr_adm_ansvar', 'gruppenr_adm_ansvar')) else: logger.debug2("No such affiliation type: %s, skipping", dta_type) if etternavn is None: logger.debug("Ikke noe navn på %s" % fnr) no_name += 1 return # TODO: If the person already exist and has conflicting data from # another source-system, some mechanism is needed to determine the # superior setting. new_person = Factory.get('Person')(db) if fnr in fnr2person_id: new_person.find(fnr2person_id[fnr]) new_person.populate(mx.DateTime.Date(year, mon, day), gender) new_person.affect_names(co.system_fs, co.name_first, co.name_last) new_person.populate_name(co.name_first, fornavn) new_person.populate_name(co.name_last, etternavn) if studentnr is not None: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr, co.externalid_studentnr) new_person.populate_external_id(co.system_fs, co.externalid_studentnr, studentnr) else: new_person.affect_external_id(co.system_fs, co.externalid_fodselsnr) new_person.populate_external_id(co.system_fs, co.externalid_fodselsnr, fnr) ad_post, ad_post_private, ad_street = _calc_address(person_info) for address_info, ad_const in ((ad_post, co.address_post), (ad_post_private, co.address_post_private), (ad_street, co.address_street)): # TBD: Skal vi slette evt. eksisterende adresse v/None? if address_info is not None: logger.debug("Populating address %s for %s", ad_const, studentnr) new_person.populate_address(co.system_fs, ad_const, **address_info) # if this is a new Person, there is no entity_id assigned to it # until written to the database. try: op = new_person.write_db() except Exception, e: logger.exception("write_db failed for person %s: %s", fnr, e) # Roll back in case of db exceptions: db.rollback() return
def next_object(self, element): def get_value(element_value): return ensure_unicode(element_value, self.encoding) def extract(element_attr): return get_value(element.get(element_attr, "")) result = HRDataPerson() # Pull out all names self._register_names(result, element) # Pull out fnr tmp = "%02d%02d%02d%05d" % tuple([ int(element.get(x)) for x in ("fodtdag", "fodtmnd", "fodtar", "personnr") ]) fnr = fodselsnr.personnr_ok(tmp) result.add_id(result.NO_SSN, fnr) # Since LT does not provide birth date directly, we extract it from fnr result.birth_date = Date(*fodselsnr.fodt_dato(fnr)) # ... and gender if fodselsnr.er_mann(fnr): result.gender = result.GENDER_MALE else: result.gender = result.GENDER_FEMALE # fi # Register address # extract = lambda y: ensure_unicode(element.get(y, ""), self.encoding) result.address = DataAddress( kind=DataAddress.ADDRESS_PRIVATE, street=(extract("adresselinje1_privatadresse"), extract("adresselinje2_privatadresse")), zip=extract("poststednr_privatadresse"), city=extract("poststednavn_privatadresse")) # Contact information and jobs # FIXME: We do not have anything more intelligent for priorities priorities = dict() for sub in element.getiterator(): if sub.tag in ( "bilag", "gjest", "tils", ): emp = self._make_employment(sub) result.add_employment(emp) elif sub.tag in ( "komm", "arbtlf", ): for contact in self._make_contact(sub, priorities): result.add_contact(contact) # od # od # Reservation rules. Roughly, all employees are not reserved, unless # they say otherwise. Everyone else *is* reserved, unless they # explicitly allow publication in catalogues. has_active = result.has_active_employments() if has_active: to_reserve = False for resv in element.findall("res"): if (resv.get("katalogkode") == "ELKAT" and resv.get("felttypekode") not in ("PRIVADR", "PRIVTLF") and resv.get("resnivakode") != "SAMTYKKE"): to_reserve = True else: to_reserve = True for resv in element.findall("res"): if (resv.get("katalogkode") == "ELKAT" and resv.get("felttypekode") not in ("PRIVADR", "PRIVTLF")): to_reserve = resv.get("resnivakode") != "SAMTYKKE" result.reserved = to_reserve if (element.get("fakultetnr_for_lonnsslip") and element.get("instituttnr_for_lonnsslip") and element.get("gruppenr_for_lonnsslip")): result.primary_ou = (cereconf.DEFAULT_INSTITUSJONSNR, extract("fakultetnr_for_lonnsslip"), extract("instituttnr_for_lonnsslip"), extract("gruppenr_for_lonnsslip")) if not (result.get_name(result.NAME_FIRST) and result.get_name(result.NAME_LAST)): raise AssertionError("Missing name for %s" % list(result.iterids())) return result
def _process_person(self, person): db = self.db const = self.const paga_nr = int(person['ansattnr']) logger.info("Processing paga_id=%r", paga_nr) try: birthdate = parse_date(person['fodselsdato']) except ValueError as e: raise SkipPerson("Invalid birth date (%s)" % (e, )) if person['kjonn'] == 'M': gender = const.gender_male else: gender = const.gender_female fnr = person['fnr'] if fnr and fnr[6:11] != '00000': try: fodselsnr.personnr_ok(fnr) except Exception as e: raise SkipPerson("Invalid fnr (%s)" % (e, )) gender_chk = const.gender_male if fodselsnr.er_kvinne(fnr): gender_chk = const.gender_female if gender_chk != gender: raise SkipPerson("Inconsistent gender (gender=%r, ssn=%r)" % (gender, gender_chk)) fnr_date = datetime.date(*(fodselsnr.fodt_dato(fnr))) if not cmp_birthdates(birthdate, fnr_date): raise SkipPerson("Inconsistent birth date (date=%r, ssn=%r)" % (birthdate, fnr_date)) # Passport data national_id_type = person['edag_id_type'] origin_country = person['country'] if not origin_country: origin_country = 'NO' if person['edag_id_nr']: # generate external id on the form: # <2-char national id>-<national_id> national_id_val = '%s-%s' % (origin_country, person['edag_id_nr']) else: national_id_val = None # Abort early if there are no valid identifiers from the source system: if not any(( fnr and fnr[6:11] != '00000', national_id_type == 'passnummer' and national_id_val, )): # TODO: Wouldn't it be enough with ansattnr? raise SkipPerson("No valid identifier (fnr, passnummer)") new_person = Factory.get('Person')(db) identifiers = [(const.externalid_paga_ansattnr, paga_nr), (const.externalid_fodselsnr, fnr)] if national_id_type == 'passnummer' and national_id_val: identifiers.append((const.externalid_pass_number, national_id_val)) for id_type, id_value in identifiers: if _populate_existing(new_person, id_type, id_value): break if not person.get('fornavn', '').strip(): raise SkipPerson("Missing first name") if not person.get('etternavn', '').strip(): raise SkipPerson("Missing last name") new_person.populate(mx.DateTime.DateFrom(birthdate), gender) new_person.affect_names(const.system_paga, const.name_first, const.name_last) new_person.affect_external_id(const.system_paga, const.externalid_fodselsnr, const.externalid_paga_ansattnr, const.externalid_pass_number) new_person.populate_name(const.name_first, person['fornavn']) new_person.populate_name(const.name_last, person['etternavn']) if fnr and fnr[6:11] != '00000': # do not import external id where external_id type is fnr and # fnr[6-11] =='00000' new_person.populate_external_id(const.system_paga, const.externalid_fodselsnr, fnr) if national_id_type == 'passnummer' and national_id_val: new_person.populate_external_id(const.system_paga, const.externalid_pass_number, national_id_val) new_person.populate_external_id(const.system_paga, const.externalid_paga_ansattnr, paga_nr) # If it's a new person, we need to call write_db() to have an entity_id # assigned to it. op = new_person.write_db() if person.get('tittel_personlig'): new_person.add_name_with_language( name_variant=const.personal_title, name_language=const.language_nb, name=person['tittel_personlig']) # work_title is set by _determine_affiliations affiliations = _determine_affiliations(self.db, self.const, new_person, person) new_person.populate_affiliation(const.system_paga) if 'fakultetnr_for_lonnsslip' in person: sted = get_sted(db, person['fakultetnr_for_lonnsslip'], person['instituttnr_for_lonnsslip'], person['gruppenr_for_lonnsslip']) if sted is not None: if sted['addr_street'] is not None: new_person.populate_address(const.system_paga, type=const.address_street, **sted['addr_street']) if sted['addr_post'] is not None: new_person.populate_address(const.system_paga, type=const.address_post, **sted['addr_post']) if 'lokasjon' in person: logger.debug( 'Populating paga_id=%r location address with ' 'source=%s, type=%s, text=%r', paga_nr, const.system_paga, const.address_location, person['lokasjon']) new_person.populate_address(source_system=const.system_paga, type=const.address_location, address_text=person['lokasjon']) else: logger.warning("No location address for paga_id=%r", paga_nr) for k, v in affiliations.items(): ou_id, aff, aff_stat = v new_person.populate_affiliation(const.system_paga, ou_id, int(aff), int(aff_stat)) self.old_affs.discard(k) c_prefs = {} new_person.populate_contact_info(const.system_paga) for c_type, value in determine_contact(const, person): c_type = int(c_type) pref = c_prefs.get(c_type, 0) new_person.populate_contact_info(const.system_paga, c_type, value, pref) c_prefs[c_type] = pref + 1 # # Also add personal/home street address if it exists in the import file # priv_addr = (person.get('adresse'), person.get('postnr'), person.get('poststed')) if any(priv_addr): logger.debug("Setting additional home address: %s %s %s", priv_addr[0], priv_addr[1], priv_addr[2]) # TODO: Address country? Should probably use const.human2constant() # country_code = new_person.get_country_code(origin_country) # TODO: Is there really a guaranteed connection between passport # origin country and home address? new_person.populate_address(const.system_paga, const.address_post_private, priv_addr[0], None, priv_addr[1], priv_addr[2], None) op2 = new_person.write_db() set_person_spreads(self.db, self.const, new_person, person) if op is None and op2 is None: logger.info("EQUAL: No change to person with paga_id=%r", paga_nr) elif op: logger.info("NEW: Created new person with paga_id=%r", paga_nr) else: logger.info("UPDATE: Updated person with paga_id=%r (%s, %s)", paga_nr, op, op2)