def parse_xml_roles(fname): """ Parse XML dump of FS roles (roller) and return a mapping structured thus: map = { K1 : [ S_1, S_2, ... S_k ], ... Kn : [ S_1, ..., S_kn ], } ... where each K_i is a key falling into one of these four categories: undakt, undenh, kursakt, evu; and each S_i is a mapping structured thus: map2 = { 'fodselsdato' : ..., 'personnr' : ..., } S_i are an attempt to mimic db_rows (output from this function is used elsewhere where such keys are required). """ result = dict() def gimme_lambda(element, data): kind = data[roles_xml_parser.target_key] if len(kind) > 1: logger.warn("Cannot decide on role kind for: %s", kind) return kind = kind[0] if kind in ("undakt", "undenh"): key = (data["institusjonsnr"], data["emnekode"], data["versjonskode"], data["terminkode"], data["arstall"], data["terminnr"]) if kind == "undakt": key = key + (data["aktivitetkode"],) elif kind in ("evu", ): logger.info("Ignoring roles pertaining to EVU-courses for now") return else: logger.warn("%s%s: Wrong role entry kind: %s; '%s'", data["fodselsdato"], data["personnr"], kind, data) return result.setdefault(key, list()).append( { "fodselsdato" : int(data["fodselsdato"]), "personnr" : int(data["personnr"]), }) roles_xml_parser(fname, gimme_lambda) for entry in result.keys(): logger.debug("Role-mapping: '%s' => '%s'" % (entry, result[entry])) return result
def parse_xml_roles(fname): """ Parse XML dump of FS roles (roller) and return a mapping structured thus: map = { K1 : [ S_1, S_2, ... S_k ], ... Kn : [ S_1, ..., S_kn ], } ... where each K_i is a key falling into one of these four categories: undakt, undenh, kursakt, evu; and each S_i is a mapping structured thus: map2 = { 'fodselsdato' : ..., 'personnr' : ..., } S_i are an attempt to mimic db_rows (output from this function is used elsewhere where such keys are required). """ result = dict() def gimme_lambda(element, data): kind = data[roles_xml_parser.target_key] if len(kind) > 1: logger.warn("Cannot decide on role kind for: %s", kind) return kind = kind[0] if kind in ("undakt", "undenh"): key = (data["institusjonsnr"], data["emnekode"], data["versjonskode"], data["terminkode"], data["arstall"], data["terminnr"]) if kind == "undakt": key = key + (data["aktivitetkode"], ) elif kind in ("evu", ): logger.info("Ignoring roles pertaining to EVU-courses for now") return else: logger.warn("%s%s: Wrong role entry kind: %s; '%s'", data["fodselsdato"], data["personnr"], kind, data) return result.setdefault(key, list()).append({ "fodselsdato": int(data["fodselsdato"]), "personnr": int(data["personnr"]), }) roles_xml_parser(fname, gimme_lambda) for entry in result.keys(): logger.debug("Role-mapping: '%s' => '%s'" % (entry, result[entry])) return result
def main(): init_globals() # Opprett objekt for "internal:hia.no:fs:{supergroup}" fs_super = fs_supergroup() # Gå igjennom alle kjente undervisningsenheter; opprett # gruppe-objekter for disse. # # La fs-supergruppe-objektet ta seg av all logikk rundt hvor mange # nivåer gruppestrukturen skal ha for undervisningsenhet-grupper, # etc. def create_UE_helper(el_name, attrs): if el_name == 'undenhet': fs_super.add('undenh', attrs) logger.info("Leser XML-fil: underv_enhet.xml") access_FS.underv_enhet_xml_parser( os.path.join(dump_dir, 'underv_enhet.xml'), create_UE_helper) # Gå igjennom alle kjente EVU-kurs; opprett gruppeobjekter for disse. def create_evukurs_helper(el_name, attrs): if (el_name == "evukurs" and attrs.get("status_aktiv") == 'J' and attrs.get("status_nettbasert_und") == 'J'): if (immediate_evu_expire and mx.DateTime.strptime( attrs.get("dato_til"), "%Y-%m-%d") < mx.DateTime.now()): logger.debug("Kurs %s-%s ekspirerte", attrs["etterutdkurskode"], attrs["kurstidsangivelsekode"]) else: fs_super.add("evu", attrs) # fi # fi # end create_evukurs_helper xmlfile = "evu_kursinfo.xml" logger.info("Leser XML-fil: %s", xmlfile) access_FS.evukurs_xml_parser(os.path.join(dump_dir, xmlfile), create_evukurs_helper) logger.info("Ferdig med %s", xmlfile) # Meld studenter inn i undervisningsenhet-gruppene def student_UE_helper(el_name, attrs): if el_name == 'student': for undenh in fs_super.list_matches_1('undenh', attrs, 'student'): undenh.add(attrs) logger.info("Leser XML-fil: student_undenh.xml") access_FS.student_undenh_xml_parser( os.path.join(dump_dir, 'student_undenh.xml'), student_UE_helper) # Meld EVU-kursdeltakere i de respektive EVU-kursgruppene. def EVU_deltaker_helper(el_name, attrs): if el_name == "person" and len(attrs.get("evu")) > 0: # Dette blir ikke fult så pent -- i merged_persons plasserer man # informasjonen om EVU-tilknytning i form av underelementer av # <person>. Dermed må ethvert EVU-underelement (de er samlet i en # liste av dict'er under nøkkelen "evu" under) "suppleres" med # fdato/pnr på eieren til det EVU-underelementet. tmp = { "fodselsdato": attrs["fodselsdato"], "personnr": attrs["personnr"], } for evuattrs in attrs["evu"]: evuattrs.update(tmp) for evukurs in fs_super.list_matches_1("evu", evuattrs, "kursdeltaker"): evukurs.add(evuattrs) # od # od # fi # end create_EVU_participant_helper xmlfile = "merged_persons.xml" logger.info("Leser XML-fil: %s", xmlfile) access_FS.deltaker_xml_parser(os.path.join(dump_dir, xmlfile), EVU_deltaker_helper) logger.info("Ferdig med %s", xmlfile) # Gå igjennom alle kjente studieprogrammer; opprett gruppeobjekter # for disse. def create_studieprog_helper(el_name, attrs): if el_name == 'studprog' and attrs.get('status_utgatt') != 'J': fs_super.add('studieprogram', attrs) logger.info("Leser XML-fil: studieprog.xml") access_FS.studieprog_xml_parser(os.path.join(dump_dir, 'studieprog.xml'), create_studieprog_helper) # Meld forelesere og studieledere inn i passende # undervisningsenhet/EVU-kurs -gruppene def rolle_helper(el_name, attrs): if el_name != 'rolle': return rolle = attrs['rollekode'] target = attrs[access_FS.roles_xml_parser.target_key] if len(target) != 1: return target = target[0] if target in ('undenh', 'stprog'): if rolle == 'FORELESER': for ue_foreleser in fs_super.list_matches( 'undenh', attrs, 'foreleser'): ue_foreleser.add(attrs) elif rolle in ('STUDILEDER', 'STUDKOORD'): for ue_studieleder in fs_super.list_matches( 'undenh', attrs, 'studieleder'): ue_studieleder.add(attrs) for stpr_studieleder in fs_super.list_matches( 'studieprogram', attrs, 'studieleder'): stpr_studieleder.add(attrs) # fi elif target in ('evu', ): if rolle == 'FORELESER': # Kan ett element tilhøre flere evukurs? for evu_foreleser in fs_super.list_matches( 'evu', attrs, "foreleser"): evu_foreleser.add(attrs) # od # fi # fi # end rolle_helper xmlfile = "roles.xml" logger.info("Leser XML-fil: %s", xmlfile) access_FS.roles_xml_parser(os.path.join(dump_dir, xmlfile), rolle_helper) logger.info("Ferdig med %s", xmlfile) # Finn alle studenter def student_studieprog_helper(el_name, attrs): if el_name == 'aktiv': for stpr in fs_super.list_matches_1('studieprogram', attrs, 'student'): stpr.add(attrs) logger.info("Leser XML-fil: person.xml") access_FS.person_xml_parser(os.path.join(dump_dir, 'person.xml'), student_studieprog_helper) logger.info("Ferdig med XML-fil: person.xml") # Write back all changes to the database fs_super.sync() if dryrun: logger.info("rolling back all changes") db.rollback() else: logger.info("committing all changes") db.commit()
def main(): init_globals() # Opprett objekt for "internal:hia.no:fs:{supergroup}" fs_super = fs_supergroup() # Gå igjennom alle kjente undervisningsenheter; opprett # gruppe-objekter for disse. # # La fs-supergruppe-objektet ta seg av all logikk rundt hvor mange # nivåer gruppestrukturen skal ha for undervisningsenhet-grupper, # etc. def create_UE_helper(el_name, attrs): if el_name == 'undenhet': fs_super.add('undenh', attrs) logger.info("Leser XML-fil: underv_enhet.xml") access_FS.underv_enhet_xml_parser( os.path.join(dump_dir, 'underv_enhet.xml'), create_UE_helper) # Gå igjennom alle kjente EVU-kurs; opprett gruppeobjekter for disse. def create_evukurs_helper(el_name, attrs): if (el_name == "evukurs" and attrs.get("status_aktiv") == 'J' and attrs.get("status_nettbasert_und") == 'J'): if (immediate_evu_expire and mx.DateTime.strptime(attrs.get("dato_til"), "%Y-%m-%d") < mx.DateTime.now()): logger.debug("Kurs %s-%s ekspirerte", attrs["etterutdkurskode"], attrs["kurstidsangivelsekode"]) else: fs_super.add("evu", attrs) # fi # fi # end create_evukurs_helper xmlfile = "evu_kursinfo.xml" logger.info("Leser XML-fil: %s", xmlfile) access_FS.evukurs_xml_parser(os.path.join(dump_dir, xmlfile), create_evukurs_helper) logger.info("Ferdig med %s", xmlfile) # Meld studenter inn i undervisningsenhet-gruppene def student_UE_helper(el_name, attrs): if el_name == 'student': for undenh in fs_super.list_matches_1('undenh', attrs, 'student'): undenh.add(attrs) logger.info("Leser XML-fil: student_undenh.xml") access_FS.student_undenh_xml_parser( os.path.join(dump_dir, 'student_undenh.xml'), student_UE_helper) # Meld EVU-kursdeltakere i de respektive EVU-kursgruppene. def EVU_deltaker_helper(el_name, attrs): if el_name == "person" and len(attrs.get("evu")) > 0: # Dette blir ikke fult så pent -- i merged_persons plasserer man # informasjonen om EVU-tilknytning i form av underelementer av # <person>. Dermed må ethvert EVU-underelement (de er samlet i en # liste av dict'er under nøkkelen "evu" under) "suppleres" med # fdato/pnr på eieren til det EVU-underelementet. tmp = { "fodselsdato" : attrs["fodselsdato"], "personnr" : attrs["personnr"], } for evuattrs in attrs["evu"]: evuattrs.update(tmp) for evukurs in fs_super.list_matches_1("evu", evuattrs, "kursdeltaker"): evukurs.add(evuattrs) # od # od # fi # end create_EVU_participant_helper xmlfile = "merged_persons.xml" logger.info("Leser XML-fil: %s", xmlfile) access_FS.deltaker_xml_parser(os.path.join(dump_dir, xmlfile), EVU_deltaker_helper) logger.info("Ferdig med %s", xmlfile) # Gå igjennom alle kjente studieprogrammer; opprett gruppeobjekter # for disse. def create_studieprog_helper(el_name, attrs): if el_name == 'studprog' and attrs.get('status_utgatt') != 'J': fs_super.add('studieprogram', attrs) logger.info("Leser XML-fil: studieprog.xml") access_FS.studieprog_xml_parser( os.path.join(dump_dir, 'studieprog.xml'), create_studieprog_helper) # Meld forelesere og studieledere inn i passende # undervisningsenhet/EVU-kurs -gruppene def rolle_helper(el_name, attrs): if el_name != 'rolle': return rolle = attrs['rollekode'] target = attrs[access_FS.roles_xml_parser.target_key] if len(target) != 1: return target = target[0] if target in ('undenh', 'stprog'): if rolle == 'FORELESER': for ue_foreleser in fs_super.list_matches('undenh', attrs, 'foreleser'): ue_foreleser.add(attrs) elif rolle in ('STUDILEDER', 'STUDKOORD'): for ue_studieleder in fs_super.list_matches('undenh', attrs, 'studieleder'): ue_studieleder.add(attrs) for stpr_studieleder in fs_super.list_matches('studieprogram', attrs, 'studieleder'): stpr_studieleder.add(attrs) # fi elif target in ('evu',): if rolle == 'FORELESER': # Kan ett element tilhøre flere evukurs? for evu_foreleser in fs_super.list_matches('evu', attrs, "foreleser"): evu_foreleser.add(attrs) # od # fi # fi # end rolle_helper xmlfile = "roles.xml" logger.info("Leser XML-fil: %s", xmlfile) access_FS.roles_xml_parser(os.path.join(dump_dir, xmlfile), rolle_helper) logger.info("Ferdig med %s", xmlfile) # Finn alle studenter def student_studieprog_helper(el_name, attrs): if el_name == 'aktiv': for stpr in fs_super.list_matches_1('studieprogram', attrs, 'student'): stpr.add(attrs) logger.info("Leser XML-fil: person.xml") access_FS.person_xml_parser( os.path.join(dump_dir, 'person.xml'), student_studieprog_helper) logger.info("Ferdig med XML-fil: person.xml") # Write back all changes to the database fs_super.sync() if dryrun: logger.info("rolling back all changes") db.rollback() else: logger.info("committing all changes") db.commit()
def collect_roles(role_file, fs_handler): """Read role data and build a suitable data structure. Extract all the interesting roles from role_file, and return a suitable representation of the roles for group building. @type role_file: basestring @param role_file: File with FS role data. Not all roles are of interest! """ logger.debug("Extracting roles from %s", role_file) role_parser = access_FS.roles_xml_parser result = dict() def slurp_role(element_name, attributes): """Look at a specific <role>-element""" if element_name != "rolle": return # what *kind* of role is this? role_kind = attributes[role_parser.target_key] if len(role_kind) != 1: # A warning about this has has already been issued return role_kind = role_kind[0] if role_kind not in fs_handler.valid_roles: logger.debug("Ignoring '%s' role for: %s", role_kind, repr(attributes)) return if not timeslot_is_valid(lower(attributes)): logger.debug("Ignoring '%s' - data too old/in the future: " "attrs=%s", role_kind, lower(attributes)) return attrs = fs_handler.fixup_attributes(attributes) if attrs["rollekode"] not in fs_handler.valid_role_codes: logger.debug("Ignoring '%s' role, role code %s: attrs=%s", role_kind, attrs["rollekode"], attrs) return logger.debug("Collecting role '%s' with %s", role_kind, repr(attributes)) group_key = fs_handler.attributes2key(role_kind, attrs) if group_key is None: return fs_handler.register_description(group_key, attrs) if group_key is None: logger.warn("Failed to create group key for role=%s/attrs=%s", role_kind, attrs) return if not fs_handler.role_is_exportable(role_kind, attrs): logger.debug("Ignoring role=%s/attrs=%s (not exportable to LMS)", role_kind, attrs) return fnr = "%06d%05d" % (int(attrs["fodselsdato"]), int(attrs["personnr"])) result.setdefault(group_key, set()).add(fnr) # end slurp_role access_FS.roles_xml_parser(role_file, slurp_role) logger.debug("Roles from %s result in %d groups", role_file, len(result)) return result
def collect_roles(role_file, fs_handler): """Read role data and build a suitable data structure. Extract all the interesting roles from role_file, and return a suitable representation of the roles for group building. @type role_file: basestring @param role_file: File with FS role data. Not all roles are of interest! """ logger.debug("Extracting roles from %s", role_file) role_parser = access_FS.roles_xml_parser result = dict() def slurp_role(element_name, attributes): """Look at a specific <role>-element""" if element_name != "rolle": return # what *kind* of role is this? role_kind = attributes[role_parser.target_key] if len(role_kind) != 1: # A warning about this has has already been issued return role_kind = role_kind[0] if role_kind not in fs_handler.valid_roles: logger.debug("Ignoring '%s' role for: %s", role_kind, repr(attributes)) return if not timeslot_is_valid(lower(attributes)): logger.debug("Ignoring '%s' - data too old/in the future: " "attrs=%s", role_kind, lower(attributes)) return attrs = fs_handler.fixup_attributes(attributes) if attrs["rollekode"] not in fs_handler.valid_role_codes: logger.debug("Ignoring '%s' role, role code %s: attrs=%s", role_kind, attrs["rollekode"], attrs) return logger.debug("Collecting role '%s' with %s", role_kind, repr(attributes)) group_key = fs_handler.attributes2key(role_kind, attrs) if group_key is None: return fs_handler.register_description(group_key, attrs) if group_key is None: logger.warn("Failed to create group key for role=%s/attrs=%s", role_kind, attrs) return if not fs_handler.role_is_exportable(role_kind, attrs): logger.debug("Ignoring role=%s/attrs=%s (not exportable to LMS)", role_kind, attrs) return fnr = "%06d%05d" % (int(attrs["fodselsdato"]), int(attrs["personnr"])) result.setdefault(group_key, set()).add(fnr) # end slurp_role access_FS.roles_xml_parser(role_file, slurp_role) logger.debug("Roles from %s result in %d groups", role_file, len(result)) return result