def parse_xml_roles(fname):
"""
Parse XML dump of FS roles (roller) and return a mapping structured thus:
map = { K1 : [ S_1, S_2, ... S_k ],
...
Kn : [ S_1, ..., S_kn ], }
... where each K_i is a key falling into one of these four categories:
undakt, undenh, kursakt, evu; and each S_i is a mapping structured thus:
map2 = { 'fodselsdato' : ...,
'personnr' : ..., }
S_i are an attempt to mimic db_rows (output from this function is used
elsewhere where such keys are required).
"""
result = dict()
def gimme_lambda(element, data):
kind = data[roles_xml_parser.target_key]
if len(kind) > 1:
logger.warn("Cannot decide on role kind for: %s", kind)
return
kind = kind[0]
if kind in ("undakt", "undenh"):
key = (data["institusjonsnr"],
data["emnekode"],
data["versjonskode"],
data["terminkode"],
data["arstall"],
data["terminnr"])
if kind == "undakt":
key = key + (data["aktivitetkode"],)
elif kind in ("evu", ):
logger.info("Ignoring roles pertaining to EVU-courses for now")
return
else:
logger.warn("%s%s: Wrong role entry kind: %s; '%s'",
data["fodselsdato"], data["personnr"], kind, data)
return
result.setdefault(key, list()).append(
{ "fodselsdato" : int(data["fodselsdato"]),
"personnr" : int(data["personnr"]), })
roles_xml_parser(fname, gimme_lambda)
for entry in result.keys():
logger.debug("Role-mapping: '%s' => '%s'" % (entry, result[entry]))
return result
def parse_xml_roles(fname):
"""
Parse XML dump of FS roles (roller) and return a mapping structured thus:
map = { K1 : [ S_1, S_2, ... S_k ],
...
Kn : [ S_1, ..., S_kn ], }
... where each K_i is a key falling into one of these four categories:
undakt, undenh, kursakt, evu; and each S_i is a mapping structured thus:
map2 = { 'fodselsdato' : ...,
'personnr' : ..., }
S_i are an attempt to mimic db_rows (output from this function is used
elsewhere where such keys are required).
"""
result = dict()
def gimme_lambda(element, data):
kind = data[roles_xml_parser.target_key]
if len(kind) > 1:
logger.warn("Cannot decide on role kind for: %s", kind)
return
kind = kind[0]
if kind in ("undakt", "undenh"):
key = (data["institusjonsnr"], data["emnekode"],
data["versjonskode"], data["terminkode"], data["arstall"],
data["terminnr"])
if kind == "undakt":
key = key + (data["aktivitetkode"], )
elif kind in ("evu", ):
logger.info("Ignoring roles pertaining to EVU-courses for now")
return
else:
logger.warn("%s%s: Wrong role entry kind: %s; '%s'",
data["fodselsdato"], data["personnr"], kind, data)
return
result.setdefault(key, list()).append({
"fodselsdato":
int(data["fodselsdato"]),
"personnr":
int(data["personnr"]),
})
roles_xml_parser(fname, gimme_lambda)
for entry in result.keys():
logger.debug("Role-mapping: '%s' => '%s'" % (entry, result[entry]))
return result
def main():
init_globals()
# Opprett objekt for "internal:hia.no:fs:{supergroup}"
fs_super = fs_supergroup()
# Gå igjennom alle kjente undervisningsenheter; opprett
# gruppe-objekter for disse.
#
# La fs-supergruppe-objektet ta seg av all logikk rundt hvor mange
# nivåer gruppestrukturen skal ha for undervisningsenhet-grupper,
# etc.
def create_UE_helper(el_name, attrs):
if el_name == 'undenhet':
fs_super.add('undenh', attrs)
logger.info("Leser XML-fil: underv_enhet.xml")
access_FS.underv_enhet_xml_parser(
os.path.join(dump_dir, 'underv_enhet.xml'), create_UE_helper)
# Gå igjennom alle kjente EVU-kurs; opprett gruppeobjekter for disse.
def create_evukurs_helper(el_name, attrs):
if (el_name == "evukurs" and attrs.get("status_aktiv") == 'J'
and attrs.get("status_nettbasert_und") == 'J'):
if (immediate_evu_expire and mx.DateTime.strptime(
attrs.get("dato_til"), "%Y-%m-%d") < mx.DateTime.now()):
logger.debug("Kurs %s-%s ekspirerte",
attrs["etterutdkurskode"],
attrs["kurstidsangivelsekode"])
else:
fs_super.add("evu", attrs)
# fi
# fi
# end create_evukurs_helper
xmlfile = "evu_kursinfo.xml"
logger.info("Leser XML-fil: %s", xmlfile)
access_FS.evukurs_xml_parser(os.path.join(dump_dir, xmlfile),
create_evukurs_helper)
logger.info("Ferdig med %s", xmlfile)
# Meld studenter inn i undervisningsenhet-gruppene
def student_UE_helper(el_name, attrs):
if el_name == 'student':
for undenh in fs_super.list_matches_1('undenh', attrs, 'student'):
undenh.add(attrs)
logger.info("Leser XML-fil: student_undenh.xml")
access_FS.student_undenh_xml_parser(
os.path.join(dump_dir, 'student_undenh.xml'), student_UE_helper)
# Meld EVU-kursdeltakere i de respektive EVU-kursgruppene.
def EVU_deltaker_helper(el_name, attrs):
if el_name == "person" and len(attrs.get("evu")) > 0:
# Dette blir ikke fult så pent -- i merged_persons plasserer man
# informasjonen om EVU-tilknytning i form av underelementer av
# <person>. Dermed må ethvert EVU-underelement (de er samlet i en
# liste av dict'er under nøkkelen "evu" under) "suppleres" med
# fdato/pnr på eieren til det EVU-underelementet.
tmp = {
"fodselsdato": attrs["fodselsdato"],
"personnr": attrs["personnr"],
}
for evuattrs in attrs["evu"]:
evuattrs.update(tmp)
for evukurs in fs_super.list_matches_1("evu", evuattrs,
"kursdeltaker"):
evukurs.add(evuattrs)
# od
# od
# fi
# end create_EVU_participant_helper
xmlfile = "merged_persons.xml"
logger.info("Leser XML-fil: %s", xmlfile)
access_FS.deltaker_xml_parser(os.path.join(dump_dir, xmlfile),
EVU_deltaker_helper)
logger.info("Ferdig med %s", xmlfile)
# Gå igjennom alle kjente studieprogrammer; opprett gruppeobjekter
# for disse.
def create_studieprog_helper(el_name, attrs):
if el_name == 'studprog' and attrs.get('status_utgatt') != 'J':
fs_super.add('studieprogram', attrs)
logger.info("Leser XML-fil: studieprog.xml")
access_FS.studieprog_xml_parser(os.path.join(dump_dir, 'studieprog.xml'),
create_studieprog_helper)
# Meld forelesere og studieledere inn i passende
# undervisningsenhet/EVU-kurs -gruppene
def rolle_helper(el_name, attrs):
if el_name != 'rolle':
return
rolle = attrs['rollekode']
target = attrs[access_FS.roles_xml_parser.target_key]
if len(target) != 1:
return
target = target[0]
if target in ('undenh', 'stprog'):
if rolle == 'FORELESER':
for ue_foreleser in fs_super.list_matches(
'undenh', attrs, 'foreleser'):
ue_foreleser.add(attrs)
elif rolle in ('STUDILEDER', 'STUDKOORD'):
for ue_studieleder in fs_super.list_matches(
'undenh', attrs, 'studieleder'):
ue_studieleder.add(attrs)
for stpr_studieleder in fs_super.list_matches(
'studieprogram', attrs, 'studieleder'):
stpr_studieleder.add(attrs)
# fi
elif target in ('evu', ):
if rolle == 'FORELESER':
# Kan ett element tilhøre flere evukurs?
for evu_foreleser in fs_super.list_matches(
'evu', attrs, "foreleser"):
evu_foreleser.add(attrs)
# od
# fi
# fi
# end rolle_helper
xmlfile = "roles.xml"
logger.info("Leser XML-fil: %s", xmlfile)
access_FS.roles_xml_parser(os.path.join(dump_dir, xmlfile), rolle_helper)
logger.info("Ferdig med %s", xmlfile)
# Finn alle studenter
def student_studieprog_helper(el_name, attrs):
if el_name == 'aktiv':
for stpr in fs_super.list_matches_1('studieprogram', attrs,
'student'):
stpr.add(attrs)
logger.info("Leser XML-fil: person.xml")
access_FS.person_xml_parser(os.path.join(dump_dir, 'person.xml'),
student_studieprog_helper)
logger.info("Ferdig med XML-fil: person.xml")
# Write back all changes to the database
fs_super.sync()
if dryrun:
logger.info("rolling back all changes")
db.rollback()
else:
logger.info("committing all changes")
db.commit()
def main():
init_globals()
# Opprett objekt for "internal:hia.no:fs:{supergroup}"
fs_super = fs_supergroup()
# Gå igjennom alle kjente undervisningsenheter; opprett
# gruppe-objekter for disse.
#
# La fs-supergruppe-objektet ta seg av all logikk rundt hvor mange
# nivåer gruppestrukturen skal ha for undervisningsenhet-grupper,
# etc.
def create_UE_helper(el_name, attrs):
if el_name == 'undenhet':
fs_super.add('undenh', attrs)
logger.info("Leser XML-fil: underv_enhet.xml")
access_FS.underv_enhet_xml_parser(
os.path.join(dump_dir, 'underv_enhet.xml'),
create_UE_helper)
# Gå igjennom alle kjente EVU-kurs; opprett gruppeobjekter for disse.
def create_evukurs_helper(el_name, attrs):
if (el_name == "evukurs" and
attrs.get("status_aktiv") == 'J' and
attrs.get("status_nettbasert_und") == 'J'):
if (immediate_evu_expire and
mx.DateTime.strptime(attrs.get("dato_til"), "%Y-%m-%d") <
mx.DateTime.now()):
logger.debug("Kurs %s-%s ekspirerte",
attrs["etterutdkurskode"],
attrs["kurstidsangivelsekode"])
else:
fs_super.add("evu", attrs)
# fi
# fi
# end create_evukurs_helper
xmlfile = "evu_kursinfo.xml"
logger.info("Leser XML-fil: %s", xmlfile)
access_FS.evukurs_xml_parser(os.path.join(dump_dir, xmlfile),
create_evukurs_helper)
logger.info("Ferdig med %s", xmlfile)
# Meld studenter inn i undervisningsenhet-gruppene
def student_UE_helper(el_name, attrs):
if el_name == 'student':
for undenh in fs_super.list_matches_1('undenh', attrs,
'student'):
undenh.add(attrs)
logger.info("Leser XML-fil: student_undenh.xml")
access_FS.student_undenh_xml_parser(
os.path.join(dump_dir, 'student_undenh.xml'),
student_UE_helper)
# Meld EVU-kursdeltakere i de respektive EVU-kursgruppene.
def EVU_deltaker_helper(el_name, attrs):
if el_name == "person" and len(attrs.get("evu")) > 0:
# Dette blir ikke fult så pent -- i merged_persons plasserer man
# informasjonen om EVU-tilknytning i form av underelementer av
# <person>. Dermed må ethvert EVU-underelement (de er samlet i en
# liste av dict'er under nøkkelen "evu" under) "suppleres" med
# fdato/pnr på eieren til det EVU-underelementet.
tmp = { "fodselsdato" : attrs["fodselsdato"],
"personnr" : attrs["personnr"], }
for evuattrs in attrs["evu"]:
evuattrs.update(tmp)
for evukurs in fs_super.list_matches_1("evu", evuattrs,
"kursdeltaker"):
evukurs.add(evuattrs)
# od
# od
# fi
# end create_EVU_participant_helper
xmlfile = "merged_persons.xml"
logger.info("Leser XML-fil: %s", xmlfile)
access_FS.deltaker_xml_parser(os.path.join(dump_dir, xmlfile),
EVU_deltaker_helper)
logger.info("Ferdig med %s", xmlfile)
# Gå igjennom alle kjente studieprogrammer; opprett gruppeobjekter
# for disse.
def create_studieprog_helper(el_name, attrs):
if el_name == 'studprog' and attrs.get('status_utgatt') != 'J':
fs_super.add('studieprogram', attrs)
logger.info("Leser XML-fil: studieprog.xml")
access_FS.studieprog_xml_parser(
os.path.join(dump_dir, 'studieprog.xml'),
create_studieprog_helper)
# Meld forelesere og studieledere inn i passende
# undervisningsenhet/EVU-kurs -gruppene
def rolle_helper(el_name, attrs):
if el_name != 'rolle':
return
rolle = attrs['rollekode']
target = attrs[access_FS.roles_xml_parser.target_key]
if len(target) != 1:
return
target = target[0]
if target in ('undenh', 'stprog'):
if rolle == 'FORELESER':
for ue_foreleser in fs_super.list_matches('undenh', attrs,
'foreleser'):
ue_foreleser.add(attrs)
elif rolle in ('STUDILEDER', 'STUDKOORD'):
for ue_studieleder in fs_super.list_matches('undenh', attrs,
'studieleder'):
ue_studieleder.add(attrs)
for stpr_studieleder in fs_super.list_matches('studieprogram',
attrs,
'studieleder'):
stpr_studieleder.add(attrs)
# fi
elif target in ('evu',):
if rolle == 'FORELESER':
# Kan ett element tilhøre flere evukurs?
for evu_foreleser in fs_super.list_matches('evu', attrs,
"foreleser"):
evu_foreleser.add(attrs)
# od
# fi
# fi
# end rolle_helper
xmlfile = "roles.xml"
logger.info("Leser XML-fil: %s", xmlfile)
access_FS.roles_xml_parser(os.path.join(dump_dir, xmlfile),
rolle_helper)
logger.info("Ferdig med %s", xmlfile)
# Finn alle studenter
def student_studieprog_helper(el_name, attrs):
if el_name == 'aktiv':
for stpr in fs_super.list_matches_1('studieprogram', attrs,
'student'):
stpr.add(attrs)
logger.info("Leser XML-fil: person.xml")
access_FS.person_xml_parser(
os.path.join(dump_dir, 'person.xml'),
student_studieprog_helper)
logger.info("Ferdig med XML-fil: person.xml")
# Write back all changes to the database
fs_super.sync()
if dryrun:
logger.info("rolling back all changes")
db.rollback()
else:
logger.info("committing all changes")
db.commit()
def collect_roles(role_file, fs_handler):
"""Read role data and build a suitable data structure.
Extract all the interesting roles from role_file, and return a suitable
representation of the roles for group building.
@type role_file: basestring
@param role_file:
File with FS role data. Not all roles are of interest!
"""
logger.debug("Extracting roles from %s", role_file)
role_parser = access_FS.roles_xml_parser
result = dict()
def slurp_role(element_name, attributes):
"""Look at a specific <role>-element"""
if element_name != "rolle":
return
# what *kind* of role is this?
role_kind = attributes[role_parser.target_key]
if len(role_kind) != 1:
# A warning about this has has already been issued
return
role_kind = role_kind[0]
if role_kind not in fs_handler.valid_roles:
logger.debug("Ignoring '%s' role for: %s",
role_kind, repr(attributes))
return
if not timeslot_is_valid(lower(attributes)):
logger.debug("Ignoring '%s' - data too old/in the future: "
"attrs=%s", role_kind, lower(attributes))
return
attrs = fs_handler.fixup_attributes(attributes)
if attrs["rollekode"] not in fs_handler.valid_role_codes:
logger.debug("Ignoring '%s' role, role code %s: attrs=%s",
role_kind, attrs["rollekode"], attrs)
return
logger.debug("Collecting role '%s' with %s",
role_kind, repr(attributes))
group_key = fs_handler.attributes2key(role_kind, attrs)
if group_key is None:
return
fs_handler.register_description(group_key, attrs)
if group_key is None:
logger.warn("Failed to create group key for role=%s/attrs=%s",
role_kind, attrs)
return
if not fs_handler.role_is_exportable(role_kind, attrs):
logger.debug("Ignoring role=%s/attrs=%s (not exportable to LMS)",
role_kind, attrs)
return
fnr = "%06d%05d" % (int(attrs["fodselsdato"]),
int(attrs["personnr"]))
result.setdefault(group_key, set()).add(fnr)
# end slurp_role
access_FS.roles_xml_parser(role_file, slurp_role)
logger.debug("Roles from %s result in %d groups", role_file, len(result))
return result
def collect_roles(role_file, fs_handler):
"""Read role data and build a suitable data structure.
Extract all the interesting roles from role_file, and return a suitable
representation of the roles for group building.
@type role_file: basestring
@param role_file:
File with FS role data. Not all roles are of interest!
"""
logger.debug("Extracting roles from %s", role_file)
role_parser = access_FS.roles_xml_parser
result = dict()
def slurp_role(element_name, attributes):
"""Look at a specific <role>-element"""
if element_name != "rolle":
return
# what *kind* of role is this?
role_kind = attributes[role_parser.target_key]
if len(role_kind) != 1:
# A warning about this has has already been issued
return
role_kind = role_kind[0]
if role_kind not in fs_handler.valid_roles:
logger.debug("Ignoring '%s' role for: %s",
role_kind, repr(attributes))
return
if not timeslot_is_valid(lower(attributes)):
logger.debug("Ignoring '%s' - data too old/in the future: "
"attrs=%s", role_kind, lower(attributes))
return
attrs = fs_handler.fixup_attributes(attributes)
if attrs["rollekode"] not in fs_handler.valid_role_codes:
logger.debug("Ignoring '%s' role, role code %s: attrs=%s",
role_kind, attrs["rollekode"], attrs)
return
logger.debug("Collecting role '%s' with %s",
role_kind, repr(attributes))
group_key = fs_handler.attributes2key(role_kind, attrs)
if group_key is None:
return
fs_handler.register_description(group_key, attrs)
if group_key is None:
logger.warn("Failed to create group key for role=%s/attrs=%s",
role_kind, attrs)
return
if not fs_handler.role_is_exportable(role_kind, attrs):
logger.debug("Ignoring role=%s/attrs=%s (not exportable to LMS)",
role_kind, attrs)
return
fnr = "%06d%05d" % (int(attrs["fodselsdato"]),
int(attrs["personnr"]))
result.setdefault(group_key, set()).add(fnr)
# end slurp_role
access_FS.roles_xml_parser(role_file, slurp_role)
logger.debug("Roles from %s result in %d groups", role_file, len(result))
return result
