def output_people(writer, sysname, personfile, ou_cache):
"""Output information about all interesting people.
A person is interesting for FRIDA, if it has active employments
(tilsetting) or active guest records (gjest). A record is considered
active, if it has a start date in the past (compared to the moment when
the script is run) and the end date is either unknown or in the future.
"""
logger.info("extracting people from %s", personfile)
phd_students = cache_phd_students()
logger.info("cached PhD students (%d people)", len(phd_students))
writer.startElement("personer")
parser = system2parser(sysname)(personfile, logger, False)
for person in parser.iter_person():
if not should_export_person(person):
continue
# NB! phd_students is updated destructively
output_person(writer, person, phd_students, ou_cache)
# process whatever is left of phd-students
output_phd_students(writer, sysname, phd_students, ou_cache)
writer.endElement("personer")
def main():
opts, args = getopt.getopt(sys.argv[1:], "s:o:f:",
["source-spec=",
"output-file=",
"faculty="])
filename = None
sources = list()
faculty = None
for option, value in opts:
if option in ("-s", "--source-spec",):
sysname, personfile = value.split(":")
sources.append((sysname, personfile))
elif option in ("-o", "--output-file",):
filename = value
elif option in ("-f", "--faculty",):
faculty = int(value)
assert 0 < faculty < 100, "Faculty is a 2-digit number"
assert filename, "Need an output file name"
assert faculty, "Need a faculty to operate on"
logger.debug("sources is %s", sources)
stream = file(filename, "w")
for system_name, filename in sources:
# Locate the appropriate Cerebrum constant
source_system = getattr(constants, system_name)
parser = system2parser(system_name)
fetch_data(parser(filename, logger, False),
source_system,
faculty,
stream)
stream.close()
def output_people(writer, sysname, personfile, ou_cache):
"""Output information about all interesting people.
A person is interesting for FRIDA, if it has active employments
(tilsetting) or active guest records (gjest). A record is considered
active, if it has a start date in the past (compared to the moment when
the script is run) and the end date is either unknown or in the future.
"""
logger.info("extracting people from %s", personfile)
phd_students = cache_phd_students()
logger.info("cached PhD students (%d people)", len(phd_students))
writer.startElement("personer")
parser = system2parser(sysname)(personfile, logger, False)
for person in parser.iter_person():
if not should_export_person(person):
continue
# NB! phd_students is updated destructively
output_person(writer, person, phd_students, ou_cache)
# process whatever is left of phd-students
output_phd_students(writer, sysname, phd_students, ou_cache)
writer.endElement("personer")
def generate_output(stream, do_employees, sysname, person_file):
"""
Create dump for UA
"""
db_person = Factory.get("Person")(db)
ou = Factory.get("OU")(db)
const = Factory.get("Constants")(db)
if do_employees:
logger.info("Extracting employee info from %s", person_file)
source_system = getattr(const, sysname)
parser = system2parser(sysname)(person_file, logger, False)
# Go through all persons in person_info_file
for xml_person in parser.iter_person():
try:
fnr = xml_person.get_id(xml_person.NO_SSN)
if fnr is None:
sapnr = xml_person.get_id(xml_person.SAP_NR)
logger.warn('Employee %s has no fnr', sapnr)
continue
db_person.find_by_external_id(const.externalid_fodselsnr, fnr,
source_system=source_system)
except Errors.NotFoundError:
logger.warn("Couldn't find person with fnr %s in db", fnr)
continue
process_employee(db_person, ou, const, xml_person, fnr, stream)
db_person.clear()
def get_bet_fritak_utv_data(sysname, person_file):
"""Finn pc-stuevakter/gruppelærere mfl. ved å parse SAP-dumpen."""
ret = {}
sap_ansattnr2pid = {}
roller_fritak = cereconf.PQUOTA_ROLLER_FRITAK
for p in person.search_external_ids(source_system=const.system_sap,
id_type=const.externalid_sap_ansattnr,
fetchall=False):
sap_ansattnr2pid[p['external_id']] = int(p['entity_id'])
# Parse person file
parser = system2parser(sysname)(person_file, logger, False)
for pers in parser.iter_person():
sap_nr = pers.get_id(pers.SAP_NR)
for employment in pers.iteremployment():
if (
employment.is_guest() and employment.is_active() and
employment.code in roller_fritak
):
if sap_nr not in sap_ansattnr2pid:
logger.warn("Unknown person (%r) from %r", sap_nr, sysname)
continue
ret[sap_ansattnr2pid[sap_nr]] = True
return ret
def main():
global dryrun
global logger
logger = Factory.get_logger("cronjob")
options, rest = getopt.getopt(sys.argv[1:], "do:", ["dryrun", "ou-file="])
dryrun = False
filename = None
for option, value in options:
if option in ("-d", "--dryrun"):
dryrun = True
elif option in (
"-o",
"--ou-file",
):
source_system, filename = value.split(":", 1)
if not filename:
logger.error("Missing OU input file")
sys.exit(1)
db = Factory.get("Database")()
db.cl_init(change_program="import_SAP")
parser = system2parser(source_system)
process_OUs(db, parser(filename, logger))
def main():
global dryrun
global logger
logger = Factory.get_logger("cronjob")
options, rest = getopt.getopt(sys.argv[1:],
"do:",
["dryrun", "ou-file="])
dryrun = False
filename = None
for option, value in options:
if option in ("-d", "--dryrun"):
dryrun = True
elif option in ("-o", "--ou-file",):
source_system, filename = value.split(":", 1)
if not filename:
logger.error("Missing OU input file")
sys.exit(1)
db = Factory.get("Database")()
db.cl_init(change_program="import_SAP")
parser = system2parser(source_system)
process_OUs(db, parser(filename, logger))
def output_ous(writer, sysname, oufile):
"""
Run through all OUs and publish the interesting ones.
An OU is interesting to FRIDA, if:
- the OU is supposed to be published (marked as such in the data source)
- it has been less than a year since the OU has been terminated.
"""
# First we build an ID cache.
ou_cache = {}
parser = system2parser(sysname)(oufile, logger, False)
for ou in parser.iter_ou():
sko = ou.get_id(ou.NO_SKO, None)
if sko:
ou_cache[sko] = ou
logger.info("Cached info on %d OUs from %s", len(ou_cache), oufile)
result = set()
writer.startElement("organisasjon")
for sko in ou_cache:
publishable_sko = find_publishable_sko(sko, ou_cache)
if not publishable_sko:
logger.info("Cannot find publishable sko starting from %s", sko)
continue
if publishable_sko in result:
continue
output_ou(writer, ou_cache[publishable_sko])
result.add(publishable_sko)
writer.endElement("organisasjon")
return ou_cache
def output_ous(writer, sysname, oufile):
"""
Run through all OUs and publish the interesting ones.
An OU is interesting to FRIDA, if:
- the OU is supposed to be published (marked as such in the data source)
- it has been less than a year since the OU has been terminated.
"""
# First we build an ID cache.
ou_cache = {}
parser = system2parser(sysname)(oufile, logger, False)
for ou in parser.iter_ou():
sko = ou.get_id(ou.NO_SKO, None)
if sko:
ou_cache[sko] = ou
logger.info("Cached info on %d OUs from %s", len(ou_cache), oufile)
result = set()
writer.startElement("organisasjon")
for sko in ou_cache:
publishable_sko = find_publishable_sko(sko, ou_cache)
if not publishable_sko:
logger.info("Cannot find publishable sko starting from %s", sko)
continue
if publishable_sko in result:
continue
output_ou(writer, ou_cache[publishable_sko])
result.add(publishable_sko)
writer.endElement("organisasjon")
return ou_cache
def main(args=None):
args = make_parser().parse_args(args)
logger.info("Start {0}".format(__file__))
LDIFutils.needs_base64 = args.needs_base64
xml_parser = system2parser('system_sap')(args.input_file, logger)
show_ou = OUSelector('ORG_OU', cereconf.OU_USAGE_SPREAD)
get_ous = OrgTree(xml_parser.iter_ou(), show_ou)
use_lang = LanguageSelector(cereconf.LDAP['pref_languages'])
aff_selector = AffSelector(
cereconf.LDAP_PERSON['affiliation_selector'])
stats = {
'seen': 0,
'excluded': 0,
'included': 0
}
with atomic_or_stdout(args.output_file) as output:
for person in xml_parser.iter_person():
stats['seen'] += 1
partial_affs = set()
for emp in iterate_employments(person, aff_selector):
aff = format_scoped_aff(emp, get_ous)
titles = [format_title(t)
for t in iterate_employment_titles(emp)
if use_lang(t.language)]
partial_affs.add('{0}#{1}'.format(aff, ';'.join(titles)))
if len(partial_affs) < 2:
# We want at least two unique employments to output person
stats['excluded'] += 1
continue
try:
identifier = get_identifier(person)
except ValueError:
logger.warn("Missing NIN: {0}".format(str(person)))
stats['excluded'] += 1
continue
stats['included'] += 1
output.write(
LDIFutils.entry_string(
identifier,
{'uioPersonPartialEmployment': set(partial_affs)},
add_rdn=False))
logger.info("persons"
" considered: {0[seen]:d},"
" included: {0[included]:d},"
" excluded: {0[excluded]:d}".format(stats))
logger.info("Done {0}".format(__file__))
def generate_output(stream, do_employees, do_students, sysname, person_file):
"""
Create dump for UA
"""
#
# We will do this in two steps -- first all the employees, then all the
# the students.
#
db_person = Factory.get("Person")(db)
ou = Factory.get("OU")(db)
const = Factory.get("Constants")(db)
if do_employees:
logger.info("Extracting employee info from %s", person_file)
source_system = getattr(const, sysname)
parser = system2parser(sysname)(person_file, logger, False)
# Go through all persons in person_info_file
for xml_person in parser.iter_person():
try:
fnr = xml_person.get_id(xml_person.NO_SSN)
db_person.find_by_external_id(const.externalid_fodselsnr,
fnr,
source_system=source_system)
except Errors.NotFoundError:
logger.warn("Couldn't find person with fnr %s in db", fnr)
continue
process_employee(db_person, ou, const, xml_person, fnr, stream)
db_person.clear()
if do_students:
logger.info("Processing all student affiliations")
for db_row in person.list_affiliations(source_system=const.system_fs,
affiliation=int(
const.affiliation_student)):
person_id = db_row.fields.person_id
try:
person.clear()
person.find(person_id)
except Cerebrum.Errors.NotFoundError:
logger.exception(
"Aiee! No person with %s exists, although "
"list_affiliations() returned it", person_id)
continue
process_student(person, ou, const, db_row, stream)
def build_tree_from_files(sources):
"""Scan sources and build the OU tree structure for that file."""
const = Factory.get("Constants")()
nodes = dict()
for (source_system, filename) in sources:
source = const.human2constant(source_system, const.AuthoritativeSystem)
parser = system2parser(str(source)) or system2parser(source_system)
if not parser:
raise RuntimeError("Cannot determine source system from %s" %
str(source_system))
parser = parser(filename, None)
for xml_ou in parser.iter_ou():
node = build_node_from_file(xml_ou)
nodes[node.node_id] = node
rs = create_root_set(nodes)
if set_has_cycles(nodes):
return set()
return rs
def generate_output(stream, do_employees, do_students, sysname, person_file):
"""
Create dump for UA
"""
#
# We will do this in two steps -- first all the employees, then all the
# the students.
#
db_person = Factory.get("Person")(db)
ou = Factory.get("OU")(db)
const = Factory.get("Constants")(db)
if do_employees:
logger.info("Extracting employee info from %s", person_file)
source_system = getattr(const, sysname)
parser = system2parser(sysname)(person_file, logger, False)
# Go through all persons in person_info_file
for xml_person in parser.iter_person():
try:
fnr = xml_person.get_id(xml_person.NO_SSN)
db_person.find_by_external_id(const.externalid_fodselsnr, fnr, source_system=source_system)
except Errors.NotFoundError:
logger.warn("Couldn't find person with fnr %s in db", fnr)
continue
process_employee(db_person, ou, const, xml_person, fnr, stream)
db_person.clear()
if do_students:
logger.info("Processing all student affiliations")
for db_row in person.list_affiliations(
source_system=const.system_fs, affiliation=int(const.affiliation_student)
):
person_id = db_row.fields.person_id
try:
person.clear()
person.find(person_id)
except Cerebrum.Errors.NotFoundError:
logger.exception(
"Aiee! No person with %s exists, although " "list_affiliations() returned it", person_id
)
continue
process_student(person, ou, const, db_row, stream)
def output_people(writer, sysname, personfile, ou_cache):
"""
Output information about all interesting people.
A person is interesting for Cristin if it has active employments
(tilsetting) or active guest records (gjest). A record is considered
active if it has a start date in the past (compared to the moment when
the script is run) and the end date is either unknown or in the future.
"""
logger.info("Extracting people from %s", personfile)
writer.startElement("personer")
parser = system2parser(sysname)(personfile, logger, fetchall=False)
for person in parser.iter_person(filter_out_sko_0000=False,
require_ou_for_assignments=False):
if not should_export_person(person):
continue
output_person(writer, person, ou_cache)
writer.endElement("personer")
def output_people(writer, sysname, personfile, ou_cache):
"""
Output information about all interesting people.
A person is interesting for Cristin if it has active employments
(tilsetting) or active guest records (gjest). A record is considered
active if it has a start date in the past (compared to the moment when
the script is run) and the end date is either unknown or in the future.
"""
logger.info("Extracting people from %s", personfile)
writer.startElement("personer")
parser = system2parser(sysname)(personfile, logger, fetchall=False)
for person in parser.iter_person(filter_out_sko_0000=False,
require_ou_for_assignments=False):
if not should_export_person(person):
continue
output_person(writer, person, ou_cache)
writer.endElement("personer")
def output_data(sysname, pfile):
"""Scan through pfile with a parser derived from sysname."""
parser = system2parser(sysname)(pfile, logger, False)
for person in parser.iter_person():
phones = person.get_contact(DataContact.CONTACT_PHONE)
if not phones:
continue
# fi
vanlig, gjest, bilag = split_emps(person.iteremployment())
if not (vanlig or gjest or bilag):
continue
# fi
vanlig = ",".join(vanlig)
gjest = ",".join(gjest)
bilag = ",".join(bilag)
phones.sort(lambda x, y: cmp(x.priority, y.priority))
print "%s@%s;%s;%s" % (",".join([x.value for x in phones]),
vanlig, gjest, bilag)
def get_bet_fritak_utv_data(sysname, person_file):
"""Finn pc-stuevakter/gruppelærere mfl. ved å parse SAP-dumpen."""
ret = {}
sap_ansattnr2pid = {}
roller_fritak = cereconf.PQUOTA_ROLLER_FRITAK
for p in pe.search_external_ids(source_system=co.system_sap,
id_type=co.externalid_sap_ansattnr,
fetchall=False):
sap_ansattnr2pid[p['external_id']] = int(p['entity_id'])
# Parse person file
parser = system2parser(sysname)(person_file, logger, False)
for pers in parser.iter_person():
sap_nr = pers.get_id(pers.SAP_NR)
for employment in pers.iteremployment():
if (employment.is_guest() and employment.is_active()
and employment.code in roller_fritak):
if sap_nr not in sap_ansattnr2pid:
logger.warn("Unknown person (%r) from %r", sap_nr, sysname)
continue
ret[sap_ansattnr2pid[sap_nr]] = True
return ret
def build_employee_cache(db, sysname, filename):
"""Build a mapping of primary account names for employees to their
employment status.
Employment status in this case is a pair of booleans, that tell whether
the person with that primary account has tilsettinger and bilag that we
need.
:Parameters:
db : a Database instance
DB connection to Cerebrum.
sysname : basestring
Name of the authoritative system whence the data comes
filename : basestring
XML file name (source file)
"""
logger.debug("Building employee cache")
# Cache *all* primary accounts. This helps us bind a primary account to an
# fnr in the XML data.
db_person = Factory.get("Person")(db)
const = Factory.get("Constants")(db)
logger.debug("Fetching all fnr->account mappings...")
fnr2uname = db_person.getdict_external_id2primary_account(
const.externalid_fodselsnr)
logger.debug("... done (%d mappings)", len(fnr2uname))
logger.debug("Fetching all passport-nr->account mappings...")
pnr2uname = db_person.getdict_external_id2primary_account(
const.externalid_pass_number)
logger.debug("... done (%d mappings)", len(pnr2uname))
# Build mapping for the employees
parser = system2parser(sysname)(filename, logger, False)
# mapping from uname to employment status
employee_cache = dict()
for xmlperson in parser.iter_person():
fnr = xmlperson.get_id(xmlperson.NO_SSN)
passport_nr = xmlperson.get_id(xmlperson.PASSNR)
if not fnr and not passport_nr:
logger.debug("Person %s has no fnr or passport-nr in XML source",
list(xmlperson.iterids()))
continue
# Everyone with bilag more recent than 180 days old is eligible
bilag = filter(lambda x: ((not x.end) or
(x.end >= (Date(*time.localtime()[:3]) -
DateTimeDeltaFromDays(180)))) and
x.kind == x.BILAG,
xmlperson.iteremployment())
# Add to cache, if found in Cerebrum either by fnr or passport-nr.
# each entry is a pair, telling whether the person has active
# tilsetting and bilag (in that order). We do not need to know *what*
# they are, only that they exist.
if fnr in fnr2uname:
employee_cache[fnr2uname[fnr]] = (xmlperson.
has_active_employments(),
bool(bilag))
elif passport_nr in pnr2uname:
employee_cache[pnr2uname[passport_nr]] = (xmlperson.
has_active_employments(),
bool(bilag))
else:
logger.debug("Cerebrum failed to find primary account for person "
"with fnr: %s, passport-nr: %s.", fnr, passport_nr)
# IVR 2007-07-13 FIXME: Is this actually useful?
del fnr2uname
del pnr2uname
logger.debug("employee_cache has %d uname->employment status mappings",
len(employee_cache))
return employee_cache
"shown online"),
"system_sap": ("SAP-elektroniske-reservasjoner",
"Internal group for people from SAP which will not be "
"shown online")
}
logger.debug("sources is %s", sources)
# Load current automatic traits (AFFILIATE_TRAITS)
if include_del:
cerebrum_traits = load_old_traits()
for system_name, filename in sources:
# Locate the appropriate Cerebrum constant
source_system = getattr(const, system_name)
parser = system2parser(system_name)
# Locate the proper reservation group
group, group_members = locate_and_build(system2group[system_name])
# Load old affiliations
cerebrum_affs = set()
if include_del:
cerebrum_affs = load_old_affiliations(source_system)
person2external = load_old_person2external(source_system)
# Read in the file, and register the information in cerebrum
if filename is not None:
parse_data(parser(filename, logger, False), source_system, group,
group_members, gen_groups, include_del and cerebrum_affs
or set(), include_del and cerebrum_traits or dict(),
def import_org_units(sources, target_system, cer_ou_tab):
"""Scan the sources and import all the OUs into Cerebrum.
:Parameters:
sources : sequence
Sequence of pairs (system_name, filename), where system_name is used
to describe the parser to use, and filename is the XML file with data.
target_system : basestring
Describes the authoritative system which is populated from sources.
cer_ou_tab : dictionary
ou_id -> sko (basestring) mapping, containing the OUs present in
Cerebrum at the start of this script. This is used to delete obsolete
OUs from Cerebrum.
"""
ou = OU_class(db)
# These are used to help build OU structure information
stedkode2ou = dict()
org_units = dict()
existing_ou_mappings = dict()
source_system = getattr(co, target_system)
perspective = source2perspective[source_system]
for system, filename in sources:
logger.debug("Processing %s data from %s", system, filename)
db_writer = XML2Cerebrum(db, source_system, logger, def_kat_merke)
parser = system2parser(system)(filename, logger, False)
for xmlou in parser.iter_ou():
formatted_sko = format_sko(xmlou)
if not formatted_sko:
logger.error("Missing sko for OU %s (names: %s). Skipped!" %
(list(xmlou.iterids()),
map(lambda (x, y): str(x) + ": " + '; '.join(map(str, y)),
xmlou.iternames())))
continue
if (xmlou.start_date and xmlou.start_date > DateTime.now()):
logger.info("OU %s is not active yet and will therefore be "
"ignored for the time being.", formatted_sko)
continue
if (xmlou.end_date and xmlou.end_date < DateTime.now()):
logger.info("OU %s is expired and some of its information "
"will no longer be maintained", formatted_sko)
else:
org_units[formatted_sko] = xmlou
if verbose:
logger.debug("Processing %s '%s'" %
(formatted_sko,
xmlou.get_name_with_lang(xmlou.NAME_SHORT,
"no", "nb", "nn", "en")))
args = (xmlou, None)
if clean_obsolete_ous:
args = (xmlou, cer_ou_tab)
status, ou_id = db_writer.store_ou(*args)
if verbose:
logger.debug("**** %s ****", status)
# Not sure why this casting to int is required for PostgreSQL
stedkode2ou[formatted_sko] = int(ou_id)
db.commit()
# Once we've registered all OUs, build and register parent information
for node in ou.get_structure_mappings(perspective):
existing_ou_mappings[int(node["ou_id"])] = node["parent_id"]
# Now populate the entire ou_structure. Note that expired OUs will not
# be processed here.
logger.info("Populate ou_structure")
for stedkode in org_units.keys():
rec_make_ou(stedkode, ou, existing_ou_mappings, org_units,
stedkode2ou, perspective)
db.commit()
def import_org_units(sources, target_system, cer_ou_tab):
"""
Scan the sources and import all the OUs into Cerebrum.
:param sources:
A sequence of pairs (system_name, filename), where:
- system_name identifies the appropriate XML file parser for the source
file.
- filename is the XML file to parse and import.
:param target_system:
The authoritative system we're importing data from.
:type cer_ou_tab: dict
:param cer_ou_tab:
*ou_id* -> *sko* mapping, containing the OUs present in
Cerebrum at the start of this script.
This is used to delete obsolete OUs from Cerebrum.
"""
ou = OU_class(db)
# These are used to help build OU structure information
stedkode2ou = dict()
org_units = dict()
existing_ou_mappings = dict()
source_system = getattr(co, target_system)
perspective = source2perspective[source_system]
for system, filename in sources:
logger.debug("Processing %s data from %s", system, filename)
db_writer = XML2Cerebrum(db, source_system, logger, def_kat_merke)
parser = system2parser(system)(filename, logger, False)
for xmlou in parser.iter_ou():
formatted_sko = format_sko(xmlou)
if not formatted_sko:
ids = list(xmlou.iterids())
names = [six.text_type(x) + ': ' + '; '.join(six.text_type(n)
for n in y)
for x, y in xmlou.iternames()]
logger.error("Missing sko for OU %r (names: %s). Skipped!",
ids, names)
continue
if (xmlou.start_date and xmlou.start_date > mx.DateTime.now()):
logger.info("OU sko=%r is not active yet and will therefore "
"be ignored for the time being.", formatted_sko)
continue
if (xmlou.end_date and xmlou.end_date < mx.DateTime.now()):
logger.info("OU sko=%r is expired and some of its information "
"will no longer be maintained", formatted_sko)
else:
org_units[formatted_sko] = xmlou
if verbose:
logger.debug("Processing sko=%r (%s)",
formatted_sko,
xmlou.get_name_with_lang(xmlou.NAME_SHORT,
"no", "nb", "nn", "en"))
args = (xmlou, None)
if clean_obsolete_ous:
args = (xmlou, cer_ou_tab)
status, ou_id = db_writer.store_ou(*args)
if verbose:
logger.debug("**** %s ****", status)
# Not sure why this casting to int is required for PostgreSQL
stedkode2ou[formatted_sko] = int(ou_id)
db.commit()
# Once we've registered all OUs, build and register parent information
for node in ou.get_structure_mappings(perspective):
existing_ou_mappings[int(node["ou_id"])] = node["parent_id"]
# Now populate the entire ou_structure. Note that expired OUs will not
# be processed here.
logger.info("Populate ou_structure")
for stedkode in org_units.keys():
rec_make_ou(stedkode, ou, existing_ou_mappings, org_units,
stedkode2ou, perspective)
db.commit()
def main(inargs=None):
parser = argparse.ArgumentParser()
parser.add_argument(
'-i',
'--input-file',
default=DEFAULT_INPUT_FILE,
help="sap2bas XML input file (default: %(default)s)",
metavar='FILE',
)
parser.add_argument(
'-o',
'--output-file',
default=DEFAULT_OUTPUT_FILE,
help="LDIF output file, or '-' for stdout (default: %(default)s)",
metavar='FILE',
)
parser.add_argument(
'-u',
'--utf8-data',
dest='needs_base64',
action='store_const',
const=LDIFutils.needs_base64_readable,
default=LDIFutils.needs_base64_safe,
help="Allow utf-8 values in ldif",
)
Cerebrum.logutils.options.install_subparser(parser)
parser.set_defaults(**{
Cerebrum.logutils.options.OPTION_LOGGER_LEVEL: 'INFO',
})
args = parser.parse_args(inargs)
Cerebrum.logutils.autoconf('cronjob', args)
logger.info('Start %s', parser.prog)
logger.debug('args: %s', repr(args))
LDIFutils.needs_base64 = args.needs_base64
xml_parser = system2parser('system_sap')(args.input_file, logger)
show_ou = OUSelector('ORG_OU', cereconf.OU_USAGE_SPREAD)
get_ous = OrgTree(xml_parser.iter_ou(), show_ou)
use_lang = LanguageSelector(cereconf.LDAP['pref_languages'])
aff_selector = AffSelector(cereconf.LDAP_PERSON['affiliation_selector'])
stats = {
'seen': 0,
'excluded': 0,
'included': 0,
}
with atomic_or_stdout(args.output_file) as output:
for person in xml_parser.iter_person():
stats['seen'] += 1
partial_affs = set()
for emp in iterate_employments(person, aff_selector):
try:
aff = format_scoped_aff(emp, get_ous)
except Exception as e:
logger.warning('Ignoring employment person=%r emp=%r: %s',
person, emp, e)
continue
titles = [
format_title(t) for t in iterate_employment_titles(emp)
if use_lang(t.language)
]
partial_affs.add('{0}#{1}'.format(aff, ';'.join(titles)))
if len(partial_affs) < 2:
# We want at least two unique employments to output person
stats['excluded'] += 1
continue
try:
identifier = get_identifier(person)
except ValueError:
logger.warn("Missing NIN: {0}".format(str(person)))
stats['excluded'] += 1
continue
stats['included'] += 1
output.write(
LDIFutils.entry_string(
identifier,
{'uioPersonPartialEmployment': list(sorted(partial_affs))},
add_rdn=False))
logger.info("persons"
" considered: {0[seen]:d},"
" included: {0[included]:d},"
" excluded: {0[excluded]:d}".format(stats))
logger.info("Done %s", parser.prog)
"system_sap":
("SAP-elektroniske-reservasjoner",
"Internal group for people from SAP which will not be "
"shown online")
}
logger.debug("sources is %s", sources)
# Load current automatic traits (AFFILIATE_TRAITS)
if include_del:
cerebrum_traits = load_old_traits()
for system_name, filename in sources:
# Locate the appropriate Cerebrum constant
source_system = getattr(const, system_name)
parser = system2parser(system_name)
# Locate the proper reservation group
group, group_members = locate_and_build(system2group[system_name])
# Load old affiliations
cerebrum_affs = set()
if include_del:
cerebrum_affs = load_old_affiliations(source_system)
person2external = load_old_person2external(source_system)
# Read in the file, and register the information in cerebrum
if filename is not None:
parse_data(parser(filename, logger, False),
source_system,
group,
def build_employee_cache(db, sysname, filename):
"""Build a mapping of primary account names for employees to their
employment status.
Employment status in this case is a pair of booleans, that tell whether
the person with that primary account has tilsettinger and bilag that we
need.
:Parameters:
db : a Database instance
DB connection to Cerebrum.
sysname : basestring
Name of the authoritative system whence the data comes
filename : basestring
XML file name (source file)
"""
logger.debug("Building employee cache")
# Cache *all* primary accounts. This helps us bind a primary account to an
# fnr in the XML data.
db_person = Factory.get("Person")(db)
const = Factory.get("Constants")(db)
logger.debug("Fetching all fnr->account mappings...")
fnr2uname = db_person.getdict_external_id2primary_account(
const.externalid_fodselsnr)
logger.debug("... done (%d mappings)", len(fnr2uname))
logger.debug("Fetching all passport-nr->account mappings...")
pnr2uname = db_person.getdict_external_id2primary_account(
const.externalid_pass_number)
logger.debug("... done (%d mappings)", len(pnr2uname))
logger.debug("Fetching all ansatt-nr->account mappings...")
anr2uname = db_person.getdict_external_id2primary_account(
const.externalid_sap_ansattnr)
logger.debug("... done (%d mappings)", len(anr2uname))
# Build mapping for the employees
parser = system2parser(sysname)(filename, logger, False)
# mapping from uname to employment status
employee_cache = dict()
for xmlperson in parser.iter_person():
fnr = xmlperson.get_id(xmlperson.NO_SSN)
passport_nr = xmlperson.get_id(xmlperson.PASSNR)
ansatt_nr = xmlperson.get_id(xmlperson.SAP_NR)
# Everyone with bilag more recent than 180 days old is eligible
bilag = filter(lambda x: ((not x.end) or
(x.end >= (Date(*time.localtime()[:3]) -
DateTimeDeltaFromDays(180)))) and
x.kind == x.BILAG,
xmlperson.iteremployment())
# Add to cache, if found in Cerebrum either by fnr, passport-nr or anr.
# each entry is a pair, telling whether the person has active
# tilsetting and bilag (in that order). We do not need to know *what*
# they are, only that they exist.
username = {anr2uname.get(ansatt_nr),
fnr2uname.get(fnr),
pnr2uname.get(passport_nr)}
username = [un for un in username if un is not None]
if not username:
logger.info("Cerebrum failed to find primary account for person "
"with ansatt-nr: %s.", ansatt_nr)
elif len(username) == 1:
employee_cache[username[0]] = (xmlperson.has_active_employments(),
bool(bilag))
else:
logger.warn("This should probably not happen, "
"different users from the same person. "
"usernames: %s", username)
# IVR 2007-07-13 FIXME: Is this actually useful?
del fnr2uname
del pnr2uname
del anr2uname
logger.debug("employee_cache has %d uname->employment status mappings",
len(employee_cache))
return employee_cache
