def main():
# legacy log configuration
Cerebrum.logutils.autoconf('cronjob', None)
try:
opts, args = getopt.getopt(
sys.argv[1:],
'd:f:o:t:a:',
['delim=', 'file=', 'out=', 'tag=', 'append='])
except getopt.GetoptError:
usage(2)
logger.info('Start')
big_xml = {}
for opt, val in opts:
if opt in ('-t', '--tag'):
tag = val
logger.debug('set tag=%r', tag)
elif opt in ('-d', '--delim'):
delim = val.split(":")
logger.debug('set delim=%r', delim)
elif opt in ('-f', '--file'):
CollectParser(val, big_xml, delim)
logger.debug('collecting data from %r with delim=%r', val, delim)
elif opt in ('-a', '--append'):
CollectParser(val, big_xml, delim, True)
logger.debug('appending data from %r with delim=%r', val, delim)
elif opt in ('-o', '--out'):
logger.debug('writing data to %r', val)
f = SimilarSizeWriter(val, "w")
f.max_pct_change = 50
xml = XMLHelper()
f.write(xml.xml_hdr + "<data>\n")
for bx_key in big_xml.keys():
bx_delim = bx_key.split("¦")
f.write("<%s %s>\n" % (
tag, " ".join(["%s=%s" % (delim[n],
xml.escape_xml_attr(bx_delim[n]))
for n in range(len(delim))])))
for tmp_tag in big_xml[bx_key]:
tmp = tmp_tag['TagName']
del(tmp_tag['TagName'])
f.write(" <%s %s/>\n" % (
tmp, " ".join(
["%s=%s" % (tk,
xml.escape_xml_attr(tmp_tag[tk]))
for tk in tmp_tag.keys()])))
f.write("</%s>\n" % tag)
f.write("</data>\n")
f.close()
logger.info('Wrote merged xml to %r', val)
logger.info('Done')
def write_xml(stream, persons):
"""
:param persons:
An iterable of person dicts.
Each person dict should contain keys from person_cols
"""
encoding = stream.encoding
xml = XMLHelper(encoding=encoding)
stream.write(xml.xml_hdr)
stream.write("<data>\n")
for person in persons:
stream.write(
xml.xmlify_dbrow(person, xml_person_attrs, 'person') + "\n")
stream.write("</data>\n")
import os
import sys
import cereconf
import Cerebrum.logutils
from Cerebrum.modules.fs.import_from_FS import ImportFromFs, set_filepath
from Cerebrum.modules.no.access_FS import make_fs
from Cerebrum.modules.xmlutils.xml_helper import XMLHelper
from Cerebrum.utils.atomicfile import FileChangeTooBigError
from Cerebrum.utils.atomicfile import SimilarSizeWriter
XML_ENCODING = 'utf-8'
logger = logging.getLogger(__name__)
xml = XMLHelper(encoding=XML_ENCODING)
def usage():
print("""Usage: %(filename)s [options]
%(doc)s
Settings:
--datadir: Override the directory where all files should be put.
Default: see cereconf.FS_DATA_DIR
Note that the datadir can be overriden by the file path
options, if these are absolute paths.
--studprog-file: Override studprog xml filename.
Default: studieprogrammer.xml
def dump_person_info(db, fname):
xml = XMLHelper()
co = Factory.get('Constants')(db)
ac = Factory.get('Account')(db)
person = Factory.get('Person')(db)
src_sys_order = {
int(co.system_fs): 1,
int(co.system_x): 2,
}
def source_system_key(row):
return src_sys_order.get(int(row['source_system']), 99)
def _fetch_names(name_type):
# Fetch persons full-name from most-significant source system
ret = {}
for row in person.list_names(variant=name_type):
pid = int(row['person_id'])
ret.setdefault(pid, []).append(row)
for pid, rows in ret.items():
rows.sort(key=source_system_key)
ret[pid] = rows[0]
assert int(name_type) == rows[0]['name_variant']
return ret
# Fetch persons full-name from most-significant source system
# pid2names = _fetch_names(co.name_full)
pid2first_name = _fetch_names(co.name_first)
pid2last_name = _fetch_names(co.name_last)
# Fetch birth-no from most-significant source system
pid2ext_ids = {}
for row in person.search_external_ids(id_type=co.externalid_fodselsnr):
pid = int(row['entity_id'])
pid2ext_ids.setdefault(pid, []).append(row)
for pid, rows in pid2ext_ids.items():
rows.sort(key=source_system_key)
pid2ext_ids[pid] = rows[0]
# Fetch birth-date
pid2birth = {}
for row in person.list_persons():
pid = int(row['person_id'])
if not row['birth_date']:
continue
pid2birth[pid] = row['birth_date'].strftime('%Y-%m-%d')
pid2ac = {}
_acid2uname = {}
# Fetch primary account
for row in ac.search():
_acid2uname[int(row['account_id'])] = row['name']
for row in ac.list_accounts_by_type(primary_only=True):
pid = int(row['person_id'])
assert pid not in pid2ac
pid2ac[pid] = _acid2uname[int(row['account_id'])]
cols = ('fnr', 'first_name', 'last_name', 'uname', 'bdate')
f = io.open(fname, "wt", encoding='utf-8')
f.write(xml.xml_hdr + u"<data>\n")
for pid, birth in pid2birth.items():
if ((pid not in pid2ext_ids) or
(not (pid in pid2first_name or
pid in pid2last_name))):
continue
row = {}
row['fnr'] = None
row['first_name'] = None
row['last_name'] = None
row['uname'] = None
row['bdate'] = None
row['fnr'] = pid2ext_ids[pid]['external_id']
row['first_name'] = pid2first_name[pid]['name']
row['last_name'] = pid2last_name[pid]['name']
if pid in pid2ac:
row['uname'] = pid2ac[pid]
row['bdate'] = pid2birth[pid]
f.write(xml.xmlify_dbrow(row, cols, 'person')) # .encode('utf-8'))
f.write(u'\n')
f.write(u"</data>\n")
f.close()