def perform_synchronization(services):
"""
Synchronize cerebrum groups with all external SERVICES.
"""
for item in services:
service = item["dbname"]
klass = get_accessor(item["accessor"])
accessor_name = item["sync_accessor"]
cerebrum_group = item["ceregroup"]
user = item["dbuser"]
db_charset = item["db_charset"]
logger.debug("Synchronizing against source %s (user: %s)",
service, user)
try:
db = Database.connect(user=user, service=service,
DB_driver=cereconf.DB_DRIVER_ORACLE)
if db_charset:
obj = klass(db, db_charset)
else:
obj = klass(db)
accessor = getattr(obj, accessor_name)
except:
type, value, tb = sys.exc_info()
logger.error("Aiee! Failed to connect to %s: %s, %s, %s",
service,
type, value,
string.join(traceback.format_tb(tb)))
else:
synchronize_group(accessor, cerebrum_group)
def generate_report():
"""Main driver for the report generation."""
xmlwriter.startDocument(encoding = "iso8859-1")
xmlwriter.startElement("document")
output_properties()
output_all_OUs()
person_info = output_people()
kull_info = prepare_kull()
ue_info = prepare_ue()
prepare_pay()
#
# All the relations
db = Database.connect(user="******", service="FSUIA.uio.no",
DB_driver=cereconf.DB_DRIVER_ORACLE)
fs = FS(db)
output_pay_relation(person_info, fs)
output_affiliations(person_info)
output_kull_relations(kull_info, person_info, fs)
output_ue_relations(ue_info, person_info, fs)
xmlwriter.endElement("document")
xmlwriter.endDocument()
def generate_report():
"""Main driver for the report generation."""
xmlwriter.startDocument(encoding="iso8859-1")
xmlwriter.startElement("document")
output_properties()
output_all_OUs()
person_info = output_people()
kull_info = prepare_kull()
ue_info = prepare_ue()
prepare_pay()
#
# All the relations
db = Database.connect(user="******",
service="FSUIA.uio.no",
DB_driver=cereconf.DB_DRIVER_ORACLE)
fs = FS(db)
output_pay_relation(person_info, fs)
output_affiliations(person_info)
output_kull_relations(kull_info, person_info, fs)
output_ue_relations(ue_info, person_info, fs)
xmlwriter.endElement("document")
xmlwriter.endDocument()
def perform_synchronization(services):
"""
Synchronize cerebrum groups with all external SERVICES.
"""
for item in services:
service = item["dbname"]
klass = get_accessor(item["accessor"])
accessor_name = item["sync_accessor"]
cerebrum_group = item["ceregroup"]
user = item["dbuser"]
db_charset = item["db_charset"]
logger.debug("Synchronizing against source %s (user: %s)", service,
user)
try:
db = Database.connect(user=user,
service=service,
DB_driver=cereconf.DB_DRIVER_ORACLE)
if db_charset:
obj = klass(db, db_charset)
else:
obj = klass(db)
accessor = getattr(obj, accessor_name)
except:
type, value, tb = sys.exc_info()
logger.error("Aiee! Failed to connect to %s: %s, %s, %s", service,
type, value, string.join(traceback.format_tb(tb)))
else:
synchronize_group(accessor, cerebrum_group)
def init_globals():
global db, const, logger, group, users_only, ou, person
db = Factory.get("Database")()
const = Factory.get("Constants")(db)
group = Factory.get("Group")(db)
person = Factory.get("Person")(db)
ou = Factory.get("OU")(db)
logger = Factory.get_logger("cronjob")
fsdb = Database.connect(user='******',
service=cereconf.FS_DATABASE_NAME,
DB_driver='cx_Oracle')
fs = FS(fsdb)
cf_dir = '/cerebrum/var/cache/Fronter'
try:
opts, args = getopt.getopt(sys.argv[1:], '',
['debug-file=', 'debug-level=',
'cf-dir=',
])
except getopt.GetoptError:
usage(1)
debug_file = os.path.join(cf_dir, "x-import.log")
debug_level = 4
set_pwd = True
users_only = False
for opt, val in opts:
if opt == '--debug-file':
debug_file = val
elif opt == '--debug-level':
debug_level = val
elif opt == '--uten-passord':
set_pwd = False
elif opt == '--cf-dir':
cf_dir = val
else:
raise ValueError, "Invalid argument: %r", (opt,)
global fronter
fronter = Fronter(db, const, logger=logger)
filename = os.path.join(cf_dir, 'test.xml')
if len(args) == 1:
filename = args[0]
elif len(args) != 0:
usage(2)
global fxml
fxml = FronterXML(filename,
cf_dir = cf_dir,
debug_file = debug_file,
debug_level = debug_level,
fronter = fronter,
include_password = set_pwd)
# Finn `uname` -> account-data for alle brukere.
global new_users
new_users = get_new_users()
def output_text(output_file):
"""
Initialize data structures and start generating the output.
"""
output_stream = MinimumSizeWriter(output_file, "w")
# 1MB is the minimum allowed size for the portal dump.
# The number is somewhat magic, but it seems sensible
output_stream.min_size = 1024*1024
db_cerebrum = Factory.get("Database")()
logger.debug(cereconf.DB_AUTH_DIR)
logger.debug(Database.__file__)
db = Database.connect(user="******",
service="FSPROD.uio.no",
DB_driver=cereconf.DB_DRIVER_ORACLE)
db_fs = FS(db)
db_person = Factory.get("Person")(db_cerebrum)
db_account = Factory.get("Account")(db_cerebrum)
constants = Factory.get("Constants")(db_cerebrum)
# FS is first. This is intentional.
lookup_order = [constants.system_fs]
for authoritative_system_name in cereconf.SYSTEM_LOOKUP_ORDER:
lookup_order.append(getattr(constants, authoritative_system_name))
rows = db_fs.portal.list_eksmeld()
logger.debug("Fetched portal information from FS")
for row in rows:
output_row(row, output_stream,
db_person, db_account, constants,
lookup_order)
output_stream.close()
def assert_connected(user, service):
global fs
if fs is None:
DB_driver = getattr(cereconf, 'DB_DRIVER_ORACLE', 'cx_Oracle')
db = Database.connect(user=user, service=service,
DB_driver=DB_driver)
fs = FS(db)
def make_report(user, report_missing, item, acc_name, *func_list):
"""
Help function to generate report stats.
"""
db_cerebrum = Factory.get("Database")()
account = Factory.get("Account")(db_cerebrum)
service = item["dbname"]
db = Database.connect(user=user, service=service,
DB_driver=cereconf.DB_DRIVER_ORACLE)
source = get_accessor(item["accessor"])(db)
accessor = getattr(source, acc_name)
stream = StringIO.StringIO()
for db_row in accessor():
#
# NB! This is not quite what we want. See comments in sanitize_group
username = string.lower(db_row["username"])
try:
account.clear()
account.find_by_name(username)
except Cerebrum.Errors.NotFoundError:
if report_missing:
stream.write("No such account in Cerebrum: %s\n" % username)
continue
for function in func_list:
message = function(account)
if message:
stream.write(message)
report_data = stream.getvalue()
stream.close()
return report_data
def main():
db_fs = Database.connect(user = "******", service = "FSPROD.uio.no",
DB_driver = cereconf.DB_DRIVER_ORACLE)
fs = FS(db_fs)
print "%-20s%-10s%20s%20s" % ("kurskode", "tidskode", "fra", "til")
print "-" * 70
for code in sys.argv[1:]:
for row in fs.evu.get_kurs_informasjon(code):
print "%-20s%-10s%20s%20s" % (row["etterutdkurskode"],
row["kurstidsangivelsekode"],
row["dato_fra"], row["dato_til"])
def main():
global LT, xml, verbose
personfile = None
stedfile = None
sid = None
user = None
fnr_update = None
try:
opts, args = getopt.getopt(sys.argv[1:],
'vs:u:f:',
['verbose', 'sid=', 'sted-file=',
'person-file=',
'fnr-update='])
except getopt.GetoptError:
usage(1)
for opt, val in opts:
if opt in ('-v', '--verbose'):
pass # not currently used
elif opt in ('-s', '--sid'):
sid = val
elif opt in ('-u',):
user = val
elif opt in ('--sted-file',):
stedfile = val
elif opt in ('--person-file',):
personfile = val
elif opt in ('-f', '--fnr-update'):
fnr_update = val
# fi
# od
if user is None or sid is None:
usage(1)
db = Database.connect(user=user, service=sid, DB_driver=cereconf.DB_DRIVER_ORACLE)
LT = LT(db)
xml = XMLHelper()
if stedfile is not None:
get_sted_info(stedfile)
# fi
if personfile is not None:
get_person_info(personfile)
# fi
if fnr_update is not None:
get_fnr_update_info(fnr_update)
def main():
db_fs = Database.connect(user = "******", service = "FSPROD.uio.no",
DB_driver = cereconf.DB_DRIVER_ORACLE)
fs = FS(db_fs)
db_cerebrum = Factory.get("Database")()
person = Factory.get("Person")(db_cerebrum)
account = Factory.get("Account")(db_cerebrum)
constants = Factory.get("Constants")(db_cerebrum)
emnekode = sys.argv[1]
for row in fs.student.get_emne_eksamensmeldinger(emnekode):
uname = fetch_primary_uname(row, person, account, constants)
print "%06d %05d %-20s%-30s --> %-10s" % (row.fodselsdato,
row.personnr,
row.etternavn,
row.fornavn,
uname)
def prepare_kull():
"""Output all kull groups.
'Kull' relationships are expressed through the group concept in the
ABC-schema. First we define all 'kull' as <group>s. Then we use <relation>
to link up people to their respective 'kull's (i.e. <group>s).
"""
db = Database.connect(user="******",
service="FSUIA.uio.no",
DB_driver=cereconf.DB_DRIVER_ORACLE)
fs = FS(db)
logger.info("Generating <group>-elements for kull")
kull_cache = dict()
for row in fs.info.list_kull():
studieprogram = str(row["studieprogramkode"])
terminkode = str(row["terminkode"])
arstall = int(row["arstall"])
internal_id = studieprogram, terminkode, arstall
if internal_id in kull_cache:
continue
# fi
xml_id = make_id(*internal_id)
name_for_humans = "Studiekull %s" % row["studiekullnavn"]
sko = make_sko(*[
row[x] for x in ("faknr_studieansv", "instituttnr_studieansv",
"gruppenr_studieansv")
])
kull_cache[internal_id] = sko
xmlwriter.startElement("group")
output_elem("groupid", xml_id,
{"groupidtype": get_group_id_type("kull")})
output_elem("description", name_for_humans)
xmlwriter.endElement("group")
# od
logger.info("Done with <group>-elements for kull (%d elements)",
len(kull_cache))
return kull_cache
def main():
db_fs = Database.connect(user = "******", service = "FSPROD.uio.no",
DB_driver = cereconf.DB_DRIVER_ORACLE)
fs = FS(db_fs)
db_cerebrum = Factory.get("Database")()
person = Factory.get("Person")(db_cerebrum)
account = Factory.get("Account")(db_cerebrum)
constants = Factory.get("Constants")(db_cerebrum)
evukode = sys.argv[1]
tidskode = sys.argv[2]
for row in fs.evu.list_kurs_deltakere(evukode, tidskode):
uname = fetch_primary_uname(row, person, account, constants)
print "%06d %05d %-20s%-30s --> %-10s" % (row['fodselsdato'],
row['personnr'],
row['etternavn'],
row['fornavn'],
uname)
def prepare_kull():
"""Output all kull groups.
'Kull' relationships are expressed through the group concept in the
ABC-schema. First we define all 'kull' as <group>s. Then we use <relation>
to link up people to their respective 'kull's (i.e. <group>s).
"""
db = Database.connect(user="******", service="FSUIA.uio.no",
DB_driver=cereconf.DB_DRIVER_ORACLE)
fs = FS(db)
logger.info("Generating <group>-elements for kull")
kull_cache = dict()
for row in fs.info.list_kull():
studieprogram = str(row["studieprogramkode"])
terminkode = str(row["terminkode"])
arstall = int(row["arstall"])
internal_id = studieprogram, terminkode, arstall
if internal_id in kull_cache:
continue
# fi
xml_id = make_id(*internal_id)
name_for_humans = "Studiekull %s" % row["studiekullnavn"]
sko = make_sko(*[row[x] for x in ("faknr_studieansv",
"instituttnr_studieansv",
"gruppenr_studieansv")])
kull_cache[internal_id] = sko
xmlwriter.startElement("group")
output_elem("groupid", xml_id,
{"groupidtype" : get_group_id_type("kull")})
output_elem("description", name_for_humans)
xmlwriter.endElement("group")
# od
logger.info("Done with <group>-elements for kull (%d elements)",
len(kull_cache))
return kull_cache
def prepare_ue():
"""Output all undervisningsenhet groups.
The procedure is the same as for kull (prepare_kull).
"""
db = Database.connect(user="******",
service="FSUIA.uio.no",
DB_driver=cereconf.DB_DRIVER_ORACLE)
fs = FS(db)
logger.info("Generating <group>-elements for UE")
ue_cache = dict()
for row in fs.undervisning.list_undervisningenheter():
id = tuple([
row[field]
for field in ("institusjonsnr", "emnekode", "versjonskode",
"terminkode", "arstall", "terminnr")
])
if id in ue_cache:
continue
# fi
xml_id = make_id(*id)
name_for_humans = "Undervisningsenhet %s" % xml_id
sko = make_sko(*[
row[x] for x in ("faknr_kontroll", "instituttnr_kontroll",
"gruppenr_kontroll")
])
ue_cache[id] = sko
xmlwriter.startElement("group")
output_elem("groupid", xml_id,
{"groupidtype": get_group_id_type("ue")})
output_elem("description", name_for_humans)
xmlwriter.endElement("group")
# od
logger.info("Done with <group>-elements for UE (%d elements)",
len(ue_cache))
return ue_cache
def prepare_ue():
"""Output all undervisningsenhet groups.
The procedure is the same as for kull (prepare_kull).
"""
db = Database.connect(user="******", service="FSUIA.uio.no",
DB_driver=cereconf.DB_DRIVER_ORACLE)
fs = FS(db)
logger.info("Generating <group>-elements for UE")
ue_cache = dict()
for row in fs.undervisning.list_undervisningenheter():
id = tuple([row[field] for field in
("institusjonsnr", "emnekode", "versjonskode",
"terminkode", "arstall", "terminnr")])
if id in ue_cache:
continue
# fi
xml_id = make_id(*id)
name_for_humans = "Undervisningsenhet %s" % xml_id
sko = make_sko(*[row[x] for x in ("faknr_kontroll",
"instituttnr_kontroll",
"gruppenr_kontroll")])
ue_cache[id] = sko
xmlwriter.startElement("group")
output_elem("groupid", xml_id,
{"groupidtype" : get_group_id_type("ue")})
output_elem("description", name_for_humans)
xmlwriter.endElement("group")
# od
logger.info("Done with <group>-elements for UE (%d elements)",
len(ue_cache))
return ue_cache
def make_report(user, report_missing, item, acc_name, *func_list):
"""
Help function to generate report stats.
"""
db_cerebrum = Factory.get("Database")()
account = Factory.get("Account")(db_cerebrum)
service = item["dbname"]
db = Database.connect(user=user,
service=service,
DB_driver=cereconf.DB_DRIVER_ORACLE)
source = get_accessor(item["accessor"])(db)
accessor = getattr(source, acc_name)
stream = StringIO.StringIO()
for db_row in accessor():
#
# NB! This is not quite what we want. See comments in sanitize_group
username = string.lower(db_row["username"])
try:
account.clear()
account.find_by_name(username)
except Cerebrum.Errors.NotFoundError:
if report_missing:
stream.write("No such account in Cerebrum: %s\n" % username)
continue
for function in func_list:
message = function(account)
if message:
stream.write(message)
report_data = stream.getvalue()
stream.close()
return report_data
def main():
global logger
logger = Factory.get_logger("cronjob")
logger.info("Starting LT sync")
try:
options, rest = getopt.getopt(sys.argv[1:],
"deuh",
["help",
"email",
"uname",
"dryrun",])
except getopt.GetoptError:
usage()
sys.exit(1)
# yrt
global dryrun
dryrun = False
email = False
uname = False
for option, value in options:
if option in ("-d", "--dryrun"):
dryrun = True
elif option in ("-h", "--help"):
usage()
sys.exit(2)
elif option in ("-e", "--email"):
logger.debug("Running sync for email")
email = True
elif option in ("-u", "--uname"):
logger.debug("Running sync for uname")
uname = True
# fi
# od
lt = LT(Database.connect(user="******", service = "LTPROD.uio.no",
DB_driver = cereconf.DB_DRIVER_ORACLE))
db = Factory.get("Database")()
person = Factory.get("Person")(db)
const = Factory.get("Constants")(db)
logger.debug("Fetching all people from LT")
lt_people = lt.GetAllPersons()
logger.debug("done")
if email:
synchronize_attribute(person.getdict_external_id2mailaddr,
lt_people,
lt.GetAllPersonsUregEmail,
lt.UpdatePriMailAddr,
lt.DeletePriMailAddr,
lt.InsertPriMailAddr,
const, lt)
# fi
if uname:
synchronize_attribute(person.getdict_external_id2primary_account,
lt_people,
lt.GetAllPersonsUregUser,
lt.UpdatePriUser,
lt.DeletePriUser,
lt.InsertPriUser,
const, lt)
elif opt in ('--fs-db-user', ):
fs_user = val
elif opt in ('--fs-db-service', ):
fs_database = val
elif opt in ('--lt-db-user', ):
lt_user = val
elif opt in ('--lt-db-service', ):
lt_database = val
elif opt in ('--dryrun', ):
dryrun = True
if not opts: # enforce atleast one argument to avoid accidential runs
usage(1)
fs = FS(user=fs_user, database=fs_database)
lt_db = Database.connect(user=lt_user,
service=lt_database,
DB_driver=cereconf.DB_DRIVER_ORACLE)
lt = LT(lt_db)
if dryrun:
fs.db.commit = fs.db.rollback
update_lt()
def usage(exitcode=0):
print """Usage: lt2fsPerson [opsjoner]
--fs-db-user name: connect with given database username (FS)
--fs-db-service name: connect to given database (FS)
--lt-db-user name: connect with given database username (LT)
--lt-db-service name: connect to given database (LT)
--dryrun : rollback changes to db
"""
def assert_connected(user="******", service="FSHIH.uio.no"):
global fs
if fs is None:
db = Database.connect(user=user, service=service,
DB_driver='cx_Oracle')
fs = FS(db)
elif opt in ('--remove'):
remove_others = True
else:
print "Unknown argument: %s" % opt
usage(1)
# cache the primary accounts
pe2primary = dict((r['person_id'], r['account_id']) for r in
acc.list_accounts_by_type(primary_only=True))
# TODO: only get fnr from system_fs?
fnr2primary = dict((r['external_id'], pe2primary[r['entity_id']]) for r in
person.list_external_ids(id_type=const.externalid_fodselsnr)
if pe2primary.has_key(r['entity_id']))
fsdb = Database.connect(user=cereconf.FS_USER,
service=cereconf.FS_DATABASE_NAME,
DB_driver='cx_Oracle')
fs = FS(fsdb)
logger.info("Updating studieprogram groups")
studieprog_grupper(fs, remove_others)
logger.info("Updating instittut groups")
institutt_grupper(remove_others)
logger.info("Updating emne/underv.enh groups")
undervisningsmelding_grupper(fs, remove_others)
logger.info("Updating vurderingsmeldinger groups")
vurderingsmelding_grupper(fs, remove_others)
usage()
elif opt in ('--fs-db-user',):
fs_user = val
elif opt in ('--fs-db-service',):
fs_database = val
elif opt in ('--lt-db-user',):
lt_user = val
elif opt in ('--lt-db-service',):
lt_database = val
elif opt in ('--dryrun',):
dryrun = True
if not opts: # enforce atleast one argument to avoid accidential runs
usage(1)
fs = FS(user=fs_user, database=fs_database)
lt_db = Database.connect(user=lt_user, service=lt_database,
DB_driver=cereconf.DB_DRIVER_ORACLE)
lt = LT(lt_db)
if dryrun:
fs.db.commit = fs.db.rollback
update_lt()
def usage(exitcode=0):
print """Usage: lt2fsPerson [opsjoner]
--fs-db-user name: connect with given database username (FS)
--fs-db-service name: connect to given database (FS)
--lt-db-user name: connect with given database username (LT)
--lt-db-service name: connect to given database (LT)
--dryrun : rollback changes to db
"""
sys.exit(exitcode)
elif option in ('-u', '--db-user',):
user = value
elif option in ('-s', '--db-service',):
service = value
elif option in ('-e', '--email',):
email = True
elif option in ('-a', '--account'):
account = True
elif option in ('-h', '--help'):
usage()
else:
print "Unknown argument: %s" % option
usage(2)
DB_driver = getattr(cereconf, 'DB_DRIVER_ORACLE', 'cx_Oracle')
fs_db = Database.connect(user=user, service=service, DB_driver=DB_driver)
fs = FS(fs_db)
db = Factory.get('Database')()
person = Factory.get('Person')(db)
const = Factory.get('Constants')(db)
if dryrun:
logger.info('This is a dryrun, changes will be rolled back')
else:
logger.info('Not a dryrun, changes will be committed')
def commit_handler():
if dryrun:
fs.db.rollback()
logger.info("Rolled back all changes")
# along with Cerebrum; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
import cerebrum_path
import re
import os
import sys
import cereconf
from Cerebrum import Database,Errors
from Utils import XMLHelper
default_personfile = "/cerebrum/var/cache/MSTAS/persons.dat"
Cerebrum = Database.connect(user="******", service="MSTAS.hiof.no",
DB_driver=cereconf.DB_DRIVER_ORACLE)
xml = XMLHelper()
LIMIT="rownum < 100" # Vært snill mot db under testing
class MSTAS(object):
"""..."""
def __init__(self, db):
self.db = db
def GetSchemaVersion(self):
return self.db.query_1("SELECT version FROM dpsys000.schema_version")
def GetPersons(self):
qry = """SELECT s_objectid, fornavn, etternavn, foedselsnr, ansatt