def taskMigration(dbi, withRBDB, prevVersion):
"""
Migrating database tasks from the old format to the new one
"""
c = Client()
for t in HelperTaskList().getTaskListInstance().getTasks():
for obj in t.listObj.values():
print console.colored(" * %s" % obj.__class__.__name__, 'blue')
if obj.__class__.__name__ == 'OfflineWebsiteCreator':
continue
if obj.__class__.__name__ == 'FoundationSync':
c.enqueue(
FoundationSyncTask(rrule.DAILY, byhour=0, byminute=0))
elif obj.__class__.__name__ == 'StatisticsUpdater':
c.enqueue(CategoryStatisticsUpdaterTask(
CategoryManager().getById('0'),
rrule.DAILY,
byhour=0, byminute=0))
elif obj.__class__.__name__ == 'sendMail':
# they have to be somewhere in the conference
alarm = t.conf.alarmList[t.id]
c.enqueue(alarm)
else:
print console.colored("WARNING: Unknown task type!", 'yellow')
if withRBDB:
DALManager.commit()
dbi.commit()
def taskMigration(dbi, withRBDB, prevVersion):
"""
Migrating database tasks from the old format to the new one
"""
c = Client()
for t in HelperTaskList().getTaskListInstance().getTasks():
for obj in t.listObj.values():
print console.colored(" * %s" % obj.__class__.__name__, 'blue')
if obj.__class__.__name__ == 'FoundationSync':
c.enqueue(
FoundationSyncTask(rrule.DAILY, byhour=0, byminute=0))
elif obj.__class__.__name__ == 'StatisticsUpdater':
c.enqueue(CategoryStatisticsUpdaterTask(
CategoryManager().getById('0'),
rrule.DAILY,
byhour=0, byminute=0))
elif obj.__class__.__name__ == 'sendMail':
# they have to be somewhere in the conference
alarm = t.conf.alarmList[t.id]
c.enqueue(alarm)
else:
raise Exception("Unknown task type!")
if withRBDB:
DALManager.commit()
dbi.commit()
def runMigration(withRBDB=False,
prevVersion=parse_version(__version__),
specified=[],
dry_run=False,
run_from=None):
global MIGRATION_TASKS
if not dry_run:
print "\nExecuting migration...\n"
dbi = DBMgr.getInstance()
print "Probing DB connection...",
# probe DB connection
dbi.startRequest()
dbi.endRequest(False)
print "DONE!\n"
if run_from:
try:
mig_tasks_names = list(t.__name__
for (__, t, __, __) in MIGRATION_TASKS)
mti = mig_tasks_names.index(run_from)
MIGRATION_TASKS = MIGRATION_TASKS[mti:]
except ValueError:
print console.colored(
"The task {0} does not exist".format(run_from), 'red')
return 1
# go from older to newer version and execute corresponding tasks
for version, task, always, never in MIGRATION_TASKS:
if never and task.__name__ not in specified:
continue
if specified and task.__name__ not in specified:
continue
if parse_version(version) > prevVersion or always:
print console.colored("#", 'green', attrs=['bold']), \
task.__doc__.replace('\n', '').replace(' ', '').strip(),
print console.colored("(%s)" % version, 'yellow')
if dry_run:
continue
dbi.startRequest()
if withRBDB:
DALManager.connect()
task(dbi, withRBDB, prevVersion)
if withRBDB:
DALManager.commit()
dbi.endRequest()
print console.colored(" DONE\n", 'green', attrs=['bold'])
if not dry_run:
print console.colored("Database Migration successful!\n",
'green',
attrs=['bold'])
def pluginReload(dbi, withRBDB, prevVersion):
"""
Reloading all plugins
"""
PluginsHolder().reloadAllPlugins()
dbi.commit()
if withRBDB:
DALManager.commit()
def pluginReload(dbi, withRBDB, prevVersion):
"""
Reloading all plugins
"""
PluginsHolder().reloadAllPlugins()
dbi.commit()
if withRBDB:
DALManager.commit()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--location', '-l', action='append', dest='locations')
args = parser.parse_args()
with DBMgr.getInstance().global_connection():
DALManager.connect()
try:
_main(args)
finally:
DALManager.disconnect()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--location', '-l', action='append', dest='locations')
args = parser.parse_args()
with DBMgr.getInstance().global_connection():
DALManager.connect()
try:
_main(args)
finally:
DALManager.disconnect()
def runMigration(withRBDB=False, prevVersion=parse_version(__version__),
specified=[], dry_run=False, run_from=None):
global MIGRATION_TASKS
if not dry_run:
print "\nExecuting migration...\n"
dbi = DBMgr.getInstance()
print "Probing DB connection...",
# probe DB connection
dbi.startRequest()
dbi.endRequest(False)
print "DONE!\n"
if run_from:
try:
mig_tasks_names = list(t.__name__ for (__, t, __, __) in MIGRATION_TASKS)
mti = mig_tasks_names.index(run_from)
MIGRATION_TASKS = MIGRATION_TASKS[mti:]
except ValueError:
print console.colored("The task {0} does not exist".format(run_from), 'red')
return 1
# go from older to newer version and execute corresponding tasks
for version, task, always, never in MIGRATION_TASKS:
if never and task.__name__ not in specified:
continue
if specified and task.__name__ not in specified:
continue
if parse_version(version) > prevVersion or always:
print console.colored("#", 'green', attrs=['bold']), \
task.__doc__.replace('\n', '').replace(' ', '').strip(),
print console.colored("(%s)" % version, 'yellow')
if dry_run:
continue
dbi.startRequest()
if withRBDB:
DALManager.connect()
task(dbi, withRBDB, prevVersion)
if withRBDB:
DALManager.commit()
dbi.endRequest()
print console.colored(" DONE\n", 'green', attrs=['bold'])
if not dry_run:
print console.colored("Database Migration successful!\n",
'green', attrs=['bold'])
def roomBlockingInit(dbi, withRBDB, prevVersion):
"""
Initializing room blocking indexes.
"""
if not withRBDB:
return
root = DALManager().getRoot()
if not root.has_key( 'RoomBlocking' ):
root['RoomBlocking'] = OOBTree()
root['RoomBlocking']['Blockings'] = IOBTree()
root['RoomBlocking']['Indexes'] = OOBTree()
root['RoomBlocking']['Indexes']['OwnerBlockings'] = OOBTree()
root['RoomBlocking']['Indexes']['DayBlockings'] = CalendarDayIndex()
root['RoomBlocking']['Indexes']['RoomBlockings'] = OOBTree()
def roomBlockingInit(dbi, withRBDB, prevVersion):
"""
Initializing room blocking indexes.
"""
if not withRBDB:
return
root = DALManager().getRoot()
if not root.has_key( 'RoomBlocking' ):
root['RoomBlocking'] = OOBTree()
root['RoomBlocking']['Blockings'] = IOBTree()
root['RoomBlocking']['Indexes'] = OOBTree()
root['RoomBlocking']['Indexes']['OwnerBlockings'] = OOBTree()
root['RoomBlocking']['Indexes']['DayBlockings'] = CalendarDayIndex()
root['RoomBlocking']['Indexes']['RoomBlockings'] = OOBTree()
def rebuildRoomReservationsIndex():
from indico.core.db import DBMgr
from MaKaC.rb_location import CrossLocationDB
from MaKaC.rb_room import RoomBase
from MaKaC.plugins.RoomBooking.default.dalManager import DALManager
from BTrees.OOBTree import OOBTree
DBMgr.getInstance().startRequest()
CrossLocationDB.connect()
root = DALManager.getRoot()
resvEx = ReservationBase()
resvEx.isConfirmed = None
allResvs = CrossLocationQueries.getReservations(resvExample=resvEx)
print "There are " + str(len(allResvs)) + " resvs and pre-resvs to index..."
c = 0
root[_ROOM_RESERVATIONS_INDEX] = OOBTree()
print "Room => Reservations Index branch created"
for resv in allResvs:
roomReservationsIndexBTree = root[_ROOM_RESERVATIONS_INDEX]
resvs = roomReservationsIndexBTree.get(resv.room.id)
if resvs == None:
resvs = [] # New list of reservations for this room
roomReservationsIndexBTree.insert(resv.room.id, resvs)
resvs.append(resv)
roomReservationsIndexBTree[resv.room.id] = resvs
c += 1
if c % 100 == 0:
print c
CrossLocationDB.commit()
CrossLocationDB.disconnect()
DBMgr.getInstance().endRequest()
def runReservationNotificationMigration(dbi, withRBDB, prevVersion):
"""
Migrate the reservation notification system.
"""
if not withRBDB:
return
# Delete old start end notification data
for i, resv in enumerate(CrossLocationQueries.getReservations()):
if hasattr(resv, '_startEndNotification'):
resv._startEndNotification = None
if i % 1000 == 0:
DALManager.commit()
# Create start notification task
Client().enqueue(RoomReservationTask(rrule.HOURLY, byminute=0, bysecond=0))
DALManager.commit()
def runReservationNotificationMigration(dbi, withRBDB, prevVersion):
"""
Migrate the reservation notification system.
"""
if not withRBDB:
return
# Delete old start end notification data
for i, resv in enumerate(CrossLocationQueries.getReservations()):
if hasattr(resv, '_startEndNotification'):
resv._startEndNotification = None
if i % 1000 == 0:
DALManager.commit()
# Create start notification task
Client().enqueue(RoomReservationTask(rrule.HOURLY, byminute=0, bysecond=0))
DALManager.commit()
def rebuildRoomReservationsIndex():
from MaKaC.common.db import DBMgr
from MaKaC.rb_location import CrossLocationDB
from MaKaC.rb_room import RoomBase
from MaKaC.plugins.RoomBooking.default.dalManager import DALManager
from BTrees.OOBTree import OOBTree
DBMgr.getInstance().startRequest()
CrossLocationDB.connect()
root = DALManager.getRoot()
resvEx = ReservationBase()
resvEx.isConfirmed = None
allResvs = CrossLocationQueries.getReservations( resvExample = resvEx )
print "There are " + str( len( allResvs ) ) + " resvs and pre-resvs to index..."
c = 0
root[_ROOM_RESERVATIONS_INDEX] = OOBTree()
print "Room => Reservations Index branch created"
for resv in allResvs:
roomReservationsIndexBTree = root[_ROOM_RESERVATIONS_INDEX]
resvs = roomReservationsIndexBTree.get( resv.room.id )
if resvs == None:
resvs = [] # New list of reservations for this room
roomReservationsIndexBTree.insert( resv.room.id, resvs )
resvs.append( resv )
roomReservationsIndexBTree[resv.room.id] = resvs
c += 1
if c % 100 == 0:
print c
CrossLocationDB.commit()
CrossLocationDB.disconnect()
DBMgr.getInstance().endRequest()
def runMigration(withRBDB=False, prevVersion=parse_version(__version__),
specified=[], dryRun=False):
if not dryRun:
print "\nExecuting migration...\n"
dbi = DBMgr.getInstance()
print "Probing DB connection...",
# probe DB connection
dbi.startRequest()
dbi.endRequest(False)
print "DONE!\n"
# go from older to newer version and execute corresponding tasks
for version, task, always, never in MIGRATION_TASKS:
if never and task.__name__ not in specified:
continue
if specified and task.__name__ not in specified:
continue
if parse_version(version) > prevVersion or always:
print console.colored("#", 'green', attrs=['bold']), \
task.__doc__.replace('\n', '').replace(' ', '').strip(),
print console.colored("(%s)" % version, 'yellow')
if dryRun:
continue
dbi.startRequest()
if withRBDB:
DALManager.connect()
task(dbi, withRBDB, prevVersion)
if withRBDB:
DALManager.commit()
dbi.endRequest()
print console.colored(" DONE\n", 'green', attrs=['bold'])
if not dryRun:
print console.colored("Database Migration successful!\n",
'green', attrs=['bold'])
def _run(args):
_setup(args)
formatter = logging.Formatter("%(asctime)s %(name)s - %(levelname)s %(filename)s:%(lineno)s: %(message)s")
root = logging.getLogger('')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
root.addHandler(handler)
dbi = DBMgr.getInstance(max_disconnect_poll=40)
dbi.startRequest()
info = HelperMaKaCInfo.getMaKaCInfoInstance()
useRBDB = info.getRoomBookingModuleActive()
if useRBDB:
DALManager.connect()
sm = SchedulerModule.getDBInstance()
t = sm.getTaskById(args.taskid)
t.plugLogger(logging.getLogger('console.run/%s' % args.taskid))
t.run()
if useRBDB:
DALManager.commit()
DALManager.disconnect()
dbi.endRequest()
def _run(args):
_setup(args)
formatter = logging.Formatter(
"%(asctime)s %(name)s - %(levelname)s %(filename)s:%(lineno)s: %(message)s"
)
root = logging.getLogger('')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
root.addHandler(handler)
dbi = DBMgr.getInstance(max_disconnect_poll=40)
dbi.startRequest()
info = HelperMaKaCInfo.getMaKaCInfoInstance()
useRBDB = info.getRoomBookingModuleActive()
if useRBDB:
DALManager.connect()
sm = SchedulerModule.getDBInstance()
t = sm.getTaskById(args.taskid)
t.plugLogger(logging.getLogger('console.run/%s' % args.taskid))
t.run()
if useRBDB:
DALManager.commit()
DALManager.disconnect()
dbi.endRequest()
def runMigration(withRBDB=False,
prevVersion=parse_version(__version__),
specified=[]):
print "\nExecuting migration...\n"
dbi = DBMgr.getInstance()
print "Probing DB connection...",
# probe DB connection
dbi.startRequest()
dbi.endRequest(False)
print "DONE!\n"
# go from older to newer version and execute corresponding tasks
for version, task, always in MIGRATION_TASKS:
if specified and task.__name__ not in specified:
continue
if parse_version(version) > prevVersion or always:
print console.colored("#", 'green', attrs=['bold']), \
task.__doc__.replace('\n', '').strip(),
print console.colored("(%s)" % version, 'yellow')
dbi.startRequest()
if withRBDB:
DALManager.connect()
task(dbi, withRBDB, prevVersion)
if withRBDB:
DALManager.commit()
dbi.endRequest()
print console.colored(" DONE\n", 'green', attrs=['bold'])
print console.colored("Database Migration successful!\n",
'green',
attrs=['bold'])
def _prepare(self):
"""
This acts as a second 'constructor', that is executed in the
context of the thread (due to database reasons)
"""
self._prepareDB()
self._dbi.startRequest()
with self._dbi.transaction() as conn:
schedMod = SchedulerModule.getDBInstance()
self._task = schedMod.getTaskById(self._taskId)
info = HelperMaKaCInfo.getMaKaCInfoInstance()
self._rbEnabled = info.getRoomBookingModuleActive()
if self._rbEnabled:
self._rbdbi = DALManager.getInstance()
self._rbdbi.connect()
else:
self._rbdbi = DALManager.dummyConnection()
# open a logging channel
self._task.plugLogger(self._logger)
def _prepare(self):
"""
This acts as a second 'constructor', that is executed in the
context of the thread (due to database reasons)
"""
self._prepareDB()
self._dbi.startRequest()
with self._dbi.transaction() as conn:
schedMod = SchedulerModule.getDBInstance()
self._task = schedMod.getTaskById(self._taskId)
info = HelperMaKaCInfo.getMaKaCInfoInstance()
self._rbEnabled = info.getRoomBookingModuleActive()
if self._rbEnabled:
self._rbdbi = DALManager.getInstance()
self._rbdbi.connect()
else:
self._rbdbi = DALManager.dummyConnection()
# open a logging channel
self._task.plugLogger(self._logger)
def pluginMigration(dbi, withRBDB, prevVersion):
"""
Adding new plugins and adapting existing ones to new name policies
"""
PLUGIN_REMAP = {
'PayPal': 'payPal',
'WorldPay': 'worldPay',
'YellowPay': 'yellowPay',
"Dummyimporter": "dummy",
"CDSInvenio": "invenio",
"CERNSearchPOST": "cern_search",
"InvenioBatchUploader": "invenio"
}
root = dbi.getDBConnection().root()
if 'plugins' in root:
ptl = []
ps = root['plugins']
for k, v in ps.iteritems():
if isinstance(v, PluginType):
ptl.append(v)
for pt in ptl:
pt.setUsable(True)
for p in pt.getPluginList(includeNonPresent=True,
includeTestPlugins=True,
includeNonActive=True):
if hasattr(p, '_Plugin__id'):
pid = p.getId()
else:
pid = p.getName()
if pid in PLUGIN_REMAP:
pid = PLUGIN_REMAP[pid]
p.setId(pid)
p.setUsable(True)
dbi.commit()
if withRBDB:
DALManager.commit()
# load new plugins, so that we can update them after
PluginsHolder().reloadAllPlugins()
dbi.commit()
if withRBDB:
DALManager.commit()
if prevVersion < parse_version("0.98b1"):
# update db for specific plugins
livesync.db.updateDBStructures(root)
dbi.commit()
if withRBDB:
DALManager.commit()
def pluginMigration(dbi, withRBDB, prevVersion):
"""
Adding new plugins and adapting existing ones to new name policies
"""
PLUGIN_REMAP = {
'PayPal': 'payPal',
'WorldPay': 'worldPay',
'YellowPay': 'yellowPay',
"Dummyimporter": "dummy",
"CDSInvenio": "invenio",
"CERNSearchPOST": "cern_search",
"InvenioBatchUploader": "invenio"
}
root = dbi.getDBConnection().root()
if 'plugins' in root:
ptl = []
ps = root['plugins']
for k, v in ps.iteritems():
if isinstance(v, PluginType):
ptl.append(v)
for pt in ptl:
pt.setUsable(True)
for p in pt.getPluginList(includeNonPresent=True,
includeTestPlugins=True,
includeNonActive=True):
if hasattr(p, '_Plugin__id'):
pid = p.getId()
else:
pid = p.getName()
if pid in PLUGIN_REMAP:
pid = PLUGIN_REMAP[pid]
p.setId(pid)
p.setUsable(True)
dbi.commit()
if withRBDB:
DALManager.commit()
# load new plugins, so that we can update them after
PluginsHolder().reloadAllPlugins()
dbi.commit()
if withRBDB:
DALManager.commit()
if prevVersion < parse_version("0.98b1"):
# update db for specific plugins
livesync.db.updateDBStructures(root)
dbi.commit()
if withRBDB:
DALManager.commit()
def runRoomDayIndexInit(dbi, withRBDB, prevVersion):
"""
Initializing room+day => reservation index.
"""
if not withRBDB:
return
root = DALManager().getRoot()
if not root.has_key('RoomDayReservationsIndex'):
root['RoomDayReservationsIndex'] = OOBTree()
for i, resv in enumerate(CrossLocationQueries.getReservations()):
resv._addToRoomDayReservationsIndex()
if i % 1000 == 0:
DALManager.commit()
DALManager.commit()
def runRoomDayIndexInit(dbi, withRBDB, prevVersion):
"""
Initializing room+day => reservation index.
"""
if not withRBDB:
return
root = DALManager().getRoot()
if not root.has_key('RoomDayReservationsIndex'):
root['RoomDayReservationsIndex'] = OOBTree()
for i, resv in enumerate(CrossLocationQueries.getReservations()):
resv._addToRoomDayReservationsIndex()
if i % 1000 == 0:
DALManager.commit()
DALManager.commit()
def conferenceMigration(dbi, withRBDB, prevVersion):
"""
Adding missing attributes to conference objects and children
"""
cdmr = displayMgr.ConfDisplayMgrRegistery()
ch = ConferenceHolder()
i = 0
from97 = prevVersion < parse_version("0.98b1")
# migrating from <=0.97.1 requires smaller granularity
for (level, obj) in console.conferenceHolderIterator(ch, deepness='subcontrib' if from97 else 'event'):
# only for conferences
if level == 'event':
if from97:
# handle sessions, that our iterator ignores
for session in obj.getSessionList():
_fixAccessController(session)
if hasattr(obj, '_Conference__alarmCounter'):
raise Exception("Conference Object %s (%s) seems to have been "
"already converted" % (obj, obj.id))
existingKeys = obj.alarmList.keys()
existingKeys.sort()
nstart = int(existingKeys[-1]) + 1 if existingKeys else 0
obj._Conference__alarmCounter = Counter(nstart)
# For each conference, take the existing tasks and
# convert them to the new object classes.
_convertAlarms(obj)
# convert registration form's "Personal Data" section to new format
obj.getRegistrationForm()._convertPersonalData()
# For each conference, fix the default style
_fixDefaultStyle(obj, cdmr)
if from97:
_fixAccessController(obj,
fixSelf=(level != 'subcontrib'))
# Convert RegistrationSessions to RegistrantSessions
if isinstance(obj, Conference):
for reg in obj.getRegistrants().itervalues():
if reg._sessions and \
isinstance(reg._sessions[0], RegistrationSession):
reg._sessions = [RegistrantSession(ses, reg) \
for ses in reg._sessions]
if i % 1000 == 999:
dbi.commit()
if withRBDB and from97:
DALManager.commit()
i += 1
dbi.commit()
if withRBDB and from97:
DALManager.commit()
def conferenceMigration(dbi, withRBDB, prevVersion):
"""
Adding missing attributes to conference objects and children
"""
cdmr = displayMgr.ConfDisplayMgrRegistery()
ch = ConferenceHolder()
i = 0
from97 = prevVersion < parse_version("0.98b1")
# migrating from <=0.97.1 requires smaller granularity
for (level, obj) in console.conferenceHolderIterator(ch, deepness='subcontrib' if from97 else 'event'):
# only for conferences
if level == 'event':
if from97:
# handle sessions, that our iterator ignores
for session in obj.getSessionList():
_fixAccessController(session)
if hasattr(obj, '_Conference__alarmCounter'):
raise Exception("Conference Object %s (%s) seems to have been "
"already converted" % (obj, obj.id))
existingKeys = obj.alarmList.keys()
existingKeys.sort()
nstart = int(existingKeys[-1]) + 1 if existingKeys else 0
obj._Conference__alarmCounter = Counter(nstart)
# For each conference, take the existing tasks and
# convert them to the new object classes.
_convertAlarms(obj)
# convert registration form's "Personal Data" section to new format
obj.getRegistrationForm()._convertPersonalData()
# For each conference, fix the default style
_fixDefaultStyle(obj, cdmr)
if from97:
_fixAccessController(obj,
fixSelf=(level != 'subcontrib'))
# Convert RegistrationSessions to RegistrantSessions
if isinstance(obj, Conference):
for reg in obj.getRegistrants().itervalues():
if reg._sessions and \
isinstance(reg._sessions[0], RegistrationSession):
reg._sessions = [RegistrantSession(ses, reg) \
for ses in reg._sessions]
if i % 1000 == 999:
dbi.commit()
if withRBDB and from97:
DALManager.commit()
i += 1
dbi.commit()
if withRBDB and from97:
DALManager.commit()
