def taskMigration(dbi, withRBDB, prevVersion): """ Migrating database tasks from the old format to the new one """ c = Client() for t in HelperTaskList().getTaskListInstance().getTasks(): for obj in t.listObj.values(): print console.colored(" * %s" % obj.__class__.__name__, 'blue') if obj.__class__.__name__ == 'OfflineWebsiteCreator': continue if obj.__class__.__name__ == 'FoundationSync': c.enqueue( FoundationSyncTask(rrule.DAILY, byhour=0, byminute=0)) elif obj.__class__.__name__ == 'StatisticsUpdater': c.enqueue(CategoryStatisticsUpdaterTask( CategoryManager().getById('0'), rrule.DAILY, byhour=0, byminute=0)) elif obj.__class__.__name__ == 'sendMail': # they have to be somewhere in the conference alarm = t.conf.alarmList[t.id] c.enqueue(alarm) else: print console.colored("WARNING: Unknown task type!", 'yellow') if withRBDB: DALManager.commit() dbi.commit()
def taskMigration(dbi, withRBDB, prevVersion): """ Migrating database tasks from the old format to the new one """ c = Client() for t in HelperTaskList().getTaskListInstance().getTasks(): for obj in t.listObj.values(): print console.colored(" * %s" % obj.__class__.__name__, 'blue') if obj.__class__.__name__ == 'FoundationSync': c.enqueue( FoundationSyncTask(rrule.DAILY, byhour=0, byminute=0)) elif obj.__class__.__name__ == 'StatisticsUpdater': c.enqueue(CategoryStatisticsUpdaterTask( CategoryManager().getById('0'), rrule.DAILY, byhour=0, byminute=0)) elif obj.__class__.__name__ == 'sendMail': # they have to be somewhere in the conference alarm = t.conf.alarmList[t.id] c.enqueue(alarm) else: raise Exception("Unknown task type!") if withRBDB: DALManager.commit() dbi.commit()
def runMigration(withRBDB=False, prevVersion=parse_version(__version__), specified=[], dry_run=False, run_from=None): global MIGRATION_TASKS if not dry_run: print "\nExecuting migration...\n" dbi = DBMgr.getInstance() print "Probing DB connection...", # probe DB connection dbi.startRequest() dbi.endRequest(False) print "DONE!\n" if run_from: try: mig_tasks_names = list(t.__name__ for (__, t, __, __) in MIGRATION_TASKS) mti = mig_tasks_names.index(run_from) MIGRATION_TASKS = MIGRATION_TASKS[mti:] except ValueError: print console.colored( "The task {0} does not exist".format(run_from), 'red') return 1 # go from older to newer version and execute corresponding tasks for version, task, always, never in MIGRATION_TASKS: if never and task.__name__ not in specified: continue if specified and task.__name__ not in specified: continue if parse_version(version) > prevVersion or always: print console.colored("#", 'green', attrs=['bold']), \ task.__doc__.replace('\n', '').replace(' ', '').strip(), print console.colored("(%s)" % version, 'yellow') if dry_run: continue dbi.startRequest() if withRBDB: DALManager.connect() task(dbi, withRBDB, prevVersion) if withRBDB: DALManager.commit() dbi.endRequest() print console.colored(" DONE\n", 'green', attrs=['bold']) if not dry_run: print console.colored("Database Migration successful!\n", 'green', attrs=['bold'])
def pluginReload(dbi, withRBDB, prevVersion): """ Reloading all plugins """ PluginsHolder().reloadAllPlugins() dbi.commit() if withRBDB: DALManager.commit()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--location', '-l', action='append', dest='locations') args = parser.parse_args() with DBMgr.getInstance().global_connection(): DALManager.connect() try: _main(args) finally: DALManager.disconnect()
def runMigration(withRBDB=False, prevVersion=parse_version(__version__), specified=[], dry_run=False, run_from=None): global MIGRATION_TASKS if not dry_run: print "\nExecuting migration...\n" dbi = DBMgr.getInstance() print "Probing DB connection...", # probe DB connection dbi.startRequest() dbi.endRequest(False) print "DONE!\n" if run_from: try: mig_tasks_names = list(t.__name__ for (__, t, __, __) in MIGRATION_TASKS) mti = mig_tasks_names.index(run_from) MIGRATION_TASKS = MIGRATION_TASKS[mti:] except ValueError: print console.colored("The task {0} does not exist".format(run_from), 'red') return 1 # go from older to newer version and execute corresponding tasks for version, task, always, never in MIGRATION_TASKS: if never and task.__name__ not in specified: continue if specified and task.__name__ not in specified: continue if parse_version(version) > prevVersion or always: print console.colored("#", 'green', attrs=['bold']), \ task.__doc__.replace('\n', '').replace(' ', '').strip(), print console.colored("(%s)" % version, 'yellow') if dry_run: continue dbi.startRequest() if withRBDB: DALManager.connect() task(dbi, withRBDB, prevVersion) if withRBDB: DALManager.commit() dbi.endRequest() print console.colored(" DONE\n", 'green', attrs=['bold']) if not dry_run: print console.colored("Database Migration successful!\n", 'green', attrs=['bold'])
def roomBlockingInit(dbi, withRBDB, prevVersion): """ Initializing room blocking indexes. """ if not withRBDB: return root = DALManager().getRoot() if not root.has_key( 'RoomBlocking' ): root['RoomBlocking'] = OOBTree() root['RoomBlocking']['Blockings'] = IOBTree() root['RoomBlocking']['Indexes'] = OOBTree() root['RoomBlocking']['Indexes']['OwnerBlockings'] = OOBTree() root['RoomBlocking']['Indexes']['DayBlockings'] = CalendarDayIndex() root['RoomBlocking']['Indexes']['RoomBlockings'] = OOBTree()
def rebuildRoomReservationsIndex(): from indico.core.db import DBMgr from MaKaC.rb_location import CrossLocationDB from MaKaC.rb_room import RoomBase from MaKaC.plugins.RoomBooking.default.dalManager import DALManager from BTrees.OOBTree import OOBTree DBMgr.getInstance().startRequest() CrossLocationDB.connect() root = DALManager.getRoot() resvEx = ReservationBase() resvEx.isConfirmed = None allResvs = CrossLocationQueries.getReservations(resvExample=resvEx) print "There are " + str(len(allResvs)) + " resvs and pre-resvs to index..." c = 0 root[_ROOM_RESERVATIONS_INDEX] = OOBTree() print "Room => Reservations Index branch created" for resv in allResvs: roomReservationsIndexBTree = root[_ROOM_RESERVATIONS_INDEX] resvs = roomReservationsIndexBTree.get(resv.room.id) if resvs == None: resvs = [] # New list of reservations for this room roomReservationsIndexBTree.insert(resv.room.id, resvs) resvs.append(resv) roomReservationsIndexBTree[resv.room.id] = resvs c += 1 if c % 100 == 0: print c CrossLocationDB.commit() CrossLocationDB.disconnect() DBMgr.getInstance().endRequest()
def runReservationNotificationMigration(dbi, withRBDB, prevVersion): """ Migrate the reservation notification system. """ if not withRBDB: return # Delete old start end notification data for i, resv in enumerate(CrossLocationQueries.getReservations()): if hasattr(resv, '_startEndNotification'): resv._startEndNotification = None if i % 1000 == 0: DALManager.commit() # Create start notification task Client().enqueue(RoomReservationTask(rrule.HOURLY, byminute=0, bysecond=0)) DALManager.commit()
def rebuildRoomReservationsIndex(): from MaKaC.common.db import DBMgr from MaKaC.rb_location import CrossLocationDB from MaKaC.rb_room import RoomBase from MaKaC.plugins.RoomBooking.default.dalManager import DALManager from BTrees.OOBTree import OOBTree DBMgr.getInstance().startRequest() CrossLocationDB.connect() root = DALManager.getRoot() resvEx = ReservationBase() resvEx.isConfirmed = None allResvs = CrossLocationQueries.getReservations( resvExample = resvEx ) print "There are " + str( len( allResvs ) ) + " resvs and pre-resvs to index..." c = 0 root[_ROOM_RESERVATIONS_INDEX] = OOBTree() print "Room => Reservations Index branch created" for resv in allResvs: roomReservationsIndexBTree = root[_ROOM_RESERVATIONS_INDEX] resvs = roomReservationsIndexBTree.get( resv.room.id ) if resvs == None: resvs = [] # New list of reservations for this room roomReservationsIndexBTree.insert( resv.room.id, resvs ) resvs.append( resv ) roomReservationsIndexBTree[resv.room.id] = resvs c += 1 if c % 100 == 0: print c CrossLocationDB.commit() CrossLocationDB.disconnect() DBMgr.getInstance().endRequest()
def runMigration(withRBDB=False, prevVersion=parse_version(__version__), specified=[], dryRun=False): if not dryRun: print "\nExecuting migration...\n" dbi = DBMgr.getInstance() print "Probing DB connection...", # probe DB connection dbi.startRequest() dbi.endRequest(False) print "DONE!\n" # go from older to newer version and execute corresponding tasks for version, task, always, never in MIGRATION_TASKS: if never and task.__name__ not in specified: continue if specified and task.__name__ not in specified: continue if parse_version(version) > prevVersion or always: print console.colored("#", 'green', attrs=['bold']), \ task.__doc__.replace('\n', '').replace(' ', '').strip(), print console.colored("(%s)" % version, 'yellow') if dryRun: continue dbi.startRequest() if withRBDB: DALManager.connect() task(dbi, withRBDB, prevVersion) if withRBDB: DALManager.commit() dbi.endRequest() print console.colored(" DONE\n", 'green', attrs=['bold']) if not dryRun: print console.colored("Database Migration successful!\n", 'green', attrs=['bold'])
def _run(args): _setup(args) formatter = logging.Formatter("%(asctime)s %(name)s - %(levelname)s %(filename)s:%(lineno)s: %(message)s") root = logging.getLogger('') handler = logging.StreamHandler() handler.setFormatter(formatter) root.addHandler(handler) dbi = DBMgr.getInstance(max_disconnect_poll=40) dbi.startRequest() info = HelperMaKaCInfo.getMaKaCInfoInstance() useRBDB = info.getRoomBookingModuleActive() if useRBDB: DALManager.connect() sm = SchedulerModule.getDBInstance() t = sm.getTaskById(args.taskid) t.plugLogger(logging.getLogger('console.run/%s' % args.taskid)) t.run() if useRBDB: DALManager.commit() DALManager.disconnect() dbi.endRequest()
def _run(args): _setup(args) formatter = logging.Formatter( "%(asctime)s %(name)s - %(levelname)s %(filename)s:%(lineno)s: %(message)s" ) root = logging.getLogger('') handler = logging.StreamHandler() handler.setFormatter(formatter) root.addHandler(handler) dbi = DBMgr.getInstance(max_disconnect_poll=40) dbi.startRequest() info = HelperMaKaCInfo.getMaKaCInfoInstance() useRBDB = info.getRoomBookingModuleActive() if useRBDB: DALManager.connect() sm = SchedulerModule.getDBInstance() t = sm.getTaskById(args.taskid) t.plugLogger(logging.getLogger('console.run/%s' % args.taskid)) t.run() if useRBDB: DALManager.commit() DALManager.disconnect() dbi.endRequest()
def runMigration(withRBDB=False, prevVersion=parse_version(__version__), specified=[]): print "\nExecuting migration...\n" dbi = DBMgr.getInstance() print "Probing DB connection...", # probe DB connection dbi.startRequest() dbi.endRequest(False) print "DONE!\n" # go from older to newer version and execute corresponding tasks for version, task, always in MIGRATION_TASKS: if specified and task.__name__ not in specified: continue if parse_version(version) > prevVersion or always: print console.colored("#", 'green', attrs=['bold']), \ task.__doc__.replace('\n', '').strip(), print console.colored("(%s)" % version, 'yellow') dbi.startRequest() if withRBDB: DALManager.connect() task(dbi, withRBDB, prevVersion) if withRBDB: DALManager.commit() dbi.endRequest() print console.colored(" DONE\n", 'green', attrs=['bold']) print console.colored("Database Migration successful!\n", 'green', attrs=['bold'])
def _prepare(self): """ This acts as a second 'constructor', that is executed in the context of the thread (due to database reasons) """ self._prepareDB() self._dbi.startRequest() with self._dbi.transaction() as conn: schedMod = SchedulerModule.getDBInstance() self._task = schedMod.getTaskById(self._taskId) info = HelperMaKaCInfo.getMaKaCInfoInstance() self._rbEnabled = info.getRoomBookingModuleActive() if self._rbEnabled: self._rbdbi = DALManager.getInstance() self._rbdbi.connect() else: self._rbdbi = DALManager.dummyConnection() # open a logging channel self._task.plugLogger(self._logger)
def pluginMigration(dbi, withRBDB, prevVersion): """ Adding new plugins and adapting existing ones to new name policies """ PLUGIN_REMAP = { 'PayPal': 'payPal', 'WorldPay': 'worldPay', 'YellowPay': 'yellowPay', "Dummyimporter": "dummy", "CDSInvenio": "invenio", "CERNSearchPOST": "cern_search", "InvenioBatchUploader": "invenio" } root = dbi.getDBConnection().root() if 'plugins' in root: ptl = [] ps = root['plugins'] for k, v in ps.iteritems(): if isinstance(v, PluginType): ptl.append(v) for pt in ptl: pt.setUsable(True) for p in pt.getPluginList(includeNonPresent=True, includeTestPlugins=True, includeNonActive=True): if hasattr(p, '_Plugin__id'): pid = p.getId() else: pid = p.getName() if pid in PLUGIN_REMAP: pid = PLUGIN_REMAP[pid] p.setId(pid) p.setUsable(True) dbi.commit() if withRBDB: DALManager.commit() # load new plugins, so that we can update them after PluginsHolder().reloadAllPlugins() dbi.commit() if withRBDB: DALManager.commit() if prevVersion < parse_version("0.98b1"): # update db for specific plugins livesync.db.updateDBStructures(root) dbi.commit() if withRBDB: DALManager.commit()
def runRoomDayIndexInit(dbi, withRBDB, prevVersion): """ Initializing room+day => reservation index. """ if not withRBDB: return root = DALManager().getRoot() if not root.has_key('RoomDayReservationsIndex'): root['RoomDayReservationsIndex'] = OOBTree() for i, resv in enumerate(CrossLocationQueries.getReservations()): resv._addToRoomDayReservationsIndex() if i % 1000 == 0: DALManager.commit() DALManager.commit()
def conferenceMigration(dbi, withRBDB, prevVersion): """ Adding missing attributes to conference objects and children """ cdmr = displayMgr.ConfDisplayMgrRegistery() ch = ConferenceHolder() i = 0 from97 = prevVersion < parse_version("0.98b1") # migrating from <=0.97.1 requires smaller granularity for (level, obj) in console.conferenceHolderIterator(ch, deepness='subcontrib' if from97 else 'event'): # only for conferences if level == 'event': if from97: # handle sessions, that our iterator ignores for session in obj.getSessionList(): _fixAccessController(session) if hasattr(obj, '_Conference__alarmCounter'): raise Exception("Conference Object %s (%s) seems to have been " "already converted" % (obj, obj.id)) existingKeys = obj.alarmList.keys() existingKeys.sort() nstart = int(existingKeys[-1]) + 1 if existingKeys else 0 obj._Conference__alarmCounter = Counter(nstart) # For each conference, take the existing tasks and # convert them to the new object classes. _convertAlarms(obj) # convert registration form's "Personal Data" section to new format obj.getRegistrationForm()._convertPersonalData() # For each conference, fix the default style _fixDefaultStyle(obj, cdmr) if from97: _fixAccessController(obj, fixSelf=(level != 'subcontrib')) # Convert RegistrationSessions to RegistrantSessions if isinstance(obj, Conference): for reg in obj.getRegistrants().itervalues(): if reg._sessions and \ isinstance(reg._sessions[0], RegistrationSession): reg._sessions = [RegistrantSession(ses, reg) \ for ses in reg._sessions] if i % 1000 == 999: dbi.commit() if withRBDB and from97: DALManager.commit() i += 1 dbi.commit() if withRBDB and from97: DALManager.commit()