def taskMigration(dbi, withRBDB, prevVersion):
"""
Migrating database tasks from the old format to the new one
"""
c = Client()
for t in HelperTaskList().getTaskListInstance().getTasks():
for obj in t.listObj.values():
print console.colored(" * %s" % obj.__class__.__name__, 'blue')
if obj.__class__.__name__ == 'FoundationSync':
c.enqueue(
FoundationSyncTask(rrule.DAILY, byhour=0, byminute=0))
elif obj.__class__.__name__ == 'StatisticsUpdater':
c.enqueue(CategoryStatisticsUpdaterTask(
CategoryManager().getById('0'),
rrule.DAILY,
byhour=0, byminute=0))
elif obj.__class__.__name__ == 'sendMail':
# they have to be somewhere in the conference
alarm = t.conf.alarmList[t.id]
c.enqueue(alarm)
else:
raise Exception("Unknown task type!")
if withRBDB:
DALManager.commit()
dbi.commit()
def taskMigration(dbi, withRBDB, prevVersion):
"""
Migrating database tasks from the old format to the new one
"""
c = Client()
for t in HelperTaskList().getTaskListInstance().getTasks():
for obj in t.listObj.values():
print console.colored(" * %s" % obj.__class__.__name__, 'blue')
if obj.__class__.__name__ == 'OfflineWebsiteCreator':
continue
if obj.__class__.__name__ == 'FoundationSync':
c.enqueue(
FoundationSyncTask(rrule.DAILY, byhour=0, byminute=0))
elif obj.__class__.__name__ == 'StatisticsUpdater':
c.enqueue(CategoryStatisticsUpdaterTask(
CategoryManager().getById('0'),
rrule.DAILY,
byhour=0, byminute=0))
elif obj.__class__.__name__ == 'sendMail':
# they have to be somewhere in the conference
alarm = t.conf.alarmList[t.id]
c.enqueue(alarm)
else:
print console.colored("WARNING: Unknown task type!", 'yellow')
if withRBDB:
DALManager.commit()
dbi.commit()
def runMigration(withRBDB=False,
prevVersion=parse_version(__version__),
specified=[],
dry_run=False,
run_from=None):
global MIGRATION_TASKS
if not dry_run:
print "\nExecuting migration...\n"
dbi = DBMgr.getInstance()
print "Probing DB connection...",
# probe DB connection
dbi.startRequest()
dbi.endRequest(False)
print "DONE!\n"
if run_from:
try:
mig_tasks_names = list(t.__name__
for (__, t, __, __) in MIGRATION_TASKS)
mti = mig_tasks_names.index(run_from)
MIGRATION_TASKS = MIGRATION_TASKS[mti:]
except ValueError:
print console.colored(
"The task {0} does not exist".format(run_from), 'red')
return 1
# go from older to newer version and execute corresponding tasks
for version, task, always, never in MIGRATION_TASKS:
if never and task.__name__ not in specified:
continue
if specified and task.__name__ not in specified:
continue
if parse_version(version) > prevVersion or always:
print console.colored("#", 'green', attrs=['bold']), \
task.__doc__.replace('\n', '').replace(' ', '').strip(),
print console.colored("(%s)" % version, 'yellow')
if dry_run:
continue
dbi.startRequest()
if withRBDB:
DALManager.connect()
task(dbi, withRBDB, prevVersion)
if withRBDB:
DALManager.commit()
dbi.endRequest()
print console.colored(" DONE\n", 'green', attrs=['bold'])
if not dry_run:
print console.colored("Database Migration successful!\n",
'green',
attrs=['bold'])
def addSuggestionsTask(dbi, withRBDB, prevVersion):
"""Add Category Suggestion Task to scheduler (Redis needed)"""
if not Config.getInstance().getRedisConnectionURL():
print console.colored(" Redis not configured, skipping", 'yellow')
return
task = CategorySuggestionTask(rrule.DAILY)
client = Client()
client.enqueue(task)
dbi.commit()
def addSuggestionsTask(dbi, withRBDB, prevVersion):
"""Add Category Suggestion Task to scheduler (Redis needed)"""
if not Config.getInstance().getRedisConnectionURL():
print console.colored(" Redis not configured, skipping", 'yellow')
return
task = CategorySuggestionTask(rrule.DAILY)
client = Client()
client.enqueue(task)
dbi.commit()
def _startFakeDB(self, zeoHost, zeoPort):
"""
Starts a temporary DB in a different port
"""
print colored("-- Starting a test DB", "cyan")
self._createNewDBFile()
self.zeoServer = TestManager._createDBServer(
os.path.join(self.dbFolder, "Data.fs"), zeoHost, zeoPort)
def add(namespace, element, name=None, doc=None):
if not name:
name = element.__name__
namespace[name] = element
if doc:
print console.colored('+ {0} : {1}'.format(name, doc), 'green')
else:
print console.colored('+ {0}'.format(name), 'green')
def main(self, testsToRun, options):
"""
Runs the main test cycle, iterating over all the TestRunners available
* testsToRun - a list of strings specifying which tests to run
* options - test options (such as verbosity...)
"""
result = False
killself = options.pop('killself', False)
TestManager._title("Starting test framework\n")
# the SMTP server will choose a free port
smtpAddr = self._startSMTPServer()
self._startManageDB()
self._setFakeConfig({"SmtpServer": smtpAddr})
if 'functional' in testsToRun:
serverAddr = self._runFakeWebServer()
baseURL = "http://{0}:{1}".format(*serverAddr)
else:
baseURL = "http://localhost:8000"
self._cfg._configVars.update({"BaseURL": baseURL})
ContextManager.set('test_env', True)
try:
for test in testsToRun:
if test in TEST_RUNNERS:
try:
result = TEST_RUNNERS[test](**options).run()
except TestOptionException, e:
TestManager._error(e)
else:
print colored(
"[ERR] Test set '%s' does not exist. "
"It has to be added in the TEST_RUNNERS variable\n",
'red') % test
finally:
# whatever happens, clean this mess up
self._stopManageDB(killself)
self._stopSMTPServer()
if killself:
# Forcefully kill ourselves. This avoids waiting for the db to shutdown (SLOW)
self._info(
'Committing suicide to avoid waiting for slow database shutdown'
)
os.kill(os.getpid(), 9)
if result:
return 0
else:
return -1
def _startFakeDB(self, zeoHost, zeoPort):
"""
Starts a temporary DB in a different port
"""
print colored("-- Starting a test DB", "cyan")
self._createNewDBFile()
self.zeoServer = TestManager._createDBServer(
os.path.join(self.dbFolder, "Data.fs"),
zeoHost, zeoPort)
def _stopFakeDB(self, killself=False):
"""
Stops the temporary DB
"""
print colored("-- Stopping test DB", "cyan")
try:
self.zeoServer.shutdown(killself)
self._removeDBFile()
except OSError, e:
print ("Problem terminating ZEO Server: " + str(e))
def print_result(self, slow=float('inf'), veryslow=float('inf')):
duration = float(self)
if duration == float('-inf'):
print colored('skipped', 'blue', attrs=['bold'])
elif duration == float('inf'):
print colored('running', 'red')
elif duration >= veryslow:
print colored(str(self), 'red', attrs=['bold'])
elif duration >= slow:
print colored(str(self), 'yellow', attrs=['bold'])
else:
print colored(str(self), 'green', attrs=['bold'])
def runMigration(withRBDB=False, prevVersion=parse_version(__version__),
specified=[], dry_run=False, run_from=None):
global MIGRATION_TASKS
if not dry_run:
print "\nExecuting migration...\n"
dbi = DBMgr.getInstance()
print "Probing DB connection...",
# probe DB connection
dbi.startRequest()
dbi.endRequest(False)
print "DONE!\n"
if run_from:
try:
mig_tasks_names = list(t.__name__ for (__, t, __, __) in MIGRATION_TASKS)
mti = mig_tasks_names.index(run_from)
MIGRATION_TASKS = MIGRATION_TASKS[mti:]
except ValueError:
print console.colored("The task {0} does not exist".format(run_from), 'red')
return 1
# go from older to newer version and execute corresponding tasks
for version, task, always, never in MIGRATION_TASKS:
if never and task.__name__ not in specified:
continue
if specified and task.__name__ not in specified:
continue
if parse_version(version) > prevVersion or always:
print console.colored("#", 'green', attrs=['bold']), \
task.__doc__.replace('\n', '').replace(' ', '').strip(),
print console.colored("(%s)" % version, 'yellow')
if dry_run:
continue
dbi.startRequest()
if withRBDB:
DALManager.connect()
task(dbi, withRBDB, prevVersion)
if withRBDB:
DALManager.commit()
dbi.endRequest()
print console.colored(" DONE\n", 'green', attrs=['bold'])
if not dry_run:
print console.colored("Database Migration successful!\n",
'green', attrs=['bold'])
def _stopFakeDB(self, killself=False):
"""
Stops the temporary DB
"""
print colored("-- Stopping test DB", "cyan")
try:
self.zeoServer.shutdown(killself)
self._removeDBFile()
except OSError, e:
print("Problem terminating ZEO Server: " + str(e))
def main(self, testsToRun, options):
"""
Runs the main test cycle, iterating over all the TestRunners available
* testsToRun - a list of strings specifying which tests to run
* options - test options (such as verbosity...)
"""
result = False
killself = options.pop('killself', False)
TestManager._title("Starting test framework\n")
# the SMTP server will choose a free port
smtpAddr = self._startSMTPServer()
if 'functional' in testsToRun:
serverAddr = self._runFakeWebServer()
baseURL = "http://{0}:{1}/indico".format(*serverAddr)
else:
baseURL = "http://localhost:8000/indico"
self._setFakeConfig({
"SmtpServer": smtpAddr,
"BaseURL": baseURL
})
self._startManageDB()
try:
for test in testsToRun:
if test in TEST_RUNNERS:
try:
result = TEST_RUNNERS[test](**options).run()
except TestOptionException, e:
TestManager._error(e)
else:
print colored("[ERR] Test set '%s' does not exist. "
"It has to be added in the TEST_RUNNERS variable\n",
'red') % test
finally:
# whatever happens, clean this mess up
self._stopManageDB(killself)
self._stopSMTPServer()
if killself:
# Forcefully kill ourselves. This avoids waiting for the db to shutdown (SLOW)
self._info('Committing suicide to avoid waiting for slow database shutdown')
os.kill(os.getpid(), 9)
if result:
return 0
else:
return -1
def _add_to_context(namespace, element, name=None, doc=None, color='green'):
if not name:
name = element.__name__
namespace[name] = element
attrs = {}
if color[-1] == '!':
color = color[:-1]
attrs['attrs'] = ['bold']
if doc:
print colored('+ {0} : {1}'.format(name, doc), color, **attrs)
else:
print colored('+ {0}'.format(name), color, **attrs)
def _add_to_context(namespace, info, element, name=None, doc=None, color='green'):
if not name:
name = element.__name__
namespace[name] = element
attrs = {}
if color[-1] == '!':
color = color[:-1]
attrs['attrs'] = ['bold']
if doc:
info.append(colored('+ {0} : {1}'.format(name, doc), color, **attrs))
else:
info.append(colored('+ {0}'.format(name), color, **attrs))
def prepare():
"""Initializes an empty database (creates tables, sets alembic rev to HEAD)"""
tables = get_all_tables(db)
if 'alembic_version' not in tables['public']:
print colored('Setting the alembic version to HEAD', 'green')
stamp()
PluginScriptDirectory.dir = os.path.join(current_app.root_path, 'core', 'plugins', 'alembic')
alembic.command.ScriptDirectory = PluginScriptDirectory
plugin_msg = cformat("%{cyan}Setting the alembic version of the %{cyan!}{}%{reset}%{cyan} "
"plugin to HEAD%{reset}")
for plugin in plugin_engine.get_active_plugins().itervalues():
if not os.path.exists(plugin.alembic_versions_path):
continue
print plugin_msg.format(plugin.name)
with plugin.plugin_context():
stamp()
# Retrieve the table list again, just in case we created unexpected hables
tables = get_all_tables(db)
tables['public'] = [t for t in tables['public'] if not t.startswith('alembic_version')]
if any(tables.viewvalues()):
print colored('Your database is not empty!', 'red')
print colored('If you just added a new table/model, create an alembic revision instead!', 'yellow')
print
print 'Tables in your database:'
for schema, schema_tables in sorted(tables.items()):
for t in schema_tables:
print cformat(' * %{cyan}{}%{reset}.%{cyan!}{}%{reset}').format(schema, t)
return
if not _require_extensions('unaccent', 'pg_trgm'):
return
print colored('Creating tables', 'green')
db.create_all()
def prepare():
"""Initializes an empty database (creates tables, sets alembic rev to HEAD)"""
tables = get_all_tables(db)
if 'alembic_version' not in tables['public']:
print colored('Setting the alembic version to HEAD', 'green')
stamp()
PluginScriptDirectory.dir = os.path.join(current_app.root_path, 'core', 'plugins', 'alembic')
alembic.command.ScriptDirectory = PluginScriptDirectory
plugin_msg = cformat("%{cyan}Setting the alembic version of the %{cyan!}{}%{reset}%{cyan} "
"plugin to HEAD%{reset}")
for plugin in plugin_engine.get_active_plugins().itervalues():
if not os.path.exists(plugin.alembic_versions_path):
continue
print plugin_msg.format(plugin.name)
with plugin.plugin_context():
stamp()
# Retrieve the table list again, just in case we created unexpected hables
tables = get_all_tables(db)
tables['public'] = [t for t in tables['public'] if not t.startswith('alembic_version')]
if any(tables.viewvalues()):
print colored('Your database is not empty!', 'red')
print colored('If you just added a new table/model, create an alembic revision instead!', 'yellow')
print
print 'Tables in your database:'
for schema, schema_tables in sorted(tables.items()):
for t in schema_tables:
print cformat(' * %{cyan}{}%{reset}.%{cyan!}{}%{reset}').format(schema, t)
return
if not _require_extensions('unaccent', 'pg_trgm'):
return
print colored('Creating tables', 'green')
db.create_all()
def __init__(self):
banner = colored('\nIndico v{} is ready for your commands!\n'.format(MaKaC.__version__),
'yellow', attrs=['bold'])
super(IndicoShell, self).__init__(banner=banner, use_bpython=False)
self._context = None
self._info = None
self._quiet = False
def redisLinkedTo(dbi, withRBDB, prevVersion):
"""Import linkedTo information into Redis"""
if not Config.getInstance().getRedisConnectionURL():
print console.colored(" Redis not configured, skipping", 'yellow')
return
with redis_client.pipeline(transaction=False) as pipe:
for i, avatar in enumerate(AvatarHolder()._getIdx().itervalues()):
avatar_links.init_links(avatar, client=pipe)
if i % 1000 == 0:
pipe.execute()
dbi.sync()
print '\r %d' % i,
sys.stdout.flush()
pipe.execute()
print '\r Done '
def redisLinkedTo(dbi, withRBDB, prevVersion):
"""Import linkedTo information into Redis"""
if not Config.getInstance().getRedisConnectionURL():
print console.colored(" Redis not configured, skipping", 'yellow')
return
with redis_client.pipeline(transaction=False) as pipe:
for i, avatar in enumerate(AvatarHolder()._getIdx().itervalues()):
avatar_links.init_links(avatar, client=pipe)
if i % 1000 == 0:
pipe.execute()
dbi.sync()
print '\r %d' % i,
sys.stdout.flush()
pipe.execute()
print '\r Done '
def __init__(self):
banner = colored('\nIndico v{} is ready for your commands!\n'.format(
MaKaC.__version__),
'yellow',
attrs=['bold'])
super(IndicoShell, self).__init__(banner=banner, use_bpython=False)
self._context = None
def chatroomIndexMigration(dbi, withRBDB, prevVersion):
"""
Migrating Chat Room index to new structure
"""
# The structure of the indexes is such that for each one self._data
# is a BTree and each node is referenced by the IndexBy___ designation,
# where ___ is the ID in question. Each node is then a TreeSet of
# ChatRoom or XMPPChatRoom objects originally orderded by ID, we need
# this to be ordered by title for effective searching / querying.
# The __cmp__ method has been altered to accommodate this new format,
# take each subnode, iterate through saving the current objects, clear the
# index and reinsert them - they will now be in the correct order.
from MaKaC.plugins.InstantMessaging.indexes import IndexByUser, IndexByConf, IndexByCRName, IndexByID
im_plugin = PluginsHolder().getPluginType('InstantMessaging')
if not im_plugin.isUsable():
print console.colored(' IM plugin not usable - jumping task',
'yellow')
return
try:
for idx in [IndexByUser(), IndexByConf(), IndexByCRName()]:
tmp_idx = defaultdict(list)
print console.colored(" * Index: " + str(idx), 'blue')
for key, node in idx._data.iteritems():
for leaf in node:
tmp_idx[key].append(leaf)
# reset index
idx._data.clear()
for accum, rooms in tmp_idx.iteritems():
for room in rooms:
# Specific handling as IndexByUser & IndexByConf have different
# arguements for tree insertion.
if isinstance(idx, IndexByUser) or isinstance(
idx, IndexByConf):
idx.index(str(accum), room)
else:
idx.index(room)
print console.colored(
"\tAll indexes have now been re-indexed and committed to the DB.",
'green')
except:
dbi.abort()
print console.colored(
"Process failed, ended abruptly, changes not committed.", 'red')
raise
def catalogMigration(dbi, withRBDB, prevVersion):
"""
Initializing/updating index catalog
"""
PluginsHolder().reloadAllPlugins(disable_if_broken=False)
skip = False
for plugin in (p for p in PluginsHolder().getList() if isinstance(p, Plugin) or isinstance(p, PluginType)):
if plugin.isActive() and not plugin.isUsable():
print console.colored(
"\r Plugin '{0}' is going to be disabled: {1}".format(
plugin.getName(),
plugin.getNotUsableReason()
), 'yellow')
skip = True
if skip and not console.yesno('\r Do you want to continue the migration anyway?'):
raise ControlledExit()
Catalog.updateDB(dbi=dbi)
def catalogMigration(dbi, prevVersion):
"""
Initializing/updating index catalog
"""
PluginsHolder().reloadAllPlugins(disable_if_broken=False)
skip = False
for plugin in (p for p in PluginsHolder().getList() if isinstance(p, Plugin) or isinstance(p, PluginType)):
if plugin.isActive() and not plugin.isUsable():
print console.colored(
"\r Plugin '{0}' is going to be disabled: {1}".format(
plugin.getName(),
plugin.getNotUsableReason()
), 'yellow')
skip = True
if skip and not console.yesno('\r Do you want to continue the migration anyway?'):
raise ControlledExit()
Catalog.updateDB(dbi=dbi)
def main():
"""
Main program cycle
"""
print console.colored("""\nThis script will migrate your Indico DB to a new version. We recommend that
this operation be executed while the web server is down, in order to avoid
concurrency problems and DB conflicts.\n\n""", 'yellow')
parser = argparse.ArgumentParser(description='Execute migration')
parser.add_argument('--with-rb', dest='useRBDB', action='store_true',
help='Use the Room Booking DB')
parser.add_argument('--run-only', dest='specified', default='',
help='Specify which step(s) to run (comma-separated)')
parser.add_argument('--prev-version', dest='prevVersion', help='Previous version of Indico (used by DB)', default=__version__)
args = parser.parse_args()
if console.yesno("Are you sure you want to execute the "
"migration now?"):
try:
return runMigration(withRBDB=args.useRBDB,
prevVersion=parse_version(args.prevVersion),
specified=filter(lambda x: x, map(lambda x: x.strip(), args.specified.split(','))))
except:
print console.colored("\nMigration failed! DB may be in "
" an inconsistent state:", 'red', attrs=['bold'])
print console.colored(traceback.format_exc(), 'red')
else:
return 1
def chatroomIndexMigration(dbi, withRBDB, prevVersion):
"""
Migrating Chat Room index to new structure
"""
# The structure of the indexes is such that for each one self._data
# is a BTree and each node is referenced by the IndexBy___ designation,
# where ___ is the ID in question. Each node is then a TreeSet of
# ChatRoom or XMPPChatRoom objects originally orderded by ID, we need
# this to be ordered by title for effective searching / querying.
# The __cmp__ method has been altered to accommodate this new format,
# take each subnode, iterate through saving the current objects, clear the
# index and reinsert them - they will now be in the correct order.
from MaKaC.plugins.InstantMessaging.indexes import IndexByUser, IndexByConf, IndexByCRName, IndexByID
im_plugin = PluginsHolder().getPluginType('InstantMessaging')
if not im_plugin.isUsable():
print console.colored(' IM plugin not usable - jumping task', 'yellow')
return
try:
for idx in [IndexByUser(), IndexByConf(), IndexByCRName()]:
tmp_idx = defaultdict(list)
print console.colored(" * Index: " + str(idx), 'blue')
for key, node in idx._data.iteritems():
for leaf in node:
tmp_idx[key].append(leaf)
# reset index
idx._data.clear()
for accum, rooms in tmp_idx.iteritems():
for room in rooms:
# Specific handling as IndexByUser & IndexByConf have different
# arguements for tree insertion.
if isinstance(idx, IndexByUser) or isinstance(idx, IndexByConf):
idx.index(str(accum), room)
else:
idx.index(room)
print console.colored("\tAll indexes have now been re-indexed and committed to the DB.", 'green')
except:
dbi.abort()
print console.colored("Process failed, ended abruptly, changes not committed.", 'red')
raise
def _require_extensions(*names):
missing = sorted(name for name in names if not has_extension(db.engine, name))
if not missing:
return True
print colored('Required Postgres extensions missing: {}'.format(', '.join(missing)), 'red')
print colored('Create them using these SQL commands (as a Postgres superuser):', 'yellow')
for name in missing:
print colored(' CREATE EXTENSION {};'.format(name), 'white', attrs={'bold': True})
return False
def _require_extensions(*names):
missing = sorted(name for name in names if not has_extension(db.engine, name))
if not missing:
return True
print colored('Required Postgres extensions missing: {}'.format(', '.join(missing)), 'red')
print colored('Create them using these SQL commands (as a Postgres superuser):', 'yellow')
for name in missing:
print colored(' CREATE EXTENSION {};'.format(name), 'white', attrs={'bold': True})
return False
def runMigration(withRBDB=False, prevVersion=parse_version(__version__),
specified=[], dryRun=False):
if not dryRun:
print "\nExecuting migration...\n"
dbi = DBMgr.getInstance()
print "Probing DB connection...",
# probe DB connection
dbi.startRequest()
dbi.endRequest(False)
print "DONE!\n"
# go from older to newer version and execute corresponding tasks
for version, task, always, never in MIGRATION_TASKS:
if never and task.__name__ not in specified:
continue
if specified and task.__name__ not in specified:
continue
if parse_version(version) > prevVersion or always:
print console.colored("#", 'green', attrs=['bold']), \
task.__doc__.replace('\n', '').replace(' ', '').strip(),
print console.colored("(%s)" % version, 'yellow')
if dryRun:
continue
dbi.startRequest()
if withRBDB:
DALManager.connect()
task(dbi, withRBDB, prevVersion)
if withRBDB:
DALManager.commit()
dbi.endRequest()
print console.colored(" DONE\n", 'green', attrs=['bold'])
if not dryRun:
print console.colored("Database Migration successful!\n",
'green', attrs=['bold'])
def prepare():
"""Initializes an empty database (creates tables, sets alembic rev to HEAD)"""
tables = get_all_tables(db)
if 'alembic_version' not in tables['public']:
print colored('Setting the alembic version to HEAD', 'green')
stamp()
# Retrieve the table list again, that way we fail if the alembic version table was not created
tables = get_all_tables(db)
tables['public'].remove('alembic_version')
if any(tables.viewvalues()):
print colored('Your database is not empty!', 'red')
print colored('If you just added a new table/model, create an alembic revision instead!', 'yellow')
print
print 'Tables in your database:'
for schema, schema_tables in sorted(tables.items()):
for t in schema_tables:
print cformat(' * %{cyan}{}%{reset}.%{cyan!}{}%{reset}').format(schema, t)
return
print colored('Creating tables', 'green')
db.create_all()
def main():
"""
Main program cycle
"""
print console.colored("""\nThis script will migrate your Indico DB to a new version. We recommend that
this operation be executed while the web server is down, in order to avoid
concurrency problems and DB conflicts.\n\n""", 'yellow')
parser = argparse.ArgumentParser(description='Execute migration')
parser.add_argument('--dry-run', '-n', dest='dry_run', action='store_true',
help='Only show which migration tasks would be executed')
parser.add_argument('--with-rb', dest='useRBDB', action='store_true',
help='Use the Room Booking DB')
parser.add_argument('--run-only', dest='specified', default='',
help='Specify which step(s) to run (comma-separated)')
parser.add_argument('--run-from', dest='run_from', default='',
help='Specify FROM which step to run (inclusive)')
parser.add_argument('--prev-version', dest='prevVersion', help='Previous version of Indico (used by DB)', default=__version__)
parser.add_argument('--profile', dest='profile', help='Use profiling during the migration', action='store_true')
args = parser.parse_args()
if args.dry_run or console.yesno("Are you sure you want to execute the migration now?"):
try:
if args.profile:
import profile, random, os
proffilename = os.path.join(Config.getInstance().getTempDir(), "migration%s.prof" % str(random.random()))
result = None
profile.runctx("""result=runMigration(withRBDB=args.useRBDB,
prevVersion=parse_version(args.prevVersion),
specified=filter(lambda x: x, map(lambda x: x.strip(), args.specified.split(','))),
run_from=args.run_from,
dry_run=args.dry_run)""",
globals(), locals(), proffilename)
return result
else:
return runMigration(withRBDB=args.useRBDB,
prevVersion=parse_version(args.prevVersion),
specified=filter(lambda x: x, map(lambda x: x.strip(), args.specified.split(','))),
run_from=args.run_from,
dry_run=args.dry_run)
except ControlledExit:
return 1
except (Exception, SystemExit, KeyboardInterrupt):
print console.colored("\nMigration failed! DB may be in "
" an inconsistent state:", 'red', attrs=['bold'])
print console.colored(traceback.format_exc(), 'red')
return -1
else:
return 1
def runMigration(withRBDB=False,
prevVersion=parse_version(__version__),
specified=[]):
print "\nExecuting migration...\n"
dbi = DBMgr.getInstance()
print "Probing DB connection...",
# probe DB connection
dbi.startRequest()
dbi.endRequest(False)
print "DONE!\n"
# go from older to newer version and execute corresponding tasks
for version, task, always in MIGRATION_TASKS:
if specified and task.__name__ not in specified:
continue
if parse_version(version) > prevVersion or always:
print console.colored("#", 'green', attrs=['bold']), \
task.__doc__.replace('\n', '').strip(),
print console.colored("(%s)" % version, 'yellow')
dbi.startRequest()
if withRBDB:
DALManager.connect()
task(dbi, withRBDB, prevVersion)
if withRBDB:
DALManager.commit()
dbi.endRequest()
print console.colored(" DONE\n", 'green', attrs=['bold'])
print console.colored("Database Migration successful!\n",
'green',
attrs=['bold'])
def main():
"""
Main program cycle
"""
print console.colored("""\nThis script will migrate your Indico DB to a new version. We recommend that
this operation be executed while the web server is down, in order to avoid
concurrency problems and DB conflicts.\n\n""", 'yellow')
parser = argparse.ArgumentParser(description='Execute migration')
parser.add_argument('--dry-run', '-n', dest='dry_run', action='store_true',
help='Only show which migration tasks would be executed')
parser.add_argument('--run-only', dest='specified', default='',
help='Specify which step(s) to run (comma-separated)')
parser.add_argument('--run-from', dest='run_from', default='',
help='Specify FROM which step to run (inclusive)')
parser.add_argument('--prev-version', dest='prevVersion', help='Previous version of Indico (used by DB)', default=__version__)
parser.add_argument('--profile', dest='profile', help='Use profiling during the migration', action='store_true')
args = parser.parse_args()
if args.dry_run or console.yesno("Are you sure you want to execute the migration now?"):
try:
if args.profile:
import profile, random, os
proffilename = os.path.join(Config.getInstance().getTempDir(), "migration%s.prof" % str(random.random()))
result = None
profile.runctx("""result=runMigration(
prevVersion=parse_version(args.prevVersion),
specified=filter(lambda x: x, map(lambda x: x.strip(), args.specified.split(','))),
run_from=args.run_from,
dry_run=args.dry_run)""",
globals(), locals(), proffilename)
return result
else:
with make_app().app_context():
return runMigration(prevVersion=parse_version(args.prevVersion),
specified=filter(None, map(str.strip, args.specified.split(','))),
run_from=args.run_from,
dry_run=args.dry_run)
except ControlledExit:
return 1
except (Exception, SystemExit, KeyboardInterrupt):
print console.colored("\nMigration failed! DB may be in an inconsistent state:", 'red', attrs=['bold'])
print console.colored(traceback.format_exc(), 'red')
return -1
else:
return 1
def prepare():
"""Initializes an empty database (creates tables, sets alembic rev to HEAD)"""
tables = get_all_tables(db)
if 'alembic_version' not in tables['public']:
print colored('Setting the alembic version to HEAD', 'green')
stamp()
# Retrieve the table list again, that way we fail if the alembic version table was not created
tables = get_all_tables(db)
tables['public'].remove('alembic_version')
if any(tables.viewvalues()):
print colored('Your database is not empty!', 'red')
print colored(
'If you just added a new table/model, create an alembic revision instead!',
'yellow')
print
print 'Tables in your database:'
for schema, schema_tables in sorted(tables.items()):
for t in schema_tables:
print cformat(
' * %{cyan}{}%{reset}.%{cyan!}{}%{reset}').format(
schema, t)
return
print colored('Creating tables', 'green')
db.create_all()
def main():
"""
Main program cycle
"""
print console.colored(
"""\nThis script will migrate your Indico DB to a new version. We recommend that
this operation be executed while the web server is down, in order to avoid
concurrency problems and DB conflicts.\n\n""", 'yellow')
parser = argparse.ArgumentParser(description='Execute migration')
parser.add_argument('--with-rb',
dest='useRBDB',
action='store_true',
help='Use the Room Booking DB')
parser.add_argument('--run-only',
dest='specified',
default='',
help='Specify which step(s) to run (comma-separated)')
parser.add_argument('--prev-version',
dest='prevVersion',
help='Previous version of Indico (used by DB)',
default=__version__)
args = parser.parse_args()
if console.yesno("Are you sure you want to execute the " "migration now?"):
try:
return runMigration(withRBDB=args.useRBDB,
prevVersion=parse_version(args.prevVersion),
specified=filter(
lambda x: x,
map(lambda x: x.strip(),
args.specified.split(','))))
except:
print console.colored(
"\nMigration failed! DB may be in "
" an inconsistent state:",
'red',
attrs=['bold'])
print console.colored(traceback.format_exc(), 'red')
else:
return 1
def get_storage(zodb_uri):
uri_parts = urlparse(str(zodb_uri))
print colored("Trying to open {}...".format(zodb_uri), 'green')
if uri_parts.scheme == 'zeo':
if uri_parts.port is None:
print colored("No ZEO port specified. Assuming 9675", 'yellow')
storage = ClientStorage((uri_parts.hostname, uri_parts.port or 9675),
username=uri_parts.username,
password=uri_parts.password,
realm=uri_parts.path[1:])
elif uri_parts.scheme in ('file', None):
storage = FileStorage.FileStorage(uri_parts.path)
else:
raise Exception("URI scheme not known: {}".format(uri_parts.scheme))
print colored("Done!", 'green')
return storage
def get_storage(zodb_uri):
uri_parts = urlparse(str(zodb_uri))
print colored("Trying to open {}...".format(zodb_uri), 'green')
if uri_parts.scheme == 'zeo':
if uri_parts.port is None:
print colored("No ZEO port specified. Assuming 9675", 'yellow')
storage = ClientStorage((uri_parts.hostname, uri_parts.port or 9675),
username=uri_parts.username,
password=uri_parts.password,
realm=uri_parts.path[1:])
elif uri_parts.scheme in ('file', None):
storage = FileStorage.FileStorage(uri_parts.path)
else:
raise Exception("URI scheme not known: {}".format(uri_parts.scheme))
print colored("Done!", 'green')
return storage
def _title(text):
"""
Prints an title
"""
print colored("-- " + str(text), 'yellow', attrs=['bold'])
def _error(message):
"""
Prints an info message
"""
print colored("-- " + str(message), 'red')
def _info(message):
"""
Prints an info message
"""
print colored("-- " + str(message), 'cyan')
def _error(cls, message):
"""
Prints an error message
"""
print colored("** %s" % message, 'red')
def _debug(message):
"""
Prints an info message
"""
print colored("-- " + str(message), 'grey')
def _error(message):
"""
Prints an info message
"""
print colored("-- " + str(message), 'red')
def _title(text):
"""
Prints an title
"""
print colored("-- " + str(text), 'yellow', attrs=['bold'])
# IPython >=1.0
from IPython.terminal.embed import InteractiveShellEmbed
except ImportError:
from IPython.frontend.terminal.embed import InteractiveShellEmbed
from IPython.config.loader import Config as IPConfig
OLD_IPYTHON = False
except ImportError:
try:
# IPython <0.12
from IPython.Shell import IPShellEmbed
OLD_IPYTHON = True
except ImportError:
import code
HAS_IPYTHON = False
SHELL_BANNER = console.colored('\nindico {0}\n'.format(MaKaC.__version__), 'yellow')
def add(namespace, element, name=None, doc=None):
if not name:
name = element.__name__
namespace[name] = element
if doc:
print console.colored('+ {0} : {1}'.format(name, doc), 'green')
else:
print console.colored('+ {0}'.format(name), 'green')
def make_indico_dispatcher(wsgi_app):
def _info(message):
"""
Prints an info message
"""
print colored("-- " + str(message), 'cyan')
def _info(cls, message):
"""
Prints an info message
"""
print colored("** %s" % message, 'blue')
def _debug(message):
"""
Prints an info message
"""
print colored("-- " + str(message), 'grey')
def _success(cls, message):
"""
Prints an info message
"""
print colored("** %s" % message, 'green')
