def user_create(grant_admin):
"""Creates new user"""
update_session_options(db)
user_type = 'user' if not grant_admin else 'admin'
while True:
email = prompt_email()
if email is None:
return
email = email.lower()
if not User.find(User.all_emails.contains(email), ~User.is_deleted, ~User.is_pending).count():
break
error('Email already exists')
first_name = prompt("First name")
last_name = prompt("Last name")
affiliation = prompt("Affiliation", '')
print()
while True:
username = prompt("Enter username").lower()
if not Identity.find(provider='indico', identifier=username).count():
break
error('Username already exists')
password = prompt_pass()
if password is None:
return
identity = Identity(provider='indico', identifier=username, password=password)
user = create_user(email, {'first_name': to_unicode(first_name), 'last_name': to_unicode(last_name),
'affiliation': to_unicode(affiliation)}, identity)
user.is_admin = grant_admin
print_user_info(user)
if prompt_bool(cformat("%{yellow}Create the new {}?").format(user_type), default=True):
db.session.add(user)
db.session.commit()
success("New {} created successfully with ID: {}".format(user_type, user.id))
def run(agent_id, force=False):
"""Runs the livesync agent"""
update_session_options(db)
if agent_id is None:
agent_list = LiveSyncAgent.find_all()
else:
agent = LiveSyncAgent.find_first(id=int(agent_id))
if agent is None:
print 'No such agent'
return
agent_list = [agent]
for agent in agent_list:
if agent.backend is None:
print cformat('Skipping agent: %{red!}{}%{reset} (backend not found)').format(agent.name)
continue
if not agent.initial_data_exported and not force:
print cformat('Skipping agent: %{red!}{}%{reset} (initial export not performed)').format(agent.name)
continue
print cformat('Running agent: %{white!}{}%{reset}').format(agent.name)
with DBMgr.getInstance().global_connection():
try:
agent.create_backend().run()
db.session.commit()
finally:
transaction.abort()
def run(agent_id, force=False):
"""Runs the livesync agent"""
update_session_options(db)
if agent_id is None:
agent_list = LiveSyncAgent.find_all()
else:
agent = LiveSyncAgent.find_first(id=int(agent_id))
if agent is None:
print 'No such agent'
return
agent_list = [agent]
for agent in agent_list:
if agent.backend is None:
print cformat(
'Skipping agent: %{red!}{}%{reset} (backend not found)'
).format(agent.name)
continue
if not agent.initial_data_exported and not force:
print cformat(
'Skipping agent: %{red!}{}%{reset} (initial export not performed)'
).format(agent.name)
continue
print cformat('Running agent: %{white!}{}%{reset}').format(agent.name)
with DBMgr.getInstance().global_connection():
try:
agent.create_backend().run()
db.session.commit()
finally:
transaction.abort()
def main(main_uri, rb_uri, sqla_uri, photo_path, drop, merged_avatars):
update_session_options(db) # get rid of the zope transaction extension
main_root, rb_root, app = setup(main_uri, rb_uri, sqla_uri)
global tz
try:
tz = pytz.timezone(main_root['MaKaCInfo']['main'].getTimezone())
except KeyError:
tz = pytz.utc
start = time.clock()
with app.app_context():
if drop:
print cformat('%{yellow!}*** DANGER')
print cformat('%{yellow!}***%{reset} '
'%{red!}ALL DATA%{reset} in your database %{yellow!}{!r}%{reset} will be '
'%{red!}PERMANENTLY ERASED%{reset}!').format(db.engine.url)
if raw_input(cformat('%{yellow!}***%{reset} To confirm this, enter %{yellow!}YES%{reset}: ')) != 'YES':
print 'Aborting'
sys.exit(1)
delete_all_tables(db)
stamp()
db.create_all()
if Location.find().count():
# Usually there's no good reason to migrate with data in the DB. However, during development one might
# comment out some migration tasks and run the migration anyway.
print cformat('%{yellow!}*** WARNING')
print cformat('%{yellow!}***%{reset} Your database is not empty, migration will most likely fail!')
if raw_input(cformat('%{yellow!}***%{reset} To confirm this, enter %{yellow!}YES%{reset}: ')) != 'YES':
print 'Aborting'
sys.exit(1)
migrate(main_root, rb_root, photo_path, merged_avatars)
print 'migration took {} seconds'.format((time.clock() - start))
def initial_export(agent_id, force=False):
"""Performs the initial data export for an agent"""
update_session_options(db)
agent = LiveSyncAgent.find_first(id=int(agent_id))
if agent is None:
print 'No such agent'
return
if agent.backend is None:
print cformat('Cannot run agent %{red!}{}%{reset} (backend not found)').format(agent.name)
return
print cformat('Selected agent: %{white!}{}%{reset} ({})').format(agent.name, agent.backend.title)
if agent.initial_data_exported and not force:
print 'The initial export has already been performed for this agent.'
print cformat('To re-run it, use %{yellow!}--force%{reset}')
return
def _iter_events():
for i, (_, event) in enumerate(conferenceHolderIterator(ConferenceHolder(), deepness='event'), 1):
yield event
if i % 1000 == 0:
# Clean local ZEO cache
transaction.abort()
with DBMgr.getInstance().global_connection():
agent.create_backend().run_initial_export(_iter_events())
agent.initial_data_exported = True
db.session.commit()
def setup(self):
update_session_options(db) # get rid of the zope transaction extension
self.app = app = IndicoFlask('indico_zodbimport')
app.config['PLUGINENGINE_NAMESPACE'] = 'indico.plugins'
app.config['PLUGINENGINE_PLUGINS'] = self.plugins
app.config['SQLALCHEMY_DATABASE_URI'] = self.sqlalchemy_uri
plugin_engine.init_app(app)
if not plugin_engine.load_plugins(app):
print cformat(
'%{red!}Could not load some plugins: {}%{reset}').format(
', '.join(plugin_engine.get_failed_plugins(app)))
sys.exit(1)
db.init_app(app)
import_all_models()
alembic_migrate.init_app(
app, db, os.path.join(app.root_path, '..', 'migrations'))
self.connect_zodb()
try:
self.tz = pytz.timezone(
getattr(self.zodb_root['MaKaCInfo']['main'], '_timezone',
'UTC'))
except KeyError:
self.tz = pytz.utc
with app.app_context():
if not self.pre_check():
sys.exit(1)
if self.destructive:
print cformat('%{yellow!}*** DANGER')
print cformat(
'%{yellow!}***%{reset} '
'%{red!}ALL DATA%{reset} in your database %{yellow!}{!r}%{reset} will be '
'%{red!}PERMANENTLY ERASED%{reset}!').format(db.engine.url)
if raw_input(
cformat(
'%{yellow!}***%{reset} To confirm this, enter %{yellow!}YES%{reset}: '
)) != 'YES':
print 'Aborting'
sys.exit(1)
delete_all_tables(db)
stamp()
db.create_all()
if self.has_data():
# Usually there's no good reason to migrate with data in the DB. However, during development one might
# comment out some migration tasks and run the migration anyway.
print cformat('%{yellow!}*** WARNING')
print cformat(
'%{yellow!}***%{reset} Your database is not empty, migration will most likely fail!'
)
if raw_input(
cformat(
'%{yellow!}***%{reset} To confirm this, enter %{yellow!}YES%{reset}: '
)) != 'YES':
print 'Aborting'
sys.exit(1)
def setup(self):
update_session_options(db) # get rid of the zope transaction extension
self.app = app = IndicoFlask("indico_zodbimport")
app.config["PLUGINENGINE_NAMESPACE"] = "indico.plugins"
app.config["PLUGINENGINE_PLUGINS"] = self.plugins
app.config["SQLALCHEMY_DATABASE_URI"] = self.sqlalchemy_uri
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = True
plugin_engine.init_app(app)
if not plugin_engine.load_plugins(app):
print(
cformat("%{red!}Could not load some plugins: {}%{reset}").format(
", ".join(plugin_engine.get_failed_plugins(app))
)
)
sys.exit(1)
db.init_app(app)
import_all_models()
alembic_migrate.init_app(app, db, os.path.join(app.root_path, "..", "migrations"))
self.connect_zodb()
try:
self.tz = pytz.timezone(getattr(self.zodb_root["MaKaCInfo"]["main"], "_timezone", "UTC"))
except KeyError:
self.tz = pytz.utc
with app.app_context():
if not self.pre_check():
sys.exit(1)
if self.destructive:
print(cformat("%{yellow!}*** DANGER"))
print(
cformat(
"%{yellow!}***%{reset} "
"%{red!}ALL DATA%{reset} in your database %{yellow!}{!r}%{reset} will be "
"%{red!}PERMANENTLY ERASED%{reset}!"
).format(db.engine.url)
)
if raw_input(cformat("%{yellow!}***%{reset} To confirm this, enter %{yellow!}YES%{reset}: ")) != "YES":
print("Aborting")
sys.exit(1)
delete_all_tables(db)
stamp()
db.create_all()
if self.has_data():
# Usually there's no good reason to migrate with data in the DB. However, during development one might
# comment out some migration tasks and run the migration anyway.
print(cformat("%{yellow!}*** WARNING"))
print(
cformat(
"%{yellow!}***%{reset} Your database is not empty, migration may fail or add duplicate " "data!"
)
)
if raw_input(cformat("%{yellow!}***%{reset} To confirm this, enter %{yellow!}YES%{reset}: ")) != "YES":
print("Aborting")
sys.exit(1)
def start(self, delay):
if self.DISABLE_ZODB_HOOK:
update_session_options(db)
self._executionDelay = delay
try:
self.run()
self.endedOn = self._getCurrentDateTime()
finally:
self.running = False
def start(self, delay):
if self.DISABLE_ZODB_HOOK:
update_session_options(db)
self._executionDelay = delay
try:
self.run()
self.endedOn = self._getCurrentDateTime()
finally:
self.running = False
def runMigration(prevVersion=parse_version(__version__),
specified=[],
dry_run=False,
run_from=None):
global MIGRATION_TASKS
if not dry_run:
print "\nExecuting migration...\n"
dbi = DBMgr.getInstance()
print "Probing DB connection...",
# probe DB connection
dbi.startRequest()
dbi.endRequest(False)
update_session_options(db)
print "DONE!\n"
if run_from:
try:
mig_tasks_names = list(t.__name__
for (__, t, __, __) in MIGRATION_TASKS)
mti = mig_tasks_names.index(run_from)
MIGRATION_TASKS = MIGRATION_TASKS[mti:]
except ValueError:
print console.colored(
"The task {0} does not exist".format(run_from), 'red')
return 1
# go from older to newer version and execute corresponding tasks
for version, task, always, never in MIGRATION_TASKS:
if never and task.__name__ not in specified:
continue
if specified and task.__name__ not in specified:
continue
if parse_version(version) > prevVersion or always:
print console.colored("#", 'green', attrs=['bold']), \
task.__doc__.replace('\n', '').replace(' ', '').strip(),
print console.colored("(%s)" % version, 'yellow')
if dry_run:
continue
dbi.startRequest()
task(dbi, prevVersion)
dbi.endRequest()
print console.colored(" DONE\n", 'green', attrs=['bold'])
if not dry_run:
print console.colored("Database Migration successful!\n",
'green',
attrs=['bold'])
def run(self, args):
# disable the zodb commit hook
update_session_options(db)
# remove the celery shell command
next(funcs for group, funcs, _ in command_classes if group == 'Main').remove('shell')
del CeleryCommand.commands['shell']
if args and args[0] == 'flower':
# Somehow flower hangs when executing it using CeleryCommand() so we simply exec it directly.
# It doesn't really need the celery config anyway (besides the broker url)
try:
import flower
except ImportError:
print cformat('%{red!}Flower is not installed')
sys.exit(1)
client_id = Config.getInstance().getFlowerClientId()
if client_id:
app = OAuthApplication.find_first(client_id=client_id)
if app is None:
print cformat('%{red!}There is no OAuth application with the client id {}.').format(client_id)
sys.exit(1)
elif 'read:user' not in app.default_scopes:
print cformat('%{red!}The {} application needs the read:user scope.').format(app.name)
sys.exit(1)
print cformat('%{green!}Only Indico admins will have access to flower.')
print cformat('%{yellow}Note that revoking admin privileges will not revoke Flower access.')
print cformat('%{yellow}To force re-authentication, restart Flower.')
auth_args = ['--auth=^Indico Admin$', '--auth_provider=indico.core.celery.flower.FlowerAuthHandler']
auth_env = {'INDICO_FLOWER_CLIENT_ID': app.client_id,
'INDICO_FLOWER_CLIENT_SECRET': app.client_secret,
'INDICO_FLOWER_AUTHORIZE_URL': url_for('oauth.oauth_authorize', _external=True),
'INDICO_FLOWER_TOKEN_URL': url_for('oauth.oauth_token', _external=True),
'INDICO_FLOWER_USER_URL': url_for('users.authenticated_user', _external=True)}
else:
print cformat('%{red!}WARNING: %{yellow!}Flower authentication is disabled.')
print cformat('%{yellow!}Having access to Flower allows one to shutdown Celery workers.')
print
auth_args = []
auth_env = {}
args = ['celery', '-b', Config.getInstance().getCeleryBroker()] + args + auth_args
env = dict(os.environ, **auth_env)
os.execvpe('celery', args, env)
elif args and args[0] == 'shell':
print cformat('%{red!}Please use `indico shell`.')
sys.exit(1)
else:
CeleryCommand(celery).execute_from_commandline(['indico celery'] + args)
def run(self, args):
# disable the zodb commit hook
update_session_options(db)
# remove the celery shell command
next(funcs for group, funcs, _ in command_classes if group == 'Main').remove('shell')
del CeleryCommand.commands['shell']
if args and args[0] == 'flower':
# Somehow flower hangs when executing it using CeleryCommand() so we simply exec it directly.
# It doesn't really need the celery config anyway (besides the broker url)
try:
import flower
except ImportError:
print cformat('%{red!}Flower is not installed')
sys.exit(1)
client_id = Config.getInstance().getFlowerClientId()
if client_id:
app = OAuthApplication.find_first(client_id=client_id)
if app is None:
print cformat('%{red!}There is no OAuth application with the client id {}.').format(client_id)
sys.exit(1)
elif 'read:user' not in app.default_scopes:
print cformat('%{red!}The {} application needs the read:user scope.').format(app.name)
sys.exit(1)
print cformat('%{green!}Only Indico admins will have access to flower.')
print cformat('%{yellow}Note that revoking admin privileges will not revoke Flower access.')
print cformat('%{yellow}To force re-authentication, restart Flower.')
auth_args = ['--auth=^Indico Admin$', '--auth_provider=indico.core.celery.flower.FlowerAuthHandler']
auth_env = {'INDICO_FLOWER_CLIENT_ID': app.client_id,
'INDICO_FLOWER_CLIENT_SECRET': app.client_secret,
'INDICO_FLOWER_AUTHORIZE_URL': url_for('oauth.oauth_authorize', _external=True),
'INDICO_FLOWER_TOKEN_URL': url_for('oauth.oauth_token', _external=True),
'INDICO_FLOWER_USER_URL': url_for('users.authenticated_user', _external=True)}
else:
print cformat('%{red!}WARNING: %{yellow!}Flower authentication is disabled.')
print cformat('%{yellow!}Having access to Flower allows one to shutdown Celery workers.')
print
auth_args = []
auth_env = {}
args = ['celery', '-b', Config.getInstance().getCeleryBroker()] + args + auth_args
env = dict(os.environ, **auth_env)
os.execvpe('celery', args, env)
elif args and args[0] == 'shell':
print cformat('%{red!}Please use `indico shell`.')
sys.exit(1)
else:
CeleryCommand(celery).execute_from_commandline(['indico celery'] + args)
def runMigration(prevVersion=parse_version(__version__), specified=[], dry_run=False, run_from=None):
global MIGRATION_TASKS
if not dry_run:
print "\nExecuting migration...\n"
dbi = DBMgr.getInstance()
print "Probing DB connection...",
# probe DB connection
dbi.startRequest()
dbi.endRequest(False)
update_session_options(db)
print "DONE!\n"
if run_from:
try:
mig_tasks_names = list(t.__name__ for (__, t, __, __) in MIGRATION_TASKS)
mti = mig_tasks_names.index(run_from)
MIGRATION_TASKS = MIGRATION_TASKS[mti:]
except ValueError:
print console.colored("The task {0} does not exist".format(run_from), 'red')
return 1
# go from older to newer version and execute corresponding tasks
for version, task, always, never in MIGRATION_TASKS:
if never and task.__name__ not in specified:
continue
if specified and task.__name__ not in specified:
continue
if parse_version(version) > prevVersion or always:
print console.colored("#", 'green', attrs=['bold']), \
task.__doc__.replace('\n', '').replace(' ', '').strip(),
print console.colored("(%s)" % version, 'yellow')
if dry_run:
continue
dbi.startRequest()
task(dbi, prevVersion)
dbi.endRequest()
print console.colored(" DONE\n", 'green', attrs=['bold'])
if not dry_run:
print console.colored("Database Migration successful!\n",
'green', attrs=['bold'])
def setup(self):
update_session_options(db) # get rid of the zope transaction extension
self.app = app = IndicoFlask('indico_zodbimport')
app.config['PLUGINENGINE_NAMESPACE'] = 'indico.plugins'
app.config['PLUGINENGINE_PLUGINS'] = self.plugins
app.config['SQLALCHEMY_DATABASE_URI'] = self.sqlalchemy_uri
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True
plugin_engine.init_app(app)
if not plugin_engine.load_plugins(app):
print(
cformat(
'%{red!}Could not load some plugins: {}%{reset}').format(
', '.join(plugin_engine.get_failed_plugins(app))))
sys.exit(1)
db.init_app(app)
import_all_models()
alembic_migrate.init_app(
app, db, os.path.join(app.root_path, '..', 'migrations'))
self.connect_zodb()
try:
self.tz = pytz.timezone(
getattr(self.zodb_root['MaKaCInfo']['main'], '_timezone',
'UTC'))
except KeyError:
self.tz = pytz.utc
with app.app_context():
if not self.pre_check():
sys.exit(1)
if self.has_data():
# Usually there's no good reason to migrate with data in the DB. However, during development one might
# comment out some migration tasks and run the migration anyway.
print(cformat('%{yellow!}*** WARNING'))
print(
cformat(
'%{yellow!}***%{reset} Your database is not empty, migration may fail or add duplicate '
'data!'))
if raw_input(
cformat(
'%{yellow!}***%{reset} To confirm this, enter %{yellow!}YES%{reset}: '
)) != 'YES':
print('Aborting')
sys.exit(1)
def setup(self):
update_session_options(db) # get rid of the zope transaction extension
self.app = app = IndicoFlask('indico_zodbimport')
app.config['PLUGINENGINE_NAMESPACE'] = 'indico.plugins'
app.config['PLUGINENGINE_PLUGINS'] = self.plugins
app.config['SQLALCHEMY_DATABASE_URI'] = self.sqlalchemy_uri
plugin_engine.init_app(app)
if not plugin_engine.load_plugins(app):
print(cformat('%{red!}Could not load some plugins: {}%{reset}').format(
', '.join(plugin_engine.get_failed_plugins(app))))
sys.exit(1)
db.init_app(app)
import_all_models()
alembic_migrate.init_app(app, db, os.path.join(app.root_path, '..', 'migrations'))
self.connect_zodb()
try:
self.tz = pytz.timezone(getattr(self.zodb_root['MaKaCInfo']['main'], '_timezone', 'UTC'))
except KeyError:
self.tz = pytz.utc
with app.app_context():
if not self.pre_check():
sys.exit(1)
if self.destructive:
print(cformat('%{yellow!}*** DANGER'))
print(cformat('%{yellow!}***%{reset} '
'%{red!}ALL DATA%{reset} in your database %{yellow!}{!r}%{reset} will be '
'%{red!}PERMANENTLY ERASED%{reset}!').format(db.engine.url))
if raw_input(cformat('%{yellow!}***%{reset} To confirm this, enter %{yellow!}YES%{reset}: ')) != 'YES':
print('Aborting')
sys.exit(1)
delete_all_tables(db)
stamp()
db.create_all()
if self.has_data():
# Usually there's no good reason to migrate with data in the DB. However, during development one might
# comment out some migration tasks and run the migration anyway.
print(cformat('%{yellow!}*** WARNING'))
print(cformat('%{yellow!}***%{reset} Your database is not empty, migration may fail or add duplicate '
'data!'))
if raw_input(cformat('%{yellow!}***%{reset} To confirm this, enter %{yellow!}YES%{reset}: ')) != 'YES':
print('Aborting')
sys.exit(1)
def run(self, args):
# disable the zodb commit hook
update_session_options(db)
# remove the celery shell command
next(funcs for group, funcs, _ in command_classes if group == 'Main').remove('shell')
del CeleryCommand.commands['shell']
if args and args[0] == 'flower':
# Somehow flower hangs when executing it using CeleryCommand() so we simply exec it directly.
# It doesn't really need the celery config anyway (besides the broker url)
os.execlp('celery', 'celery', '-b', Config.getInstance().getCeleryBroker(),
*args)
elif args and args[0] == 'shell':
print cformat('%{red!}Please use `indico shell`.')
sys.exit(1)
else:
CeleryCommand(celery).execute_from_commandline(['indico celery'] + args)
def database(app, postgresql):
"""Creates a test database which is destroyed afterwards
Used only internally, if you need to access the database use `db` instead to ensure
your modifications are not persistent!
"""
app.config['SQLALCHEMY_DATABASE_URI'] = postgresql
configure_db(app)
update_session_options(db_)
if 'INDICO_TEST_DATABASE_URI' in os.environ and os.environ.get('INDICO_TEST_DATABASE_HAS_TABLES') == '1':
yield db_
return
with app.app_context():
db_.create_all()
yield db_
db_.session.remove()
with app.app_context():
delete_all_tables(db_)
def database(app, postgresql):
"""Creates a test database which is destroyed afterwards
Used only internally, if you need to access the database use `db` instead to ensure
your modifications are not persistent!
"""
app.config['SQLALCHEMY_DATABASE_URI'] = postgresql
configure_db(app)
update_session_options(db_)
if 'INDICO_TEST_DATABASE_URI' in os.environ and os.environ.get('INDICO_TEST_DATABASE_HAS_TABLES') == '1':
yield db_
return
with app.app_context():
db_.create_all()
yield db_
db_.session.remove()
with app.app_context():
delete_all_tables(db_)
def initial_export(agent_id, force=False):
"""Performs the initial data export for an agent"""
update_session_options(db)
agent = LiveSyncAgent.find_first(id=int(agent_id))
if agent is None:
print 'No such agent'
return
if agent.backend is None:
print cformat(
'Cannot run agent %{red!}{}%{reset} (backend not found)').format(
agent.name)
return
print cformat('Selected agent: %{white!}{}%{reset} ({})').format(
agent.name, agent.backend.title)
if agent.initial_data_exported and not force:
print 'The initial export has already been performed for this agent.'
print cformat('To re-run it, use %{yellow!}--force%{reset}')
return
agent.create_backend().run_initial_export(Event.find(is_deleted=False))
agent.initial_data_exported = True
db.session.commit()
if event:
contribs = contribs.filter(db.m.Contribution.event_id == event)
elif category:
contribs = contribs.join(db.m.Event).filter(db.m.Event.category_chain_overlaps(category))
if log:
log.write('<table style="width: 100%;">')
for contrib in contribs:
if '<html>' in unicode(contrib.description):
click.echo(click.style('[HTML DOCUMENT] ', fg='red', bold=True) + repr(contrib))
else:
migrate_description(contrib, verbose, log)
if log:
log.write('</table>')
if html_log:
log.seek(0)
html_log.write(HTML_TPL.format(log.read()))
if not dry_run:
db.session.commit()
if __name__ == '__main__':
update_session_options(db)
with make_app().app_context():
with DBMgr.getInstance().global_connection():
main()
elif category:
contribs = contribs.join(db.m.Event).filter(
db.m.Event.category_chain.contains([category]))
if log:
log.write('<table style="width: 100%;">')
for contrib in contribs:
if '<html>' in unicode(contrib.description):
click.echo(
click.style('[HTML DOCUMENT] ', fg='red', bold=True) +
repr(contrib))
else:
migrate_description(contrib, verbose, log)
if log:
log.write('</table>')
if html_log:
log.seek(0)
html_log.write(HTML_TPL.format(log.read()))
if not dry_run:
db.session.commit()
if __name__ == '__main__':
update_session_options(db)
with make_app().app_context():
with DBMgr.getInstance().global_connection():
main()
def migrate(self):
update_session_options(db, {'expire_on_commit': False})
self.janitor_user = User.get_one(self.janitor_user_id)
self.migrate_global_templates()
self.migrate_event_templates()
