def db_reset(
load: str = None, dir: str = BACKUP_PATH, structure: bool = True, data: bool = False
):
"""Reset database, with options to load fresh data."""
if not app.debug:
prompt = (
"This deletes all data and resets the structure on %s.\nDo you want to continue?"
% app.db.engine
)
if not click.confirm(prompt):
click.echo("I did nothing.")
return
from flexmeasures.data.scripts.data_gen import reset_db
current_version = migrate.current()
reset_db(app.db)
migrate.stamp(current_version)
if load:
if not data and not structure:
click.echo("Neither --data nor --structure given ... loading nothing.")
return
from flexmeasures.data.scripts.data_gen import load_tables
load_tables(app.db, load, structure, data, dir)
def setup_database(app):
"""Prepare the database. Create tables, run migrations etc."""
def _pre_alembic_db():
""" Checks if we are migrating from a pre-alembic ihatemoney
"""
con = db.engine.connect()
tables_exist = db.engine.dialect.has_table(con, "project")
alembic_setup = db.engine.dialect.has_table(con, "alembic_version")
return tables_exist and not alembic_setup
sqlalchemy_url = app.config.get("SQLALCHEMY_DATABASE_URI")
if sqlalchemy_url.startswith("sqlite:////tmp"):
warnings.warn(
"The database is currently stored in /tmp and might be lost at "
"next reboot.")
db.init_app(app)
db.app = app
Migrate(app, db)
migrations_path = os.path.join(app.root_path, "migrations")
if _pre_alembic_db():
with app.app_context():
# fake the first migration
stamp(migrations_path, revision="b9a10d5d63ce")
# auto-execute migrations on runtime
with app.app_context():
upgrade(migrations_path)
def configure_db(app):
models.db.init_app(app)
log = logging.getLogger('ara.webapp.configure_db')
log.debug('Setting up database...')
if app.config.get('ARA_AUTOCREATE_DATABASE'):
with app.app_context():
migrations = app.config['DB_MIGRATIONS']
flask_migrate.Migrate(app, models.db, directory=migrations)
config = app.extensions['migrate'].migrate.get_config(migrations)
# Verify if the database tables have been created at all
inspector = Inspector.from_engine(models.db.engine)
if len(inspector.get_table_names()) == 0:
log.info('Initializing new DB from scratch')
flask_migrate.upgrade(directory=migrations)
# Get current alembic head revision
script = ScriptDirectory.from_config(config)
head = script.get_current_head()
# Get current revision, if available
connection = models.db.engine.connect()
context = MigrationContext.configure(connection)
current = context.get_current_revision()
if not current:
log.info('Unstable DB schema, stamping original revision')
flask_migrate.stamp(directory=migrations,
revision='da9459a1f71c')
if head != current:
log.info('DB schema out of date, upgrading')
flask_migrate.upgrade(directory=migrations)
def setup_database(app):
"""Prepare the database. Create tables, run migrations etc."""
def _pre_alembic_db():
""" Checks if we are migrating from a pre-alembic ihatemoney
"""
con = db.engine.connect()
tables_exist = db.engine.dialect.has_table(con, 'project')
alembic_setup = db.engine.dialect.has_table(con, 'alembic_version')
return tables_exist and not alembic_setup
db.init_app(app)
db.app = app
Migrate(app, db)
migrations_path = os.path.join(app.root_path, 'migrations')
if _pre_alembic_db():
with app.app_context():
# fake the first migration
stamp(migrations_path, revision='b9a10d5d63ce')
# auto-execute migrations on runtime
with app.app_context():
upgrade(migrations_path)
def initialize_db(credentials):
with app.app_context():
populate_data = True
inspector = reflection.Inspector.from_engine(db.engine)
table_name = 'events'
table_names = inspector.get_table_names()
print("[LOG] Existing tables:")
print("[LOG] " + ','.join(table_names))
if table_name not in table_names:
print("[LOG] Table not found. Attempting creation")
try:
db.create_all()
stamp()
except Exception:
populate_data = False
print(
"[LOG] Could not create tables. Either database does not exist or tables already created"
)
if populate_data:
credentials = credentials.split(":")
admin_email = os.environ.get('SUPER_ADMIN_EMAIL',
credentials[0])
admin_password = os.environ.get('SUPER_ADMIN_PASSWORD',
credentials[1])
create_super_admin(admin_email, admin_password)
populate()
else:
print(
"[LOG] Tables already exist. Skipping data population & creation."
)
def create_tables():
"""Create the database tables."""
from redash.models import db
db.create_all()
# Need to mark current DB as up to date
stamp()
def create_tables():
"""Create the database tables."""
from redash.models import db
db.create_all()
# Need to mark current DB as up to date
stamp()
def create():
""" Initialize the database by creating the necessary tables and indices """
# create all tables and indices
db.create_all()
# create alembic version table
stamp()
def init(user, password, pip_user, pip_pwd):
"""Init database and create an admin user"""
from flask_migrate import stamp
from extrapypi.extensions import db, migrate
from extrapypi import models
click.echo("Creating database...")
db.create_all()
click.echo("Database created")
click.echo("Creating user %s" % user)
pwd = custom_app_context.hash(password)
user = models.User(username=user,
email="*****@*****.**",
password_hash=pwd,
role='admin')
pip_pwd = custom_app_context.hash(pip_pwd)
pip_usr = models.User(username=pip_user,
email="*****@*****.**",
password_hash=pip_pwd,
role="installer")
db.session.add(pip_usr)
db.session.add(user)
db.session.commit()
click.echo("User %s created" % user.username)
click.echo("User %s created" % pip_usr.username)
stamp(directory=migrate.directory)
def prepare():
"""Initializes an empty database (creates tables, sets alembic rev to HEAD)"""
tables = get_all_tables(db)
if 'alembic_version' not in tables['public']:
print colored('Setting the alembic version to HEAD', 'green')
stamp()
PluginScriptDirectory.dir = os.path.join(current_app.root_path, 'core', 'plugins', 'alembic')
alembic.command.ScriptDirectory = PluginScriptDirectory
plugin_msg = cformat("%{cyan}Setting the alembic version of the %{cyan!}{}%{reset}%{cyan} "
"plugin to HEAD%{reset}")
for plugin in plugin_engine.get_active_plugins().itervalues():
if not os.path.exists(plugin.alembic_versions_path):
continue
print plugin_msg.format(plugin.name)
with plugin.plugin_context():
stamp()
# Retrieve the table list again, just in case we created unexpected hables
tables = get_all_tables(db)
tables['public'] = [t for t in tables['public'] if not t.startswith('alembic_version')]
if any(tables.viewvalues()):
print colored('Your database is not empty!', 'red')
print colored('If you just added a new table/model, create an alembic revision instead!', 'yellow')
print
print 'Tables in your database:'
for schema, schema_tables in sorted(tables.items()):
for t in schema_tables:
print cformat(' * %{cyan}{}%{reset}.%{cyan!}{}%{reset}').format(schema, t)
return
if not _require_extensions('unaccent', 'pg_trgm'):
return
print colored('Creating tables', 'green')
db.create_all()
def prepare():
"""Initializes an empty database (creates tables, sets alembic rev to HEAD)"""
tables = get_all_tables(db)
if 'alembic_version' not in tables['public']:
print colored('Setting the alembic version to HEAD', 'green')
stamp()
PluginScriptDirectory.dir = os.path.join(current_app.root_path, 'core', 'plugins', 'alembic')
alembic.command.ScriptDirectory = PluginScriptDirectory
plugin_msg = cformat("%{cyan}Setting the alembic version of the %{cyan!}{}%{reset}%{cyan} "
"plugin to HEAD%{reset}")
for plugin in plugin_engine.get_active_plugins().itervalues():
if not os.path.exists(plugin.alembic_versions_path):
continue
print plugin_msg.format(plugin.name)
with plugin.plugin_context():
stamp()
# Retrieve the table list again, just in case we created unexpected hables
tables = get_all_tables(db)
tables['public'] = [t for t in tables['public'] if not t.startswith('alembic_version')]
if any(tables.viewvalues()):
print colored('Your database is not empty!', 'red')
print colored('If you just added a new table/model, create an alembic revision instead!', 'yellow')
print
print 'Tables in your database:'
for schema, schema_tables in sorted(tables.items()):
for t in schema_tables:
print cformat(' * %{cyan}{}%{reset}.%{cyan!}{}%{reset}').format(schema, t)
return
if not _require_extensions('unaccent', 'pg_trgm'):
return
print colored('Creating tables', 'green')
db.create_all()
def main(main_uri, rb_uri, sqla_uri, photo_path, drop, merged_avatars):
update_session_options(db) # get rid of the zope transaction extension
main_root, rb_root, app = setup(main_uri, rb_uri, sqla_uri)
global tz
try:
tz = pytz.timezone(main_root['MaKaCInfo']['main'].getTimezone())
except KeyError:
tz = pytz.utc
start = time.clock()
with app.app_context():
if drop:
print cformat('%{yellow!}*** DANGER')
print cformat('%{yellow!}***%{reset} '
'%{red!}ALL DATA%{reset} in your database %{yellow!}{!r}%{reset} will be '
'%{red!}PERMANENTLY ERASED%{reset}!').format(db.engine.url)
if raw_input(cformat('%{yellow!}***%{reset} To confirm this, enter %{yellow!}YES%{reset}: ')) != 'YES':
print 'Aborting'
sys.exit(1)
delete_all_tables(db)
stamp()
db.create_all()
if Location.find().count():
# Usually there's no good reason to migrate with data in the DB. However, during development one might
# comment out some migration tasks and run the migration anyway.
print cformat('%{yellow!}*** WARNING')
print cformat('%{yellow!}***%{reset} Your database is not empty, migration will most likely fail!')
if raw_input(cformat('%{yellow!}***%{reset} To confirm this, enter %{yellow!}YES%{reset}: ')) != 'YES':
print 'Aborting'
sys.exit(1)
migrate(main_root, rb_root, photo_path, merged_avatars)
print 'migration took {} seconds'.format((time.clock() - start))
def upgradedb():
"""Upgrades the existing database to the latest schema and adds the
default configuration items if they are missing"""
alembic_version_table_exists = db.engine.dialect.has_table(
db.session.connection(), 'alembic_version')
if not alembic_version_table_exists:
virtual_domains_table_exists = db.engine.dialect.has_table(
db.session.connection(), 'virtual_domains')
virtual_users_table_exists = db.engine.dialect.has_table(
db.session.connection(), 'virtual_users')
virtual_aliases_table_exists = db.engine.dialect.has_table(
db.session.connection(), 'virtual_aliases')
# If the alembic_version table doesn't exist and the virtual_* tables
# exist, that means the database is in the default state after
# following the mail server guide on Linode or DigitalOcean.
if virtual_domains_table_exists and virtual_users_table_exists \
and virtual_aliases_table_exists:
# This marks the first revision as complete, which is the revision
# that creates the virtual_* tables
flask_migrate.stamp(revision='bcc85aaa7896')
flask_migrate.upgrade()
add_default_configuration_settings()
def setup_database(app):
"""Prepare the database. Create tables, run migrations etc."""
def _pre_alembic_db():
""" Checks if we are migrating from a pre-alembic ihatemoney
"""
con = db.engine.connect()
tables_exist = db.engine.dialect.has_table(con, 'project')
alembic_setup = db.engine.dialect.has_table(con, 'alembic_version')
return tables_exist and not alembic_setup
sqlalchemy_url = app.config.get('SQLALCHEMY_DATABASE_URI')
if sqlalchemy_url.startswith('sqlite:////tmp'):
warnings.warn(
'The database is currently stored in /tmp and might be lost at '
'next reboot.'
)
db.init_app(app)
db.app = app
Migrate(app, db)
migrations_path = os.path.join(app.root_path, 'migrations')
if _pre_alembic_db():
with app.app_context():
# fake the first migration
stamp(migrations_path, revision='b9a10d5d63ce')
# auto-execute migrations on runtime
with app.app_context():
upgrade(migrations_path)
def setup(self):
update_session_options(db) # get rid of the zope transaction extension
self.app = app = IndicoFlask('indico_zodbimport')
app.config['PLUGINENGINE_NAMESPACE'] = 'indico.plugins'
app.config['PLUGINENGINE_PLUGINS'] = self.plugins
app.config['SQLALCHEMY_DATABASE_URI'] = self.sqlalchemy_uri
plugin_engine.init_app(app)
if not plugin_engine.load_plugins(app):
print cformat(
'%{red!}Could not load some plugins: {}%{reset}').format(
', '.join(plugin_engine.get_failed_plugins(app)))
sys.exit(1)
db.init_app(app)
import_all_models()
alembic_migrate.init_app(
app, db, os.path.join(app.root_path, '..', 'migrations'))
self.connect_zodb()
try:
self.tz = pytz.timezone(
getattr(self.zodb_root['MaKaCInfo']['main'], '_timezone',
'UTC'))
except KeyError:
self.tz = pytz.utc
with app.app_context():
if not self.pre_check():
sys.exit(1)
if self.destructive:
print cformat('%{yellow!}*** DANGER')
print cformat(
'%{yellow!}***%{reset} '
'%{red!}ALL DATA%{reset} in your database %{yellow!}{!r}%{reset} will be '
'%{red!}PERMANENTLY ERASED%{reset}!').format(db.engine.url)
if raw_input(
cformat(
'%{yellow!}***%{reset} To confirm this, enter %{yellow!}YES%{reset}: '
)) != 'YES':
print 'Aborting'
sys.exit(1)
delete_all_tables(db)
stamp()
db.create_all()
if self.has_data():
# Usually there's no good reason to migrate with data in the DB. However, during development one might
# comment out some migration tasks and run the migration anyway.
print cformat('%{yellow!}*** WARNING')
print cformat(
'%{yellow!}***%{reset} Your database is not empty, migration will most likely fail!'
)
if raw_input(
cformat(
'%{yellow!}***%{reset} To confirm this, enter %{yellow!}YES%{reset}: '
)) != 'YES':
print 'Aborting'
sys.exit(1)
def create():
""" Initialize the database by creating the necessary tables and indices """
# create all tables and indices
db.create_all()
# create alembic version table
stamp()
def stamp_db():
"Upgrade database schema."
from zou.app import app
with app.app_context():
import zou
directory = os.path.join(os.path.dirname(zou.__file__), "migrations")
flask_migrate.stamp(directory=directory)
def _reset_db():
from flask_migrate import upgrade, stamp
db.drop_all()
db.engine.execute('DROP TABLE IF EXISTS alembic_version;')
db.create_all()
stamp()
upgrade()
def setup(self):
update_session_options(db) # get rid of the zope transaction extension
self.app = app = IndicoFlask("indico_zodbimport")
app.config["PLUGINENGINE_NAMESPACE"] = "indico.plugins"
app.config["PLUGINENGINE_PLUGINS"] = self.plugins
app.config["SQLALCHEMY_DATABASE_URI"] = self.sqlalchemy_uri
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = True
plugin_engine.init_app(app)
if not plugin_engine.load_plugins(app):
print(
cformat("%{red!}Could not load some plugins: {}%{reset}").format(
", ".join(plugin_engine.get_failed_plugins(app))
)
)
sys.exit(1)
db.init_app(app)
import_all_models()
alembic_migrate.init_app(app, db, os.path.join(app.root_path, "..", "migrations"))
self.connect_zodb()
try:
self.tz = pytz.timezone(getattr(self.zodb_root["MaKaCInfo"]["main"], "_timezone", "UTC"))
except KeyError:
self.tz = pytz.utc
with app.app_context():
if not self.pre_check():
sys.exit(1)
if self.destructive:
print(cformat("%{yellow!}*** DANGER"))
print(
cformat(
"%{yellow!}***%{reset} "
"%{red!}ALL DATA%{reset} in your database %{yellow!}{!r}%{reset} will be "
"%{red!}PERMANENTLY ERASED%{reset}!"
).format(db.engine.url)
)
if raw_input(cformat("%{yellow!}***%{reset} To confirm this, enter %{yellow!}YES%{reset}: ")) != "YES":
print("Aborting")
sys.exit(1)
delete_all_tables(db)
stamp()
db.create_all()
if self.has_data():
# Usually there's no good reason to migrate with data in the DB. However, during development one might
# comment out some migration tasks and run the migration anyway.
print(cformat("%{yellow!}*** WARNING"))
print(
cformat(
"%{yellow!}***%{reset} Your database is not empty, migration may fail or add duplicate " "data!"
)
)
if raw_input(cformat("%{yellow!}***%{reset} To confirm this, enter %{yellow!}YES%{reset}: ")) != "YES":
print("Aborting")
sys.exit(1)
def dbcreate():
with app.app_context(
): # Création d'un contexte pour utiliser les fonction GetAll,ExecSQL qui mémorisent
g.db = None
db.create_all()
from flask_migrate import stamp
stamp(revision='head')
database.ExecSQL("""create view objects as select oh.*,ofi.*
from obj_head oh left join obj_field ofi on oh.objid=ofi.objfid""")
def create_tables():
"""Create the database tables."""
from redash.models import db
_wait_for_db_connection(db)
db.create_all()
# Need to mark current DB as up to date
stamp()
def init_db(url):
"""
Initialise a new database.
"""
if "postgresql" in url:
try:
# Attempt to connect to an existing database using provided credentials
engine = create_engine(url)
engine.connect().close()
except OperationalError as conn_err:
# Connection failed, so connect to default postgres DB and create new megaqc db and user
config_url = make_url(url)
postgres_url = copy(config_url)
# Default db settings
postgres_url.database = "postgres"
postgres_url.username = "******"
postgres_url.password = None
default_engine = create_engine(postgres_url, isolation_level="AUTOCOMMIT")
conn = default_engine.raw_connection()
# conn.autocommit = True
# We use separate transactions here so that a failure in one doesn't affect the other
with conn.cursor() as cur:
print("Initializing the postgres user")
postgres_create_user(
config_url.username,
conn=conn,
cur=cur,
password=config_url.password,
)
with conn.cursor() as cur:
print("Initializing the postgres database")
postgres_create_database(
conn=conn,
cur=cur,
database=config_url.database,
user=config_url.username,
)
# Ue engine with newly created db / user, if it fails again something bigger wrong
engine = create_engine(url)
engine.connect().close()
else:
engine = create_engine(url)
"""Initializes the database."""
db.metadata.bind = engine
db.metadata.create_all()
# Tell alembic that we're at the latest migration, since we just created everything from scratch
stamp()
print("Initialized the database.")
def deploy():
"""Run deployment tasks."""
try:
upgrade()
except:
db.create_all()
stamp()
Role.insert_roles()
User.add_self_follows()
def reset_migrations():
"Set the database schema revision to first one."
from zou.app import app
with app.app_context():
import zou
directory = os.path.join(os.path.dirname(zou.__file__), "migrations")
flask_migrate.stamp(directory=directory, revision="base")
def post_install() -> None:
if not os.geteuid() == 0:
sys.exit('Script must be run as root')
app = create_app(parse_options())
config_path = os.path.join('/', 'etc', 'gitlab-tools', 'config.yml')
configuration = {}
if os.path.isfile(config_path):
with open(config_path) as f:
loaded_data = yaml.load(f)
if isinstance(loaded_data, dict):
configuration.update(loaded_data)
if not configuration.get('USER') and OPTIONS['--user']:
app.config['USER'] = configuration['USER'] = OPTIONS['--user']
# Generate database and config if nothing is specified
if 'SQLALCHEMY_DATABASE_URI' not in configuration or not configuration[
'SQLALCHEMY_DATABASE_URI']:
database_path = 'sqlite:///{}/gitlab-tools.db'.format(
get_home_dir(app.config['USER']))
configuration['SQLALCHEMY_DATABASE_URI'] = database_path
# We need to set DB config to make stamp work
app.config['SQLALCHEMY_DATABASE_URI'] = configuration[
'SQLALCHEMY_DATABASE_URI']
# Create empty database
with app.app_context():
db.create_all()
with app.app_context():
stamp()
# Generate secret key
if 'SECRET_KEY' not in configuration or not configuration['SECRET_KEY']:
app.config['SECRET_KEY'] = configuration[
'SECRET_KEY'] = random_password()
# Set port and host
if 'HOST' not in configuration or not configuration['HOST']:
configuration['HOST'] = '0.0.0.0'
if 'PORT' not in configuration or not configuration['PORT']:
configuration['PORT'] = 80
# Write new configuration
with open(config_path, 'w') as f:
yaml.dump(configuration,
f,
default_flow_style=False,
allow_unicode=True)
def create_tables():
"""Create the database tables."""
from redash.models import db
_wait_for_db_connection(db)
# To create triggers for searchable models, we need to call configure_mappers().
sqlalchemy.orm.configure_mappers()
db.create_all()
# Need to mark current DB as up to date
stamp()
def create_tables():
"""Create the database tables."""
from redash.models import db
_wait_for_db_connection(db)
# To create triggers for searchable models, we need to call configure_mappers().
sqlalchemy.orm.configure_mappers()
db.create_all()
# Need to mark current DB as up to date
stamp()
def initdb(ctx, purge):
"""Initialize the database and all tables."""
if purge:
ctx.invoke(drop)
echo('Initializing database...', nl=False)
db_exists = exists(join(basedir, 'tracker.db'))
db.create_all()
if not db_exists:
stamp()
echo('done')
def initialize_db():
with app.app_context():
populate_data = True
try:
db.drop_all()
db.create_all()
stamp()
except Exception:
populate_data = False
print("[LOG] Could not create tables. Either database does not exist or tables already created")
if populate_data:
populate()
def stamp_db(revision):
"Set the database schema revision to current one."
from zou.app import app
with app.app_context():
import zou
directory = os.path.join(os.path.dirname(zou.__file__), "migrations")
if revision is None:
flask_migrate.stamp(directory=directory)
else:
flask_migrate.stamp(directory=directory, revision=revision)
def create_db():
"""Create and populate the initial database.
The database connection must have been previously initialzed via `haas.model.init_db`.
"""
with app.app_context():
db.create_all()
for head in paths.keys():
# Record the version of each branch. Each extension which uses the
# database will have its own branch.
stamp(revision=head)
get_network_allocator().populate()
db.session.commit()
def prepare_db(empty=False, root_path=None, verbose=True):
"""Initialize an empty database (create tables, set alembic rev to HEAD)."""
if not _require_pg_version('9.6'):
return
if not _require_encoding('UTF8'):
return
if not _require_extensions('unaccent', 'pg_trgm'):
return
root_path = root_path or current_app.root_path
tables = get_all_tables(db)
if 'alembic_version' not in tables['public']:
if verbose:
print(cformat('%{green}Setting the alembic version to HEAD'))
stamp(directory=os.path.join(root_path, 'migrations'),
revision='heads')
PluginScriptDirectory.dir = os.path.join(root_path, 'core', 'plugins',
'alembic')
alembic.command.ScriptDirectory = PluginScriptDirectory
plugin_msg = cformat(
"%{cyan}Setting the alembic version of the %{cyan!}{}%{reset}%{cyan} "
"plugin to HEAD%{reset}")
for plugin in plugin_engine.get_active_plugins().itervalues():
if not os.path.exists(plugin.alembic_versions_path):
continue
if verbose:
print(plugin_msg.format(plugin.name))
with plugin.plugin_context():
stamp(revision='heads')
# Retrieve the table list again, just in case we created unexpected tables
tables = get_all_tables(db)
tables['public'] = [
t for t in tables['public'] if not t.startswith('alembic_version')
]
if any(tables.viewvalues()):
if verbose:
print(cformat('%{red}Your database is not empty!'))
print(
cformat(
'%{yellow}If you just added a new table/model, create an alembic revision instead!'
))
print()
print('Tables in your database:')
for schema, schema_tables in sorted(tables.items()):
for t in schema_tables:
print(
cformat(
' * %{cyan}{}%{reset}.%{cyan!}{}%{reset}').format(
schema, t))
return
create_all_tables(db, verbose=verbose, add_initial_data=(not empty))
def deploy():
"""Run deployment tasks"""
from flask_migrate import upgrade, stamp
try:
# upgrade the database.
upgrade()
except:
# I forgot to run `flask db migrate` at the beginning of the project,
# so I have to init the database like this.
db.create_all()
stamp()
if os.system("pybabel compile -d flog/translations"):
raise RuntimeError("Error: Compiling failed.")
def clear_db():
"Drop all tables from database"
from zou.app import app
with app.app_context():
import zou
print("Deleting database and tables...")
dbhelpers.drop_all()
print("Database and tables deleted.")
directory = os.path.join(os.path.dirname(zou.__file__), "migrations")
flask_migrate.stamp(directory=directory, revision="base")
def reset():
"""Reset database data and re-create tables from data model."""
if not app.debug:
prompt = (
"This deletes all data and re-creates the tables on %s.\nDo you want to continue?"
% app.db.engine
)
if not click.confirm(prompt):
click.echo("I did nothing.")
return
from flexmeasures.data.scripts.data_gen import reset_db
current_version = migrate.current()
reset_db(app.db)
migrate.stamp(current_version)
def create(force=False):
"""Create tables if the database has not been configured yet."""
# Fail if there's an alembic version set
engine = db.get_engine(flask.current_app)
conn = engine.connect()
context = MigrationContext.configure(conn)
current_rev = context.get_current_revision()
alembic_config = flask.current_app.extensions["migrate"].migrate.get_config(directory=migrate_path)
script = ScriptDirectory.from_config(alembic_config)
latest_rev = script.get_current_head()
if current_rev == latest_rev and not force:
print(u"You need to run 'evesrp -c config.py db migrate' to " u"migrate to the latest database schema.")
else:
db.create_all()
if current_rev is None:
stamp()
def migration(event, context):
with create_app().app_context():
# db commands can exit with os.exit(1) - we _have_ to catch that
with wrap_io(catch=BaseException) as (out, err):
upgrade()
if "Please use the 'init' command" in err():
init()
upgrade()
if "Target database is not up to date" in err():
stamp()
upgrade()
return {"stdout": out(), "stderr": err()}
def setup(self):
update_session_options(db) # get rid of the zope transaction extension
self.app = app = IndicoFlask('indico_zodbimport')
app.config['PLUGINENGINE_NAMESPACE'] = 'indico.plugins'
app.config['PLUGINENGINE_PLUGINS'] = self.plugins
app.config['SQLALCHEMY_DATABASE_URI'] = self.sqlalchemy_uri
plugin_engine.init_app(app)
if not plugin_engine.load_plugins(app):
print(cformat('%{red!}Could not load some plugins: {}%{reset}').format(
', '.join(plugin_engine.get_failed_plugins(app))))
sys.exit(1)
db.init_app(app)
import_all_models()
alembic_migrate.init_app(app, db, os.path.join(app.root_path, '..', 'migrations'))
self.connect_zodb()
try:
self.tz = pytz.timezone(getattr(self.zodb_root['MaKaCInfo']['main'], '_timezone', 'UTC'))
except KeyError:
self.tz = pytz.utc
with app.app_context():
if not self.pre_check():
sys.exit(1)
if self.destructive:
print(cformat('%{yellow!}*** DANGER'))
print(cformat('%{yellow!}***%{reset} '
'%{red!}ALL DATA%{reset} in your database %{yellow!}{!r}%{reset} will be '
'%{red!}PERMANENTLY ERASED%{reset}!').format(db.engine.url))
if raw_input(cformat('%{yellow!}***%{reset} To confirm this, enter %{yellow!}YES%{reset}: ')) != 'YES':
print('Aborting')
sys.exit(1)
delete_all_tables(db)
stamp()
db.create_all()
if self.has_data():
# Usually there's no good reason to migrate with data in the DB. However, during development one might
# comment out some migration tasks and run the migration anyway.
print(cformat('%{yellow!}*** WARNING'))
print(cformat('%{yellow!}***%{reset} Your database is not empty, migration may fail or add duplicate '
'data!'))
if raw_input(cformat('%{yellow!}***%{reset} To confirm this, enter %{yellow!}YES%{reset}: ')) != 'YES':
print('Aborting')
sys.exit(1)
def upgradedb():
"""Upgrades the existing database to the latest schema and adds the
default configuration items if they are missing"""
alembic_version_table_exists = db.engine.dialect.has_table(db.session.connection(), 'alembic_version')
if not alembic_version_table_exists:
virtual_domains_table_exists = db.engine.dialect.has_table(db.session.connection(), 'virtual_domains')
virtual_users_table_exists = db.engine.dialect.has_table(db.session.connection(), 'virtual_users')
virtual_aliases_table_exists = db.engine.dialect.has_table(db.session.connection(), 'virtual_aliases')
# If the alembic_version table doesn't exist and the virtual_* tables exist, that means the database is
# in the default state after following the mail server guide on Linode or DigitalOcean.
if virtual_domains_table_exists and virtual_users_table_exists and virtual_aliases_table_exists:
# This marks the first revision as complete, which is the revision that creates the virtual_* tables
flask_migrate.stamp(revision='bcc85aaa7896')
flask_migrate.upgrade()
add_default_configuration_settings()
def prepare():
"""Initializes an empty database (creates tables, sets alembic rev to HEAD)"""
tables = get_all_tables(db)
if 'alembic_version' not in tables['public']:
print colored('Setting the alembic version to HEAD', 'green')
stamp()
# Retrieve the table list again, that way we fail if the alembic version table was not created
tables = get_all_tables(db)
tables['public'].remove('alembic_version')
if any(tables.viewvalues()):
print colored('Your database is not empty!', 'red')
print colored('If you just added a new table/model, create an alembic revision instead!', 'yellow')
print
print 'Tables in your database:'
for schema, schema_tables in sorted(tables.items()):
for t in schema_tables:
print cformat(' * %{cyan}{}%{reset}.%{cyan!}{}%{reset}').format(schema, t)
return
print colored('Creating tables', 'green')
db.create_all()
def configure_db(app):
"""
0.10 is the first version of ARA that ships with a stable database schema.
We can identify a database that originates from before this by checking if
there is an alembic revision available.
If there is no alembic revision available, assume we are running the first
revision which contains the latest state of the database prior to this.
"""
models.db.init_app(app)
log = logging.getLogger('ara.webapp.configure_db')
log.debug('Setting up database...')
if app.config.get('ARA_AUTOCREATE_DATABASE'):
with app.app_context():
migrations = app.config['DB_MIGRATIONS']
flask_migrate.Migrate(app, models.db, directory=migrations)
config = app.extensions['migrate'].migrate.get_config(migrations)
# Verify if the database tables have been created at all
inspector = Inspector.from_engine(models.db.engine)
if len(inspector.get_table_names()) == 0:
log.info('Initializing new DB from scratch')
flask_migrate.upgrade(directory=migrations)
# Get current alembic head revision
script = ScriptDirectory.from_config(config)
head = script.get_current_head()
# Get current revision, if available
connection = models.db.engine.connect()
context = MigrationContext.configure(connection)
current = context.get_current_revision()
if not current:
log.info('Unstable DB schema, stamping original revision')
flask_migrate.stamp(directory=migrations,
revision='da9459a1f71c')
if head != current:
log.info('DB schema out of date, upgrading')
flask_migrate.upgrade(directory=migrations)
def prepare_db(empty=False, root_path=None, verbose=True):
"""Initialize an empty database (create tables, set alembic rev to HEAD)."""
if not _require_pg_version('9.6'):
return
if not _require_encoding('UTF8'):
return
if not _require_extensions('unaccent', 'pg_trgm'):
return
root_path = root_path or current_app.root_path
tables = get_all_tables(db)
if 'alembic_version' not in tables['public']:
if verbose:
print(cformat('%{green}Setting the alembic version to HEAD'))
stamp(directory=os.path.join(root_path, 'migrations'), revision='heads')
PluginScriptDirectory.dir = os.path.join(root_path, 'core', 'plugins', 'alembic')
alembic.command.ScriptDirectory = PluginScriptDirectory
plugin_msg = cformat("%{cyan}Setting the alembic version of the %{cyan!}{}%{reset}%{cyan} "
"plugin to HEAD%{reset}")
for plugin in plugin_engine.get_active_plugins().itervalues():
if not os.path.exists(plugin.alembic_versions_path):
continue
if verbose:
print(plugin_msg.format(plugin.name))
with plugin.plugin_context():
stamp(revision='heads')
# Retrieve the table list again, just in case we created unexpected tables
tables = get_all_tables(db)
tables['public'] = [t for t in tables['public'] if not t.startswith('alembic_version')]
if any(tables.viewvalues()):
if verbose:
print(cformat('%{red}Your database is not empty!'))
print(cformat('%{yellow}If you just added a new table/model, create an alembic revision instead!'))
print()
print('Tables in your database:')
for schema, schema_tables in sorted(tables.items()):
for t in schema_tables:
print(cformat(' * %{cyan}{}%{reset}.%{cyan!}{}%{reset}').format(schema, t))
return
create_all_tables(db, verbose=verbose, add_initial_data=(not empty))
def initialize_db(credentials):
with app.app_context():
populate_data = True
inspector = reflection.Inspector.from_engine(db.engine)
table_name = 'events'
table_names = inspector.get_table_names()
print("[LOG] Existing tables:")
print("[LOG] " + ','.join(table_names))
if table_name not in table_names:
print("[LOG] Table not found. Attempting creation")
try:
db.create_all()
stamp()
except Exception:
populate_data = False
print("[LOG] Could not create tables. Either database does not exist or tables already created")
if populate_data:
credentials = credentials.split(":")
create_super_admin(credentials[0], credentials[1])
populate()
else:
print("[LOG] Tables already exist. Skipping data population & creation.")
import logging, sys, os
sys.path.insert(0,'.')
print(os.getcwd())
log = logging.getLogger(__name__)
from nightowl.app import app, db
from flask_migrate import stamp
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
ctx = app.app_context()
ctx.push()
try:
curver = db.engine.execute("select version_num FROM alembic_version").fetchone()
except:
curver = None
basever = '98cbec7f8cc9'
if curver is None:
print("Flask migrate was never initialized. Setting version to {}".format(basever))
stamp(revision=basever)
else:
print("Current db version is {}".format(curver[0]))
def create():
database.db.create_all()
stamp(revision='head')
email = input("Enter email for super_admin : ")
if not re.match(r'[^@]+@[^@]+\.[^@]+', email):
print('\nInvalid email address\n')
continue
ask_email = False
if not password:
ask_password = True
while ask_password:
password = getpass.getpass("Enter password for super_admin : ")
if len(password) < 4:
print('\nPassword should have minimum 4 characters')
continue
repassword = getpass.getpass("Enter your password again to confirm : ")
if password != repassword:
print('\nPassword did not match')
continue
ask_password = False
create_super_admin(email, password)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("email", nargs='?', help="The email for super_admin.", default='')
parser.add_argument("password", nargs='?', help="The password for super_admin.", default='')
parsed = parser.parse_args()
with current_app.app_context():
db.create_all()
stamp()
create_default_user(parsed.email, parsed.password)
populate()
