def build_database():
""" Applies the schema to the database. Run this command once to build the database. """
engine = get_engine()
schema = config_get('database', 'schema', raise_exception=False)
if schema:
print('Schema set in config, trying to create schema:', schema)
try:
engine.execute(CreateSchema(schema))
except Exception as e:
print('Cannot create schema, please validate manually if schema creation is needed, continuing:', e)
models.register_models(engine)
# Put the database under version control
alembic_cfg = Config(config_get('alembic', 'cfg'))
command.stamp(alembic_cfg, "head")
def run(revision):
this_file_directory = os.path.dirname(__file__)
root_directory = os.path.join(this_file_directory, '..')
alembic_directory = os.path.join(root_directory, 'alembic')
ini_path = os.path.join(root_directory, 'alembic.ini')
print('ini_path: ', ini_path)
config = Config(ini_path)
config.set_main_option('script_location', alembic_directory)
#prepare and run the command
sql = False
tag = None
# command.stamp(config, revision, sql=sql, tag=tag)
#upgrade command
command.upgrade(config, revision, sql=sql, tag=tag)
def update_db(update=True):
logging.info("Checking for Database Updates...")
db_connection = DatabaseConnection(
database=options.sql_database,
hostname=options.sql_host,
port=options.sql_port,
username=options.sql_user,
password=options.sql_password,
dialect=options.sql_dialect,
)
alembic_cfg = Config("alembic/alembic.ini")
alembic_cfg.attributes["configure_logger"] = False
alembic_cfg.set_main_option("sqlalchemy.url", unquote(str(db_connection)))
if update:
command.upgrade(alembic_cfg, "head")
else:
command.stamp(alembic_cfg, "head")
def migrate_db_with_non_testing_migrations(self, destination):
if destination not in ["head", "base"]:
raise TypeError("Only head & base are accepted as destination "
"values.")
# Set the alembic script directory location
self.alembic_dpath = os.path.join(self.migr_method_dir_path,
utils.ALEMBIC_REL_PATH)
alembic_cfg = Config()
alembic_cfg.set_main_option('script_location', self.alembic_dpath)
# Undo all previous real migration of the database
with sa.engine.begin() as connection:
alembic_cfg.attributes['connection'] = connection
if destination == "head":
command.upgrade(alembic_cfg, "head")
else:
command.downgrade(alembic_cfg, "base")
def initdb(self):
# this will create all tables. It will NOT delete any tables or data.
# Instead, it will raise when something can't be created.
# TODO: explicitly check if the database is empty
Base.metadata.create_all(
self.engine, checkfirst=False
) # checkfirst False will raise when it finds an existing table
from alembic.config import Config
from alembic import command
alembic_cfg = Config(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
"sql_migrations", "alembic.ini"))
with self.engine.begin() as connection:
alembic_cfg.attributes['connection'] = connection
# mark the version table, "stamping" it with the most recent rev:
command.stamp(alembic_cfg, "head")
def upgradedb():
"""
Upgrade the database.
"""
# alembic adds significant import time, so we import it lazily
from alembic import command
from alembic.config import Config
log.info("Creating tables")
current_dir = os.path.dirname(os.path.abspath(__file__))
package_dir = os.path.normpath(os.path.join(current_dir, '..'))
directory = os.path.join(package_dir, 'migrations')
config = Config(os.path.join(package_dir, 'alembic.ini'))
config.set_main_option('script_location', directory.replace('%', '%%'))
config.set_main_option('sqlalchemy.url', settings.SQL_ALCHEMY_CONN.replace('%', '%%'))
command.upgrade(config, 'heads')
add_default_pool_if_not_exists()
def get_alembic_conf():
"""
This function returns the alembic configuration file contents by doing
the necessary updates in the 'script_location' name.
:return: The alembic configuration.
"""
# Constructing the alembic full path & getting the configuration
import os
dir_path = os.path.dirname(os.path.realpath(__file__))
alembic_fpath = os.path.join(dir_path, ALEMBIC_FILENAME)
alembic_cfg = Config(alembic_fpath)
# Set the alembic script directory location
alembic_dpath = os.path.join(dir_path, ALEMBIC_REL_PATH)
alembic_cfg.set_main_option('script_location', alembic_dpath)
return alembic_cfg
def _perform_appless_action(self, pluggable, opts):
from alembic.config import Config
repository = self._pluggable_repository(pluggable)
if repository is None:
return
tablename = self._pluggable_tablename(pluggable)
print("\n%s Migrations" % pluggable)
print("\tRepository '%s'" % repository)
print("\tVersioning Table '%s'" % tablename)
alembic_cfg = Config()
alembic_cfg.set_main_option("script_location", repository)
command = getattr(self, 'command_%s' % opts.command)
command(opts, {'alembic_cfg': alembic_cfg, 'tablename': tablename})
def test_migrations_check_revision_hashes(self):
config = Config()
config.set_main_option("script_location",
str(FARADAY_BASE / "migrations"))
script = ScriptDirectory.from_config(config)
alembic_hashes = []
for revision in script.walk_revisions():
alembic_hashes.append(revision.revision)
migrations_hashes = []
for migration in (FARADAY_BASE / 'migrations' /
'versions').glob('*.py'):
filename = migration.name
migrations_hashes.append(filename.split('_')[0])
assert set(alembic_hashes) == set(migrations_hashes)
def create_tables(db_url, alembic_ini=None, acls=None, debug=False):
""" Create the tables in the database using the information from the
url obtained.
:arg db_url, URL used to connect to the database. The URL contains
information with regards to the database engine, the host to
connect to, the user and password and the database name.
ie: <engine>://<user>:<password>@<host>/<dbname>
:kwarg alembic_ini, path to the alembic ini file. This is necessary
to be able to use alembic correctly, but not for the unit-tests.
:kwarg debug, a boolean specifying wether we should have the verbose
output of sqlalchemy or not.
:return a session that can be used to query the database.
"""
engine = create_engine(db_url, echo=debug, client_encoding='utf8')
from pagure.ui.plugins import get_plugin_tables
get_plugin_tables()
BASE.metadata.create_all(engine)
# engine.execute(collection_package_create_view(driver=engine.driver))
if db_url.startswith('sqlite:'):
# Ignore the warning about con_record
# pylint: disable=W0613
def _fk_pragma_on_connect(dbapi_con, con_record): # pragma: no cover
''' Tries to enforce referential constraints on sqlite. '''
dbapi_con.execute('pragma foreign_keys=ON')
sa.event.listen(engine, 'connect', _fk_pragma_on_connect)
if alembic_ini is not None: # pragma: no cover
# then, load the Alembic configuration and generate the
# version table, "stamping" it with the most recent rev:
# Ignore the warning missing alembic
# pylint: disable=F0401
from alembic.config import Config
from alembic import command
alembic_cfg = Config(alembic_ini)
command.stamp(alembic_cfg, "head")
scopedsession = scoped_session(sessionmaker(bind=engine))
BASE.metadata.bind = scopedsession
# Insert the default data into the db
create_default_status(scopedsession, acls=acls)
return scopedsession
def create_user():
from walkoff.serverdb import add_user, User, Role, initialize_default_resources_admin, \
initialize_default_resources_guest
from sqlalchemy_utils import database_exists, create_database
if not database_exists(db.engine.url):
create_database(db.engine.url)
db.create_all()
alembic_cfg = Config(walkoff.config.Config.ALEMBIC_CONFIG,
ini_section="walkoff",
attributes={'configure_logger': False})
# This is necessary for a flask database
connection = db.engine.connect()
context = MigrationContext.configure(connection)
script = ScriptDirectory.from_config(alembic_cfg)
context.stamp(script, "head")
# Setup admin and guest roles
initialize_default_resources_admin()
initialize_default_resources_guest()
# Setup admin user
admin_role = Role.query.filter_by(id=1).first()
admin_user = User.query.filter_by(username="******").first()
if not admin_user:
add_user(username='******', password='******', roles=[1])
elif admin_role not in admin_user.roles:
admin_user.roles.append(admin_role)
db.session.commit()
apps = set(helpers.list_apps(walkoff.config.Config.APPS_PATH)) - set([
_app.name for _app in
current_app.running_context.execution_db.session.query(App).all()
])
current_app.logger.debug('Found new apps: {0}'.format(apps))
for app_name in apps:
current_app.running_context.execution_db.session.add(
App(name=app_name, devices=[]))
db.session.commit()
current_app.running_context.execution_db.session.commit()
reschedule_all_workflows()
current_app.logger.handlers = logging.getLogger('server').handlers
def main(argv, _db_master=None, _heka_client=None):
parser = argparse.ArgumentParser(prog=argv[0],
description='Initialize Ichnaea database')
parser.add_argument('--alembic_ini',
help='Path to the alembic migration config.')
parser.add_argument('--location_ini',
help='Path to the ichnaea app config.')
parser.add_argument('--initdb',
action='store_true',
help='Initialize database')
args = parser.parse_args(argv[1:])
if args.initdb:
# Either use explicit config file location or fallback
# on environment variable or finally file in current directory
if not args.location_ini:
location_ini = os.environ.get('ICHNAEA_CFG', 'ichnaea.ini')
else:
location_ini = args.location_ini
location_ini = os.path.abspath(location_ini)
location_cfg = read_config(filename=location_ini)
# Either use explicit config file location or fallback
# to a file in the same directory as the ichnaea.ini
if not args.alembic_ini:
alembic_ini = os.path.join(os.path.dirname(location_ini),
'alembic.ini')
else:
alembic_ini = args.alembic_ini
alembic_ini = os.path.abspath(alembic_ini)
alembic_cfg = Config(alembic_ini)
alembic_section = alembic_cfg.get_section('alembic')
if _db_master is None:
db_master = Database(alembic_section['sqlalchemy.url'])
else:
db_master = _db_master
configure_heka(location_ini, _heka_client=_heka_client)
engine = db_master.engine
create_schema(engine, alembic_cfg, location_cfg)
else:
parser.print_help()
def test_db_metadata_differences(models, settings):
db_name = settings['db_name']
# first we drop anything there might be
dropdb(db_name)
# then we create a clean DB from the metadata
createdb(db_name)
metadata = models.metadata
engine = engine_from_config(settings)
metadata.bind = engine
metadata.create_all(engine, tables=[table for name, table
in metadata.tables.items() if not name.startswith('test_')])
# and store the results
create_all_result = dumpdb(db_name)
engine.dispose()
# now we do it again, but this time using migrations
dropdb(db_name)
createdb(db_name)
config = Config()
config.set_main_option('script_location', 'backrest:migrations')
script = ScriptDirectory.from_config(config)
connection = engine.connect()
environment = EnvironmentContext(config, script,
starting_rev='base', destination_rev='head')
context = MigrationContext.configure(connection)
def upgrade(rev, context):
return script._upgrade_revs('head', rev)
context._migrations_fn = upgrade
environment._migration_context = context
with environment.begin_transaction():
environment.run_migrations()
# we drop alembic_version to avoid it showing up in the diff
engine.execute('DROP TABLE alembic_version;')
# we store these results
alembic_result = dumpdb(db_name)
del context
del environment
connection.close()
del connection
engine.dispose()
# now we check whether there are differences and output them if there are
diff = unified_diff(alembic_result, create_all_result)
assert alembic_result == create_all_result, \
'Migration output differs:\n' + ''.join(diff)
def make_extension_config(extension_module_name):
config = Config(get_base_config_file())
# Record the current `extension_module_name` in the config to make it
# available to `ScriptDirectory` and `EnvironmentContext`
config.set_main_option('extension_module_name', extension_module_name)
module = get_extension_module(extension_module_name)
# If the extension module contains a `migrations/env.py`, then use that,
# otherwise use `ggrc/migrations/env.py`
module_script_location = get_extension_migrations_dir(module)
if os.path.exists(os.path.join(module_script_location, 'env.py')):
script_location = module_script_location
else:
script_location = get_base_migrations_dir()
config.set_main_option('script_location', script_location)
# Specify location of `versions` directory to be independent of `env.py`
module_versions_location = os.path.join(module_script_location, 'versions')
config.set_main_option('versions_location', module_versions_location)
return config
def alembic_set_stamp_head(config_migration_path):
# set the paths values
this_file_directory = os.path.dirname(os.path.abspath(inspect.stack()[0][1]))
root_directory = os.path.join(this_file_directory, '../config')
alembic_directory = os.path.join(this_file_directory, 'alembic')
ini_path = config_migration_path # os.path.join(root_directory, 'alembic.ini')
# create Alembic config and feed it with paths
config = Config(ini_path)
config.set_main_option('script_location', alembic_directory)
config.cmd_opts = argparse.Namespace() # arguments stub
config.attributes['configure_logger'] = False
# prepare and run the command
revision = 'head'
# upgrade command
command.upgrade(config, revision)
def database():
# Locate the testing config for Alembic
config = Config(
os.path.join(os.path.dirname(__file__), "../alembic.tests.ini"))
# Set the migration secret key here
if not os.environ.get("SECRET_KEY", None):
os.environ["SECRET_KEY"] = "test"
# Start up the in-memory database instance
db_engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(db_engine)
db_session = Session(bind=db_engine)
# Mark it as up-to-date with migrations
command.stamp(config, "head")
return db_session
def setup_alembic_config(url):
"setting up generic config for alembic migrations"
directory = 'ether_sql/migrations'
config = Config(os.path.join(directory, 'alembic.ini'))
config.set_main_option('script_location', directory)
config.cmd_opts = argparse.Namespace() # arguments stub
x_arg = 'url=' + url
if not hasattr(config.cmd_opts, 'x'):
if x_arg is not None:
setattr(config.cmd_opts, 'x', [])
if isinstance(x_arg, list) or isinstance(x_arg, tuple):
for x in x_arg:
config.cmd_opts.x.append(x)
else:
config.cmd_opts.x.append(x_arg)
else:
setattr(config.cmd_opts, 'x', None)
return config
def initialize_db(config_path: str, alembic_config_path: Optional[str] = None) -> None:
"""Load the app config and create the database tables."""
db_session = get_session_from_config(config_path)
engine = db_session.bind
create_tables(engine)
run_sql_scripts_in_dir("sql/init/", engine)
# if an Alembic config file wasn't specified, assume it's alembic.ini in the same
# directory
if not alembic_config_path:
path = os.path.split(config_path)[0]
alembic_config_path = os.path.join(path, "alembic.ini")
# mark current Alembic revision in db so migrations start from this point
alembic_cfg = Config(alembic_config_path)
command.stamp(alembic_cfg, "head")
def make_alembic_config(cmd_opts: Union[Namespace, SimpleNamespace],
base_path: Path = PROJECT_PATH) -> Config:
if not os.path.isabs(cmd_opts.config):
cmd_opts.config = str(base_path / cmd_opts.config)
config = Config(file_=cmd_opts.config,
ini_section=cmd_opts.name,
cmd_opts=cmd_opts)
# Подменяем путь alembic на абсолютный
alembic_location = config.get_main_option('script_location')
if not os.path.isabs(alembic_location):
config.set_main_option('script_location',
str(base_path / alembic_location))
if cmd_opts.pg_url:
config.set_main_option('sqlalchemy.url', cmd_opts.pg_url)
return config
def make_alembic_config(cmd_opts: Union[Namespace, SimpleNamespace],
base_path: str = PROJECT_PATH) -> Config:
# Replace path to alembic.ini file to absolute
if not os.path.isabs(cmd_opts.config):
cmd_opts.config = os.path.join(base_path, cmd_opts.config)
config = Config(file_=cmd_opts.config, ini_section=cmd_opts.name,
cmd_opts=cmd_opts)
# Replace path to alembic folder to absolute
alembic_location = config.get_main_option('script_location')
if not os.path.isabs(alembic_location):
config.set_main_option('script_location',
os.path.join(base_path, alembic_location))
if cmd_opts.pg_url:
config.set_main_option('sqlalchemy.url', cmd_opts.pg_url)
return config
def migrate(database_uri):
database._engine = None
with create_session() as db:
engine = db.bind
alembic_cfg = Config(str(HERE / "alembic.ini"))
alembic_cfg.set_main_option("script_location",
str(HERE / "migrations"))
alembic_cfg.set_main_option("sqlalchemy.url", database_uri)
alembic_cfg.attributes["configure_logger"] = True
context = MigrationContext.configure(db.connection())
head = context.get_current_revision()
if head is None:
metadata.create_all(engine)
command.stamp(alembic_cfg, "head")
else:
command.upgrade(alembic_cfg, "head")
database._engine = None
def __db_alembic_setup(self):
"""Создает среду алембика."""
migrations_path = Path(Path.home() / ccfg.ALL_CONFIGS_FOLDER)
if not migrations_path.exists():
migrations_path.mkdir()
alembic_config = Config()
alembic_config.set_main_option("script_location", "migrations")
alembic_config.set_main_option(
"url",
'sqlite:///' + self.config.restore_value(ccfg.DATABASE_FILE_KEY))
self.alembic_script = ScriptDirectory.from_config(alembic_config)
self.alembic_env = EnvironmentContext(alembic_config,
self.alembic_script)
self.alembic_env.configure(connection=self.engine.connect(),
target_metadata=canc.Base.metadata,
fn=self.__db_upgrade)
self.alembic_context = self.alembic_env.get_context()
def init_db():
import os
from . import models
from alembic.config import Config
from alembic import command
connection = engine.connect()
migration_context = MigrationContext.configure(connection)
os.chdir('watchtogether')
alembic_cfg = Config('alembic.ini')
if not migration_context.get_current_revision():
Base.query = db_session.query_property()
Base.metadata.create_all(bind=engine)
command.stamp(alembic_cfg, 'head')
else:
command.upgrade(alembic_cfg, 'head')
os.chdir('..')
def command(self):
self._load_config()
model.Session.commit()
alembic_ini = os.path.normpath(
os.path.join(__file__, '../../../alembic.ini')
)
self.alembic_cfg = Config(alembic_ini)
self.alembic_cfg.set_section_option(
'alembic', 'sqlalchemy.url', config.get('sqlalchemy.url')
)
cmd_name = (self.args[0] if self.args else '').replace('-', '_')
cmd = getattr(self, cmd_name, None)
if cmd is None:
return self.usage
return cmd()
def __init__(self):
"""
"""
from domogik.common import sql_schema
from domogik.common import database
from sqlalchemy import create_engine, MetaData, Table
from sqlalchemy import __version__ as sqla_version
from alembic.config import Config
from alembic import command
info(u"SQLAlchemy version is : {0}".format(sqla_version))
self.db_backup_file = "{0}/domogik-{1}.sql".format(tempfile.gettempdir(), int(time.time()))
self.alembic_cfg = Config("alembic.ini")
self._db = database.DbHelper()
self._url = self._db.get_url_connection_string()
self._engine = create_engine(self._url)
def notify_db(notify_api, request):
Migrate(notify_api, db)
Manager(db, MigrateCommand)
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
ALEMBIC_CONFIG = os.path.join(BASE_DIR, 'migrations')
config = Config(ALEMBIC_CONFIG + '/alembic.ini')
config.set_main_option("script_location", ALEMBIC_CONFIG)
with notify_api.app_context():
upgrade(config, 'head')
def teardown():
db.session.remove()
db.drop_all()
db.engine.execute("drop table alembic_version")
db.get_engine(notify_api).dispose()
request.addfinalizer(teardown)
def test_migration_graph_simple():
"""Test migration graph for simple case"""
config = Config()
config.set_main_option("script_location", "tests.graph:migrations")
config.set_main_option("phases", "before-deploy after-deploy final")
config.set_main_option("default-phase", "after-deploy")
config.set_main_option("sqlalchemy.url", "sqlite:///")
config.set_main_option("script-attributes", "some_attribute")
dotfile = generate_migration_graph(config)
expected = (
u"digraph revisions {\n"
"\t\"1\" [label=\"1\\n- some_attribute: some-base-value\"];\n"
"\t\"simple\" [label=\"simple\\n- some_attribute: some-value\"];\n"
"\n"
"\t\"simple\" -> \"1\";\n"
"}")
assert dotfile == expected
def get_alembic_conf():
alembic_cfg = Config()
alembic_cfg.set_main_option("script_location", "migrations")
alembic_cfg.set_main_option("sqlalchemy.url", DATABASE_URL)
alembic_cfg.config_file_name = os.path.join("migrations", 'alembic.ini')
if os.path.isdir('migrations') is False:
click.echo(click.style("Initiating alembic...", fg='bright_blue'))
alembic.init(alembic_cfg, 'migrations')
with open('migrations/env.py', 'r+') as f:
content = f.read()
content = content.replace(
'target_metadata = None',
f'from {MAIN_MODULE_NAME} import db\ntarget_metadata = db.metadata'
)
f.seek(0)
f.write(content)
return alembic_cfg
def create_test_database():
"""
Create a clean database on every test case.
For safety, we should abort if a database already exists.
We use the `sqlalchemy_utils` package here for a few helpers in
consistently creating and dropping the database.
"""
dburl = apisecrets.DATABASE_URL
create_engine(dburl)
assert not database_exists(
dburl), 'Test database already exists. Aborting tests.'
create_database(dburl)
alembic_config = Config("alembic.ini")
command.upgrade(alembic_config, "head")
# command.history(alembic_config, indicate_current=True)
yield
drop_database(dburl)
async def turbulette_setup(project_settings, create_db):
conf_module = reload(import_module("turbulette.conf"))
setup(project_settings.__name__, database=True)
async with conf.db.with_bind(bind=project_settings.DB_DSN) as engine:
settings_file = Path(find_spec(project_settings.__name__).origin)
alembic_config = (settings_file.parent / "alembic.ini").as_posix()
script_location = (settings_file.parent / "alembic").as_posix()
config = Config(file_=alembic_config)
config.set_main_option("sqlalchemy.url", str(project_settings.DB_DSN))
config.set_main_option("script_location", script_location)
upgrade(config, "heads")
cache = getattr(import_module("turbulette.cache"), "cache")
await cache.connect()
yield conf_module
if cache.is_connected:
await cache.disconnect()
await engine.close()