def check_version(ev):
""" ApplicationCreated event handler """
if not Version.__table__.exists():
return
versions = dict((v.package, v.version_num)
for v in ptah.get_session().query(Version).all())
packages = ptah.get_cfg_storage(MIGRATION_ID).keys()
has_steps = False
log = logging.getLogger('ptah.alembic')
for pkg in packages:
version = versions.get(pkg)
script = ScriptDirectory(pkg)
for sc in script.walk_revisions():
if sc.is_head:
if sc.revision != version:
has_steps = True
log.error(
"Package '%s' current revision: '%s', head: '%s'", pkg,
version, sc.revision)
break
if has_steps:
config.shutdown()
log.error("Please run `ptah-migrate` script. Stopping...")
raise SystemExit(1)
def upgrade(pkg, sql=False):
"""Upgrade to a later version."""
from alembic.config import Config
from alembic.environment import EnvironmentContext
if ':' in pkg:
pkg, rev = pkg.split(':', 1)
else:
rev = 'head'
script = ScriptDirectory(pkg)
env = EnvironmentContext(Config(''), script)
conn = ptah.get_base().metadata.bind.connect()
def upgrade(revision, context):
return script._upgrade_revs(rev, revision)
env.configure(
connection=conn,
fn=upgrade,
as_sql=sql,
starting_rev=None,
destination_rev=rev,
)
mc = env._migration_context
env._migration_context = MigrationContext(pkg, conn.dialect, conn, mc.opts)
with env:
try:
with env.begin_transaction():
env.run_migrations()
finally:
conn.close()
def test_alembic_has_single_head(session):
"""
Avoid unintentional branches in the migration history.
"""
migrations_dir = current_app.extensions['migrate'].directory
heads = ScriptDirectory(migrations_dir).get_heads()
assert len(heads) == 1
def get_current_head_version(graph):
"""
Returns the current head version.
"""
script_dir = ScriptDirectory(
"/", version_locations=[graph.metadata.get_path("migrations")])
return script_dir.get_current_head()
def test_args(self):
script = ScriptDirectory(staging_directory,
file_template="%(rev)s_%(slug)s_"
"%(year)s_%(month)s_"
"%(day)s_%(hour)s_"
"%(minute)s_%(second)s")
create_date = datetime.datetime(2012, 7, 25, 15, 8, 5)
eq_(
script._rev_path("12345", "this is a message", create_date),
"%s/versions/12345_this_is_a_"
"message_2012_7_25_15_8_5.py" % staging_directory)
def get_migration_scripts_heads() -> List[Text]:
"""Get the head revisions from all migration scripts.
Returns:
List of all head revisions, read from the migration scripts.
"""
from alembic.script import ScriptDirectory
from rasax.community.database.schema_migrations.alembic import ALEMBIC_DIR
script_dir = ScriptDirectory(ALEMBIC_DIR)
return script_dir.get_heads()
def revision(pkg, rev=None, message=None):
"""Create a new revision file."""
script = ScriptDirectory(pkg)
revs = [sc.revision for sc in script.walk_revisions()]
if not rev:
rev = alembic.util.rev_id()
if rev in revs:
raise KeyError('Revision already exists')
return script.generate_revision(rev, message, True).revision
def main(migrations_path):
migrations = ScriptDirectory(migrations_path)
heads = detect_heads(migrations)
if len(heads) > 1:
print("Migrations directory has multiple heads due to branching: {}".
format(heads),
file=sys.stderr)
sys.exit(1)
for version in version_history(migrations):
print("{:35} {}".format(*version))
def init():
"""
Prepare directory with alembic.ini, mako-files and directory for
migrations. Part of functional was copied from original Alembic Init
but changed some things with config
"""
if os.access(config.alembic_dir, os.F_OK):
raise util.CommandError("Directory {} already exists".format(
config.alembic_dir))
template_dir = os.path.join(config.template_path +
config.template_name)
if not os.access(template_dir, os.F_OK):
raise util.CommandError("No such template {}".format(template_dir))
util.status(
"Creating directory {}".format(os.path.abspath(
config.alembic_dir)), os.makedirs, config.alembic_dir)
versions = os.path.join(config.alembic_dir, 'versions')
util.status("Creating directory %s" % os.path.abspath(versions),
os.makedirs, versions)
script = ScriptDirectory(config.alembic_dir)
dirs = os.listdir(template_dir)
dirs += [
'versions/create_table_alembic_version_history.py',
]
for file_ in dirs:
file_path = os.path.join(template_dir, file_)
if file_ == 'alembic.ini.mako':
config_file = os.path.abspath('alembic.ini')
if os.access(config_file, os.F_OK):
util.msg(
"File {} already exists, skipping".format(config_file))
else:
script._generate_template(
template_dir + '/alembic.ini.mako',
os.path.join(config.alembic_dir, 'alembic.ini'),
script_location=config.alembic_dir)
elif os.path.isfile(file_path):
output_file = os.path.join(config.alembic_dir, file_)
script._copy_file(file_path, output_file)
util.msg("Please edit configuration/connection/logging "
"settings in {} before proceeding.".format(
os.path.join(config.alembic_dir, 'alembic.ini')))
def main(migrations_path):
migrations = ScriptDirectory(migrations_path)
branch_points = get_branch_points(migrations)
heads = migrations.get_heads()
if not branch_points:
print("Migrations are ordered")
elif len(branch_points) == 1 and len(heads) == 2:
fix_branch_point(migrations, branch_points[0], heads)
else:
print("Found {} branch points and {} heads, can't fix automatically".format(
[bp.revision for bp in branch_points], heads))
sys.exit(1)
def _test_tz(self, timezone_arg, given, expected):
script = ScriptDirectory(_get_staging_directory(),
file_template="%(rev)s_%(slug)s_"
"%(year)s_%(month)s_"
"%(day)s_%(hour)s_"
"%(minute)s_%(second)s",
timezone=timezone_arg)
with mock.patch(
"alembic.script.base.datetime",
mock.Mock(datetime=mock.Mock(utcnow=lambda: given,
now=lambda: given))):
create_date = script._generate_create_date()
eq_(create_date, expected)
def from_config(config):
"""Instantiate a ScriptDirectory from the given config using generic
resource finder.
"""
script_location = config.get_main_option('script_location')
if script_location is None:
raise RuntimeError("No 'script_location' key "
"found in configuration.")
import nxdrive
nxdrive_path = os.path.dirname(nxdrive.__file__)
alembic_path = nxdrive_path.replace('nxdrive', script_location)
return ScriptDirectory(find_resource_dir(script_location, alembic_path))
def check_db():
""" Checks the database revision against the known alembic migrations. """
from inbox.ignition import main_engine
inbox_db_engine = main_engine(pool_size=1, max_overflow=0)
# top-level, with setup.sh
alembic_ini_path = os.environ.get("ALEMBIC_INI_PATH",
_absolute_path('../../alembic.ini'))
alembic_cfg = alembic_config(alembic_ini_path)
alembic_basedir = os.path.dirname(alembic_ini_path)
alembic_script_dir = os.path.join(
alembic_basedir, alembic_cfg.get_main_option("script_location"))
assert os.path.isdir(alembic_script_dir), \
'Must have migrations directory at {}'.format(alembic_script_dir)
try:
inbox_db_engine.dialect.has_table(inbox_db_engine, 'alembic_version')
except sqlalchemy.exc.OperationalError:
sys.exit("Databases don't exist! Run bin/create-db")
if inbox_db_engine.dialect.has_table(inbox_db_engine, 'alembic_version'):
res = inbox_db_engine.execute(
'SELECT version_num from alembic_version')
current_revision = [r for r in res][0][0]
assert current_revision, \
'Need current revision in alembic_version table...'
script = ScriptDirectory(alembic_script_dir)
head_revision = script.get_current_head()
log.info('Head database revision: {0}'.format(head_revision))
log.info('Current database revision: {0}'.format(current_revision))
# clean up a ton (8) of idle database connections
del script
gc.collect()
if current_revision != head_revision:
raise Exception(
'Outdated database! Migrate using `alembic upgrade head`')
else:
log.info('[OK] Database scheme matches latest')
else:
raise Exception('Un-stamped database! Run `bin/create-db`. bailing.')
def init(config, directory, template='generic'):
"""Initialize a new scripts directory."""
if os.access(directory, os.F_OK):
raise util.CommandError("Directory %s already exists" % directory)
template_dir = os.path.join(config.get_template_directory(),
template)
if not os.access(template_dir, os.F_OK):
raise util.CommandError("No such template %r" % template)
util.status("Creating directory %s" % os.path.abspath(directory),
os.makedirs, directory)
versions = os.path.join(directory, 'versions')
util.status("Creating directory %s" % os.path.abspath(versions),
os.makedirs, versions)
script = ScriptDirectory(directory)
for file_ in os.listdir(template_dir):
file_path = os.path.join(template_dir, file_)
if file_ == 'alembic.ini.mako':
config_file = os.path.abspath(config.config_file_name)
if os.access(config_file, os.F_OK):
util.msg("File %s already exists, skipping" % config_file)
else:
script._generate_template(
file_path,
config_file,
script_location=directory
)
elif os.path.isfile(file_path):
output_file = os.path.join(directory, file_)
script._copy_file(
file_path,
output_file
)
util.msg("Please edit configuration/connection/logging "\
"settings in %r before proceeding." % config_file)
def update_versions(registry):
packages = []
for item in registry.introspector.get_category(MIGRATION_ID, ()):
intr = item['introspectable']
if not intr['force']:
packages.append(intr['package'])
session = ptah.get_session()
for pkg in packages:
item = session.query(Version).filter(Version.package == pkg).first()
if item is not None:
continue
script = ScriptDirectory(pkg)
revs = [sc for sc in script.walk_revisions()]
# set head as version
for sc in revs:
if sc.is_head:
session.add(Version(package=pkg, version_num=sc.revision))
break
from logging import getLogger
from flask import current_app
from sqlalchemy import create_engine
from alembic.script import ScriptDirectory
from alembic.environment import MigrationContext
from alembic.operations import Operations
from ..base import db, Model
logger = getLogger(__name__)
alembic_version_table = "alembic_version"
script = ScriptDirectory(os.path.dirname(__file__))
def order_columns(sql):
r = re.compile(r'CREATE TABLE ?["a-z_\.]*? \s*\([^;]+\);', re.MULTILINE)
for match in r.findall(sql):
match_lines = match.splitlines()
sorted_lines = copy(match_lines)
sorted_lines = sorted_lines[1:-1]
sorted_lines[-1] = "%s," % sorted_lines[-1]
sorted_lines.sort()
sorted_lines.insert(0, match_lines[0])
sorted_lines.append(match_lines[-1])
def get_script_directory():
path = OsInteraction.resource_path('alembic')
logger.debug('alembic path: %s' % path)
script_directory = ScriptDirectory(path)
return script_directory
#!/usr/bin/env python3
from pathlib import Path
from alembic.script import ScriptDirectory
MIGRATION_DIR = Path("src/meltano/migrations")
LOCK_PATH = MIGRATION_DIR.joinpath("db.lock")
scripts = ScriptDirectory(str(MIGRATION_DIR))
with LOCK_PATH.open("w") as lock:
HEAD = scripts.get_current_head()
lock.write(HEAD)
print(f"Meltano database frozen at {HEAD}.")
def init(self, directory, template="mampoer_generic", package=False):
"""Initialize a new scripts directory.
:param template: string name of the migration environment template to
use.
:param package: when True, write ``__init__.py`` files into the
environment location as well as the versions/ location.
.. versionadded:: 1.2
"""
if os.access(directory, os.F_OK) and os.listdir(directory):
raise util.CommandError(
"Directory %s already exists and is not empty" % directory)
template_dir = os.path.join(self.get_template_directory(), template)
if not os.access(template_dir, os.F_OK):
raise util.CommandError("No such template %r" % template)
if not os.access(directory, os.F_OK):
util.status(
"Creating directory %s" % os.path.abspath(directory),
os.makedirs,
directory,
)
versions = os.path.join(directory, "versions")
util.status(
"Creating directory %s" % os.path.abspath(versions),
os.makedirs,
versions,
)
script = ScriptDirectory(directory)
for file_ in os.listdir(template_dir):
file_path = os.path.join(template_dir, file_)
if file_ == "mampoer.ini.mako":
config_file = os.path.abspath(self.config.config_file_name)
if os.access(config_file, os.F_OK):
util.msg("File %s already exists, skipping" % config_file)
else:
script._generate_template(file_path,
config_file,
script_location=directory)
elif os.path.isfile(file_path):
output_file = os.path.join(directory, file_)
script._copy_file(file_path, output_file)
if package:
for path in [
os.path.join(os.path.abspath(directory), "__init__.py"),
os.path.join(os.path.abspath(versions), "__init__.py"),
]:
file_ = util.status("Adding %s" % path, open, path, "w")
file_.close()
util.msg("Please edit configuration/connection/logging "
"settings in %r before proceeding." % config_file)
