def current(directory=None, verbose=False, head_only=False):
"""Display the current revision for each database."""
config = current_app.extensions['migrate'].migrate.get_config(directory)
if alembic_version >= (0, 7, 0):
command.current(config, verbose=verbose, head_only=head_only)
else:
command.current(config)
def main(argv, _db_master=None):
parser = argparse.ArgumentParser(
prog=argv[0], description='Initialize Ichnaea database')
parser.add_argument('--initdb', action='store_true',
help='Initialize database')
args = parser.parse_args(argv[1:])
if args.initdb:
from ichnaea import config
# make sure content models are imported
from ichnaea.content import models # NOQA
conf = config()
db_master = Database(conf.get('ichnaea', 'db_master'))
engine = db_master.engine
with engine.connect() as conn:
trans = conn.begin()
_Model.metadata.create_all(engine)
trans.commit()
# Now stamp the latest alembic version
from alembic.config import Config
from alembic import command
import os
ini = os.environ.get('ICHNAEA_CFG', 'ichnaea.ini')
alembic_ini = os.path.join(os.path.split(ini)[0], 'alembic.ini')
alembic_cfg = Config(alembic_ini)
command.stamp(alembic_cfg, "head")
command.current(alembic_cfg)
else:
parser.print_help()
def create_schema(engine, alembic_cfg, location_cfg): # pragma: no cover
old_version = False
with engine.connect() as conn:
trans = conn.begin()
stmt = text('select version_num from alembic_version')
try:
result = conn.execute(stmt).fetchall()
if len(result):
old_version = True
except ProgrammingError:
pass
if not old_version:
_Model.metadata.create_all(engine)
add_api_key(conn)
add_export_config(conn)
add_users(conn, location_cfg)
trans.commit()
# Now stamp the latest alembic version
if not old_version:
command.stamp(alembic_cfg, 'head')
command.current(alembic_cfg)
def current(context, directory='migrations', verbose=False, head_only=False):
"""Display the current revision for each database."""
config = _get_config(directory)
if alembic_version >= (0, 7, 0):
command.current(config, verbose=verbose, head_only=head_only)
else:
command.current(config)
def current(directory=None, verbose=False, head_only=False):
"""Display the current revision for each database."""
app = current_app()
config = app.extra['migrate'].migrate.get_config(directory)
if alembic_version >= (0, 7, 0):
command.current(config, verbose=verbose, head_only=head_only)
else:
command.current(config)
def test_head_rev_pre_context(self):
env_file_fixture("""
assert context.get_head_revision() == '%s'
""" % c)
command.upgrade(self.cfg, b, sql=True)
command.downgrade(self.cfg, "%s:%s" % (b, a), sql=True)
command.stamp(self.cfg, b, sql=True)
command.current(self.cfg)
def test_head_rev_pre_context(self):
env_file_fixture("""
assert context.get_head_revision() == '%s'
""" % c)
command.upgrade(self.cfg, b, sql=True)
command.downgrade(self.cfg, "%s:%s" % (b, a), sql=True)
command.stamp(self.cfg, b, sql=True)
command.current(self.cfg)
def current(self,
directory=None,
verbose=False,
head_only=False,
**kwargs): # pragma: no cover
"""Display the current revision for each database."""
config = _get_config(directory)
command.current(config, verbose=verbose, head_only=head_only)
def cli_db_current(verbose):
"""
Display the current revision for a database.
"""
alembic_cfg = AlembicConfig("alembic.ini")
alembic_cfg.set_main_option("script_location",
str(Path() / "db_migrations"))
command.current(config=alembic_cfg, verbose=verbose)
def version():
parser = argparse.ArgumentParser(
description='Print out the actual version of the database')
parser.add_argument('dburl', help='Database URL for SQLAlchemy')
args = parser.parse_args()
repositories.sa_pixort_data(url=args.dburl)
command.current(_get_config(args.dburl))
def test_head_rev_post_context(self):
env_file_fixture("""
context.configure(dialect_name='sqlite')
assert context.get_head_revision() == '%s'
""" % c)
command.upgrade(self.cfg, b, sql=True)
command.downgrade(self.cfg, "%s:%s" % (b, a), sql=True)
command.stamp(self.cfg, b, sql=True)
command.current(self.cfg)
def test_head_rev_pre_context_multihead(self):
d, e, f = multi_heads_fixture(self.cfg, a, b, c)
env_file_fixture("""
assert set(context.get_head_revisions()) == set(('%s', '%s', '%s', ))
""" % (e, f, c))
command.upgrade(self.cfg, e, sql=True)
command.downgrade(self.cfg, "%s:%s" % (e, b), sql=True)
command.stamp(self.cfg, c, sql=True)
command.current(self.cfg)
def test_starting_rev_post_context(self):
env_file_fixture("""
context.configure(dialect_name='sqlite', starting_rev='x')
assert context.get_starting_revision_argument() == 'x'
""")
command.upgrade(self.cfg, a, sql=True)
command.downgrade(self.cfg, "%s:%s" % (b, a), sql=True)
command.current(self.cfg)
command.stamp(self.cfg, a)
def test_starting_rev_post_context(self):
env_file_fixture("""
context.configure(dialect_name='sqlite', starting_rev='x')
assert context.get_starting_revision_argument() == 'x'
""")
command.upgrade(self.cfg, a, sql=True)
command.downgrade(self.cfg, "%s:%s" % (b, a), sql=True)
command.current(self.cfg)
command.stamp(self.cfg, a)
def test_head_rev_post_context(self):
env_file_fixture("""
context.configure(dialect_name='sqlite')
assert context.get_head_revision() == '%s'
""" % c)
command.upgrade(self.cfg, b, sql=True)
command.downgrade(self.cfg, "%s:%s" % (b, a), sql=True)
command.stamp(self.cfg, b, sql=True)
command.current(self.cfg)
def current():
import os
import settings
from alembic.config import Config
from alembic.command import current
alembic_ini_path = os.path.join(settings.BASE_DIR, 'alembic.ini')
alembic_cfg = Config(alembic_ini_path)
current(alembic_cfg)
def get_current_version(alembic_cfg):
captured_text = None
def print_stdout(text, *arg):
nonlocal captured_text
captured_text = text
alembic_cfg.print_stdout = print_stdout
command.current(alembic_cfg)
return captured_text
def current(context,
directory='migrations',
verbose=False,
head_only=False):
"""Display the current revision for each database."""
config = _get_config(directory)
if alembic_version >= (0, 7, 0):
command.current(config, verbose=verbose, head_only=head_only)
else:
command.current(config)
def get_current_database_revision():
captured_text = None
def print_stdout(text, *arg, **kwargs):
nonlocal captured_text
captured_text = text
alembic_config = _get_alembic_config()
alembic_config.print_stdout = print_stdout
command.current(alembic_config)
return captured_text
def test_head_rev_pre_context_multihead(self):
d, e, f = multi_heads_fixture(self.cfg, a, b, c)
env_file_fixture(
"""
assert set(context.get_head_revisions()) == set(('%s', '%s', '%s', ))
"""
% (e, f, c)
)
command.upgrade(self.cfg, e, sql=True)
command.downgrade(self.cfg, "%s:%s" % (e, b), sql=True)
command.stamp(self.cfg, c, sql=True)
command.current(self.cfg)
def current(config, verbose: bool):
"""Display the current revision for each database."""
bot = Bot(config)
directory = os.path.join('yui', 'models', 'migrations')
c = Config(os.path.join(directory, 'alembic.ini'))
c.set_main_option('script_location', directory)
c.set_main_option('sqlalchemy.url', bot.config.DATABASE_URL)
c.attributes['Base'] = bot.orm_base
command.current(c, verbose=verbose)
def get_current_database_version(input_cfg):
captured_text = []
def print_stdout(text, *arg):
nonlocal captured_text
captured_text.append(text)
input_cfg.print_stdout = print_stdout
command.current(input_cfg)
if not captured_text:
return None
else:
return captured_text[0]
def run(parser, options, args):
if len(args) not in (0, 1):
parser.error('Bad number of parameters')
# Read the configuration of the application
try:
application = args[0]
except IndexError:
application = 'kansha'
cfg = _build_alembic_config()
_set_sqlalchemy_uri(cfg, application, parser.error)
command.current(cfg, verbose=options.verbose)
def run(parser, options, args):
if len(args) not in (0, 1):
parser.error('Bad number of parameters')
# Read the configuration of the application
try:
application = args[0]
except IndexError:
application = 'kansha'
cfg = _build_alembic_config()
_set_sqlalchemy_uri(cfg, application, parser.error)
command.current(cfg, verbose=options.verbose)
def migrations(action: str, database: str, revision: Optional[str], **_):
if database == "":
print("A database must be specified")
return
cfg = Config("./alembic.ini")
cfg.attributes["sqlalchemy.url"] = database
try:
if action == "run":
command.upgrade(cfg, "head" if revision is None else revision)
elif action == "reset":
command.downgrade(cfg, "base" if revision is None else revision)
elif action == "status":
command.current(cfg)
except CommandError as e:
print(f"Failed to run command: {e}")
def current(self, verbose=False):
"""
Return hottest Revision Object
:param verbose: additional information
:return: last Revision Object
"""
if verbose:
return command.current(self.init_config, verbose=True)
return self.context.get_current_revision()
def create_schema(engine, alembic_cfg, location_cfg):
old_version = False
with engine.connect() as conn:
trans = conn.begin()
stmt = text('select version_num from alembic_version')
try:
result = conn.execute(stmt).fetchall()
if len(result):
old_version = True
except ProgrammingError:
pass
if not old_version:
_Model.metadata.create_all(engine)
add_test_api_key(conn)
add_users(conn, location_cfg)
trans.commit()
# Now stamp the latest alembic version
if not old_version:
command.stamp(alembic_cfg, 'head')
command.current(alembic_cfg)
def upgrade(
context,
directory='migrations',
revision='head',
sql=False,
tag=None,
x_arg=None,
app=None,
backup=True,
):
"""Upgrade to a later version"""
_db_filepath = current_app.config.get('SQLALCHEMY_DATABASE_PATH', None)
_db_filepath_backup = '%s.backup' % (_db_filepath, )
if backup:
if os.path.exists(_db_filepath):
log.info('Pre-upgrade Sqlite3 database backup')
log.info('\tDatabase : %r' % (_db_filepath, ))
log.info('\tBackup : %r' % (_db_filepath_backup, ))
shutil.copy2(_db_filepath, _db_filepath_backup)
config = _get_config(directory, x_arg=x_arg)
try:
command.upgrade(config, revision, sql=sql, tag=tag)
command.current(config)
except Exception:
if os.path.exists(_db_filepath_backup):
log.error('Rolling back Sqlite3 database to backup')
shutil.copy2(_db_filepath_backup, _db_filepath)
log.error('...restored')
log.critical('Database upgrade failed', exc_info=True)
finally:
if os.path.exists(_db_filepath_backup):
log.info('Deleting database backup %r' % (_db_filepath_backup, ))
os.remove(_db_filepath_backup)
log.info('...deleted')
def upgrade_database(tag, sql, revision):
"""Upgrades database schema to newest version."""
from sqlalchemy_utils import database_exists, create_database
alembic_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "alembic.ini")
alembic_cfg = AlembicConfig(alembic_path)
if not database_exists(str(config.SQLALCHEMY_DATABASE_URI)):
create_database(str(config.SQLALCHEMY_DATABASE_URI))
Base.metadata.create_all(engine)
alembic_command.stamp(alembic_cfg, "head")
else:
if not alembic_command.current(alembic_cfg):
Base.metadata.create_all(engine)
alembic_command.stamp(alembic_cfg, "head")
else:
alembic_command.upgrade(alembic_cfg, revision, sql=sql, tag=tag)
click.secho("Success.", fg="green")
def db_version(database='main', context=None):
"""Display the current database version."""
if database not in ('main', 'api'):
raise exception.Invalid('%s is not a valid database' % database)
repository = _find_migrate_repo(database)
engine = _get_engine(database, context=context)
migrate_version = None
if _is_database_under_migrate_control(engine, repository):
migrate_version = migrate_api.db_version(engine, repository)
alembic_version = None
if _is_database_under_alembic_control(engine):
alembic_version = alembic_api.current(engine)
return alembic_version or migrate_version
def test_heads_upg(self):
command.stamp(self.cfg, (self.b2.revision,))
command.upgrade(self.cfg, (self.b3.revision))
with self._assert_lines(["a2", "b3"]):
command.current(self.cfg)
def test_two_heads(self):
command.stamp(self.cfg, ())
command.stamp(self.cfg, (self.a1.revision, self.b1.revision))
with self._assert_lines(["a1", "b1"]):
command.current(self.cfg)
def test_no_current(self):
command.stamp(self.cfg, ())
with self._assert_lines([]):
command.current(self.cfg)
def run(self, runtime):
command.current(self.config)
def current(config, verbose=False, head_only=False):
"""Display the current revision for each database."""
if alembic_version >= (0, 7, 0):
command.current(config, verbose=verbose, head_only=head_only)
else:
command.current(config)
def current(directory = None):
"Display the current revision for each database."
config = _get_config(directory)
command.current(config)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-d', "--direction", action="store", default=None,
dest="direction", help='-d up (or down) default = up',
required=False)
#
# db type
#
parser.add_argument('-n', "--number", action="store",
dest="number", help='-n 1 (default = 1)',
default="1", required=False)
parser.add_argument('-r', "--revision", action="store",
dest="revision", help='-r 6669c0b62b06 (take the unique part from a migration id)',
default=None, required=False)
parser.add_argument('-c', "--current", action="store_true",
dest="current", help='Display the current revision for a database.',
default=False, required=False)
parser.add_argument('-l', "--list", action="store_true",
dest="history", help='List changeset scripts in chronological order.',
default=False, required=False)
args = parser.parse_args()
#
# show some args
#
#print("all args: ", args)
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=sa_exc.SAWarning)
alembic_cfg = Config(os.path.join(os.path.dirname(__file__), "alembic.ini"))
# show the current version
if args.current:
print(50*"-")
print("current version: ")
print(50*"-")
command.current(alembic_cfg, verbose=True)
sys.exit()
# show all changes
if args.history:
print(50*"-")
print("revision history: ")
print(50*"-")
command.history(alembic_cfg, rev_range=None, verbose=False)
sys.exit()
#
# really migrate
#
res = "NOTHING"
if args.direction == "up":
# upgrade
res = migrate("up", args.revision, args.number)
elif args.direction == "down":
# downgrade
res = migrate("down", args.revision, args.number)
else:
print("Error: ")
print("You must at least give a direction info up / down to migrate:")
print(50*"-")
print(" Change history ")
print(50*"-")
command.history(alembic_cfg, rev_range=None, verbose=False)
#print(" res type: " + str(type(res)))
#print(str(res))
#print(" res dir: " + str(dir(res)))
sys.exit()
def current():
""" Display revisions for each database. """
config = _get_config()
return command.current(config)
def current(self, directory=None, verbose=False, head_only=False, **kwargs): # pragma: no cover
"""Display the current revision for each database."""
config = _get_config(directory)
command.current(config, verbose=verbose, head_only=head_only)
def current(self):
"""Display the current revision for each database.
"""
config = self.alembic_config()
alembic_command.current(config)
return True
def getCurrentDatabaseRevision(cls, databaseUrl, config =None):
if not config:
config = cls.getConfig(databaseUrl)
return alembicCmd.current(config)
def current(directory=None, verbose=False, head_only=False):
"""Display the current revision for each database."""
config = current_app.extensions['migrate'].migrate.get_config(directory)
command.current(config, verbose=verbose, head_only=head_only)
def current():
"Display the current revision for each database."
config = _get_config()
command.current(config)
def current(app, verbose=False):
"""Display the current revision for each database."""
command.current(app.migrations_config, verbose=verbose)
def test_no_current(self):
command.stamp(self.cfg, ())
with self._assert_lines([]):
command.current(self.cfg)
def test_plain_current(self):
command.stamp(self.cfg, ())
command.stamp(self.cfg, self.a3.revision)
with self._assert_lines(["a3"]):
command.current(self.cfg)
def current(verbose, sql_url=None):
alembic_cfg = init_config(sql_url)
alembic_command.current(alembic_cfg, verbose=verbose)
def test_two_heads(self):
command.stamp(self.cfg, ())
command.stamp(self.cfg, (self.a1.revision, self.b1.revision))
with self._assert_lines(["a1", "b1"]):
command.current(self.cfg)
def run(self, runtime):
command.current(self.config)
def test_heads_one_is_dependent(self):
command.stamp(self.cfg, ())
command.stamp(self.cfg, (self.b2.revision,))
with self._assert_lines(["a2", "b2"]):
command.current(self.cfg)
def current(verbose, config=None):
alembic_cfg = config or init_config()
alembic_command.current(alembic_cfg, verbose=verbose)
def test_heads_upg(self):
command.stamp(self.cfg, (self.b2.revision,))
command.upgrade(self.cfg, (self.b3.revision))
with self._assert_lines(["a2", "b3"]):
command.current(self.cfg)
def test_doesnt_create_alembic_version(self):
command.current(self.cfg)
engine = self.bind
with engine.connect() as conn:
is_false(_connectable_has_table(conn, "alembic_version", None))
def current(opt, config):
command.current(config, verbose=opt["--verbose"])
def test_plain_current(self):
command.stamp(self.cfg, ())
command.stamp(self.cfg, self.a3.revision)
with self._assert_lines(["a3"]):
command.current(self.cfg)
def current(verbose, sql_url=None):
alembic_cfg = init_config(sql_url)
alembic_command.current(alembic_cfg, verbose=verbose)
def test_heads_one_is_dependent(self):
command.stamp(self.cfg, ())
command.stamp(self.cfg, (self.b2.revision,))
with self._assert_lines(["a2", "b2"]):
command.current(self.cfg)
def current(app, *, verbose: 'use more verbose output' = False):
"""Display the current revision for a database."""
alembic.current(_get_config(app), verbose=verbose)
def cli(ctx):
"""
Run setup after a fresh Vagrant installation.
"""
log = logging.getLogger('ipsv.setup')
assert isinstance(ctx, Context)
lock_path = os.path.join(ctx.config.get('Paths', 'Data'), 'setup.lck')
if os.path.exists(lock_path):
raise Exception('Setup is locked, please remove the setup lock file to continue')
# Create our package directories
p = Echo('Creating IPS Vagrant system directories...')
dirs = ['/etc/ipsv', ctx.config.get('Paths', 'Data'), ctx.config.get('Paths', 'Log'),
ctx.config.get('Paths', 'NginxSitesAvailable'), ctx.config.get('Paths', 'NginxSitesEnabled'),
ctx.config.get('Paths', 'NginxSSL')]
for d in dirs:
if not os.path.exists(d):
os.makedirs(d, 0o755)
p.done()
p = Echo('Copying IPS Vagrant configuration files...')
with open('/etc/ipsv/ipsv.conf', 'w+') as f:
ctx.config.write(f)
p.done()
# Set up alembic
alembic_cfg = Config(os.path.join(ctx.basedir, 'alembic.ini'))
alembic_cfg.set_main_option("script_location", os.path.join(ctx.basedir, 'migrations'))
alembic_cfg.set_main_option("sqlalchemy.url", "sqlite:////{path}"
.format(path=os.path.join(ctx.config.get('Paths', 'Data'), 'sites.db')))
command.current(alembic_cfg)
command.downgrade(alembic_cfg, 'base')
command.upgrade(alembic_cfg, 'head')
# Update the system
p = Echo('Updating package cache...')
cache = apt.Cache()
cache.update()
cache.open(None)
p.done()
p = Echo('Upgrading system packages...')
cache.upgrade()
cache.commit()
p.done()
# Install our required packages
requirements = ['nginx', 'php5-fpm', 'php5-curl', 'php5-gd', 'php5-imagick', 'php5-json', 'php5-mysql',
'php5-readline', 'php5-apcu']
for requirement in requirements:
# Make sure the package is available
p = Echo('Marking package {pkg} for installation'.format(pkg=requirement))
if requirement not in cache:
log.warn('Required package {pkg} not available'.format(pkg=requirement))
p.done(p.FAIL)
continue
# Mark the package for installation
cache[requirement].mark_install()
p.done()
log.info('Committing package cache')
p = Echo('Downloading and installing packages...')
cache.commit()
p.done()
# Disable the default server block
p = Echo('Configuring Nginx...')
default_available = os.path.join(ctx.config.get('Paths', 'NginxSitesAvailable'), 'default')
default_enabled = os.path.join(ctx.config.get('Paths', 'NginxSitesEnabled'), 'default')
if os.path.isfile(default_available):
os.remove(default_available)
if os.path.islink(default_enabled):
os.unlink(default_enabled)
p.done()
# Restart Nginx
FNULL = open(os.devnull, 'w')
p = Echo('Restarting Nginx...')
subprocess.check_call(['service', 'nginx', 'restart'], stdout=FNULL, stderr=subprocess.STDOUT)
p.done()
# php5-fpm configuration
p = Echo('Configuring php5-fpm...')
if os.path.isfile('/etc/php5/fpm/pool.d/www.conf'):
os.remove('/etc/php5/fpm/pool.d/www.conf')
fpm_config = FpmPoolConfig().template
with open('/etc/php5/fpm/pool.d/ips.conf', 'w') as f:
f.write(fpm_config)
p.done()
# Restart php5-fpm
p = Echo('Restarting php5-fpm...')
subprocess.check_call(['service', 'php5-fpm', 'restart'], stdout=FNULL, stderr=subprocess.STDOUT)
p.done()
# Copy the man pages and rebuild the manual database
p = Echo('Writing manual pages...')
man_path = os.path.join(ctx.basedir, 'man', 'ipsv.1')
sys_man_path = '/usr/local/share/man/man1'
if not os.path.exists(sys_man_path):
os.makedirs(sys_man_path)
shutil.copyfile(man_path, os.path.join(sys_man_path, 'ipsv.1'))
subprocess.check_call(['mandb'], stdout=FNULL, stderr=subprocess.STDOUT)
# Enable the welcome message
log.debug('Writing welcome message')
wm_header = '## DO NOT REMOVE :: AUTOMATICALLY GENERATED BY IPSV ##'
wm_remove = False
# Remove old profile data
for line in fileinput.input('/etc/profile', inplace=True):
# Header / footer match?
if line == wm_header:
# Footer match (Stop removing)
if wm_remove:
wm_remove = False
continue
# Header match (Start removing)
wm_remove = True
continue
# Removing lines?
if wm_remove:
continue
# Print line and continue as normal
sys.stdout.write(line)
# Write new profile data
with open('/etc/profile', 'a') as f:
f.write("\n" + wm_header + "\n")
fl_lock_path = os.path.join(ctx.config.get('Paths', 'Data'), 'first_login.lck')
f.write('if [ ! -f "{lp}" ]; then'.format(lp=fl_lock_path) + "\n")
f.write(' less "{wp}"'.format(wp=os.path.join(ctx.basedir, 'WELCOME.rst')) + "\n")
f.write(' sudo touch "{lp}"'.format(lp=fl_lock_path) + "\n")
f.write('fi' + "\n")
f.write(wm_header + "\n")
p.done()
log.debug('Writing setup lock file')
with open(os.path.join(ctx.config.get('Paths', 'Data'), 'setup.lck'), 'w') as f:
f.write('1')
def do_version(config):
return command.current(config)