def _expected_heads():
cfg_path = join(dirname(__file__), 'migrations', 'alembic.ini')
cfg = Config(cfg_path)
_configure_alembic(cfg)
cfg.set_main_option('script_location', dirname(cfg_path))
script_dir = ScriptDirectory.from_config(cfg)
return set(script_dir.get_heads())
def bootstrap_db(config_uri=None, engine=None, with_migration=True):
"""Bring a blank database to a functional state."""
if engine is None:
engine = create_engine(config_uri)
db.configure(bind=engine)
if with_migration:
context = MigrationContext.configure(engine.connect())
db_version = context.get_current_revision()
if db_version:
sys.stderr.write('Database already initialized. Bailing out.\n')
sys.exit(2)
config = Config(config_uri)
app_cfg = config.get_section('app:main')
script_dir = ScriptDirectory.from_config(config)
heads = script_dir.get_heads()
if len(heads) > 1:
sys.stderr.write('Error: migration scripts have more than one '
'head.\nPlease resolve the situation before '
'attempting to bootstrap the database.\n')
sys.exit(2)
metadata.create_all(engine)
with transaction.manager:
School(name=app_cfg.get('sortie.school', 'Default School')).save()
# Clean up the sccoped session to allow a later app instantiation.
db.remove()
if with_migration and heads:
command.stamp(config, 'head')
def __init__(self, url=u'sqlite:////tmp/janitoo_db.sqlite', pkg_name='janitoo_db', ep_name='janitoo', conf_file=None, **kwargs):
"""
"""
self.pkg_name = pkg_name
self.ep_name = ep_name
if conf_file is None:
src = os.path.join(pkg_resources.resource_filename(pkg_resources.Requirement.parse(self.pkg_name), 'config'), u'alembic_template.conf')
else:
src = conf_file
options = JNTOptions({'conf_file':conf_file})
options.load()
alembic = options.get_options('database')
url = alembic['sqlalchemy.url']
file_ = os.path.join(tempfile.gettempdir(), u'jntal_%s.conf')%(''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(26)))
shutil.copyfile(src, file_)
alConfig.__init__(self, file_=file_, ini_section='database', **kwargs)
config_path = pkg_resources.resource_filename(pkg_resources.Requirement.parse(self.pkg_name), 'config')
self.set_main_option("script_location", os.path.join(config_path, 'alembic'))
self.set_main_option("sqlalchemy.url", url)
version_locations = u"%s/alembic/versions %s/models/%s"%(config_path, config_path, self.ep_name)
for entrypoint in pkg_resources.iter_entry_points(group='janitoo.models'):
pkg = entrypoint.module_name.split('.')[0]
config_path = pkg_resources.resource_filename(pkg_resources.Requirement.parse(pkg), 'config')
version_locations += u" %s/models/%s"%(config_path, entrypoint.name)
self.set_main_option("version_locations", version_locations)
def main():
log = get_logger()
argparser = argparse.ArgumentParser('irco-import')
argparser.add_argument('-v', '--verbose', action='store_true')
argparser.add_argument('database')
args = argparser.parse_args()
sentry.context.merge({
'tags': {
'command': 'irco-upgrade',
},
'extra': {
'parsed_arguments': args.__dict__,
}
})
log.info('arguments_parsed', args=args)
config = Config()
config.set_main_option('script_location', 'irco:migrations')
config.set_main_option('sqlalchemy.url', args.database)
command.upgrade(config, 'head', sql=False, tag=None)
def alembic_revision(log_url):
"""local task to merge changes from glottologcurator available to the production
site via an alembic migration script.
pulls the changelog from glottologcurator and create a new alembic revision with it.
"""
user = raw_input('HTTP Basic auth user for glottologcurator: ')
password = getpass('HTTP Basic auth password for glottologcurator: ')
kw = {}
if user and password:
kw['auth'] = (user, password)
changes = requests.get(log_url, **kw).json()
config = Config()
config.set_main_option("script_location", path('.').joinpath('migrations'))
scriptdir = ScriptDirectory.from_config(config)
script = scriptdir.generate_revision(
rev_id(), "Glottolog Curator", refresh=True,
upgrades="""\
# from glottologcurator
conn = op.get_bind()
for sql, params in [
%s
]:
conn.execute(sql, params)
""" % '\n'.join(u' ("""{0}""", {1}),'.format(*event) for event in changes['events']))
print('new alembic migration script created:')
print(script.path)
def main(argv=sys.argv):
args = parser.parse_args(argv[1:])
setup_logging(args.config)
app_settings = get_appsettings(args.config)
alembic_cfg = Config(args.config)
blame = alembic_cfg.get_main_option('blame')
engine = create_engine(alembic_cfg.get_main_option('sqlalchemy.url'))
apps = aslist(app_settings['occams.apps'])
errors = []
assert blame, 'Need to blame someone!'
with engine.begin() as connection:
connection.info['blame'] = blame
for app in apps:
try:
module = import_module(app)
except ImportError:
errors.append('{}: Unable to import'.format(app))
continue
else:
if hasattr(module, 'initdb'):
module.initdb(connection)
else:
errors.append('{}: Does not have "initdb"'.format(app))
for error in errors:
print(error)
alembic_cfg.attributes['connection'] = connection
command.stamp(alembic_cfg, 'heads')
def create_schema(self, set_alembic_version=True):
"""Creates the 'mediatum' schema.
:param set_alembic_version: Stamp database with current alembic revision information. Defaults to True.
Can be disabled if a schema for testing is going to be created.
"""
s = self.session
logg.info("creating DB schema...")
s.execute("CREATE SCHEMA mediatum")
s.commit()
try:
self.create_all()
if set_alembic_version:
# create alembic version table and set current alembic version to head
from alembic.config import Config
from alembic import command
alembic_cfg = Config(os.path.join(config.basedir, "alembic.ini"))
alembic_cfg.attributes["running_in_mediatum"] = True
command.stamp(alembic_cfg, "head")
s.commit()
logg.info("commited database structure")
except:
# I tried to use a transaction to enclose everything, but sqlalchemy (?) fails when the schema is created within the transaction
# solution: just drop the schema it if something fails after schema creation
s.execute("DROP SCHEMA mediatum CASCADE")
raise
def perform_migratons(config_name):
''' If fails, then we should revert to previous version of SLUG running on Heroku
link: http://stackoverflow.com/questions/24622170/using-alembic-api-from-inside-application-code
'''
db_url = configuration[config_name].SQLALCHEMY_DATABASE_URI
alembic_config = AlembicConfig('.\\AlertWeb\\alembic.ini')
alembic_config.set_main_option('sqlalchemy.url', db_url)
alembic_config.set_main_option('script_location', '.\\AlertWeb\\migrations')
script_dir = ScriptDirectory.from_config(alembic_config)
head_revision = script_dir.get_current_head()
current_revision = get_current_revision(db_url)
def upgrade(rev, context):
print(rev)
return script_dir._upgrade_revs(head_revision, rev)
#script_dir.
# Facade for migration context.
with EnvironmentContext(alembic_config,
script_dir,
as_sql=False,
fn=upgrade,
starting_rev=current_revision,
destination_rev=head_revision,
tag=None
):
script_dir.run_env()
def db_migration(request):
print("Doing db setup")
app_env_var_mock = mock.patch.dict('gds_metrics.os.environ', {'PROMETHEUS_METRICS_PATH': '/_metrics'})
app_env_var_mock.start()
app = create_app('test')
Migrate(app, db)
Manager(db, MigrateCommand)
ALEMBIC_CONFIG = os.path.join(os.path.dirname(__file__), '../migrations/alembic.ini')
config = Config(ALEMBIC_CONFIG)
config.set_main_option("script_location", "migrations")
with app.app_context():
upgrade(config, 'head')
print("Done db setup")
def teardown():
app = create_app('test')
with app.app_context():
db.session.remove()
db.engine.execute("drop sequence suppliers_supplier_id_seq cascade")
db.drop_all()
db.engine.execute("drop table alembic_version")
insp = inspect(db.engine)
for enum in insp.get_enums():
db.Enum(name=enum['name']).drop(db.engine)
db.get_engine(app).dispose()
app_env_var_mock.stop()
request.addfinalizer(teardown)
def init_config(basedir):
alembic_dir = os.path.join(basedir, 'alembic')
alembic_cfg = Config()
alembic_cfg.set_main_option('sqlalchemy.url', get_db_url())
alembic_cfg.set_main_option('script_location', alembic_dir)
command.upgrade(alembic_cfg, "head")
def main(args): # pragma: no cover
"""local task to make changes from glottologcurator available to the production
site via an alembic migration script.
pulls the changelog from glottologcurator and creates a new alembic revision with it.
"""
kw = {}
if args.http_user and args.http_password:
kw['auth'] = (args.http_user, args.http_password)
changes = requests.get(args.log_url, **kw).json()
config = Config()
config.set_main_option("script_location", args.migrations_dir)
scriptdir = ScriptDirectory.from_config(config)
script = scriptdir.generate_revision(
rev_id(), "Glottolog Curator", refresh=True,
upgrades="""\
# from glottologcurator
conn = op.get_bind()
for sql, params in [
%s
]:
conn.execute(sql, params)
""" % '\n'.join(u' ("""{0}""", {1}),'.format(
event[0], parse_json_with_datetime(event[1])) for event in changes['events']))
args.log.info('new alembic migration script created:')
args.log.info(script.path)
args.log.info('run "alembic upgrade head" to merge changes')
def setupPackage():
os.environ['MONGO_URI'] = 'mongodb://localhost'
os.environ['MONGO_DB_NAME'] = 'royal_example'
os.environ['MONGO_DB_PREFIX'] = ''
# sqla extentsion setup.
global engine
alembic_config = Config()
alembic_config.set_main_option('script_location',
'example/ext/sqla/db')
alembic_config.set_main_option('sqlalchemy.url', mysql_uri)
engine = create_engine(mysql_uri)
try:
command.downgrade(alembic_config, 'base')
except:
log.exception("Migration downgrade failed, clearing all tables")
metadata = MetaData(engine)
metadata.reflect()
for table in metadata.tables.values():
for fk in table.foreign_keys:
engine.execute(DropConstraint(fk.constraint))
metadata.drop_all()
command.upgrade(alembic_config, 'head')
def db_create():
"""Create the tables and do alembic stuff"""
print 'db at: {0}'.format(
app.config['SQLALCHEMY_DATABASE_URI'])
try:
db.engine.execute('select * from project')
print 'Database already exists with tables.'
return
except (OperationalError, ProgrammingError):
# An operational error here means that the "project" table
# doesn't exist so we should create things!
pass
print 'Creating {0}....'.format(
app.config['SQLALCHEMY_DATABASE_URI'])
db.create_all()
from alembic.config import Config
from alembic import command
alembic_cfg = Config('alembic.ini')
alembic_cfg.set_main_option("sqlalchemy.url",
app.config["SQLALCHEMY_DATABASE_URI"])
command.stamp(alembic_cfg, 'head')
print 'Done.'
def db_session2(request):
sql_str = os.environ.get("DB_STRING2", "sqlite://")
engine = create_engine(sql_str)
engine.echo = True
# pyramid way
maker = sessionmaker(bind=engine)
Base.metadata.bind = engine
Base.metadata.drop_all(engine)
engine.execute("DROP TABLE IF EXISTS alembic_ziggurat_foundations_version")
if sql_str.startswith("sqlite"):
# sqlite will not work with alembic
Base.metadata.create_all(engine)
else:
alembic_cfg = Config()
alembic_cfg.set_main_option(
"script_location", "ziggurat_foundations:migrations"
)
alembic_cfg.set_main_option("sqlalchemy.url", sql_str)
command.upgrade(alembic_cfg, "head")
session = maker()
def teardown():
session.rollback()
session.close()
request.addfinalizer(teardown)
return session
def alembic_config(self):
alembic_cfg = Config()
alembic_cfg.set_section_option(
'alembic', 'script_location', 'alembic')
alembic_cfg.set_section_option(
'alembic', 'sqlalchemy.url', str(self.db.engine.url))
return alembic_cfg
def db(request, monkeypatch):
"""Session-wide test database."""
if os.path.exists(TESTDB_PATH):
os.unlink(TESTDB_PATH)
from wordbook.infra import config
from wordbook.infra import db
monkeypatch.setattr(config, 'SQLALCHEMY_ENGINE', 'sqlite:///' + TESTDB_PATH)
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
_engine = create_engine(
'sqlite:///' + TESTDB_PATH,
echo=config.SQLALCHEMY_ECHO,
echo_pool=config.SQLALCHEMY_ECHO_POOL)
db_session = sessionmaker(bind=_engine)
def get_session():
return db_session()
monkeypatch.setattr(db, 'get_session', get_session)
alembic_config = Config('alembic.ini')
alembic_config.set_section_option('alembic', 'sqlalchemy.url', 'sqlite:///' + TESTDB_PATH)
def teardown():
downgrade(alembic_config, '18554c40c9e')
os.unlink(TESTDB_PATH)
upgrade(alembic_config, 'head')
request.addfinalizer(teardown)
return
def run_alembic_migration(self):
""" Migrate to latest Alembic revision if not up-to-date. """
def migrate_if_required(rev, context):
rev = script.get_revision(rev)
if not (rev and rev.is_head):
migration_required = True
return []
migration_required = False
config = Config(os.path.join(os.path.dirname(__file__), 'alembic.ini'))
config.set_section_option('alembic', 'script_location',
os.path.join(os.path.dirname(__file__), 'migrations'))
config.set_main_option('sqlalchemy.url',
'sqlite:///' + self.filepath)
script = ScriptDirectory.from_config(config)
with EnvironmentContext(config, script, fn=migrate_if_required):
script.run_env()
if migration_required:
logging.info('Migrating SQLite database to latest revision')
alembic.command.upgrade(config, 'head')
else:
logging.info('SQLite database is on the latest revision')
def upgradedb():
logging.info("Creating tables")
package_dir = os.path.abspath(os.path.dirname(__file__))
directory = os.path.join(package_dir, "migrations")
config = Config(os.path.join(package_dir, "alembic.ini"))
config.set_main_option("script_location", directory)
config.set_main_option("sqlalchemy.url", conf.get("core", "SQL_ALCHEMY_CONN"))
command.upgrade(config, "head")
def _create_database():
directory = os.path.join(os.path.dirname(__file__), '../migrations')
database_config = AlembicConfig(os.path.join(
directory,
'alembic.ini'
))
database_config.set_main_option('script_location', directory)
command.upgrade(database_config, 'head', sql=False, tag=None)
def actionUpgradeDB(self, version="head"):
from alembic.config import Config
from alembic import command
alembic_cfg = Config()
alembic_cfg.set_main_option("script_location", "alembic")
alembic_cfg.set_main_option("url", config.DB_URL)
command.upgrade(alembic_cfg, version)
def upgrade():
from alembic.config import Config
alembic_cfg = Config()
scripts_path = os.path.join(os.path.dirname(__file__), "migrations")
alembic_cfg.set_main_option("script_location", scripts_path)
from alembic.command import upgrade
return upgrade(alembic_cfg, revision="head")
def upgradedb():
logging.info("Creating tables")
package_dir = os.path.abspath(os.path.dirname(__file__))
directory = os.path.join(package_dir, 'migrations')
config = Config(os.path.join(package_dir, 'alembic.ini'))
config.set_main_option('script_location', directory)
config.set_main_option('sqlalchemy.url',
configuration.get('core', 'SQL_ALCHEMY_CONN'))
command.upgrade(config, 'heads')
def _get_config():
"""
Return an alembic config.
:return: config
"""
config = Config('alembic.ini')
config.set_main_option('script_location', 'migrations')
return config
def db_migrations(app):
alembic_ini = os.path.join(config.project_dir, 'alembic.ini')
alembic_config = AlembicConfig(alembic_ini)
alembic_config.attributes['testing'] = True
upgrade(alembic_config, 'head')
yield
database.db.session.remove()
database.db.drop_all(app=app)
stamp(alembic_config, 'base')
def main(argv=sys.argv):
config_uri = argv[1]
options = parse_vars(argv[2:])
setup_logging(config_uri)
alembic_cfg = Config(config_uri)
alembic_cfg.file_config = configparser.SafeConfigParser(options)
alembic_cfg.file_config.read([alembic_cfg.config_file_name])
command.upgrade(alembic_cfg, "head")
def migration_downgrade(commit):
"""Migrate database structure"""
from alembic.config import Config
from alembic.command import downgrade
config = Config(os.path.normpath(os.path.abspath(__file__) + "/../alembic.ini"))
config.set_main_option("script_location", "alembic")
downgrade(config, commit)
def actionCreateMigration(self, message=None, auto=True):
from alembic.config import Config
from alembic import command
if not message:
self.parser.error("Please add comment")
alembic_cfg = Config()
alembic_cfg.set_main_option("script_location", "alembic")
alembic_cfg.set_main_option("url", config.DB_URL)
command.revision(alembic_cfg, message=message, autogenerate=auto)
def actionDowngradeDB(self, version=None):
from alembic.config import Config
from alembic import command
if not version:
self.parser.error("Downgrade DB requires version for migration")
alembic_cfg = Config()
alembic_cfg.set_main_option("script_location", "alembic")
alembic_cfg.set_main_option("url", config.DB_URL)
command.downgrade(alembic_cfg, version)
def create_alembic_config(**kwargs):
"""Returns an `alembic.config.Config` object configured for uber.
"""
kwargs['file_'] = alembic_ini_path
alembic_config = AlembicConfig(**kwargs)
# Override settings from "alembic.ini"
alembic_config.set_main_option('script_location', script_location)
alembic_config.set_main_option(
'version_locations', version_locations_option)
return alembic_config
def actionCreateDB(self):
from models import metadata, engine
from alembic.config import Config
from alembic import command
metadata.create_all(engine)
alembic_cfg = Config()
alembic_cfg.set_main_option("script_location", "alembic")
alembic_cfg.set_main_option("url", config.DB_URL)
command.stamp(alembic_cfg, "head")
def run(version):
alembic_cfg = Config()
alembic_cfg.set_main_option('script_location',
f'{irrd_root}/irrd/storage/alembic')
command.downgrade(alembic_cfg, version)
print(f'Downgrade successful, or already on this version.')
app = create_app(config)
app.app_context().push()
client = app.test_client
db.drop_all()
with db.engine.connect().execution_options(autocommit=True) as conn:
conn.execute('DROP TABLE IF EXISTS alembic_version;')
db.session.commit()
if os.environ.get('TEST_USE_HOST') is not None:
alembicini_loc = "alembic.ini"
if not os.path.isfile(alembicini_loc):
alembicini_loc = "../../alembic.ini"
alembic_cfg = Config(alembicini_loc)
command.upgrade(alembic_cfg, "head")
else:
db.create_all()
db.session.commit()
def reset():
# Clear table data
meta = db.metadata
for table in reversed(meta.sorted_tables):
db.session.execute(table.delete())
db.session.commit()
# Load devdb
devdb_location = "devdb.sql"
def _make_config(self, location):
cfg = Config()
cfg.set_main_option("script_location", location)
cfg.set_main_option("sqlalchemy.url", get_settings()['sqlalchemy.url'])
return cfg
import alembic
import names
from alembic.config import Config
from flask_migrate import Migrate, MigrateCommand
from flask_script import Manager
from oh_queue import app, socketio
from oh_queue.models import db, Assignment, ConfigEntry, Location, Ticket, TicketStatus, User
migrate = Migrate(app, db)
manager = Manager(app)
manager.add_command('db', MigrateCommand)
alembic_cfg = Config('migrations/alembic.ini')
def not_in_production(f):
@functools.wraps(f)
def wrapper(*args, **kwargs):
if app.config.get('ENV') == 'prod':
print('this commend should not be run in production. Aborting')
sys.exit(1)
return f(*args, **kwargs)
return wrapper
@manager.command
@not_in_production
"vanuatu": "Vanuatu Environment Data Portal",
"solomonislands": "Solomon Islands Environment Data Portal",
"nauru": "Nauru Environment Data Portal",
"rmi": "Republic of Marshall Islands Environment Data Portal",
"tuvalu": "Tuvalu Environment Data Portal",
"fiji": "Fiji Environment Data Portal",
"tonga": "Tonga Environment Data Portal",
"samoa": "Samoa Environment Data Portal",
"niue": "Niue Environment Data Portal",
"cookislands": "Cook Islands Environment Data Portal",
"kiribati": "Kiribati Environment Data Portal",
}
alembic_ini = os.path.normpath(os.path.join(__file__, '../../../alembic.ini'))
alembic_cfg = Config(alembic_ini)
alembic_cfg.set_section_option('alembic', 'sqlalchemy.url',
config.get('sqlalchemy.url'))
def get_commnads():
return [spc]
@click.group()
def spc():
pass
@spc.command('db_upgrade')
def db_upgrade():
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-d',
"--direction",
action="store",
default=None,
dest="direction",
help='-d up (or down) default = up',
required=False)
#
# db type
#
parser.add_argument('-n',
"--number",
action="store",
dest="number",
help='-n 1 (default = 1)',
default="1",
required=False)
parser.add_argument(
'-r',
"--revision",
action="store",
dest="revision",
help='-r 6669c0b62b06 (take the unique part from a migration id)',
default=None,
required=False)
parser.add_argument('-c',
"--current",
action="store_true",
dest="current",
help='Display the current revision for a database.',
default=False,
required=False)
parser.add_argument('-l',
"--list",
action="store_true",
dest="history",
help='List changeset scripts in chronological order.',
default=False,
required=False)
args = parser.parse_args()
#
# show some args
#
#print("all args: ", args)
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=sa_exc.SAWarning)
alembic_cfg = Config(
os.path.join(os.path.dirname(__file__), "alembic.ini"))
# show the current version
if args.current:
print(50 * "-")
print("current version: ")
print(50 * "-")
command.current(alembic_cfg, verbose=True)
sys.exit()
# show all changes
if args.history:
print(50 * "-")
print("revision history: ")
print(50 * "-")
command.history(alembic_cfg, rev_range=None, verbose=False)
sys.exit()
#
# really migrate
#
res = "NOTHING"
if args.direction == "up":
# upgrade
res = migrate("up", args.revision, args.number)
elif args.direction == "down":
# downgrade
res = migrate("down", args.revision, args.number)
else:
print("Error: ")
print(
"You must at least give a direction info up / down to migrate:"
)
print(50 * "-")
print(" Change history ")
print(50 * "-")
command.history(alembic_cfg, rev_range=None, verbose=False)
#print(" res type: " + str(type(res)))
#print(str(res))
#print(" res dir: " + str(dir(res)))
sys.exit()
if 'path' in step:
kwargs['path'] = step['path']
yield dict(position=index,
code=step['script'],
type=step['type'],
**kwargs)
def push_data(url, username, password, alembic_config, case=None, show=False):
"""Push migration data.
:args: command line arguments namespace object
:raises: Exception if PDT replied with an error
"""
config = Config(alembic_config)
for migration in get_migrations_data(config):
if case == migration['attributes']['case_id'] or not case:
print(
'Got migration data for migration: {migration[revision]}, case: {migration[attributes][case_id]}'
.format(migration=migration))
call_url = '{0}/api/migrations/'.format(url)
data = {
"uid":
migration['revision'],
"parent":
migration['down_revision'],
"case": {
"id": str(migration['attributes']['case_id'])
},
"pre_deploy_steps":
import os
import sys
import time
from alembic.config import Config
LOCALDEV_PORT = 8080
NAME = 'MovieDisco'
ROOT_URL_PROD = 'http://moviedisco.baylaunch.com'
ROOT_URL_DEV = 'http://localhost:%d' % LOCALDEV_PORT
IS_REAL_PROD = os.environ.get('COMMIT') is not None
IS_LIKE_PROD = (IS_REAL_PROD or (os.environ.get('PRODUCTION') == '1')
or (len(sys.argv) >= 2 and sys.argv[1] == 'prod'))
ALEMBIC_CONFIG_FILE = 'alembic_prod.ini' if IS_LIKE_PROD else 'alembic.ini'
ALEMBIC_CONFIG = Config(ALEMBIC_CONFIG_FILE)
DB_URL = ALEMBIC_CONFIG.get_main_option('sqlalchemy.url')
CONFIG = {
'VERSION': os.environ.get('COMMIT')
or str(int(time.time() * 1000)), # cache-buster
'SQLALCHEMY_DATABASE_URI': DB_URL,
'ROOT_URL_WITH_SLASH': (ROOT_URL_PROD if IS_LIKE_PROD else ROOT_URL_DEV) +
'/', # for auto_context
}
def main():
from docassemble.webapp.database import dbprefix
if dbprefix.startswith('postgresql') and not daconfig.get(
'force text to varchar upgrade', False):
do_varchar_upgrade = False
else:
do_varchar_upgrade = True
with app.app_context():
if daconfig.get('use alembic', True):
if do_varchar_upgrade:
changed = False
if db.engine.has_table(dbtableprefix + 'userdict'):
db.session.query(UserDict).filter(
db.func.length(UserDict.filename) > 255).delete(
synchronize_session=False)
changed = True
if db.engine.has_table(dbtableprefix + 'userdictkeys'):
db.session.query(UserDictKeys).filter(
db.func.length(UserDictKeys.filename) > 255).delete(
synchronize_session=False)
changed = True
if db.engine.has_table(dbtableprefix + 'chatlog'):
db.session.query(ChatLog).filter(
db.func.length(ChatLog.filename) > 255).delete(
synchronize_session=False)
changed = True
if db.engine.has_table(dbtableprefix + 'uploads'):
db.session.query(Uploads).filter(
db.func.length(Uploads.filename) > 255).delete(
synchronize_session=False)
db.session.query(Uploads).filter(
db.func.length(Uploads.yamlfile) > 255).delete(
synchronize_session=False)
changed = True
if db.engine.has_table(dbtableprefix + 'objectstorage'):
db.session.query(ObjectStorage).filter(
db.func.length(ObjectStorage.key) > 1024).delete(
synchronize_session=False)
changed = True
if db.engine.has_table(dbtableprefix + 'speaklist'):
db.session.query(SpeakList).filter(
db.func.length(SpeakList.filename) > 255).delete(
synchronize_session=False)
changed = True
if db.engine.has_table(dbtableprefix + 'shortener'):
db.session.query(Shortener).filter(
db.func.length(Shortener.filename) > 255).delete(
synchronize_session=False)
db.session.query(Shortener).filter(
db.func.length(Shortener.key) > 255).delete(
synchronize_session=False)
changed = True
if db.engine.has_table(dbtableprefix + 'machinelearning'):
db.session.query(MachineLearning).filter(
db.func.length(MachineLearning.key) > 1024).delete(
synchronize_session=False)
db.session.query(MachineLearning).filter(
db.func.length(MachineLearning.group_id) > 1024
).delete(synchronize_session=False)
changed = True
if db.engine.has_table(dbtableprefix + 'globalobjectstorage'):
db.session.query(GlobalObjectStorage).filter(
db.func.length(GlobalObjectStorage.key) > 1024).delete(
synchronize_session=False)
changed = True
if changed:
db.session.commit()
packagedir = pkg_resources.resource_filename(
pkg_resources.Requirement.parse('docassemble.webapp'),
'docassemble/webapp')
if not os.path.isdir(packagedir):
sys.exit("path for running alembic could not be found")
from alembic.config import Config
from alembic import command
alembic_cfg = Config(os.path.join(packagedir, 'alembic.ini'))
alembic_cfg.set_main_option("sqlalchemy.url",
alchemy_connection_string())
alembic_cfg.set_main_option("script_location",
os.path.join(packagedir, 'alembic'))
if not db.engine.has_table(dbtableprefix + 'alembic_version'):
start_time = time.time()
sys.stderr.write("Creating alembic stamp\n")
command.stamp(alembic_cfg, "head")
sys.stderr.write("Done creating alembic stamp after " +
str(time.time() - start_time) + " seconds\n")
if db.engine.has_table(dbtableprefix + 'user'):
start_time = time.time()
sys.stderr.write("Creating alembic stamp\n")
sys.stderr.write("Running alembic upgrade\n")
command.upgrade(alembic_cfg, "head")
sys.stderr.write("Done running alembic upgrade after " +
str(time.time() - start_time) + " seconds\n")
#db.drop_all()
start_time = time.time()
try:
sys.stderr.write("Trying to create tables\n")
db.create_all()
except:
sys.stderr.write(
"Error trying to create tables; trying a second time.\n")
try:
db.create_all()
except:
sys.stderr.write(
"Error trying to create tables; trying a third time.\n")
db.create_all()
sys.stderr.write("Finished creating tables after " +
str(time.time() - start_time) + " seconds.\n")
populate_tables()
db.engine.dispose()
from gavel.dialects.db.structures import (
store_all, )
import gavel.dialects.tptp.parser as build_tptp
from gavel.dialects.tptp.compiler import TPTPCompiler
from gavel.dialects.db.compiler import DBCompiler
from gavel.dialects.tptp.parser import Problem
from gavel.dialects.tptp.parser import TPTPParser, TPTPProblemParser
from gavel.prover.hets.interface import HetsProve, HetsSession, HetsEngine
from gavel.prover.registry import get_prover
from gavel.selection.selector import Sine
from alembic import command
from alembic.config import Config
ROOT_DIR = os.path.dirname(__file__)
alembic_cfg = Config(os.path.join(ROOT_DIR, "alembic.ini"))
alembic_cfg.set_main_option("script_location",
os.path.join(ROOT_DIR, "alembic"))
@click.group()
def db():
pass
@click.command()
def migrate_db():
"""Create tables for storage of formulas"""
command.upgrade(alembic_cfg, "head")
def handle_createdb(self, superuser_normaldb_pg_url, superuser_pg_url,
normal_user_pg_url, database_name):
"""Handles creating the database and populating it with critical stuff
:arg string superuser_normaldb_pg_url: super creds for working db
:arg string superuser_pg_url: superuser creds for overall db
:arg string normal_user_pg_url: normal user creds
:arg string database_name: the name of the database to create
:returns: True if everything worked
"""
self.config.logger.info('create database section with %s',
superuser_pg_url)
with PostgreSQLAlchemyManager(superuser_pg_url,
self.config.logger,
autocommit=False) as db:
db.create_database(database_name)
if self.config.no_roles:
self.config.logger.info("Skipping role creation")
else:
db.create_roles(self.config)
db.commit()
# database extensions section
self.config.logger.info('database extensions section with %s',
superuser_normaldb_pg_url)
with PostgreSQLAlchemyManager(superuser_normaldb_pg_url,
self.config.logger,
autocommit=False) as db:
db.setup_extensions()
db.grant_public_schema_ownership(self.config.database_username)
db.commit()
# database schema section
if self.config.no_schema:
self.config.logger.info("not adding a schema")
return True
alembic_cfg = Config(self.config.alembic_config)
alembic_cfg.set_main_option('sqlalchemy.url', normal_user_pg_url)
self.config.logger.info('database schema section with %s',
normal_user_pg_url)
with PostgreSQLAlchemyManager(normal_user_pg_url,
self.config.logger,
autocommit=False) as db:
# Order matters below
db.turn_function_body_checks_off()
db.load_raw_sql('types')
db.load_raw_sql('procs')
# We need to commit to make a type visible for table creation
db.commit()
db.create_tables()
db.commit()
if not self.config.get('no_staticdata'):
self.import_staticdata(db)
db.commit()
command.stamp(alembic_cfg, "heads")
db.session.close()
# database owner section
self.config.logger.info('database extensions section with %s',
superuser_normaldb_pg_url)
with PostgreSQLAlchemyManager(superuser_normaldb_pg_url,
self.config.logger,
autocommit=False) as db:
db.set_table_owner(self.config.database_username)
db.set_default_owner(database_name, self.config.database_username)
db.set_grants(self.config) # config has user lists
return True
def execute(self):
from alembic.config import Config
from alembic import command
create_all()
alembic_cfg = Config(get_config('alembic.alembic_ini'))
command.stamp(alembic_cfg, "head")
# -*- coding: utf-8 -*-
from datetime import date, timedelta
from os import environ
from os.path import dirname, join
from unittest import main, TestCase
from alembic import command
from alembic.config import Config
from simplejson import loads
environ['environment'] = 'test'
command.upgrade(
Config(join(dirname(__file__), 'alembic', 'settings', 'test.ini')), 'head')
from serve import application, engine, session_
from modules import models
models.populate(engine, session_)
class TestCase_(TestCase):
def create_app(self):
return application
class TestCase1(TestCase_):
email = '*****@*****.**'
password = '******'
def setup_alembic_config():
alembic_cfg = Config("alembic.ini")
command.stamp(alembic_cfg, "head")
# This line sets up loggers basically.
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = None
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
from alembic.config import Config
alembic_cfg = Config()
connection_string = os.environ['PROD_CONNECTION_STRING']
alembic_cfg.set_main_option("sqlalchemy.url", connection_string)
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
def upgrade_db(config_uri):
alembic_cfg = Config(config_uri)
command.upgrade(alembic_cfg, "head")
def check_config(self) -> Optional[Config]:
if not os.path.isfile(f"{self.project_url}/{self.config_file}"):
config = None
else:
config = Config(self.config_file)
return config
from sqlalchemy.orm import scoped_session
from sqlalchemy.orm import sessionmaker
from alembic.config import Config
import pkg_resources
#
# スクリプトを実行するディレクトリ設定
#
base_path = os.path.dirname(os.path.abspath(__file__))
base_path = os.path.abspath(os.path.join(base_path, '../'))
os.chdir(base_path)
#
# SQL エンジンの取得
#
config = Config(pkg_resources.resource_filename('eyed', 'alembic.ini'))
Engine = engine_from_config(config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
#
# SQL接続用セッションオブジェクト化
#
session_factory = sessionmaker(bind=Engine)
Session = scoped_session(session_factory)
#
# セッションの取得
#
class SessionFactory:
const.MYSQL_PORT, SAE_DATABASE)
app.config['SQLALCHEMY_POOL_RECYCLE'] = 10
elif RUNTIME_ENV in ("local", ):
LOCAL_DATABASE = "test"
app.secret_key = "ME@deepgully"
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///%s.db' % LOCAL_DATABASE
#app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql://*****:*****@127.0.0.1:3306/%s'%LOCAL_DATABASE
elif RUNTIME_ENV in ("gae", "gae_dev"):
app.secret_key = "ME@deepgully+GAE"
if RUNTIME_ENV in ("bae", "local"):
from alembic.config import Config
MIGRATE_CFG = Config("alembic.ini")
MIGRATE_CFG.set_section_option("alembic", "sqlalchemy.url",
app.config['SQLALCHEMY_DATABASE_URI'])
app.config['MIGRATE_CFG'] = MIGRATE_CFG
app.config["SiteTitle"] = "ME@deepgully"
app.config["SiteSubTitle"] = ""
app.config["OwnerEmail"] = "*****@*****.**"
app.config["DefaultPassword"] = "******"
######################################
## User
######################################
from flask.ext.login import LoginManager
login_manager = LoginManager()
from alembic.config import Config
def seek_alembic_ini():
while True:
current_path = getcwd()
if path.isfile('alembic.ini'):
return path.join(current_path, 'alembic.ini')
elif not path.isfile('main.py'):
current_path = path.join(current_path, '..')
else:
raise RuntimeError('alembic.ini not found')
cfg = Config(seek_alembic_ini())
def upgrade_latest(generate_data=False):
fun = lambda: command.upgrade(cfg, 'head')
if generate_data:
return run_with_dummy_data(fun)
return fun()
def downgrade_one():
return command.downgrade(cfg, '-1')
def upgrade_to_revision(revision: str, generate_data=False):
fun = lambda: command.upgrade(cfg, revision)
from __future__ import with_statement
from alembic import context
from sqlalchemy import engine_from_config, pool
from logging.config import fileConfig
import os, django, sys
sys.path.append('.')
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "oeplatform.settings")
django.setup()
from api.connection import get_connection_string, _get_engine
from alembic.config import Config
alembic_cfg = Config()
alembic_cfg.set_main_option("url", get_connection_string())
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
import os
import pytest
import sys
root = os.path.abspath(os.path.join(os.path.dirname(__file__)))
if root not in sys.path:
sys.path.insert(0, root)
from alembic.config import Config
from alembic import command
from sqlalchemy import event
from sqlalchemy.orm import Session
alembic_cfg = Config(os.path.join(root, 'alembic.ini'))
from changes.config import create_app, db
from changes.storage.mock import FileStorageCache
@pytest.fixture(scope='session')
def session_config(request):
db_name = 'test_changes'
return {
'db_name': db_name,
# TODO(dcramer): redis db is shared
'redis_db': '9',
}
@pytest.fixture(scope='session')
def alembic_config(base_dir, db_string):
config = Config()
config.set_main_option('version_locations', versions_dir(base_dir))
config.set_main_option('script_location', scripts_dir(base_dir))
config.set_main_option('url', db_string)
return config
def apply_migrations():
"""Applies all alembic migrations."""
config = Config(TestConfig.ALEMBIC_INI)
upgrade(config, 'head')
import os
from app import create_app
from alembic.config import Config
from alembic import command
alembic_cfg = Config("./alembic.ini")
alembic_cfg.set_main_option('sqlalchemy.url', os.environ['DATABASE_URI'])
command.upgrade(alembic_cfg, "head")
(app, _) = create_app("prod")
from logging.config import fileConfig
import django
from alembic import context
from alembic.config import Config
from sqlalchemy import engine_from_config, pool
import dataedit.structures
from api.connection import _get_engine, get_connection_string
from base.structures import metadata as target_metadata
sys.path.append(".")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "oeplatform.settings")
django.setup()
alembic_cfg = Config()
db_url = get_connection_string()
db_url = db_url.replace('%', '%%')
alembic_cfg.set_main_option("url", db_url)
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
def run_migrations():
config = Config(file_="alembic.ini", ini_section="alembic")
config.set_main_option("script_location",
os.path.join(ROOT_PATH, "alembic"))
command.upgrade(config, "head")
qv2 = Supporter(member=u2, proposition=q1_counter, submitter=True)
s.add(qv2)
qv3 = Supporter(member=u1, proposition=q2)
s.add(qv3)
qv6 = Supporter(member=u1, proposition=q3)
s.add(qv6)
qv7 = Supporter(member=u1, proposition=q4)
s.add(qv7)
qv8 = Supporter(member=u1, proposition=q5)
s.add(qv8)
qv4 = Supporter(member=u2,
proposition=q1,
status=SupporterStatus.RETRACTED)
s.add(qv4)
qv5 = Supporter(member=u2, proposition=q2, status=SupporterStatus.EXPIRED)
s.add(qv5)
qv1 = ArgumentVote(member=u1, relation=arg1_rel, weight=1)
s.add(qv1)
qv2 = ArgumentVote(member=u1, relation=arg2_rel, weight=-1)
s.add(qv2)
qv3 = ArgumentVote(member=u2, relation=arg1_rel, weight=-1)
s.add(qv3)
transaction.commit()
logg.info("committed")
alembic_cfg = Config("./alembic.ini")
command.stamp(alembic_cfg, "head")
import os
import pytest
import shutil
import subprocess
from alembic import command
from alembic.config import Config
from freight.config import create_app, db
from freight.constants import PROJECT_ROOT
ALEMBIC_CONFIG = Config(os.path.join(PROJECT_ROOT, 'alembic.ini'))
os.environ['FREIGHT_CONF'] = os.path.join(PROJECT_ROOT, 'tests', 'config.py')
@pytest.fixture(autouse=True, scope='session')
def app(request):
app = create_app(REDIS_URL='redis://localhost/9')
app_context = app.test_request_context()
app_context.push()
return app
def pytest_runtest_teardown(item):
from redis import StrictRedis
client = StrictRedis(db=9)
client.flushdb()
@pytest.fixture(autouse=True)
def _upgrade_database():
directory = os.path.join(os.path.dirname(__file__), '../migrations')
database_config = AlembicConfig(os.path.join(directory, 'alembic.ini'))
database_config.set_main_option('script_location', directory)
command.upgrade(database_config, 'head', sql=False, tag=None)
