def main(argv=sys.argv):
args = parser.parse_args(argv[1:])
setup_logging(args.config)
app_settings = get_appsettings(args.config)
alembic_cfg = Config(args.config)
blame = alembic_cfg.get_main_option('blame')
engine = create_engine(alembic_cfg.get_main_option('sqlalchemy.url'))
apps = aslist(app_settings['occams.apps'])
errors = []
assert blame, 'Need to blame someone!'
with engine.begin() as connection:
connection.info['blame'] = blame
for app in apps:
try:
module = import_module(app)
except ImportError:
errors.append('{}: Unable to import'.format(app))
continue
else:
if hasattr(module, 'initdb'):
module.initdb(connection)
else:
errors.append('{}: Does not have "initdb"'.format(app))
for error in errors:
print(error)
alembic_cfg.attributes['connection'] = connection
command.stamp(alembic_cfg, 'heads')
def alembic_cfg() -> Config:
cfg = Config("alembic.ini")
version_locations = cfg.get_main_option("version_locations")
cfg.set_main_option(
"version_locations",
f"{version_locations} {orchestrator_module_location}/{migration_dir}/versions/schema"
)
logger.info("Version Locations",
locations=cfg.get_main_option("version_locations"))
return cfg
def make_alembic_config(cmd_opts: Namespace,
base_path: str = BASE_PATH) -> Config:
"""
Create custom alembic config.
:param cmd_opts: Default config
:type cmd_opts: Namespace
:param base_path: Path of config file
:type base_path: str
:return: Alembic configuration
:rtype: Config
"""
# Make path to config absolute
if not os.path.isabs(cmd_opts.config):
cmd_opts.config = os.path.join(base_path, cmd_opts.config)
config = Config(file_=cmd_opts.config, cmd_opts=cmd_opts)
# Make path to scripts absolute
alembic_location = config.get_main_option("script_location")
if not os.path.isabs(alembic_location):
config.set_main_option("script_location",
os.path.join(base_path, alembic_location))
# Set alembic db from application config
config.set_main_option("sqlalchemy.url", AppConfig().db_url)
return config
def make_alembic_config(options: Namespace,
base_path: str = PROJECT_PATH) -> Config:
"""
Создает объект конфигурации alembic на основе аргументов командной строки,
и настраивает его (меняет относительные пути на абсолютные).
:param options: аргументы командной строки
:param base_path: путь, относительного которого формируются относительные пути
:return: объект конфигурации alembic
"""
# Если указан относительный путь до alembic.ini, то добавляем в начало base_path
# (формируем абсолютный путь)
if not os.path.isabs(options.config):
options.config = os.path.join(base_path, options.config)
# Создаем объект конфигурации Alembic
config = Config(file_=options.config,
ini_section=options.name,
cmd_opts=options)
if options.db_url:
# Меняем значение sqlalchemy.url из конфига Alembic
config.set_main_option("sqlalchemy.url", options.db_url)
# Подменяем путь до папки с alembic (требуется, чтобы alembic мог найти env.py, шаблон для
# генерации миграций и сами миграции)
alembic_location = config.get_main_option("script_location")
if not os.path.isabs(alembic_location):
config.set_main_option("script_location",
os.path.join(base_path, alembic_location))
return config
def start_db_migration(self) -> None:
"""
Runs alembic to upgrade the selected database to the current conversion head. This ensures that the database
schema version matches the python ORM version.
:return: Nothing
"""
self.close_last_session()
local_dir = os.path.dirname(
os.path.abspath(inspect.getfile(
inspect.currentframe()))) # script directory
alembic_cfg = Config(
os.path.abspath(os.path.join(local_dir, "alembic.ini")))
# Alembic has problems, when called inside a library,
# therefore we change the current alembic folder with its absolute path
alembic_cfg.set_main_option(
"script_location",
os.path.abspath(
os.path.join(
local_dir,
alembic_cfg.get_main_option("script_location",
"alembic"))))
alembic_cfg.set_main_option("sqlalchemy.url", self.__connection)
command.upgrade(alembic_cfg, "head")
def check_current_head(self) -> bool:
"""
Checks if the selected database schema version matches the python source ORM schema version.
If false please run start_db_migration to update the database.
:return: True if both version are matching, else False
"""
local_dir = os.path.dirname(
os.path.abspath(inspect.getfile(
inspect.currentframe()))) # script directory
alembic_cfg = Config(
os.path.abspath(os.path.join(local_dir, "alembic.ini")))
# Alembic has problems, when called inside a library,
# therefore we change the current folder with its absolute path
alembic_cfg.set_main_option(
"script_location",
os.path.abspath(
os.path.join(
local_dir,
alembic_cfg.get_main_option("script_location",
"alembic"))))
directory = script.ScriptDirectory.from_config(alembic_cfg)
with self.__engine.begin() as connection:
context = migration.MigrationContext.configure(connection)
return set(context.get_current_heads()) == set(
directory.get_heads())
def migrate_to_head():
'''
Trying to update database to head revision.
If alembic unable to recognize current revision (In case when database version is newer than backend)
then do nothing.
'''
config_file = Path(__file__).parent / 'alembic.ini'
config = Config(config_file)
# mindsdb can runs not from project directory
script_location_abc = config_file.parent / config.get_main_option('script_location')
config.set_main_option('script_location', str(script_location_abc))
script = ScriptDirectory.from_config(config)
with db.engine.begin() as conn:
mc = MigrationContext.configure(conn)
cur_revision = mc.get_current_revision()
try:
script.revision_map.get_revision(cur_revision)
except ResolutionError:
print('!!! Database version higher than application !!!')
return
upgrade(config=config, revision='head')
def run_migrations_offline(config: Config, target_metadata: MetaData) -> None:
"""
Run migrations in 'offline' mode.
This configures the context with just a URL and not an Engine, though an
Engine is acceptable here as well. By skipping the Engine creation we
don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the script output.
"""
url = config.get_main_option("sqlalchemy.url")
# RNC
context.configure(
url=url,
target_metadata=target_metadata,
render_as_batch=
True, # for SQLite mode; http://stackoverflow.com/questions/30378233 # noqa
literal_binds=True,
version_table=ALEMBIC_VERSION_TABLE,
compare_type=custom_compare_type,
# ... http://blog.code4hire.com/2017/06/setting-up-alembic-to-detect-the-column-length-change/ # noqa
# ... https://eshlox.net/2017/08/06/alembic-migration-for-string-length-change/ # noqa
# process_revision_directives=writer,
process_revision_directives=process_revision_directives,
)
with context.begin_transaction():
context.run_migrations()
def search_models():
""" running search models function in .alembic/env.py """
from alembic.config import Config
alembic_cfg = Config("alembic.ini")
main_dir = alembic_cfg.get_main_option("script_location")
main_dir = _set_unix_path(main_dir)
if path.exists(main_dir):
import importlib.util
sp_path = path.join(main_dir, "search_models.py")
spec = importlib.util.spec_from_file_location("search_models", sp_path)
script = importlib.util.module_from_spec(spec)
spec.loader.exec_module(script)
# f._search_models()
message = "[I] 搜索完成。"
else:
message = "[W] 没找到 .alembic 文件夹。"
print(Fore.YELLOW + message.strip() + "\n")
return
def initConn(inifile):
"""
Creates connection information from a config.ini file
:param inifile: name of the ini file for database connection
:return: connection parameters for sqlalchemy engine
"""
# create the alembic config object from inifile
conf = Config(inifile)
# create the sqlurl and ssl certs from the inifile
sqlurl = conf.get_main_option('sqlurl')
sslrootcert = conf.get_main_option('sslrootcert')
sslcert = conf.get_main_option('sslcert')
sslkey = conf.get_main_option('sslkey')
# return sqlurl and ssl certs
return sqlurl, sslrootcert, sslcert, sslkey
def get_alembic_config(mm, ini_file=None, ini_section='migrations', stdout=sys.stdout):
appcfg = mm.cfg.get_settings()
cfg = Config(file_=ini_file, ini_section=ini_section, stdout=stdout)
if cfg.get_main_option('script_location') is None:
cfg.set_main_option('script_location', 'netprofile:alembic')
if cfg.get_main_option('sqlalchemy.url') is None:
cfg.set_main_option('sqlalchemy.url', appcfg.get('sqlalchemy.url'))
if cfg.get_main_option('output_encoding') is None:
cfg.set_main_option('output_encoding', 'utf-8')
migration_paths = []
for mod in mm.modules.values():
if mod.dist and mod.dist.location:
path = os.path.join(mod.dist.location, 'migrations')
if os.path.isdir(path):
migration_paths.append(path)
if len(migration_paths) > 0:
cfg.set_main_option('version_locations', ' '.join(migration_paths))
cfg.attributes['mm'] = mm
return cfg
def temp_db() -> Generator[None, None, None]:
"""Create new test db for testing session."""
create_engine(TEST_DB)
db_exists = database_exists(TEST_DB)
if db_exists:
drop_database(TEST_DB)
create_database(TEST_DB) # Create the test database.
config = Config('alembic.ini') # Run the migrations.
config.set_main_option('sqlalchemy.url', TEST_DB)
assert config.get_main_option('sqlalchemy.url') == TEST_DB
command.upgrade(config, 'head')
yield # Run the tests.
drop_database(TEST_DB)
def get_alembic_config(mm, ini_file=None, ini_section='migrations',
stdout=sys.stdout):
appcfg = mm.cfg.get_settings()
cfg = Config(file_=ini_file, ini_section=ini_section, stdout=stdout)
if cfg.get_main_option('script_location') is None:
cfg.set_main_option('script_location', 'netprofile:alembic')
if cfg.get_main_option('sqlalchemy.url') is None:
cfg.set_main_option('sqlalchemy.url', appcfg.get('sqlalchemy.url'))
if cfg.get_main_option('output_encoding') is None:
cfg.set_main_option('output_encoding', 'utf-8')
migration_paths = []
for mod in mm.modules.values():
if mod.dist and mod.dist.location:
path = os.path.join(mod.dist.location, 'migrations')
if os.path.isdir(path):
migration_paths.append(path)
if len(migration_paths) > 0:
cfg.set_main_option('version_locations', ' '.join(migration_paths))
cfg.attributes['mm'] = mm
return cfg
def show(args):
'''Show the current schema information.'''
alembic_cfg = Config(args.config)
sa_url = alembic_cfg.get_main_option('sqlalchemy.url')
engine = sa.create_engine(sa_url)
with engine.begin() as connection:
context = MigrationContext.configure(connection)
current_rev = context.get_current_revision()
script = ScriptDirectory.from_config(alembic_cfg)
heads = script.get_heads()
head_rev = heads[0] if len(heads) > 0 else None
print(f'Current database revision: {current_rev}')
print(f'The head revision of available migrations: {head_rev}')
def main(argv=sys.argv):
args = parser.parse_args(argv[1:])
setup_logging(args.config)
alembic_cfg = Config(args.config)
blame = alembic_cfg.get_main_option('blame')
engine = create_engine(alembic_cfg.get_main_option('sqlalchemy.url'))
assert blame, 'Need to blame someone!'
with engine.begin() as connection:
connection.execute(
sa.text('SET LOCAL "application.name" = :param'), param='occams'
)
connection.execute(
sa.text('SET LOCAL "application.user" = :param'), param=blame
)
Base.metadata.create_all(connection)
alembic_cfg.attributes['connection'] = connection
command.stamp(alembic_cfg, 'heads')
def configure(self, options, config):
super(AlembicAttrib, self).configure(options, config)
ini = options.ini if options.ini else "alembic.ini"
echo = True if options.echo else False
config = Config(ini)
url = config.get_main_option("sqlalchemy.url")
engine = create_engine(url, echo=echo)
connection = engine.connect()
context = MigrationContext.configure(connection)
script = ScriptDirectory.from_config(config)
self.current_revision = context.get_current_revision()
self.revisions = [script.revision for script in script.iterate_revisions(self.current_revision, None)]
def main(argv=sys.argv):
args = parser.parse_args(argv[1:])
setup_logging(args.config)
alembic_cfg = Config(args.config)
engine = create_engine(alembic_cfg.get_main_option('sqlalchemy.url'))
blame = models.get_blame_from_url(engine.url)
with engine.begin() as connection:
models.set_pg_locals(connection, 'initdb', blame)
Base.metadata.create_all(connection)
alembic_cfg.attributes['connection'] = connection
command.stamp(alembic_cfg, 'heads')
def make_alembic_config(cmd_opts: Union[Namespace, SimpleNamespace],
base_path: str = PROJECT_PATH) -> Config:
if not os.path.isabs(cmd_opts.config):
cmd_opts.config = os.path.join(base_path, cmd_opts.config)
config = Config(file_=cmd_opts.config, ini_section=cmd_opts.name,
cmd_opts=cmd_opts)
alembic_location = config.get_main_option("script_location")
if not os.path.isabs(alembic_location):
config.set_main_option("script_location",
os.path.join(base_path, alembic_location))
if cmd_opts.pg_url:
config.set_main_option("sqlalchemy.url", cmd_opts.pg_url)
return config
def make_alembic_config(cmd_opts: Union[Namespace, SimpleNamespace],
base_path: Path = PROJECT_PATH) -> Config:
if not os.path.isabs(cmd_opts.config):
cmd_opts.config = str(base_path / cmd_opts.config)
config = Config(file_=cmd_opts.config,
ini_section=cmd_opts.name,
cmd_opts=cmd_opts)
# Подменяем путь alembic на абсолютный
alembic_location = config.get_main_option('script_location')
if not os.path.isabs(alembic_location):
config.set_main_option('script_location',
str(base_path / alembic_location))
if cmd_opts.pg_url:
config.set_main_option('sqlalchemy.url', cmd_opts.pg_url)
return config
def make_alembic_config(cmd_opts: Union[Namespace, SimpleNamespace],
base_path: str = PROJECT_PATH) -> Config:
# Replace path to alembic.ini file to absolute
if not os.path.isabs(cmd_opts.config):
cmd_opts.config = os.path.join(base_path, cmd_opts.config)
config = Config(file_=cmd_opts.config, ini_section=cmd_opts.name,
cmd_opts=cmd_opts)
# Replace path to alembic folder to absolute
alembic_location = config.get_main_option('script_location')
if not os.path.isabs(alembic_location):
config.set_main_option('script_location',
os.path.join(base_path, alembic_location))
if cmd_opts.pg_url:
config.set_main_option('sqlalchemy.url', cmd_opts.pg_url)
return config
def get_alembic_config():
"""
Возвращает alembic.config.Config, который отличается от оригинального
только заменённым именем БД, чтобы все манипуляции с данными происходили в БД для тестов.
:return: alembic.config.Config
"""
config = Config('alembic.ini')
sqlalchemy_url = config.get_main_option('sqlalchemy.url')
# Подменяем db_url для тестовой БД.
# Подразумевается, что тестовая БД от продуктовой отличается только названием,
# находится на том же сервере и доступна для того же пользователя, что и продуктовая.
last_slash_index = sqlalchemy_url.rfind('/')
new_db_url = sqlalchemy_url[:last_slash_index +
1] + app.config.DB_SETTINGS['database']
config.set_main_option('sqlalchemy.url', new_db_url)
return config
def initialize_db(dbconnection):
alembic_cfg = Config(DEFAULT_ALEMBIC_INI_PATH)
db = alembic_cfg.get_main_option("sqlalchemy.url", dbconnection)
alembic_cfg.set_main_option("sqlalchemy.url", db)
engine = sa.engine_from_config(
alembic_cfg.get_section(alembic_cfg.config_ini_section),
prefix='sqlalchemy.')
node.create_tables(engine)
groups.create_tables(engine)
public.create_tables(engine)
xfeatures.create_tables(engine)
quotaholder_serials.create_tables(engine)
# then, load the Alembic configuration and generate the
# version table, "stamping" it with the most recent rev:
command.stamp(alembic_cfg, "head")
def initialize_db():
alembic_cfg = Config(DEFAULT_ALEMBIC_INI_PATH)
db = alembic_cfg.get_main_option("sqlalchemy.url",
PITHOS_BACKEND_DB_CONNECTION)
alembic_cfg.set_main_option("sqlalchemy.url", db)
engine = sa.engine_from_config(alembic_cfg.get_section(
alembic_cfg.config_ini_section),
prefix='sqlalchemy.')
node.create_tables(engine)
groups.create_tables(engine)
public.create_tables(engine)
xfeatures.create_tables(engine)
# then, load the Alembic configuration and generate the
# version table, "stamping" it with the most recent rev:
command.stamp(alembic_cfg, "head")
def patch_env():
from alembic.config import Config
from alembic.script.base import ScriptDirectory
alembic_cfg = Config("alembic.ini")
main_dir = alembic_cfg.get_main_option("script_location")
main_dir = _set_unix_path(main_dir)
env_path = path.join(main_dir, "env.py")
with open(env_path, "r") as rf:
_origin_f = rf.read()
with open(env_path, "w") as wf:
_new_f = _origin_f.replace(_origin_line, _new_line)
wf.write(_new_f)
script = ScriptDirectory(main_dir)
script._copy_file(path.join(_package_dir, sp_search_models),
path.join(main_dir, "search_models.py"))
def oneshot(cli_ctx, alembic_config):
'''
Set up your database with one-shot schema migration instead of
iterating over multiple revisions if there is no existing database.
It uses alembic.ini to configure database connection.
Reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
#building-an-up-to-date-database-from-scratch
'''
with cli_ctx.logger:
alembic_cfg = Config(alembic_config)
sa_url = alembic_cfg.get_main_option('sqlalchemy.url')
engine = sa.create_engine(sa_url)
engine.execute('CREATE EXTENSION IF NOT EXISTS "uuid-ossp";')
with engine.begin() as connection:
context = MigrationContext.configure(connection)
current_rev = context.get_current_revision()
if current_rev is None:
# For a fresh clean database, create all from scratch.
# (it will raise error if tables already exist.)
log.info('Detected a fresh new database.')
log.info('Creating tables...')
with engine.begin() as connection:
alembic_cfg.attributes['connection'] = connection
metadata.create_all(engine, checkfirst=False)
log.info('Stamping alembic version to head...')
command.stamp(alembic_cfg, 'head')
else:
# If alembic version info is already available, perform incremental upgrade.
log.info('Detected an existing database.')
log.info('Performing schema upgrade to head...')
with engine.begin() as connection:
alembic_cfg.attributes['connection'] = connection
command.upgrade(alembic_cfg, 'head')
log.info(
"If you don't need old migrations, delete them and set "
"\"down_revision\" value in the earliest migration to \"None\".")
def main():
alembic = CommandLine()
options = alembic.parser.parse_args()
# Если указан относительный путь (alembic.ini), добавляем в начало
# абсолютный путь до приложения
if not os.path.isabs(options.config):
options.config = os.path.join(PROJECT_PATH, options.config)
# Создаем объект конфигурации Alembic
config = Config(file_=options.config,
ini_section=options.name,
cmd_opts=options)
# Подменяем путь до папки с alembic на абсолютный (требуется, чтобы alembic
# мог найти env.py, шаблон для генерации миграций и сами миграции)
alembic_location = config.get_main_option('script_location')
if not os.path.isabs(alembic_location):
config.set_main_option('script_location',
os.path.join(PROJECT_PATH, alembic_location))
# Запускаем команду alembic
exit(alembic.run_cmd(config, options))
def make_alembic_config(cmd_opts: Union[Namespace, SimpleNamespace],
base_path: str = PROJECT_PATH) -> Config:
"""
Creates alembic configuration object based on command line arguments,
replaces relatives pathes with absolute ones.
"""
# Replace relative path to alembic.ini with absolute one
if not os.path.isabs(cmd_opts.config):
cmd_opts.config = os.path.join(base_path, cmd_opts.config)
config = Config(file_=cmd_opts.config,
ini_section=cmd_opts.name,
cmd_opts=cmd_opts)
# Replace relative path to directory with alembic with absolute one
alembic_location = config.get_main_option('script_location')
if not os.path.isabs(alembic_location):
config.set_main_option('script_location',
os.path.join(base_path, alembic_location))
if cmd_opts.pg_url:
config.set_main_option('sqlalchemy.url', cmd_opts.pg_url)
return config
def make_alembic_config(cmd_opts: Union[Namespace, SimpleNamespace],
base_path: str = PROJECT_PATH) -> Config:
"""
Создает объект конфигурации alembic на основе аргументов командной строки,
подменяет относительные пути на абсолютные.
"""
# Подменяем путь до файла alembic.ini на абсолютный
if not os.path.isabs(cmd_opts.config):
cmd_opts.config = os.path.join(base_path, cmd_opts.config)
config = Config(file_=cmd_opts.config,
ini_section=cmd_opts.name,
cmd_opts=cmd_opts)
# Подменяем путь до папки с alembic на абсолютный
alembic_location = config.get_main_option('script_location')
if not os.path.isabs(alembic_location):
config.set_main_option('script_location',
os.path.join(base_path, alembic_location))
if cmd_opts.pg_url:
config.set_main_option('sqlalchemy.url', cmd_opts.pg_url)
return config
class Database():
def __init__(self, config, database_url, before_rev, after_rev):
self.before_rev = before_rev
self.after_rev = after_rev
self.alembic_cfg = Config(config)
self.alembic_cfg.set_main_option("sqlalchemy.url", database_url)
command.upgrade(self.alembic_cfg, before_rev)
self.engine = create_engine(
self.alembic_cfg.get_main_option("sqlalchemy.url"))
self.metadata = MetaData()
self.metadata.reflect(bind=self.engine)
def insert(self, data):
connection = self.engine.connect()
for table_name, table_data in data.items():
table = self.metadata.tables[table_name]
for entry in table_data:
for column in table.c:
if not column.nullable and column.name not in entry:
entry[column.name.split(".")[-1]] = generate_fake_data(
column.type)
print(entry)
connection.execute(table.insert().values(**entry))
connection.close()
def migrate(self):
command.upgrade(self.alembic_cfg, self.after_rev)
self.metadata = MetaData()
self.metadata.reflect(bind=self.engine)
def get_table(self, table_name):
return self.metadata.tables[table_name]
def connection(self):
return self.engine.connect()
def oneshot(args):
'''Set up your database with one-shot schema migration instead of
iterating over multiple revisions.
It uses alembic.ini to configure database connection.
Reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
#building-an-up-to-date-database-from-scratch
'''
from alembic.config import Config
from alembic import command
alembic_cfg = Config(args.config)
sa_url = alembic_cfg.get_main_option('sqlalchemy.url')
log.info('Creating tables...')
engine = sa.create_engine(sa_url)
engine.execute('CREATE EXTENSION IF NOT EXISTS "uuid-ossp";')
# it will raise error if tables already exist.
metadata.create_all(engine, checkfirst=False)
log.info(f'Stamping alembic version to {args.schema_version}...')
command.stamp(alembic_cfg, args.schema_version)
log.info("If you don't need old migrations, delete them and set "
"\"down_revision\" value in the earliest migration to \"None\".")
class MakeMigrations(DataBaseCommands):
def create_model_branch(self,
apps: typing.List[typing.Union[str]]) -> None:
for app in apps:
os.environ["CRAX_CURRENT"] = app
version_path = self.create_version_dir(app)
try:
self.config.set_main_option("version_locations", version_path)
revision(
self.config,
branch_label=f"{self.db_name}/{app}",
autogenerate=True,
version_path=version_path,
message=self.args.message,
)
except CommandError as e:
sys.stderr.write(
f"Failed to create initial migrations: {e} \n")
break
def get_revisions(self) -> dict:
revisions = {}
for app in self.applications:
alembic_script = ScriptDirectory.from_config(self.config)
try:
rev = alembic_script.get_revisions(
f"{self.db_name}/{app}@head")
if rev:
label = list(rev[0].branch_labels)[0]
revisions.update(
{label: [rev[0].revision, rev[0].down_revision]})
except (CommandError, KeyError):
pass
return revisions
def wipe_existing_versions(self) -> None:
engine = create_engine(self.default_connection, echo=False)
session = sessionmaker()
_session = session(bind=engine)
try:
_session.execute("delete from alembic_version")
_session.commit()
except (ProgrammingError, OperationalError):
sys.stdout.write(
"Failed to delete alembic versions. If you had an errors, please check it out. \n"
)
def check_migrations_exist(self, app: str) -> Optional[bool]:
if app != "crax.auth":
app_dir = f"{self.project_url}/{app}"
else:
module = "crax.auth"
spl_module = module.split(".")
auth_path = __import__(module, fromlist=spl_module)
app_dir = auth_path.__path__[0]
if "models.py" in os.listdir(app_dir):
if len(self.databases) > 1:
try:
app_content = os.listdir(
f"{app_dir}/{self.migrations_dir}")
check_path = self.db_name
except FileNotFoundError:
return False
else:
try:
app_content = os.listdir(app_dir)
check_path = self.migrations_dir
except FileNotFoundError: # pragma: no cover
return False
if check_path in app_content:
return True
else:
return False
def delete_crax_migrations(self, module: str) -> None:
spl_module = module.split(".")
auth_path = __import__(module, fromlist=spl_module)
path = f"{auth_path.__path__[0]}/{self.migrations_dir}"
if os.path.exists(path):
shutil.rmtree(path)
def check_deleted(self) -> None:
dir_exists = os.path.exists(f"{self.project_url}/{self.alembic_dir}")
if self.config is not None and dir_exists:
if len(self.databases) > 1:
migration_dirs = [
f"{self.project_url}/{app}/{self.migrations_dir}/{self.db_name}"
for app in self.applications if app != "crax.auth"
]
else:
migration_dirs = [
f"{self.project_url}/{app}/{self.migrations_dir}"
for app in self.applications if app != "crax.auth"
]
engine = create_engine(self.default_connection, echo=False)
session = sessionmaker()
_session = session(bind=engine)
try:
existing_versions = [
x
for x in _session.execute("select * from alembic_version")
]
not_existing_dirs = any(
[not os.path.exists(x) for x in migration_dirs])
if existing_versions and not_existing_dirs:
self.delete_crax_migrations("crax.auth")
if os.path.isfile(f"{self.project_url}/alembic.ini"):
self.write_config("alembic", "crax_migrated",
"migrated")
self.wipe_existing_versions()
except (ProgrammingError, OperationalError):
sys.stdout.write(
"Failed to get alembic versions. If you had an errors, please check it out. \n"
)
def make_migrations(self) -> None:
os.environ["CRAX_ONLINE"] = "true"
dir_exists = os.path.exists(f"{self.project_url}/{self.alembic_dir}")
if self.config is None or not dir_exists:
if os.path.isfile(f"{self.project_url}/alembic.ini"):
os.remove(f"{self.project_url}/alembic.ini")
if os.path.exists(f"{self.project_url}/{self.alembic_dir}"):
shutil.rmtree(f"{self.project_url}/{self.alembic_dir}")
self.delete_crax_migrations("crax.auth")
self.wipe_existing_versions()
self.config = Config("alembic.ini")
init(self.config, self.alembic_dir, package=True)
shutil.copy(env.__file__,
f"{self.project_url}/{self.alembic_dir}/env.py")
self.create_model_branch(self.applications)
self.create_db_map()
else:
self.check_deleted()
migrated = self.config.get_main_option("crax_migrated")
if migrated == "not migrated":
sys.stderr.write(
"You have unapplied migrations. Please run migrate command first. \n"
)
sys.exit(1)
db_map = self.config.get_main_option("crax_db_map")
if db_map:
try:
map_dict = json.loads(
self.config.get_main_option("crax_db_map"))
current_db_map = map_dict[self.db_name]
if isinstance(current_db_map, dict):
os.environ["CRAX_DB_TABLES"] = json.dumps(
current_db_map)
except KeyError:
pass
for app in self.applications:
os.environ["CRAX_CURRENT"] = app
version_dir = self.create_version_dir(app)
exists = self.check_migrations_exist(app)
message = self.get_option("message")
if exists is not None:
if exists is False:
revision(
self.config,
branch_label=f"{self.db_name}/{app}",
head="base",
version_path=version_dir,
autogenerate=True,
message=message,
)
if self.check_branch_exists(f"{self.db_name}/{app}"):
revisions = self.get_revisions()
if revisions:
try:
stamp(
self.config,
revisions[f"{self.db_name}/{app}"][0],
)
except (KeyError, IndexError):
stamp(self.config,
f"{self.db_name}/{app}@head")
else:
revision(
self.config,
head=f"{self.db_name}/{app}@head",
version_path=version_dir,
autogenerate=True,
message=message,
)
if self.check_branch_exists(f"{self.db_name}/{app}"):
revisions = self.get_revisions()
if revisions:
try:
stamp(
self.config,
revisions[f"{self.db_name}/{app}"][0],
)
except (KeyError, IndexError):
stamp(self.config,
f"{self.db_name}/{app}@head")
else:
sys.stdout.write(
f"No models found in application {app}. Skipping. \n")
revisions = self.get_revisions()
for app in self.applications:
if revisions:
if self.check_branch_exists(f"{self.db_name}/{app}"):
try:
conf_latest = self.config.get_main_option(
"crax_latest_revisions", None)
if conf_latest:
latest = json.loads(conf_latest)
app_latest = latest[f"{self.db_name}/{app}"]
if revisions[f"{self.db_name}/{app}"][
0] != app_latest:
if (revisions[f"{self.db_name}/{app}"][1]
is not None):
stamp(
self.config,
revisions[f"{self.db_name}/{app}"]
[1],
)
else:
stamp(self.config,
f"{self.db_name}/{app}@base")
except (KeyError, IndexError, CommandError) as e:
branch = f"{self.db_name}/{app}"
if branch in revisions:
stamp(self.config,
f"{self.db_name}/{app}@base")
else:
sys.stderr.write(
f"Failed to stamp: {e}. Please check it out. \n"
)
self.write_config("alembic", "crax_migrated", "not migrated")
from com.app_utils import load_config_secrets
LOCALDEV_PORT = 8080
NAME = 'Sitemap'
ROOT_URL_PROD = 'http://sitemap.baylaunch.com'
ROOT_URL_DEV = 'http://localhost:%d' % LOCALDEV_PORT
IS_REAL_PROD = os.environ.get('COMMIT') is not None
IS_LIKE_PROD = (IS_REAL_PROD or (os.environ.get('PRODUCTION') == '1')
or (len(sys.argv) >= 2 and sys.argv[1] == 'prod'))
ALEMBIC_CONFIG_FILE = 'alembic_prod.ini' if IS_LIKE_PROD else 'alembic.ini'
ALEMBIC_CONFIG = Config(ALEMBIC_CONFIG_FILE)
DB_URL = ALEMBIC_CONFIG.get_main_option('sqlalchemy.url')
VERSION = os.environ.get('COMMIT') or str(int(
time.time() * 1000)) # cache-busts Angular templates
CONFIG = {
'SQLALCHEMY_DATABASE_URI': DB_URL,
'ROOT_URL_WITH_SLASH': (ROOT_URL_PROD if IS_LIKE_PROD else ROOT_URL_DEV) +
'/', # for auto_context
# render
'RENDER_URL':
'http://localhost:%s' % (os.environ.get('RENDER_PORT') or 3000),
#'RENDER_PROD_ONLY': True,
#'LOG_SQL': True,
def create_db(params):
params = process_connection_values(params)
db_name = params["database"]
username = params["username"]
params["username"] = MASTER_USER
params["password"] = MASTER_PASSWORD
master_params = params.copy()
master_params["database"] = MASTER_DB
engine = connect(master_params)
engine.execute('COMMIT')
sql = 'CREATE DATABASE "{}";'.format(db_name)
try:
engine.execute(sql)
except Exception as e:
print sql, e
# TODO: This will only run for the first time and fail in all other cases.
# Maybe test before instead?
sql = 'CREATE ROLE {user} LOGIN PASSWORD "{user}" VALID UNTIL "infinity";'.format(
user=username)
try:
engine.execute(sql)
except Exception as e:
pass
engine = connect(params)
# TODO: Alembic (and sqlalchemy for that matter) don't like schemas. We should
# figure out a way to add these later
config = load_flask_config()
models = config.get("models", [])
if not models:
raise Exception("This app has no models defined in config")
for model_module_name in models:
log.info("Building models from %s", model_module_name)
models = importlib.import_module(model_module_name)
models.ModelBase.metadata.create_all(engine)
# stamp the db with the latest alembic upgrade version
from drift.flaskfactory import _find_app_root
approot = _find_app_root()
ini_path = os.path.join(approot, "alembic.ini")
alembic_cfg = Config(ini_path)
script_path = os.path.join(
os.path.split(os.path.abspath(ini_path))[0], "alembic")
alembic_cfg.set_main_option("script_location", script_path)
db_names = alembic_cfg.get_main_option('databases')
connection_string = format_connection_string(params)
alembic_cfg.set_section_option(db_names, "sqlalchemy.url",
connection_string)
command.stamp(alembic_cfg, "head")
for schema in SCHEMAS:
# Note that this does not automatically grant on tables added later
sql = '''
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA "{schema}" TO {user};
GRANT USAGE, SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA "{schema}" TO {user};
GRANT ALL ON SCHEMA "{schema}" TO {user};'''.format(
schema=schema, user=username)
try:
engine.execute(sql)
except Exception as e:
print sql, e
return db_name
def parse_config():
alembic_cfg = Config(alembic_ini_file_path)
app_ini_file_name = alembic_cfg.get_main_option('pylons_config_file')
app_ini_file_dir = os.path.dirname(alembic_cfg.config_file_name)
app_config_uri = 'config:%s' % app_ini_file_name
return appconfig(app_config_uri, 'thelma', relative_to=app_ini_file_dir)