def downgradedb(self):
with app.app_context():
downgrade(directory='migrations',
revision='-1',
sql=False,
tag=None)
return "db downgraded"
def tearDown(self):
db.session.remove()
with self.app.app_context():
# Rollback all DB migrations
downgrade(revision='base')
super().tearDown()
def setUp(self):
super(DbTest, self).setUp()
# Cleaning DB. It useful in case of tests failure
directory = os.path.join(os.path.dirname(__file__),
'..', 'api', 'db', 'migrations')
with app.app_context():
flask_migrate.downgrade(directory=directory)
flask_migrate.upgrade(directory=directory)
def seed(filename, drop_all):
with open(filename) as f:
data = json.load(f)
if drop_all:
flask_migrate.downgrade(revision="base")
db.drop_all()
flask_migrate.upgrade(revision="head")
tags = data.get("tags")
if tags:
db.session.add_all([Tag(name=tag) for tag in tags])
posts = data.get("posts")
if posts:
def create_post(post):
tags = []
if "tags" in post:
tags = [
Tag.query.filter(Tag.name == tag).one()
for tag in post["tags"]
]
return Post(title=post.get("title"),
summary=post.get("summary"),
content=post.get("content"),
tags=tags)
posts = map(create_post, posts)
db.session.add_all(posts)
sites = data.get("sites")
if sites:
db.session.add_all([Site(name=site) for site in sites])
subjects = data.get("questionSubjects")
if subjects:
db.session.add_all([Subject(name=subject) for subject in subjects])
questions = data.get("questions")
if questions:
def create_question(question):
site = Site.query.filter(Site.name == question.get("site")).one()
grade = Grade[question.get("grade").upper()]
subject = Subject.query.filter(
Subject.name == question.get("subject")).one()
return Question(site=site,
grade=grade,
specialty=question.get("specialty"),
subject=subject,
text=question.get("text"))
questions = map(create_question, questions)
db.session.add_all(questions)
db.session.commit()
def downgrade_migrations():
"""debug only"""
from flask_migrate import downgrade
if os.path.exists(os.getenv('MIGRATIONS', basedir + '/migrations')):
downgrade(directory=os.getenv('MIGRATIONS', basedir + '/migrations'))
print 'downgrade database :' + os.getenv('MIGRATIONS',
basedir + '/migrations')
else:
print 'Can\'t find :' + os.getenv('MIGRATIONS',
basedir + '/migrations')
def fresh_db_fixture():
"""Ensure a fresh DB with migrations for tests."""
local_db()
app = create_app()
with app.app_context():
upgrade()
yield
with app.app_context():
downgrade(revision='base')
if path.exists(environ['DB_HOST']):
unlink(environ['DB_HOST'])
def free_the_zoo(zoo_url):
confirm = str(input('Are you sure you want to clear and re-migrate the database? (y/N): ')).strip()
if confirm == 'y':
init_zoo_db(zoo_url)
if flask_migrate.current() is not None:
flask_migrate.downgrade(tag='base')
flask_migrate.upgrade()
migrate_models()
def db(app):
"""Session-wide test database."""
_db.app = app
# Apply migrations (creating tables)
Migrate(app, _db)
upgrade(revision="head")
yield _db
# Undo migrations (removing tables)
downgrade(revision="base")
def test_create_empty_index_image_request(app, auth_env, client, db):
total_requests = 20
empty_index_revision = 'e16a8cd2e028'
# flask_login.current_user is used in RequestAdd.from_json and RequestRm.from_json,
# which requires a request context
with app.test_request_context(environ_base=auth_env):
# Generate some data to verify migration
data = {
'from_index': 'quay.io/namespace/index_image:latest',
'binary_image': 'quay.io/namespace/binary_image:latest',
}
request = RequestCreateEmptyIndex.from_json(data)
db.session.add(request)
for i in range(total_requests):
request_class = random.choice(
(RequestAdd, RequestRm, RequestCreateEmptyIndex))
if request_class == RequestAdd:
data = {
'binary_image': 'quay.io/namespace/binary_image:latest',
'bundles': [f'quay.io/namespace/bundle:{i}'],
'from_index': f'quay.io/namespace/repo:{i}',
}
request = RequestAdd.from_json(data)
elif request_class == RequestRm:
data = {
'binary_image': 'quay.io/namespace/binary_image:latest',
'operators': [f'operator-{i}'],
'from_index': f'quay.io/namespace/repo:{i}',
}
request = RequestRm.from_json(data)
elif request_class == RequestCreateEmptyIndex:
data = {
'from_index': f'quay.io/namespace/index_image:{i}',
'binary_image': 'quay.io/namespace/binary_image:latest',
}
request = RequestCreateEmptyIndex.from_json(data)
if i % 5 == 0:
# Simulate failed request
request.add_state('failed', 'Failed due to an unknown error')
db.session.add(request)
db.session.commit()
expected_rv_json = client.get(
f'/api/v1/builds?per_page={total_requests}&verbose=true').json
flask_migrate.downgrade(revision=empty_index_revision)
flask_migrate.upgrade()
actual_rv_json = client.get(
f'/api/v1/builds?per_page={total_requests}&verbose=true').json
assert expected_rv_json == actual_rv_json
def downgrade_db(revision):
"""
Downgrade db to previous revision of the database schema
(for development only). For revision you can use an hash or a relative migration identifier.
"""
from zou.app import app
with app.app_context():
import zou
directory = os.path.join(os.path.dirname(zou.__file__), "migrations")
flask_migrate.downgrade(directory=directory, revision=revision)
def trigger_migration_downgrade(migration_path):
"""
Optional function to trigger a database migration downgrade 1 version.
Causes program to exit!
Parameters
----------
migration_path : str
Migration Path
"""
downgrade(directory=migration_path)
exit(0)
def setUp(self):
super(DbTest, self).setUp()
# Cleaning all changes from the previous test
db.session.rollback()
directory = self.get_migrations_dir()
with app.app_context():
try:
flask_migrate.downgrade(directory=directory,
revision='base')
except CommandError as e:
app.logger.debug("DB migration downgrade failed: %s", e)
self.clean_db()
flask_migrate.upgrade(directory=directory)
def test_abort_when_downgrading_from_regenerate_bundle_request(
app, auth_env, client, db):
"""Verify downgrade is prevented if "regenerate-bundle" requests exist."""
total_requests = 20
# flask_login.current_user is used in Request*.from_json which requires a request context
with app.test_request_context(environ_base=auth_env):
# Always add a RequestRegenerateBundle to ensure sufficient test data is available
data = {'from_bundle_image': 'quay.io/namespace/bundle-image:latest'}
request = RequestRegenerateBundle.from_json(data)
db.session.add(request)
# One request was already added, let's add the remaining ones
for i in range(total_requests - 1):
request_class = random.choice(
(RequestAdd, RequestRm, RequestRegenerateBundle))
if request_class == RequestAdd:
data = {
'binary_image': 'quay.io/namespace/binary_image:latest',
'bundles': [f'quay.io/namespace/bundle:{i}'],
'from_index': f'quay.io/namespace/repo:{i}',
}
request = RequestAdd.from_json(data)
elif request_class == RequestRm:
data = {
'binary_image': 'quay.io/namespace/binary_image:latest',
'operators': [f'operator-{i}'],
'from_index': f'quay.io/namespace/repo:{i}',
}
request = RequestRm.from_json(data)
else:
data = {
'from_bundle_image':
'quay.io/namespace/bundle-image:latest'
}
request = RequestRegenerateBundle.from_json(data)
db.session.add(request)
db.session.add(request)
db.session.commit()
# flask_migrate raises a SystemExit exception regardless of what's raised from the
# downgrade function. This exception doesn't hold a reference to the RuntimeError
# we expect from the downgrade function in the migration script. The best we can
# do is catch the SystemExit exception.
with pytest.raises(SystemExit):
flask_migrate.downgrade(revision=INITIAL_DB_REVISION)
def bdd(init_flask_db):
"""
All tests under this module will have a database
with an up to date scheme
At the end of the module the database scheme will be downgraded and upgraded again
in the next module to test the database migrations
"""
with app.app_context():
flask_migrate.Migrate(app, db)
migration_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'migrations')
flask_migrate.upgrade(directory=migration_dir)
yield
with app.app_context():
flask_migrate.downgrade(revision='base', directory=migration_dir)
def seeddb(initdb=False, cleanupdb=False, orders=20):
"""Seed fake data into the database.
To create new database use the '-i' or '--initdb' option.
To clear up the existing database use the '-c' or '--cleanupdb' option.
"""
if initdb:
print("Creating database...")
upgrade()
if cleanupdb:
print("Cleaning up database...")
downgrade(revision='base')
upgrade()
seeded_data = seed_test_data(orders=int(orders))
print("Created and saved test data into database: %r" % seeded_data)
def client():
app = create_app()
path = os.path.dirname(os.path.abspath(__name__))
app.config['SQLALCHEMY_DATABASE_URI'] = f'sqlite:///{path}/test.db'
app.config['TESTING'] = True
# import pdb; pdb.set_trace()
with app.test_client() as client:
with app.app_context():
flask_migrate.downgrade(revision="base")
flask_migrate.upgrade(revision='head')
for dashboard in get_dashboards():
db.session.add(dashboard)
for visualization in get_visualizations():
db.session.add(visualization)
client.secret = app.config['CAIPIRINHA_CONFIG']['secret']
db.session.commit()
yield client
def bdd(init_flask_db):
"""
All tests under this module will have a database
with an up to date scheme
At the end of the module the database scheme will be downgraded and upgraded again
in the next module to test the database migrations
"""
with app.app_context():
flask_migrate.Migrate(app, db)
migration_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'migrations')
flask_migrate.upgrade(directory=migration_dir)
yield
with app.app_context():
flask_migrate.downgrade(revision='base', directory=migration_dir)
def seeddb(initdb=False, cleanupdb=False, orders=20):
"""Seed fake data into the database.
To create new database use the '-i' or '--initdb' option.
To clear up the existing database use the '-c' or '--cleanupdb' option.
"""
if initdb:
print("Creating database...")
upgrade()
if cleanupdb:
print("Cleaning up database...")
downgrade(revision='base')
upgrade()
seeded_data = seed_test_data(orders=int(orders))
print("Created and saved test data into database: %r" % seeded_data)
def test_clean_db(self):
# Crashing alembic versions history
db.session.execute("UPDATE alembic_version SET version_num='x'")
db.session.commit()
migrations_dir = self.get_migrations_dir()
with app.app_context():
# Checking migrations are broken
self.assertRaises(
CommandError, flask_migrate.downgrade,
directory=migrations_dir, revision='base'
)
self.clean_db()
# Checking migrations flow is fixed
flask_migrate.downgrade(directory=migrations_dir, revision='base')
flask_migrate.upgrade(directory=migrations_dir)
def database(action, revision, message):
with webui.app.app_context():
# creates folder /migrations with necessary files for migrations
if action == "init":
init_migration()
# creates migration with current state of models
elif action == "migrate":
comment = str(message) if message else ""
migrate(message=comment)
# upgrades current state to latest version of database or to defined revision
elif action == "upgrade":
if revision:
upgrade(revision=revision)
else:
upgrade()
# downgrades current state to previous version of database or to defined revision
elif action == "downgrade":
if revision:
downgrade(revision=revision)
else:
downgrade()
# shows current migration
elif action == "current":
return current()
# shows history of migrations until current migration
elif action == "history":
return history()
# creates migration manually with comment = message
elif action == "create-migration":
comment = str(message) if message else ""
return manual_migrate(message=comment)
def test_migrate_to_polymorphic_requests(app, auth_env, client, db):
total_requests = 20
# flask_login.current_user is used in RequestAdd.from_json and RequestRm.from_json,
# which requires a request context
with app.test_request_context(environ_base=auth_env):
# Generate some data to verify migration
for i in range(total_requests):
if random.choice((True, False)):
data = {
'binary_image': 'quay.io/namespace/binary_image:latest',
'bundles': [f'quay.io/namespace/bundle:{i}'],
'from_index': f'quay.io/namespace/repo:{i}',
}
request = RequestAdd.from_json(data)
else:
data = {
'binary_image': 'quay.io/namespace/binary_image:latest',
'operators': [f'operator-{i}'],
'from_index': f'quay.io/namespace/repo:{i}',
}
request = RequestRm.from_json(data)
if i % 5 == 0:
# Simulate failed request
request.add_state('failed', 'Failed due to an unknown error')
db.session.add(request)
db.session.commit()
expected_rv_json = client.get(
f'/api/v1/builds?per_page={total_requests}&verbose=true').json
flask_migrate.downgrade(revision=INITIAL_DB_REVISION)
flask_migrate.upgrade()
actual_rv_json = client.get(
f'/api/v1/builds?per_page={total_requests}&verbose=true').json
assert expected_rv_json == actual_rv_json
def remove_migrations():
"""Remove all alembic migrations."""
downgrade(revision="base", sql=False)
def tearDown(self):
db.session.remove()
downgrade(x_arg='data=true', revision='base')
def tearDown(self):
db.session.remove()
downgrade(x_arg="data=true", revision="base")
def db_downgrade(c, dir=DEFAULT_MIGRATION_DIR):
"""
Downgrade database to a specific revision.
"""
with app.app_context():
migrate.downgrade(directory=dir)
def cleanupdb():
"""Clear up the database through migration."""
downgrade(revision='base')
upgrade()
def test_migrate(self):
# run upgrades from initial to head.
upgrade()
# run downgrades from head to base.
downgrade(revision="base")
def init_database(app):
with app.app_context():
flask_migrate.upgrade(revision='head')
yield
db.session.remove()
flask_migrate.downgrade(revision='base')
def tearDown(self):
flask_migrate.downgrade()
self.ctx.pop()
def downgrade(revision='-1'):
with app.app_context():
return flask_migrate.downgrade('migrates', revision)
def database(app):
flask_migrate.upgrade(revision='head')
yield
flask_migrate.downgrade(revision='base')
def down():
flask_migrate.downgrade()
def teardown_db(feature):
with chaos.app.app_context():
flask_migrate.downgrade(revision='base', directory=migration_dir)
def downgrade(args):
"""Downgrade database schema."""
flask_migrate.downgrade(DIRECTORY, args['revision'])
def reset():
shutil.rmtree(app.config['IMAGE_FOLDER'])
shutil.rmtree(app.config['IMAGE_CACHE_FOLDER'])
downgrade(revision='base')
upgrade()
def cleanupdb():
"""Clear up the database through migration."""
downgrade(revision='base')
upgrade()
def db_downgrade(c, dir=DEFAULT_MIGRATION_DIR):
"""
Downgrade database to a specific revision.
"""
with app.app_context():
migrate.downgrade(directory=dir)
def teardown():
downgrade(directory='migrations', revision='base')
def clean_db():
"""Delete database tables."""
downgrade(revision="base")
def downgrade_database(self, target='base'):
plugindir = current_app.extensions['plugin_manager'].plugin_folder
downgrade(directory=os.path.join(plugindir, 'migrations'), revision=self.settings_key + '@' + target)
