def create_test_database(main_app):
SQLALCHEMY_DATABASE_URI = main_app.config.get("SQLALCHEMY_DATABASE_URI")
print("CREATE SQLALCHEMY_DATABASE_URI ", SQLALCHEMY_DATABASE_URI)
if database_exists(SQLALCHEMY_DATABASE_URI):
drop_database(SQLALCHEMY_DATABASE_URI)
create_database(SQLALCHEMY_DATABASE_URI)
def base_app():
"""Flask application fixture."""
instance_path = tempfile.mkdtemp()
base_app = Flask(__name__, instance_path=instance_path)
base_app.config.update(
ACCOUNTS_USE_CELERY=False,
LOGIN_DISABLED=False,
SECRET_KEY='testing_key',
SQLALCHEMY_DATABASE_URI=os.getenv('SQLALCHEMY_DATABASE_URI',
'sqlite://'),
TEST_USER_EMAIL='*****@*****.**',
TEST_USER_PASSWORD='******',
TESTING=True,
WTF_CSRF_ENABLED=False,
)
Babel(base_app)
Mail(base_app)
Menu(base_app)
InvenioDB(base_app)
InvenioAccounts(base_app)
base_app.register_blueprint(accounts_blueprint)
with base_app.app_context():
if str(db.engine.url) != "sqlite://" and \
not database_exists(str(db.engine.url)):
create_database(str(db.engine.url))
db.create_all()
yield base_app
with base_app.app_context():
drop_database(str(db.engine.url))
shutil.rmtree(instance_path)
def _db(app, request):
"""Session-wide test database."""
if os.path.exists(TESTDB_PATH):
os.unlink(TESTDB_PATH)
if not request.config.getoption('--postgresql'):
psef.models.db.create_all()
else:
db_name, generated = get_database_name(request)
if generated:
psef.models.db.create_all()
manage.app = app
manage.seed_force(psef.models.db)
manage.test_data(psef.models.db)
assert psef.models.Permission.query.all()
psef.permissions.database_permissions_sanity_check(app)
try:
yield psef.models.db
finally:
if request.config.getoption('--postgresql'):
db_name, generated = get_database_name(request)
if generated:
orm.session.close_all_sessions()
psef.models.db.engine.dispose()
drop_database(db_name)
else:
os.unlink(TESTDB_PATH)
def initialize_database(exist_ok: bool = False, drop_existing: bool = False):
db_url = engine.url
typer.echo(f"Using database at {db_url}")
if database_exists(db_url):
if drop_existing:
with wrap_echo("Dropping database"):
drop_database(db_url)
elif not exist_ok:
typer.echo(
f"Database already exists, aborting.\n"
f"Use --exist-ok if you are sure the database is uninitialized and contains no data.\n"
f"Use --drop-existing if you want to recreate it.",
err=True,
)
return
with wrap_echo("Creating database"):
create_database(db_url)
pass
with wrap_echo("Creating metadata"):
base.metadata.create_all()
pass
typer.echo("Database initialization complete.")
def populate(bulk_data, test_data, posts, topics, force, initdb):
"""Creates the necessary tables and groups for FlaskBB."""
if force:
click.secho("[+] Recreating database...", fg="cyan")
drop_database(db.engine.url)
# do not initialize the db if -i is passed
if not initdb:
upgrade_database()
if initdb:
click.secho("[+] Initializing database...", fg="cyan")
upgrade_database()
if test_data:
click.secho("[+] Adding some test data...", fg="cyan")
create_test_data()
if bulk_data:
timer = time.time()
topic_count, post_count = insert_bulk_data(int(topics), int(posts))
elapsed = time.time() - timer
click.secho("[+] It took {} seconds to create {} topics and {} posts"
.format(elapsed, topic_count, post_count), fg="cyan")
# this just makes the most sense for the command name; use -i to
# init the db as well
if not test_data:
click.secho("[+] Populating the database with some defaults...",
fg="cyan")
create_default_groups()
create_default_settings()
def drop_db(section="test"):
"""Cleanup
"""
LOG.info(f"Drop database")
CONF = make_conf(section)
if database_exists(CONF):
drop_database(CONF)
def create_ticclat_database(delete_existing=False):
"""
Create the TICCLAT database.
Sets the proper encoding settings and uses the schema to create tables.
"""
# db = MySQLdb.connect(user=user, passwd=passwd, host=host)
# engine = create_engine(f"mysql://{user}:{passwd}@{host}/{dbname}?charset=utf8mb4")
engine = get_engine(without_database=True)
connection = engine.connect()
db_name = get_db_name()
try:
connection.execute(
f"CREATE DATABASE {db_name} CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;"
)
except MySQLdb.ProgrammingError as exception:
if database_exists(engine.url):
if not delete_existing:
raise Exception(
f"Database `{db_name}` already exists, delete it first before recreating."
)
drop_database(engine.url)
connection.execute(
f"CREATE DATABASE {db_name} CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;"
)
else:
raise exception
connection.close()
engine = get_engine()
# create tables
Base.metadata.create_all(engine)
def base_app():
"""Flask application fixture."""
instance_path = tempfile.mkdtemp()
base_app = Flask(__name__, instance_path=instance_path)
base_app.config.update(
ACCOUNTS_USE_CELERY=False,
LOGIN_DISABLED=False,
SECRET_KEY='testing_key',
SQLALCHEMY_DATABASE_URI=os.getenv('SQLALCHEMY_DATABASE_URI',
'sqlite://'),
TEST_USER_EMAIL='*****@*****.**',
TEST_USER_PASSWORD='******',
TESTING=True,
WTF_CSRF_ENABLED=False,
)
Babel(base_app)
Mail(base_app)
Menu(base_app)
InvenioDB(base_app)
InvenioAccounts(base_app)
base_app.register_blueprint(accounts_blueprint)
with base_app.app_context():
if str(db.engine.url) != "sqlite://" and \
not database_exists(str(db.engine.url)):
create_database(str(db.engine.url))
db.create_all()
yield base_app
with base_app.app_context():
drop_database(str(db.engine.url))
shutil.rmtree(instance_path)
def tearDownClass(cls):
cls.transaction.rollback()
cls.connection.close()
cls.engine.dispose()
if cls.create:
drop_database(cls.engine.url)
super(StoreTestCase, cls).tearDownClass()
def xtest_db_is_automatically_migrated(disk_store):
db_conn_str = 'postgresql://localhost/test_provenance_automigrate'
if sql_utils.database_exists(db_conn_str):
sql_utils.drop_database(db_conn_str)
sql_utils.create_database(db_conn_str)
repo = r.PostgresRepo(db_conn_str,
disk_store,
read=True,
write=True,
delete=True,
create_db=False,
upgrade_db=True)
p.set_default_repo(repo)
@p.provenance()
def calculate(a, b):
return a + b
# make sure it all works
assert calculate(1, 2) == 3
p.set_default_repo(None)
sql_utils.drop_database(db_conn_str)
def test_alembic_and_db_create_match(clean_app):
"""Check that alembic recipes and alembic models are in sync."""
with clean_app.app_context():
ext = clean_app.extensions['invenio-db']
if db.engine.name == 'sqlite':
raise pytest.skip('Upgrades are not supported on SQLite.')
# Make sure that alembic upgrades can be run now
ext.alembic.upgrade()
constraints = get_all_constraints(clean_app)
alembic_constraint_names = [x[0] for x in constraints]
# Recreate the database with alembic only (without migrating)
db.session.remove()
drop_database(db.engine.url)
db.engine.dispose()
create_database(db.engine.url)
db.create_all()
# Check that the resulting state is in sync with alembic metaData
assert not ext.alembic.compare_metadata()
# Check that the constraints are the same. This is needed because
# alembic.compare_metadata does not check all constraints.
constraints = get_all_constraints(clean_app)
db_create_constraint_names = [x[0] for x in constraints]
assert set(alembic_constraint_names) == set(db_create_constraint_names)
def teardown():
with app.app_context():
db_url = str(db.engine.url)
db.session.close()
if db_url != "sqlite://":
drop_database(db_url)
shutil.rmtree(instance_path)
def teardown():
with base_app.app_context():
db.session.close()
if str(db.engine.url) != 'sqlite://':
drop_database(str(db.engine.url))
print('Path: ' + instance_path)
shutil.rmtree(instance_path)
def clean_app(request, base_app):
"""Application with database and elasticsearch cleaned."""
with base_app.app_context():
try:
db.session.remove()
drop_database(db.engine.url)
except ProgrammingError:
pass
create_database(db.engine.url)
# reset elasticsearch
for deleted in current_search.delete(ignore=[404]):
pass
# reset queues
current_queues.delete()
current_queues.declare()
yield base_app
def finalize():
with base_app.app_context():
db.session.remove()
drop_database(db.engine.url)
# Dispose the engine in order to close all connections. This is
# needed for sqlite in memory databases.
db.engine.dispose()
current_queues.delete()
request.addfinalizer(finalize)
return base_app
def test_alembic_and_db_create_match(clean_app):
"""Check that alembic recipes and alembic models are in sync."""
with clean_app.app_context():
ext = clean_app.extensions['invenio-db']
if db.engine.name == 'sqlite':
raise pytest.skip('Upgrades are not supported on SQLite.')
# Make sure that alembic upgrades can be run now
ext.alembic.upgrade()
constraints = get_all_constraints(clean_app)
alembic_constraint_names = [x[0] for x in constraints]
# Recreate the database with alembic only (without migrating)
db.session.remove()
drop_database(db.engine.url)
db.engine.dispose()
create_database(db.engine.url)
db.create_all()
# Check that the resulting state is in sync with alembic metaData
assert not ext.alembic.compare_metadata()
# Check that the constraints are the same. This is needed because
# alembic.compare_metadata does not check all constraints.
constraints = get_all_constraints(clean_app)
db_create_constraint_names = [x[0] for x in constraints]
assert set(alembic_constraint_names) == set(db_create_constraint_names)
def tearDownClass(cls):
cls.transaction.rollback()
cls.connection.close()
cls.engine.dispose()
if cls.create:
drop_database(cls.engine.url)
super(StoreTestCase, cls).tearDownClass()
def clean_app(request, base_app):
"""Application with database and elasticsearch cleaned."""
with base_app.app_context():
try:
db.session.remove()
drop_database(db.engine.url)
except ProgrammingError:
pass
create_database(db.engine.url)
# reset elasticsearch
for deleted in current_search.delete(ignore=[404]):
pass
# reset queues
current_queues.delete()
current_queues.declare()
yield base_app
def finalize():
with base_app.app_context():
db.session.remove()
drop_database(db.engine.url)
# Dispose the engine in order to close all connections. This is
# needed for sqlite in memory databases.
db.engine.dispose()
current_queues.delete()
request.addfinalizer(finalize)
return base_app
def install(welcome, force, username, email, password, group):
"""Installs flaskbb. If no arguments are used, an interactive setup
will be run.
"""
click.secho("[+] Installing FlaskBB...", fg="cyan")
if database_exists(db.engine.url):
if force or click.confirm(click.style(
"Existing database found. Do you want to delete the old one and "
"create a new one?", fg="magenta")
):
drop_database(db.engine.url)
else:
sys.exit(0)
create_database(db.engine.url)
upgrade_database()
click.secho("[+] Creating default settings...", fg="cyan")
create_default_groups()
create_default_settings()
click.secho("[+] Creating admin user...", fg="cyan")
prompt_save_user(username, email, password, group)
if welcome:
click.secho("[+] Creating welcome forum...", fg="cyan")
create_welcome_forum()
click.secho("[+] Compiling translations...", fg="cyan")
compile_translations()
click.secho("[+] FlaskBB has been successfully installed!",
fg="green", bold=True)
def initialize_database(exist_ok: bool = False, drop_existing: bool = False):
db_url = engine.url
typer.echo(f"Using database at {db_url}")
if database_exists(db_url):
if drop_existing:
with wrap_echo("Dropping database"):
drop_database(db_url)
elif not exist_ok:
typer.echo(
f"Database already exists, aborting.\n"
f"Use --exist-ok if you are sure the database is uninitialized and contains no data.\n"
f"Use --drop-existing if you want to recreate it.",
err=True,
)
return
with wrap_echo("Creating database"):
create_database(db_url)
pass
with engine.connect() as con:
with wrap_echo("Installing pgcrypto extension"):
con.execute("CREATE EXTENSION IF NOT EXISTS pgcrypto;")
pass
with wrap_echo("Creating metadata"):
base.metadata.create_all()
pass
typer.echo("Database initialization complete.")
def app():
"""Flask application fixture."""
instance_path = tempfile.mkdtemp()
app = Flask('testapp', instance_path=instance_path)
app.config.update(
CELERY_ALWAYS_EAGER=True,
CELERY_TASK_ALWAYS_EAGER=True,
CELERY_CACHE_BACKEND='memory',
CELERY_EAGER_PROPAGATES_EXCEPTIONS=True,
CELERY_TASK_EAGER_PROPAGATES=True,
CELERY_RESULT_BACKEND='cache',
JSONSCHEMAS_HOST='inveniosoftware.org',
TESTING=True,
SECRET_KEY='CHANGE_ME',
SQLALCHEMY_DATABASE_URI=os.environ.get('SQLALCHEMY_DATABASE_URI',
'sqlite:///test.db'),
SQLALCHEMY_TRACK_MODIFICATIONS=True,
SERVER_NAME='app',
OAISERVER_ID_PREFIX='oai:inveniosoftware.org:recid/',
OAISERVER_QUERY_PARSER_FIELDS=["title_statement"],
OAISERVER_RECORD_INDEX='_all',
OAISERVER_REGISTER_SET_SIGNALS=True,
)
if not hasattr(app, 'cli'):
from flask_cli import FlaskCLI
FlaskCLI(app)
InvenioDB(app)
FlaskCeleryExt(app)
InvenioJSONSchemas(app)
InvenioRecords(app)
InvenioPIDStore(app)
InvenioMARC21(app)
client = Elasticsearch(hosts=[os.environ.get('ES_HOST', 'localhost')])
search = InvenioSearch(app, client=client)
search.register_mappings('records', 'data')
InvenioIndexer(app)
InvenioOAIServer(app)
app.register_blueprint(blueprint)
with app.app_context():
if str(db.engine.url) != 'sqlite://' and \
not database_exists(str(db.engine.url)):
create_database(str(db.engine.url))
db.create_all()
list(search.delete(ignore=[404]))
list(search.create())
search.flush_and_refresh('_all')
with app.app_context():
yield app
with app.app_context():
db.session.close()
if str(db.engine.url) != 'sqlite://':
drop_database(str(db.engine.url))
list(search.delete(ignore=[404]))
search.client.indices.delete("*-percolators")
shutil.rmtree(instance_path)
def app(request):
"""Flask application fixture."""
instance_path = tempfile.mkdtemp()
app = Flask('testapp', instance_path=instance_path)
app.config.update(
TESTING=True,
SERVER_NAME='localhost:5000',
SQLALCHEMY_DATABASE_URI=os.environ.get('SQLALCHEMY_DATABASE_URI',
'sqlite:///test.db'),
SQLALCHEMY_TRACK_MODIFICATIONS=True,
RECORDS_REST_ENDPOINTS=config.RECORDS_REST_ENDPOINTS,
# No permission checking
RECORDS_REST_DEFAULT_CREATE_PERMISSION_FACTORY=None,
RECORDS_REST_DEFAULT_READ_PERMISSION_FACTORY=None,
RECORDS_REST_DEFAULT_UPDATE_PERMISSION_FACTORY=None,
RECORDS_REST_DEFAULT_DELETE_PERMISSION_FACTORY=None,
RECORDS_REST_DEFAULT_SEARCH_INDEX=ES_INDEX,
RECORDS_REST_SORT_OPTIONS={
ES_INDEX: dict(year=dict(fields=['year'], ))
},
)
app.config['RECORDS_REST_ENDPOINTS']['recid']['search_class'] = TestSearch
# update the application with the configuration provided by the test
if hasattr(request, 'param') and 'config' in request.param:
app.config.update(**request.param['config'])
FlaskCLI(app)
InvenioDB(app)
InvenioREST(app)
InvenioRecords(app)
InvenioPIDStore(app)
InvenioSearch(app)
InvenioAccess(app)
InvenioRecordsREST(app)
with app.app_context():
# Setup app
if not database_exists(str(db.engine.url)) and \
app.config['SQLALCHEMY_DATABASE_URI'] != 'sqlite://':
create_database(db.engine.url)
db.drop_all()
db.create_all()
if current_search_client.indices.exists(ES_INDEX):
current_search_client.indices.delete(ES_INDEX)
current_search_client.indices.create(ES_INDEX)
prepare_indexing(app)
with app.app_context():
# Yield app in request context
with app.test_request_context():
yield app
with app.app_context():
# Teardown app
db.drop_all()
if app.config['SQLALCHEMY_DATABASE_URI'] != 'sqlite://':
drop_database(db.engine.url)
shutil.rmtree(instance_path)
def teardown():
with app.app_context():
drop_database(str(db.engine.url))
# Delete sessions in kvsession store
if hasattr(app, 'kvsession_store'):
for key in app.kvsession_store.iter_keys():
app.kvsession_store.delete(key)
shutil.rmtree(instance_path)
def teardown():
with app.app_context():
drop_database(str(db.engine.url))
# Delete sessions in kvsession store
if hasattr(app, 'kvsession_store') and \
isinstance(app.kvsession_store, RedisStore):
app.kvsession_store.redis.flushall()
shutil.rmtree(app.instance_path)
def db(app):
"""Database fixture."""
if not database_exists(str(db_.engine.url)):
create_database(str(db_.engine.url))
db_.create_all()
yield db_
db_.session.remove()
drop_database(str(db_.engine.url))
def teardown():
with app.app_context():
drop_database(str(db.engine.url))
# Delete sessions in kvsession store
if hasattr(app, 'kvsession_store') and \
isinstance(app.kvsession_store, RedisStore):
app.kvsession_store.redis.flushall()
shutil.rmtree(app.instance_path)
def engine(dsn):
if database_exists(dsn):
drop_database(dsn)
create_database(dsn)
engine = create_engine(dsn)
engine.execute('CREATE EXTENSION postgis')
GeonameBase.metadata.create_all(bind=engine)
return engine
def db(app):
"""Database fixture."""
if not database_exists(str(db_.engine.url)):
create_database(str(db_.engine.url))
db_.create_all()
yield db_
db_.session.remove()
drop_database(str(db_.engine.url))
def reset_db():
if sgdb_in(['MySQL', 'MariaDB']):
url = Configuration.get('get_url')()
if database_exists(url):
drop_database(url)
db_template_name = Configuration.get('db_template_name', None)
create_database(url, template=db_template_name)
def finalize():
with base_app.app_context():
db.session.remove()
drop_database(db.engine.url)
# Dispose the engine in order to close all connections. This is
# needed for sqlite in memory databases.
db.engine.dispose()
current_queues.delete()
def engine(dsn):
if database_exists(dsn):
drop_database(dsn)
create_database(dsn)
engine = create_engine(dsn)
engine.execute('CREATE EXTENSION postgis')
GeonameBase.metadata.create_all(bind=engine)
return engine
def teardown():
with app.app_context():
drop_database(str(db.engine.url))
# Delete sessions in kvsession store
if hasattr(app, 'kvsession_store'):
for key in app.kvsession_store.iter_keys():
app.kvsession_store.delete(key)
shutil.rmtree(instance_path)
def assert_clean(engine, database):
try:
drop_database(engine.url)
log.warning(f"Dropping database {database}")
except:
log.info(f"Database {database} does not exist")
log.info(f"Creating database {database}")
create_database(engine.url)
def destroy():
"""Drop database repository."""
click.secho('Dropping database {0}...'.format(_db.engine.url),
bold=True, fg='yellow')
drop_database(_db.engine.url)
click.secho('Database dropped!', bold=True, fg='green')
def finalize():
with base_app.app_context():
db.session.remove()
drop_database(db.engine.url)
# Dispose the engine in order to close all connections. This is
# needed for sqlite in memory databases.
db.engine.dispose()
current_queues.delete()
def test_db():
"""Test database backend."""
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get(
'SQLALCHEMY_DATABASE_URI', 'sqlite:///test.db'
)
FlaskCLI(app)
InvenioDB(app)
InvenioRecords(app)
with app.app_context():
create_database(db.engine.url)
db.create_all()
assert len(db.metadata.tables) == 3
data = {'title': 'Test'}
from invenio_records.models import RecordMetadata as RM
# Create a record
with app.app_context():
assert RM.query.count() == 0
record_uuid = Record.create(data).id
db.session.commit()
assert RM.query.count() == 1
db.session.commit()
# Retrieve created record
with app.app_context():
record = Record.get_record(record_uuid)
assert record.dumps() == data
with pytest.raises(NoResultFound):
Record.get_record(uuid.uuid4())
record['field'] = True
record = record.patch([
{'op': 'add', 'path': '/hello', 'value': ['world']}
])
assert record['hello'] == ['world']
record.commit()
db.session.commit()
with app.app_context():
record2 = Record.get_record(record_uuid)
assert record2.model.version_id == 2
assert record2['field']
assert record2['hello'] == ['world']
db.session.commit()
# Cannot commit record without model (i.e. Record.create_record)
with app.app_context():
record3 = Record({'title': 'Not possible'})
with pytest.raises(RecordNotCommitableError):
record3.commit()
with app.app_context():
db.drop_all()
drop_database(db.engine.url)
def db_engine(db_conn_str):
if sql_utils.database_exists(db_conn_str):
sql_utils.drop_database(db_conn_str)
sql_utils.create_database(db_conn_str)
engine = create_engine(db_conn_str, json_serializer=r.Encoder().encode)
Base.metadata.create_all(engine)
return engine
def db_drop():
# db teardown - drop all
config.main([
'-c', alembic_cfg, '-n', schema_name, '-x',
'dbconf={0}'.format(db_conf), '-x',
'adapter={0}'.format(db_adapter), 'downgrade', 'base'
])
# drop db incl. alembic tables
drop_database(alchemy_url)
def db(flask_app):
database.app = flask_app
if functions.database_exists(database.engine.url):
functions.drop_database(database.engine.url)
functions.create_database(database.engine.url)
database.create_all()
yield database
database.session.remove()
database.drop_all()
def drop_db_if_exist(self):
if self.environment != 'test':
raise "Drop DB? Not sure in %s env. Do it manually!" % self.environment
try:
drop_database(self.db_url)
except sqlalchemy.exc.ProgrammingError as e:
db_not_exist = 'database "%s" does not exist' % self.db_name in str(e)
if not db_not_exist:
raise e
def create_tables(rm: ResourceManager):
engine = rm.db.get_bind()
db_url = engine.url
if database_exists(db_url):
drop_database(db_url)
create_database(db_url)
rm.db.execute('CREATE EXTENSION IF NOT EXISTS "uuid-ossp";')
rm.db.commit()
Base.metadata.create_all(bind=engine)
def app():
"""Flask application fixture."""
instance_path = tempfile.mkdtemp()
app = Flask('testapp', instance_path=instance_path)
app.config.update(
CELERY_ALWAYS_EAGER=True,
CELERY_CACHE_BACKEND='memory',
CELERY_EAGER_PROPAGATES_EXCEPTIONS=True,
CELERY_RESULT_BACKEND='cache',
JSONSCHEMAS_HOST='inveniosoftware.org',
TESTING=True,
SECRET_KEY='CHANGE_ME',
SQLALCHEMY_DATABASE_URI=os.environ.get('SQLALCHEMY_DATABASE_URI',
'sqlite://'),
SQLALCHEMY_TRACK_MODIFICATIONS=True,
SERVER_NAME='app',
OAISERVER_RECORD_INDEX='_all',
# Disable set signals because the celery tasks cannot be run
# synchronously
OAISERVER_REGISTER_SET_SIGNALS=False,
SEARCH_ELASTIC_KEYWORD_MAPPING={None: ['_all']},
)
if not hasattr(app, 'cli'):
from flask_cli import FlaskCLI
FlaskCLI(app)
InvenioDB(app)
FlaskCeleryExt(app)
InvenioJSONSchemas(app)
InvenioRecords(app)
InvenioPIDStore(app)
InvenioMARC21(app)
client = Elasticsearch(hosts=[os.environ.get('ES_HOST', 'localhost')])
search = InvenioSearch(app, client=client)
search.register_mappings('records', 'data')
InvenioIndexer(app)
InvenioOAIServer(app)
app.register_blueprint(blueprint)
with app.app_context():
if str(db.engine.url) != 'sqlite://' and \
not database_exists(str(db.engine.url)):
create_database(str(db.engine.url))
db.create_all()
list(search.create(ignore=[400]))
sleep(5)
with app.app_context():
yield app
with app.app_context():
db.session.close()
if str(db.engine.url) != 'sqlite://':
drop_database(str(db.engine.url))
list(search.delete(ignore=[404]))
shutil.rmtree(instance_path)
def database(app):
"""Ensure that the database schema is created."""
if not database_exists(str(db_.engine.url)):
create_database(str(db_.engine.url))
db_.drop_all()
db_.create_all()
yield db_
drop_database(str(db_.engine.url))
def test_migrations_upgrade():
with pytest.raises(OperationalError):
User.query.all()
# ensure that the database is created
create_database(db.engine.url)
alembic.upgrade()
assert len(User.query.all()) == 0
drop_database(db.engine.url)
def db(app, request):
"""Setup database."""
if not database_exists(str(db_.engine.url)):
create_database(str(db_.engine.url))
db_.create_all()
yield db_
drop_database(str(db_.engine.url))
# Delete sessions in kvsession store
if hasattr(app, 'kvsession_store') and \
isinstance(app.kvsession_store, RedisStore):
app.kvsession_store.redis.flushall()
def db(app):
"""Database fixture."""
if not database_exists(str(db_.engine.url)) and \
app.config['SQLALCHEMY_DATABASE_URI'] != 'sqlite://':
create_database(db_.engine.url)
db_.create_all()
yield db_
db_.session.remove()
db_.session.close()
if str(db_.engine.url) != 'sqlite://':
drop_database(str(db_.engine.url))
def init(user='******', password='', yes_i_know=False):
"""Initialize database and user."""
from invenio.ext.sqlalchemy.utils import initialize_database_user
from invenio.utils.text import wrap_text_in_a_box, wait_for_user
from sqlalchemy_utils.functions import database_exists, create_database, \
drop_database
from sqlalchemy.engine.url import URL
from sqlalchemy import create_engine
# Step 0: confirm deletion
wait_for_user(wrap_text_in_a_box(
"WARNING: You are going to destroy your database tables! Run first"
" `inveniomanage database drop`."
))
# Step 1: create URI to connect admin user
cfg = current_app.config
SQLALCHEMY_DATABASE_URI = URL(
cfg.get('CFG_DATABASE_TYPE', 'mysql'),
username=user,
password=password,
host=cfg.get('CFG_DATABASE_HOST'),
database=cfg.get('CFG_DATABASE_NAME'),
port=cfg.get('CFG_DATABASE_PORT'),
)
# Step 2: drop the database if already exists
if database_exists(SQLALCHEMY_DATABASE_URI):
drop_database(SQLALCHEMY_DATABASE_URI)
print('>>> Database has been dropped.')
# Step 3: create the database
create_database(SQLALCHEMY_DATABASE_URI, encoding='utf8')
print('>>> Database has been created.')
# Step 4: setup connection with special user
engine = create_engine(SQLALCHEMY_DATABASE_URI)
engine.connect()
# Step 5: grant privileges for the user
initialize_database_user(
engine=engine,
database_name=current_app.config['CFG_DATABASE_NAME'],
database_user=current_app.config['CFG_DATABASE_USER'],
database_pass=current_app.config['CFG_DATABASE_PASS'],
)
print('>>> Database user has been initialized.')
def validate_database_schema(app, alembic):
"""Check that the database schema is as expected after a full migration."""
# Check that the resulting state is in sync with sqlalchemy's MetaData
assert not alembic.compare_metadata()
# list all existing constraints
current_constraints = get_all_constraints(app)
current_constraint_names = [x[1] for x in current_constraints]
# Recreate the database with alembic only (without migrating)
db.session.remove()
drop_database(db.engine.url)
db.engine.dispose()
create_database(db.engine.url)
alembic.upgrade()
# Check that the constraints are the same. This is needed because
# alembic.compare_metadata does not check all constraints.
expected_constraints = get_all_constraints(app)
expected_constraint_names = [x[1] for x in expected_constraints]
assert set(current_constraint_names) == set(expected_constraint_names)
def test_init():
app = Flask('demo')
app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get(
'SQLALCHEMY_DATABASE_URI', 'sqlite:///test.db'
)
FlaskCLI(app)
InvenioDB(app, entrypoint_name=False)
class Demo(db.Model):
__tablename__ = 'demo'
pk = sa.Column(sa.Integer, primary_key=True)
class Demo2(db.Model):
__tablename__ = 'demo2'
pk = sa.Column(sa.Integer, primary_key=True)
with app.app_context():
create_database(db.engine.url)
db.create_all()
assert len(db.metadata.tables) == 2
db.drop_all()
drop_database(db.engine.url)
def update_database():
"""Insert all cards in database"""
new_version = fetch.mtgjson_version()
current_version = get_version()
if current_version is not None and new_version == current_version:
return
# TODO: create the new database in a different file and move to the
# current file when the prices are updated
if database_exists(config.DATABASE['db.url']):
drop_database(engine.url)
db_connection = engine.connect()
schema.create_database_schema()
cards = fetch.all_cards()
for name, card in cards.items():
insert_card(name, card)
from database import price_grabber
price_grabber.make_all_cards_list()
update_version(db_connection, new_version)
def test_init():
"""Test extension initialization."""
app = Flask('demo')
app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get(
'SQLALCHEMY_DATABASE_URI', 'sqlite:///test.db'
)
FlaskCLI(app)
InvenioDB(app, entry_point_group=False)
class Demo(db.Model):
__tablename__ = 'demo'
pk = sa.Column(sa.Integer, primary_key=True)
class Demo2(db.Model):
__tablename__ = 'demo2'
pk = sa.Column(sa.Integer, primary_key=True)
fk = sa.Column(sa.Integer, sa.ForeignKey(Demo.pk))
with app.app_context():
create_database(db.engine.url)
db.create_all()
assert len(db.metadata.tables) == 2
# Test foreign key constraint checking
d1 = Demo()
db.session.add(d1)
d2 = Demo2(fk=d1.pk)
db.session.add(d2)
db.session.commit()
# Fails fk check
d3 = Demo2(fk=10)
db.session.add(d3)
pytest.raises(IntegrityError, db.session.commit)
db.session.rollback()
db.drop_all()
drop_database(db.engine.url)
def app(request):
"""Flask application fixture."""
# Set temporary instance path for sqlite
instance_path = tempfile.mkdtemp()
app = Flask('testapp', instance_path=instance_path)
app.config.update(
SQLALCHEMY_DATABASE_URI=os.environ.get(
'SQLALCHEMY_DATABASE_URI', 'sqlite:///test.db'),
CELERY_ALWAYS_EAGER=True,
CELERY_RESULT_BACKEND="cache",
CELERY_CACHE_BACKEND="memory",
CELERY_EAGER_PROPAGATES_EXCEPTIONS=True,
OPENAIRE_OAI_LOCAL_SOURCE='invenio_openaire/data/oaire_local.sqlite',
SEARCH_AUTOINDEX=[],
TESTING=True,
)
FlaskCLI(app)
InvenioDB(app)
InvenioRecords(app)
InvenioCelery(app)
InvenioPIDStore(app)
InvenioOpenAIRE(app)
InvenioSearch(app)
InvenioJSONSchemas(app)
with app.app_context():
if not database_exists(str(db.engine.url)):
create_database(str(db.engine.url))
db.drop_all()
db.create_all()
yield app
drop_database(str(db.engine.url))
shutil.rmtree(instance_path)
def teardown():
drop_database(str(db.engine.url))
shutil.rmtree(instance_path)
def base_app(request):
"""Flask application fixture."""
instance_path = tempfile.mkdtemp()
def init_app(app_):
app_.config.update(
CELERY_ALWAYS_EAGER=True,
CELERY_CACHE_BACKEND='memory',
CELERY_EAGER_PROPAGATES_EXCEPTIONS=True,
CELERY_RESULT_BACKEND='cache',
JSONSCHEMAS_URL_SCHEME='http',
SECRET_KEY='CHANGE_ME',
SECURITY_PASSWORD_SALT='CHANGE_ME_ALSO',
SQLALCHEMY_DATABASE_URI=os.environ.get(
'SQLALCHEMY_DATABASE_URI', 'sqlite:///test.db'),
SQLALCHEMY_TRACK_MODIFICATIONS=True,
SQLALCHEMY_ECHO=False,
TESTING=True,
WTF_CSRF_ENABLED=False,
DEPOSIT_SEARCH_API='/api/search',
SECURITY_PASSWORD_HASH='plaintext',
SECURITY_PASSWORD_SCHEMES=['plaintext'],
SECURITY_DEPRECATED_PASSWORD_SCHEMES=[],
OAUTHLIB_INSECURE_TRANSPORT=True,
OAUTH2_CACHE_TYPE='simple',
)
app_.url_map.converters['pid'] = PIDConverter
Babel(app_)
FlaskCeleryExt(app_)
Breadcrumbs(app_)
OAuth2Provider(app_)
InvenioDB(app_)
InvenioAccounts(app_)
InvenioAccess(app_)
InvenioIndexer(app_)
InvenioJSONSchemas(app_)
InvenioOAuth2Server(app_)
InvenioFilesREST(app_)
InvenioPIDStore(app_)
InvenioRecords(app_)
InvenioSearch(app_)
api_app = Flask('testapiapp', instance_path=instance_path)
init_app(api_app)
InvenioREST(api_app)
InvenioOAuth2ServerREST(api_app)
InvenioRecordsREST(api_app)
InvenioDepositREST(api_app)
app = Flask('testapp', instance_path=instance_path)
init_app(app)
app.register_blueprint(accounts_blueprint)
app.register_blueprint(oauth2server_settings_blueprint)
InvenioAssets(app)
InvenioSearchUI(app)
InvenioRecordsUI(app)
InvenioDeposit(app)
app.wsgi_app = DispatcherMiddleware(app.wsgi_app, {
'/api': api_app.wsgi_app
})
with app.app_context():
if str(db.engine.url) != 'sqlite://' and \
not database_exists(str(db.engine.url)):
create_database(str(db.engine.url))
db.create_all()
yield app
with app.app_context():
if str(db.engine.url) != 'sqlite://':
drop_database(str(db.engine.url))
shutil.rmtree(instance_path)
def teardown():
with app.app_context():
drop_database(str(db.engine.url))
shutil.rmtree(instance_path)
def teardown():
with app.app_context():
drop_database(str(db.engine.url))
def dropAll(url):
drop_database(url)
def finalize():
with app.app_context():
db.drop_all()
if app.config['SQLALCHEMY_DATABASE_URI'] != 'sqlite://':
drop_database(db.engine.url)
shutil.rmtree(instance_path)
def destroy():
"""Drop database."""
click.secho('Destroying database {0}'.format(_db.engine.url),
fg='red', bold=True)
drop_database(_db.engine.url)
