def fulltext(args):
# we must initialize all node types to import fulltexts
init.full_init()
nid_mod_or_all = args.nid_mod_or_all.lower()
remove_versioning()
if nid_mod_or_all == "all":
import_count = utils.search.import_fulltexts(args.overwrite)
logg.info("loaded fulltexts for %s nodes", import_count)
elif nid_mod_or_all.startswith("mod"):
mod_n, mod_i = [int(x) for x in nid_mod_or_all.split(" ")[1:]]
import_count = utils.search.import_fulltexts(args.overwrite, mod_n, mod_i)
logg.info("loaded fulltexts for %s nodes with id mod %s == %s", import_count, mod_n, mod_i)
else:
nid = int(nid_mod_or_all)
node = q(Node).get(nid)
if node is None:
logg.warn("node # %s not found!", nid)
return
imported = utils.search.import_node_fulltext(node, args.overwrite)
if imported:
logg.info("loaded fulltext for node # %s", nid)
else:
logg.info("nothing imported for node # %s", nid)
def teardown_method(self, method):
remove_versioning()
self.db.drop_all()
self.context.pop()
self.context = None
self.client = None
self.app = None
def fulltext(args):
# we must initialize all node types to import fulltexts
init.full_init()
nid_mod_or_all = args.nid_mod_or_all.lower()
remove_versioning()
if nid_mod_or_all == "all":
import_count = utils.search.import_fulltexts(args.overwrite)
logg.info("loaded fulltexts for %s nodes", import_count)
elif nid_mod_or_all.startswith("mod"):
mod_n, mod_i = [int(x) for x in nid_mod_or_all.split(" ")[1:]]
import_count = utils.search.import_fulltexts(args.overwrite, mod_n,
mod_i)
logg.info("loaded fulltexts for %s nodes with id mod %s == %s",
import_count, mod_n, mod_i)
else:
nid = int(nid_mod_or_all)
node = q(Node).get(nid)
if node is None:
logg.warn("node # %s not found!", nid)
return
imported = utils.search.import_node_fulltext(node, args.overwrite)
if imported:
logg.info("loaded fulltext for node # %s", nid)
else:
logg.info("nothing imported for node # %s", nid)
def test_disabled_versioning_with_custom_table(db, app, versioning, tables):
"""Test SQLAlchemy-Continuum table loading."""
app.config['DB_VERSIONING'] = versioning
class EarlyClass(db.Model):
__versioned__ = {}
pk = db.Column(db.Integer, primary_key=True)
idb = InvenioDB(app,
entry_point_group=None,
db=db,
versioning_manager=VersioningManager())
with app.app_context():
db.drop_all()
db.create_all()
before = len(db.metadata.tables)
ec = EarlyClass()
ec.pk = 1
db.session.add(ec)
db.session.commit()
assert tables == len(db.metadata.tables)
db.drop_all()
if versioning:
remove_versioning(manager=idb.versioning_manager)
def test_disabled_versioning_with_custom_table(db, app, versioning, tables):
"""Test SQLAlchemy-Continuum table loading."""
app.config['DB_VERSIONING'] = versioning
class EarlyClass(db.Model):
__versioned__ = {}
pk = db.Column(db.Integer, primary_key=True)
idb = InvenioDB(app, entry_point_group=None, db=db)
with app.app_context():
db.drop_all()
db.create_all()
before = len(db.metadata.tables)
ec = EarlyClass()
ec.pk = 1
db.session.add(ec)
db.session.commit()
assert tables == len(db.metadata.tables)
db.drop_all()
if versioning:
from sqlalchemy_continuum import remove_versioning
remove_versioning(manager=idb.versioning_manager)
def test_versioning(db, app):
"""Test SQLAlchemy-Continuum enabled versioning."""
from sqlalchemy_continuum import VersioningManager
app.config['DB_VERSIONING'] = True
idb = InvenioDB(app,
entry_point_group='invenio_db.models_b',
db=db,
versioning_manager=VersioningManager())
with app.app_context():
assert 4 == len(db.metadata.tables)
db.create_all()
from demo.versioned_b import UnversionedArticle, VersionedArticle
original_name = 'original_name'
versioned = VersionedArticle()
unversioned = UnversionedArticle()
versioned.name = original_name
unversioned.name = original_name
db.session.add(versioned)
db.session.add(unversioned)
db.session.commit()
assert unversioned.name == versioned.name
modified_name = 'modified_name'
versioned.name = modified_name
unversioned.name = modified_name
db.session.commit()
assert unversioned.name == modified_name
assert versioned.name == modified_name
assert versioned.versions[0].name == original_name
assert versioned.versions[1].name == versioned.name
versioned.versions[0].revert()
db.session.commit()
assert unversioned.name == modified_name
assert versioned.name == original_name
versioned.versions[1].revert()
db.session.commit()
assert unversioned.name == versioned.name
with app.app_context():
db.drop_all()
from sqlalchemy_continuum import remove_versioning
remove_versioning(manager=idb.versioning_manager)
def test_versioning(db, app):
"""Test SQLAlchemy-Continuum enabled versioning."""
from sqlalchemy_continuum import VersioningManager
app.config["DB_VERSIONING"] = True
idb = InvenioDB(app, entry_point_group="invenio_db.models_b", db=db, versioning_manager=VersioningManager())
with app.app_context():
assert 4 == len(db.metadata.tables)
db.create_all()
from demo.versioned_b import UnversionedArticle, VersionedArticle
original_name = "original_name"
versioned = VersionedArticle()
unversioned = UnversionedArticle()
versioned.name = original_name
unversioned.name = original_name
db.session.add(versioned)
db.session.add(unversioned)
db.session.commit()
assert unversioned.name == versioned.name
modified_name = "modified_name"
versioned.name = modified_name
unversioned.name = modified_name
db.session.commit()
assert unversioned.name == modified_name
assert versioned.name == modified_name
assert versioned.versions[0].name == original_name
assert versioned.versions[1].name == versioned.name
versioned.versions[0].revert()
db.session.commit()
assert unversioned.name == modified_name
assert versioned.name == original_name
versioned.versions[1].revert()
db.session.commit()
assert unversioned.name == versioned.name
with app.app_context():
db.drop_all()
from sqlalchemy_continuum import remove_versioning
remove_versioning(manager=idb.versioning_manager)
def teardown_method(self, method):
remove_versioning()
self.db.session.remove()
self.db.drop_all()
self.db.session.close_all()
self.db.engine.dispose()
self.context.pop()
self.context = None
self.client = None
self.app = None
def teardown_method(self, method):
remove_versioning()
QueryPool.queries = []
versioning_manager.reset()
self.session.close_all()
self.session.expunge_all()
self.drop_tables()
self.engine.dispose()
self.connection.close()
def teardown_method(self, method):
remove_versioning()
self.db.session.remove()
self.db.drop_all()
self.db.session.close_all()
self.db.engine.dispose()
self.context.pop()
self.context = None
self.client = None
self.app = None
def teardown_method(self, method):
remove_versioning()
QueryPool.queries = []
versioning_manager.reset()
self.session.close_all()
self.session.expunge_all()
self.drop_tables()
self.engine.dispose()
self.connection.close()
def versions(s):
# we really must commit before running version migration or nodes created earlier will be lost
s.commit()
from migration import version_migration
# all node classes must be defined for versioning, stub them if some plugins are missing, for example
init.check_undefined_nodeclasses(stub_undefined_nodetypes=True)
version_migration.reset_version_data()
version_migration.fix_versioning_attributes()
version_migration.insert_migrated_version_nodes(version_migration.all_version_nodes())
version_migration.finish()
remove_versioning()
def versions(s):
# we really must commit before running version migration or nodes created earlier will be lost
s.commit()
from migration import version_migration
# all node classes must be defined for versioning, stub them if some plugins are missing, for example
init.check_undefined_nodeclasses(stub_undefined_nodetypes=True)
version_migration.reset_version_data()
version_migration.fix_versioning_attributes()
version_migration.insert_migrated_version_nodes(
version_migration.all_version_nodes())
version_migration.finish()
remove_versioning()
def test_versioning_without_versioned_tables(db, app):
"""Test SQLAlchemy-Continuum without versioned tables."""
app.config['DB_VERSIONING'] = True
idb = InvenioDB(app,
db=db,
entry_point_group=None,
versioning_manager=VersioningManager())
with app.app_context():
assert 'transaction' in db.metadata.tables
remove_versioning(manager=idb.versioning_manager)
def test_db_create_alembic_upgrade(app, db):
"""Test that 'db create/drop' and 'alembic create' are compatible.
It also checks that "alembic_version" table is processed properly
as it is normally created by alembic and not by sqlalchemy.
"""
app.config['DB_VERSIONING'] = True
ext = InvenioDB(app,
entry_point_group=None,
db=db,
versioning_manager=VersioningManager())
with app.app_context():
try:
if db.engine.name == 'sqlite':
raise pytest.skip('Upgrades are not supported on SQLite.')
db.drop_all()
runner = CliRunner()
script_info = ScriptInfo(create_app=lambda info: app)
# Check that 'db create' creates the same schema as
# 'alembic upgrade'.
result = runner.invoke(db_cmd, ['create', '-v'], obj=script_info)
assert result.exit_code == 0
assert db.engine.has_table('transaction')
assert ext.alembic.migration_context._has_version_table()
# Note that compare_metadata does not detect additional sequences
# and constraints.
assert not ext.alembic.compare_metadata()
ext.alembic.upgrade()
assert db.engine.has_table('transaction')
ext.alembic.downgrade(target='96e796392533')
assert db.engine.table_names() == ['alembic_version']
# Check that 'db drop' removes all tables, including
# 'alembic_version'.
ext.alembic.upgrade()
result = runner.invoke(db_cmd, ['drop', '-v', '--yes-i-know'],
obj=script_info)
assert result.exit_code == 0
assert len(db.engine.table_names()) == 0
ext.alembic.upgrade()
db.drop_all()
drop_alembic_version_table()
assert len(db.engine.table_names()) == 0
finally:
drop_database(str(db.engine.url))
remove_versioning(manager=ext.versioning_manager)
create_database(str(db.engine.url))
def test_versioning_without_versioned_tables(db, app):
"""Test SQLAlchemy-Continuum without versioned tables."""
from sqlalchemy_continuum import VersioningManager
app.config['DB_VERSIONING'] = True
idb = InvenioDB(app, db=db, entry_point_group=None,
versioning_manager=VersioningManager())
with app.app_context():
assert 'transaction' in db.metadata.tables
from sqlalchemy_continuum import remove_versioning
remove_versioning(manager=idb.versioning_manager)
def test_versioning_model_classname(db, app):
"""Test the versioning model utilities."""
class FooClass(db.Model):
__versioned__ = {}
pk = db.Column(db.Integer, primary_key=True)
app.config["DB_VERSIONING"] = True
idb = InvenioDB(app)
manager = idb.versioning_manager
manager.options["use_module_name"] = True
assert versioning_model_classname(manager, FooClass) == "Test_UtilsFooClassVersion"
manager.options["use_module_name"] = False
assert versioning_model_classname(manager, FooClass) == "FooClassVersion"
assert versioning_models_registered(manager, db.Model)
remove_versioning(manager=manager)
def test_db_create_alembic_upgrade(app, db):
"""Test that 'db create/drop' and 'alembic create' are compatible.
It also checks that "alembic_version" table is processed properly
as it is normally created by alembic and not by sqlalchemy.
"""
app.config["DB_VERSIONING"] = True
ext = InvenioDB(
app, entry_point_group=None, db=db, versioning_manager=VersioningManager()
)
with app.app_context():
try:
if db.engine.name == "sqlite":
raise pytest.skip("Upgrades are not supported on SQLite.")
db.drop_all()
runner = app.test_cli_runner()
# Check that 'db create' creates the same schema as
# 'alembic upgrade'.
result = runner.invoke(db_cmd, ["create", "-v"])
assert result.exit_code == 0
assert has_table(db.engine, "transaction")
assert ext.alembic.migration_context._has_version_table()
# Note that compare_metadata does not detect additional sequences
# and constraints.
assert not ext.alembic.compare_metadata()
ext.alembic.upgrade()
assert has_table(db.engine, "transaction")
ext.alembic.downgrade(target="96e796392533")
assert inspect(db.engine).get_table_names() == ["alembic_version"]
# Check that 'db drop' removes all tables, including
# 'alembic_version'.
ext.alembic.upgrade()
result = runner.invoke(db_cmd, ["drop", "-v", "--yes-i-know"])
assert result.exit_code == 0
assert len(inspect(db.engine).get_table_names()) == 0
ext.alembic.upgrade()
db.drop_all()
drop_alembic_version_table()
assert len(inspect(db.engine).get_table_names()) == 0
finally:
drop_database(str(db.engine.url))
remove_versioning(manager=ext.versioning_manager)
create_database(str(db.engine.url))
def teardown_method(self, method):
self.session.rollback()
uow_leaks = versioning_manager.units_of_work
session_map_leaks = versioning_manager.session_connection_map
remove_versioning()
QueryPool.queries = []
versioning_manager.reset()
self.session.close_all()
self.session.expunge_all()
self.drop_tables()
self.engine.dispose()
self.connection.close()
assert not uow_leaks
assert not session_map_leaks
def test_create_file_version(monkeypatch, first_version_node, middle_version_node, current_version_node):
from sqlalchemy_continuum import version_class
from core import File
NodeToFileVersion = version_class(NodeToFile)
FileVersion = version_class(File)
def _version_class(clz):
if clz == NodeToFile:
return NodeToFileVersion
elif clz == File:
return FileVersion
monkeypatch.setattr(version_migration, "version_class", _version_class)
Transaction = versioning_manager.transaction_cls
remove_versioning()
first = first_version_node
middle = middle_version_node
current = current_version_node
first_files = [FileFactory()]
first.files.extend(first_files)
middle_files = [FileFactory()]
middle.files.extend(middle_files)
current_files = [FileFactory()]
current.files.extend(current_files)
transaction = Transaction()
version_migration.create_file_versions(first, middle, transaction)
transaction2 = Transaction()
version_migration.create_file_versions(middle, current, transaction2)
def test_naming_convention(db, app):
"""Test naming convention."""
from sqlalchemy_continuum import remove_versioning
ext = InvenioDB(app, entry_point_group=False, db=db)
cfg = dict(
DB_VERSIONING=True,
DB_VERSIONING_USER_MODEL=None,
SQLALCHEMY_DATABASE_URI=app.config['SQLALCHEMY_DATABASE_URI'],
)
with app.app_context():
if db.engine.name == 'sqlite':
raise pytest.skip('Upgrades are not supported on SQLite.')
def model_factory(base):
"""Create test models."""
class Master(base):
__tablename__ = 'master'
pk = sa.Column(sa.Integer, primary_key=True)
name = sa.Column(sa.String(100), unique=True)
city = sa.Column(sa.String(100), index=True)
active = sa.Column(sa.Boolean(name='active'), server_default='1')
class Slave(base):
__tablename__ = 'slave'
pk = sa.Column(sa.Integer, primary_key=True)
fk = sa.Column(sa.Integer, sa.ForeignKey(Master.pk))
code = sa.Column(sa.Integer, index=True, unique=True)
source = sa.Column(sa.String(100))
__table_args__ = (
sa.Index(None, source),
# do not add anything after
getattr(base, '__table_args__', {}))
return Master, Slave
source_db = shared.SQLAlchemy(metadata=shared.MetaData(
naming_convention={
'ix': 'source_ix_%(table_name)s_%(column_0_label)s',
'uq': 'source_uq_%(table_name)s_%(column_0_name)s',
'ck': 'source_ck_%(table_name)s_%(constraint_name)s',
'fk': 'source_fk_%(table_name)s_%(column_0_name)s_'
'%(referred_table_name)s',
'pk': 'source_pk_%(table_name)s',
}), )
source_app = Flask('source_app')
source_app.config.update(**cfg)
source_models = model_factory(source_db.Model)
source_ext = InvenioDB(
source_app,
entry_point_group=False,
db=source_db,
versioning_manager=VersioningManager(),
)
with source_app.app_context():
source_db.metadata.bind = source_db.engine
source_db.create_all()
source_ext.alembic.stamp('dbdbc1b19cf2')
assert not source_ext.alembic.compare_metadata()
source_constraints = set([
cns for model in source_models
for cns in list(model.__table__.constraints) +
list(model.__table__.indexes)
])
remove_versioning(manager=source_ext.versioning_manager)
target_db = shared.SQLAlchemy(metadata=shared.MetaData(
naming_convention=shared.NAMING_CONVENTION))
target_app = Flask('target_app')
target_app.config.update(**cfg)
target_models = model_factory(target_db.Model)
target_ext = InvenioDB(
target_app,
entry_point_group=False,
db=target_db,
versioning_manager=VersioningManager(),
)
with target_app.app_context():
target_db.metadata.bind = target_db.engine
assert target_ext.alembic.compare_metadata()
target_ext.alembic.upgrade('35c1075e6360')
assert not target_ext.alembic.compare_metadata()
target_db.drop_all()
target_constraints = set([
cns.name for model in source_models
for cns in list(model.__table__.constraints) +
list(model.__table__.indexes)
])
remove_versioning(manager=target_ext.versioning_manager)
assert source_constraints ^ target_constraints
def test_versioning(
native_versioning,
versioning_strategy,
property_mod_tracking
):
transaction_column_name = 'transaction_id'
end_transaction_column_name = 'end_transaction_id'
plugins = [TransactionChangesPlugin(), TransactionMetaPlugin()]
if property_mod_tracking:
plugins.append(PropertyModTrackerPlugin())
transaction_cls = TransactionFactory()
user_cls = None
Model = declarative_base()
options = {
'create_models': True,
'native_versioning': native_versioning,
'base_classes': (Model, ),
'strategy': versioning_strategy,
'transaction_column_name': transaction_column_name,
'end_transaction_column_name': end_transaction_column_name,
}
make_versioned(options=options)
dns = 'postgres://postgres@localhost/sqlalchemy_continuum_test'
versioning_manager.plugins = plugins
versioning_manager.transaction_cls = transaction_cls
versioning_manager.user_cls = user_cls
engine = create_engine(dns)
# engine.echo = True
connection = engine.connect()
class Article(Model):
__tablename__ = 'article'
__versioned__ = copy(options)
id = sa.Column(sa.Integer, autoincrement=True, primary_key=True)
name = sa.Column(sa.Unicode(255), nullable=False)
content = sa.Column(sa.UnicodeText)
description = sa.Column(sa.UnicodeText)
class Tag(Model):
__tablename__ = 'tag'
__versioned__ = copy(options)
id = sa.Column(sa.Integer, autoincrement=True, primary_key=True)
article_id = sa.Column(sa.Integer, sa.ForeignKey(Article.id))
article = sa.orm.relationship(Article, backref='tags')
sa.orm.configure_mappers()
Model.metadata.create_all(connection)
Session = sessionmaker(bind=connection)
session = Session(autoflush=False)
session.execute('CREATE EXTENSION IF NOT EXISTS hstore')
Model.metadata.create_all(connection)
start = time()
for i in range(20):
for i in range(20):
session.add(Article(name=u'Article', tags=[Tag(), Tag()]))
session.commit()
print 'Testing with:'
print ' native_versioning=%r' % native_versioning
print ' versioning_strategy=%r' % versioning_strategy
print ' property_mod_tracking=%r' % property_mod_tracking
print colored('%r seconds' % (time() - start), 'red')
Model.metadata.drop_all(connection)
remove_versioning()
versioning_manager.reset()
session.close_all()
session.expunge_all()
Model.metadata.drop_all(connection)
engine.dispose()
connection.close()
def test_versioning(native_versioning, versioning_strategy,
property_mod_tracking):
transaction_column_name = 'transaction_id'
end_transaction_column_name = 'end_transaction_id'
plugins = [TransactionChangesPlugin(), TransactionMetaPlugin()]
if property_mod_tracking:
plugins.append(PropertyModTrackerPlugin())
transaction_cls = TransactionFactory()
user_cls = None
Model = declarative_base()
options = {
'create_models': True,
'native_versioning': native_versioning,
'base_classes': (Model, ),
'strategy': versioning_strategy,
'transaction_column_name': transaction_column_name,
'end_transaction_column_name': end_transaction_column_name,
}
make_versioned(options=options)
dns = 'postgres://postgres@localhost/sqlalchemy_continuum_test'
versioning_manager.plugins = plugins
versioning_manager.transaction_cls = transaction_cls
versioning_manager.user_cls = user_cls
engine = create_engine(dns)
# engine.echo = True
class Article(Model):
__tablename__ = 'article'
__versioned__ = copy(options)
id = sa.Column(sa.Integer, autoincrement=True, primary_key=True)
name = sa.Column(sa.Unicode(255), nullable=False)
content = sa.Column(sa.UnicodeText)
description = sa.Column(sa.UnicodeText)
class Tag(Model):
__tablename__ = 'tag'
__versioned__ = copy(options)
id = sa.Column(sa.Integer, autoincrement=True, primary_key=True)
article_id = sa.Column(sa.Integer, sa.ForeignKey(Article.id))
article = sa.orm.relationship(Article, backref='tags')
sa.orm.configure_mappers()
connection = engine.connect()
Model.metadata.create_all(connection)
Session = sessionmaker(bind=connection)
session = Session(autoflush=False)
session.execute('CREATE EXTENSION IF NOT EXISTS hstore')
Model.metadata.create_all(connection)
start = time()
for i in range(20):
for i in range(20):
session.add(Article(name=u'Article', tags=[Tag(), Tag()]))
session.commit()
print 'Testing with:'
print ' native_versioning=%r' % native_versioning
print ' versioning_strategy=%r' % versioning_strategy
print ' property_mod_tracking=%r' % property_mod_tracking
print colored('%r seconds' % (time() - start), 'red')
Model.metadata.drop_all(connection)
remove_versioning()
versioning_manager.reset()
session.close_all()
session.expunge_all()
Model.metadata.drop_all(connection)
engine.dispose()
connection.close()