def fulltext(args): # we must initialize all node types to import fulltexts init.full_init() nid_mod_or_all = args.nid_mod_or_all.lower() remove_versioning() if nid_mod_or_all == "all": import_count = utils.search.import_fulltexts(args.overwrite) logg.info("loaded fulltexts for %s nodes", import_count) elif nid_mod_or_all.startswith("mod"): mod_n, mod_i = [int(x) for x in nid_mod_or_all.split(" ")[1:]] import_count = utils.search.import_fulltexts(args.overwrite, mod_n, mod_i) logg.info("loaded fulltexts for %s nodes with id mod %s == %s", import_count, mod_n, mod_i) else: nid = int(nid_mod_or_all) node = q(Node).get(nid) if node is None: logg.warn("node # %s not found!", nid) return imported = utils.search.import_node_fulltext(node, args.overwrite) if imported: logg.info("loaded fulltext for node # %s", nid) else: logg.info("nothing imported for node # %s", nid)
def teardown_method(self, method): remove_versioning() self.db.drop_all() self.context.pop() self.context = None self.client = None self.app = None
def test_disabled_versioning_with_custom_table(db, app, versioning, tables): """Test SQLAlchemy-Continuum table loading.""" app.config['DB_VERSIONING'] = versioning class EarlyClass(db.Model): __versioned__ = {} pk = db.Column(db.Integer, primary_key=True) idb = InvenioDB(app, entry_point_group=None, db=db, versioning_manager=VersioningManager()) with app.app_context(): db.drop_all() db.create_all() before = len(db.metadata.tables) ec = EarlyClass() ec.pk = 1 db.session.add(ec) db.session.commit() assert tables == len(db.metadata.tables) db.drop_all() if versioning: remove_versioning(manager=idb.versioning_manager)
def test_disabled_versioning_with_custom_table(db, app, versioning, tables): """Test SQLAlchemy-Continuum table loading.""" app.config['DB_VERSIONING'] = versioning class EarlyClass(db.Model): __versioned__ = {} pk = db.Column(db.Integer, primary_key=True) idb = InvenioDB(app, entry_point_group=None, db=db) with app.app_context(): db.drop_all() db.create_all() before = len(db.metadata.tables) ec = EarlyClass() ec.pk = 1 db.session.add(ec) db.session.commit() assert tables == len(db.metadata.tables) db.drop_all() if versioning: from sqlalchemy_continuum import remove_versioning remove_versioning(manager=idb.versioning_manager)
def test_versioning(db, app): """Test SQLAlchemy-Continuum enabled versioning.""" from sqlalchemy_continuum import VersioningManager app.config['DB_VERSIONING'] = True idb = InvenioDB(app, entry_point_group='invenio_db.models_b', db=db, versioning_manager=VersioningManager()) with app.app_context(): assert 4 == len(db.metadata.tables) db.create_all() from demo.versioned_b import UnversionedArticle, VersionedArticle original_name = 'original_name' versioned = VersionedArticle() unversioned = UnversionedArticle() versioned.name = original_name unversioned.name = original_name db.session.add(versioned) db.session.add(unversioned) db.session.commit() assert unversioned.name == versioned.name modified_name = 'modified_name' versioned.name = modified_name unversioned.name = modified_name db.session.commit() assert unversioned.name == modified_name assert versioned.name == modified_name assert versioned.versions[0].name == original_name assert versioned.versions[1].name == versioned.name versioned.versions[0].revert() db.session.commit() assert unversioned.name == modified_name assert versioned.name == original_name versioned.versions[1].revert() db.session.commit() assert unversioned.name == versioned.name with app.app_context(): db.drop_all() from sqlalchemy_continuum import remove_versioning remove_versioning(manager=idb.versioning_manager)
def test_versioning(db, app): """Test SQLAlchemy-Continuum enabled versioning.""" from sqlalchemy_continuum import VersioningManager app.config["DB_VERSIONING"] = True idb = InvenioDB(app, entry_point_group="invenio_db.models_b", db=db, versioning_manager=VersioningManager()) with app.app_context(): assert 4 == len(db.metadata.tables) db.create_all() from demo.versioned_b import UnversionedArticle, VersionedArticle original_name = "original_name" versioned = VersionedArticle() unversioned = UnversionedArticle() versioned.name = original_name unversioned.name = original_name db.session.add(versioned) db.session.add(unversioned) db.session.commit() assert unversioned.name == versioned.name modified_name = "modified_name" versioned.name = modified_name unversioned.name = modified_name db.session.commit() assert unversioned.name == modified_name assert versioned.name == modified_name assert versioned.versions[0].name == original_name assert versioned.versions[1].name == versioned.name versioned.versions[0].revert() db.session.commit() assert unversioned.name == modified_name assert versioned.name == original_name versioned.versions[1].revert() db.session.commit() assert unversioned.name == versioned.name with app.app_context(): db.drop_all() from sqlalchemy_continuum import remove_versioning remove_versioning(manager=idb.versioning_manager)
def teardown_method(self, method): remove_versioning() self.db.session.remove() self.db.drop_all() self.db.session.close_all() self.db.engine.dispose() self.context.pop() self.context = None self.client = None self.app = None
def teardown_method(self, method): remove_versioning() QueryPool.queries = [] versioning_manager.reset() self.session.close_all() self.session.expunge_all() self.drop_tables() self.engine.dispose() self.connection.close()
def versions(s): # we really must commit before running version migration or nodes created earlier will be lost s.commit() from migration import version_migration # all node classes must be defined for versioning, stub them if some plugins are missing, for example init.check_undefined_nodeclasses(stub_undefined_nodetypes=True) version_migration.reset_version_data() version_migration.fix_versioning_attributes() version_migration.insert_migrated_version_nodes(version_migration.all_version_nodes()) version_migration.finish() remove_versioning()
def versions(s): # we really must commit before running version migration or nodes created earlier will be lost s.commit() from migration import version_migration # all node classes must be defined for versioning, stub them if some plugins are missing, for example init.check_undefined_nodeclasses(stub_undefined_nodetypes=True) version_migration.reset_version_data() version_migration.fix_versioning_attributes() version_migration.insert_migrated_version_nodes( version_migration.all_version_nodes()) version_migration.finish() remove_versioning()
def test_versioning_without_versioned_tables(db, app): """Test SQLAlchemy-Continuum without versioned tables.""" app.config['DB_VERSIONING'] = True idb = InvenioDB(app, db=db, entry_point_group=None, versioning_manager=VersioningManager()) with app.app_context(): assert 'transaction' in db.metadata.tables remove_versioning(manager=idb.versioning_manager)
def test_db_create_alembic_upgrade(app, db): """Test that 'db create/drop' and 'alembic create' are compatible. It also checks that "alembic_version" table is processed properly as it is normally created by alembic and not by sqlalchemy. """ app.config['DB_VERSIONING'] = True ext = InvenioDB(app, entry_point_group=None, db=db, versioning_manager=VersioningManager()) with app.app_context(): try: if db.engine.name == 'sqlite': raise pytest.skip('Upgrades are not supported on SQLite.') db.drop_all() runner = CliRunner() script_info = ScriptInfo(create_app=lambda info: app) # Check that 'db create' creates the same schema as # 'alembic upgrade'. result = runner.invoke(db_cmd, ['create', '-v'], obj=script_info) assert result.exit_code == 0 assert db.engine.has_table('transaction') assert ext.alembic.migration_context._has_version_table() # Note that compare_metadata does not detect additional sequences # and constraints. assert not ext.alembic.compare_metadata() ext.alembic.upgrade() assert db.engine.has_table('transaction') ext.alembic.downgrade(target='96e796392533') assert db.engine.table_names() == ['alembic_version'] # Check that 'db drop' removes all tables, including # 'alembic_version'. ext.alembic.upgrade() result = runner.invoke(db_cmd, ['drop', '-v', '--yes-i-know'], obj=script_info) assert result.exit_code == 0 assert len(db.engine.table_names()) == 0 ext.alembic.upgrade() db.drop_all() drop_alembic_version_table() assert len(db.engine.table_names()) == 0 finally: drop_database(str(db.engine.url)) remove_versioning(manager=ext.versioning_manager) create_database(str(db.engine.url))
def test_versioning_without_versioned_tables(db, app): """Test SQLAlchemy-Continuum without versioned tables.""" from sqlalchemy_continuum import VersioningManager app.config['DB_VERSIONING'] = True idb = InvenioDB(app, db=db, entry_point_group=None, versioning_manager=VersioningManager()) with app.app_context(): assert 'transaction' in db.metadata.tables from sqlalchemy_continuum import remove_versioning remove_versioning(manager=idb.versioning_manager)
def test_versioning_model_classname(db, app): """Test the versioning model utilities.""" class FooClass(db.Model): __versioned__ = {} pk = db.Column(db.Integer, primary_key=True) app.config["DB_VERSIONING"] = True idb = InvenioDB(app) manager = idb.versioning_manager manager.options["use_module_name"] = True assert versioning_model_classname(manager, FooClass) == "Test_UtilsFooClassVersion" manager.options["use_module_name"] = False assert versioning_model_classname(manager, FooClass) == "FooClassVersion" assert versioning_models_registered(manager, db.Model) remove_versioning(manager=manager)
def test_db_create_alembic_upgrade(app, db): """Test that 'db create/drop' and 'alembic create' are compatible. It also checks that "alembic_version" table is processed properly as it is normally created by alembic and not by sqlalchemy. """ app.config["DB_VERSIONING"] = True ext = InvenioDB( app, entry_point_group=None, db=db, versioning_manager=VersioningManager() ) with app.app_context(): try: if db.engine.name == "sqlite": raise pytest.skip("Upgrades are not supported on SQLite.") db.drop_all() runner = app.test_cli_runner() # Check that 'db create' creates the same schema as # 'alembic upgrade'. result = runner.invoke(db_cmd, ["create", "-v"]) assert result.exit_code == 0 assert has_table(db.engine, "transaction") assert ext.alembic.migration_context._has_version_table() # Note that compare_metadata does not detect additional sequences # and constraints. assert not ext.alembic.compare_metadata() ext.alembic.upgrade() assert has_table(db.engine, "transaction") ext.alembic.downgrade(target="96e796392533") assert inspect(db.engine).get_table_names() == ["alembic_version"] # Check that 'db drop' removes all tables, including # 'alembic_version'. ext.alembic.upgrade() result = runner.invoke(db_cmd, ["drop", "-v", "--yes-i-know"]) assert result.exit_code == 0 assert len(inspect(db.engine).get_table_names()) == 0 ext.alembic.upgrade() db.drop_all() drop_alembic_version_table() assert len(inspect(db.engine).get_table_names()) == 0 finally: drop_database(str(db.engine.url)) remove_versioning(manager=ext.versioning_manager) create_database(str(db.engine.url))
def teardown_method(self, method): self.session.rollback() uow_leaks = versioning_manager.units_of_work session_map_leaks = versioning_manager.session_connection_map remove_versioning() QueryPool.queries = [] versioning_manager.reset() self.session.close_all() self.session.expunge_all() self.drop_tables() self.engine.dispose() self.connection.close() assert not uow_leaks assert not session_map_leaks
def test_create_file_version(monkeypatch, first_version_node, middle_version_node, current_version_node): from sqlalchemy_continuum import version_class from core import File NodeToFileVersion = version_class(NodeToFile) FileVersion = version_class(File) def _version_class(clz): if clz == NodeToFile: return NodeToFileVersion elif clz == File: return FileVersion monkeypatch.setattr(version_migration, "version_class", _version_class) Transaction = versioning_manager.transaction_cls remove_versioning() first = first_version_node middle = middle_version_node current = current_version_node first_files = [FileFactory()] first.files.extend(first_files) middle_files = [FileFactory()] middle.files.extend(middle_files) current_files = [FileFactory()] current.files.extend(current_files) transaction = Transaction() version_migration.create_file_versions(first, middle, transaction) transaction2 = Transaction() version_migration.create_file_versions(middle, current, transaction2)
def test_naming_convention(db, app): """Test naming convention.""" from sqlalchemy_continuum import remove_versioning ext = InvenioDB(app, entry_point_group=False, db=db) cfg = dict( DB_VERSIONING=True, DB_VERSIONING_USER_MODEL=None, SQLALCHEMY_DATABASE_URI=app.config['SQLALCHEMY_DATABASE_URI'], ) with app.app_context(): if db.engine.name == 'sqlite': raise pytest.skip('Upgrades are not supported on SQLite.') def model_factory(base): """Create test models.""" class Master(base): __tablename__ = 'master' pk = sa.Column(sa.Integer, primary_key=True) name = sa.Column(sa.String(100), unique=True) city = sa.Column(sa.String(100), index=True) active = sa.Column(sa.Boolean(name='active'), server_default='1') class Slave(base): __tablename__ = 'slave' pk = sa.Column(sa.Integer, primary_key=True) fk = sa.Column(sa.Integer, sa.ForeignKey(Master.pk)) code = sa.Column(sa.Integer, index=True, unique=True) source = sa.Column(sa.String(100)) __table_args__ = ( sa.Index(None, source), # do not add anything after getattr(base, '__table_args__', {})) return Master, Slave source_db = shared.SQLAlchemy(metadata=shared.MetaData( naming_convention={ 'ix': 'source_ix_%(table_name)s_%(column_0_label)s', 'uq': 'source_uq_%(table_name)s_%(column_0_name)s', 'ck': 'source_ck_%(table_name)s_%(constraint_name)s', 'fk': 'source_fk_%(table_name)s_%(column_0_name)s_' '%(referred_table_name)s', 'pk': 'source_pk_%(table_name)s', }), ) source_app = Flask('source_app') source_app.config.update(**cfg) source_models = model_factory(source_db.Model) source_ext = InvenioDB( source_app, entry_point_group=False, db=source_db, versioning_manager=VersioningManager(), ) with source_app.app_context(): source_db.metadata.bind = source_db.engine source_db.create_all() source_ext.alembic.stamp('dbdbc1b19cf2') assert not source_ext.alembic.compare_metadata() source_constraints = set([ cns for model in source_models for cns in list(model.__table__.constraints) + list(model.__table__.indexes) ]) remove_versioning(manager=source_ext.versioning_manager) target_db = shared.SQLAlchemy(metadata=shared.MetaData( naming_convention=shared.NAMING_CONVENTION)) target_app = Flask('target_app') target_app.config.update(**cfg) target_models = model_factory(target_db.Model) target_ext = InvenioDB( target_app, entry_point_group=False, db=target_db, versioning_manager=VersioningManager(), ) with target_app.app_context(): target_db.metadata.bind = target_db.engine assert target_ext.alembic.compare_metadata() target_ext.alembic.upgrade('35c1075e6360') assert not target_ext.alembic.compare_metadata() target_db.drop_all() target_constraints = set([ cns.name for model in source_models for cns in list(model.__table__.constraints) + list(model.__table__.indexes) ]) remove_versioning(manager=target_ext.versioning_manager) assert source_constraints ^ target_constraints
def test_versioning( native_versioning, versioning_strategy, property_mod_tracking ): transaction_column_name = 'transaction_id' end_transaction_column_name = 'end_transaction_id' plugins = [TransactionChangesPlugin(), TransactionMetaPlugin()] if property_mod_tracking: plugins.append(PropertyModTrackerPlugin()) transaction_cls = TransactionFactory() user_cls = None Model = declarative_base() options = { 'create_models': True, 'native_versioning': native_versioning, 'base_classes': (Model, ), 'strategy': versioning_strategy, 'transaction_column_name': transaction_column_name, 'end_transaction_column_name': end_transaction_column_name, } make_versioned(options=options) dns = 'postgres://postgres@localhost/sqlalchemy_continuum_test' versioning_manager.plugins = plugins versioning_manager.transaction_cls = transaction_cls versioning_manager.user_cls = user_cls engine = create_engine(dns) # engine.echo = True connection = engine.connect() class Article(Model): __tablename__ = 'article' __versioned__ = copy(options) id = sa.Column(sa.Integer, autoincrement=True, primary_key=True) name = sa.Column(sa.Unicode(255), nullable=False) content = sa.Column(sa.UnicodeText) description = sa.Column(sa.UnicodeText) class Tag(Model): __tablename__ = 'tag' __versioned__ = copy(options) id = sa.Column(sa.Integer, autoincrement=True, primary_key=True) article_id = sa.Column(sa.Integer, sa.ForeignKey(Article.id)) article = sa.orm.relationship(Article, backref='tags') sa.orm.configure_mappers() Model.metadata.create_all(connection) Session = sessionmaker(bind=connection) session = Session(autoflush=False) session.execute('CREATE EXTENSION IF NOT EXISTS hstore') Model.metadata.create_all(connection) start = time() for i in range(20): for i in range(20): session.add(Article(name=u'Article', tags=[Tag(), Tag()])) session.commit() print 'Testing with:' print ' native_versioning=%r' % native_versioning print ' versioning_strategy=%r' % versioning_strategy print ' property_mod_tracking=%r' % property_mod_tracking print colored('%r seconds' % (time() - start), 'red') Model.metadata.drop_all(connection) remove_versioning() versioning_manager.reset() session.close_all() session.expunge_all() Model.metadata.drop_all(connection) engine.dispose() connection.close()
def test_versioning(native_versioning, versioning_strategy, property_mod_tracking): transaction_column_name = 'transaction_id' end_transaction_column_name = 'end_transaction_id' plugins = [TransactionChangesPlugin(), TransactionMetaPlugin()] if property_mod_tracking: plugins.append(PropertyModTrackerPlugin()) transaction_cls = TransactionFactory() user_cls = None Model = declarative_base() options = { 'create_models': True, 'native_versioning': native_versioning, 'base_classes': (Model, ), 'strategy': versioning_strategy, 'transaction_column_name': transaction_column_name, 'end_transaction_column_name': end_transaction_column_name, } make_versioned(options=options) dns = 'postgres://postgres@localhost/sqlalchemy_continuum_test' versioning_manager.plugins = plugins versioning_manager.transaction_cls = transaction_cls versioning_manager.user_cls = user_cls engine = create_engine(dns) # engine.echo = True class Article(Model): __tablename__ = 'article' __versioned__ = copy(options) id = sa.Column(sa.Integer, autoincrement=True, primary_key=True) name = sa.Column(sa.Unicode(255), nullable=False) content = sa.Column(sa.UnicodeText) description = sa.Column(sa.UnicodeText) class Tag(Model): __tablename__ = 'tag' __versioned__ = copy(options) id = sa.Column(sa.Integer, autoincrement=True, primary_key=True) article_id = sa.Column(sa.Integer, sa.ForeignKey(Article.id)) article = sa.orm.relationship(Article, backref='tags') sa.orm.configure_mappers() connection = engine.connect() Model.metadata.create_all(connection) Session = sessionmaker(bind=connection) session = Session(autoflush=False) session.execute('CREATE EXTENSION IF NOT EXISTS hstore') Model.metadata.create_all(connection) start = time() for i in range(20): for i in range(20): session.add(Article(name=u'Article', tags=[Tag(), Tag()])) session.commit() print 'Testing with:' print ' native_versioning=%r' % native_versioning print ' versioning_strategy=%r' % versioning_strategy print ' property_mod_tracking=%r' % property_mod_tracking print colored('%r seconds' % (time() - start), 'red') Model.metadata.drop_all(connection) remove_versioning() versioning_manager.reset() session.close_all() session.expunge_all() Model.metadata.drop_all(connection) engine.dispose() connection.close()