def check(self): """Report any pending database upgrades. An exit code of 3 indicates db expand is needed, see stdout output. An exit code of 4 indicates db migrate is needed, see stdout output. An exit code of 5 indicates db contract is needed, see stdout output. """ engine = db_api.get_engine() self._validate_engine(engine) curr_heads = alembic_migrations.get_current_alembic_heads() expand_heads = alembic_migrations.get_alembic_branch_head( db_migration.EXPAND_BRANCH) contract_heads = alembic_migrations.get_alembic_branch_head( db_migration.CONTRACT_BRANCH) if (contract_heads in curr_heads): print(_('Database is up to date. No upgrades needed.')) sys.exit() elif ((not expand_heads) or (expand_heads not in curr_heads)): print( _('Your database is not up to date. ' 'Your first step is to run `glance-manage db expand`.')) sys.exit(3) elif data_migrations.has_pending_migrations(db_api.get_engine()): print( _('Your database is not up to date. ' 'Your next step is to run `glance-manage db migrate`.')) sys.exit(4) elif ((not contract_heads) or (contract_heads not in curr_heads)): print( _('Your database is not up to date. ' 'Your next step is to run `glance-manage db contract`.')) sys.exit(5)
def migrate(self, online_migration=True): """Run the data migration phase of a database migration.""" if online_migration: self._validate_engine(db_api.get_engine()) curr_heads = alembic_migrations.get_current_alembic_heads() contract_head = alembic_migrations.get_alembic_branch_head( db_migration.CONTRACT_BRANCH) if (contract_head in curr_heads): print(_('Database is up to date. No migrations needed.')) sys.exit() expand_head = alembic_migrations.get_alembic_branch_head( db_migration.EXPAND_BRANCH) if expand_head not in curr_heads: sys.exit( _('Data migration did not run. Data migration cannot be ' 'run before database expansion. Run database ' 'expansion first using "glance-manage db expand"')) if data_migrations.has_pending_migrations(db_api.get_engine()): rows_migrated = data_migrations.migrate(db_api.get_engine()) print(_('Migrated %s rows') % rows_migrated) else: print(_('Database migration is up to date. No migration needed.'))
def setUp(self): """Establish a clean test environment""" super(TestRegistryV2Client, self).setUp() db_api.get_engine() self.context = context.RequestContext(is_admin=True) uuid1_time = timeutils.utcnow() uuid2_time = uuid1_time + datetime.timedelta(seconds=5) self.FIXTURES = [ self.get_extra_fixture( id=UUID1, name='fake image #1', is_public=False, disk_format='ami', container_format='ami', size=13, virtual_size=26, properties={'type': 'kernel'}, location="swift://*****:*****@acct/container/obj.tar.0", created_at=uuid1_time), self.get_extra_fixture(id=UUID2, name='fake image #2', properties={}, size=19, virtual_size=38, location="file:///tmp/glance-tests/2", created_at=uuid2_time) ] self.destroy_fixtures() self.create_fixtures() self.client = rclient.RegistryClient("0.0.0.0")
def setUp(self): """Establish a clean test environment""" super(TestRegistryV2Client, self).setUp() db_api.get_engine() self.context = context.RequestContext(is_admin=True) uuid1_time = timeutils.utcnow() uuid2_time = uuid1_time + datetime.timedelta(seconds=5) self.FIXTURES = [ self.get_extra_fixture( id=UUID1, name="fake image #1", is_public=False, disk_format="ami", container_format="ami", size=13, virtual_size=26, properties={"type": "kernel"}, location="swift://*****:*****@acct/container/obj.tar.0", created_at=uuid1_time, ), self.get_extra_fixture( id=UUID2, name="fake image #2", properties={}, size=19, virtual_size=38, location="file:///tmp/glance-tests/2", created_at=uuid2_time, ), ] self.destroy_fixtures() self.create_fixtures() self.client = rclient.RegistryClient("0.0.0.0")
def setUp(self): """Establish a clean test environment""" super(TestRegistryV1Client, self).setUp() db_api.get_engine() self.context = context.RequestContext(is_admin=True) self.FIXTURES = [ self.get_fixture( id=UUID1, name='fake image #1', is_public=False, disk_format='ami', container_format='ami', size=13, location="swift://*****:*****@acct/container/obj.tar.0", properties={'type': 'kernel'}), self.get_fixture(id=UUID2, name='fake image #2', properties={}, size=19, location="file:///tmp/glance-tests/2") ] self.destroy_fixtures() self.create_fixtures() self.client = rclient.RegistryClient("0.0.0.0")
def sync(self, version=None, current_version=None): """ Place a database under migration control and upgrade/downgrade it, creating first if necessary. """ if current_version not in (None, "None"): migration.db_version_control(db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, version=current_version) migration.db_sync(db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, version)
def sync(self, version=None, current_version=None): """ Place a database under migration control and upgrade/downgrade it, creating first if necessary. """ if current_version not in (None, 'None'): migration.db_version_control(db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, version=current_version) migration.db_sync(db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, version)
def setUp(self): super(TestRegistryRPC, self).setUp() self.mapper = routes.Mapper() self.api = test_utils.FakeAuthMiddleware(rserver.API(self.mapper), is_admin=True) uuid1_time = timeutils.utcnow() uuid2_time = uuid1_time + datetime.timedelta(seconds=5) self.FIXTURES = [ {'id': UUID1, 'name': 'fake image #1', 'status': 'active', 'disk_format': 'ami', 'container_format': 'ami', 'is_public': False, 'created_at': uuid1_time, 'updated_at': uuid1_time, 'deleted_at': None, 'deleted': False, 'checksum': None, 'min_disk': 0, 'min_ram': 0, 'size': 13, 'locations': [{'url': "file:///%s/%s" % (self.test_dir, UUID1), 'metadata': {}}], 'properties': {'type': 'kernel'}}, {'id': UUID2, 'name': 'fake image #2', 'status': 'active', 'disk_format': 'vhd', 'container_format': 'ovf', 'is_public': True, 'created_at': uuid2_time, 'updated_at': uuid2_time, 'deleted_at': None, 'deleted': False, 'checksum': None, 'min_disk': 5, 'min_ram': 256, 'size': 19, 'locations': [{'url': "file:///%s/%s" % (self.test_dir, UUID2), 'metadata': {}}], 'properties': {}}] self.context = glance.context.RequestContext(is_admin=True) db_api.setup_db_env() db_api.get_engine() self.destroy_fixtures() self.create_fixtures()
def get_current_alembic_heads(): """Return current heads (if any) from the alembic migration table""" engine = db_api.get_engine() with engine.connect() as conn: context = alembic_migration.MigrationContext.configure(conn) heads = context.get_current_heads() return heads
def get_current_alembic_heads(): """Return current heads (if any) from the alembic migration table""" engine = db_api.get_engine() with engine.connect() as conn: context = alembic_migration.MigrationContext.configure(conn) heads = context.get_current_heads() def update_alembic_version(old, new): """Correct alembic head in order to upgrade DB using EMC method. :param:old: Actual alembic head :param:new: Expected alembic head to be updated """ meta = MetaData(engine) alembic_version = Table('alembic_version', meta, autoload=True) alembic_version.update().values(version_num=new).where( alembic_version.c.version_num == old).execute() if ("pike01" in heads): update_alembic_version("pike01", "pike_contract01") elif ("ocata01" in heads): update_alembic_version("ocata01", "ocata_contract01") heads = context.get_current_heads() return heads
def get_current_alembic_heads(): """Return current heads (if any) from the alembic migration table""" engine = db_api.get_engine() with engine.connect() as conn: context = alembic_migration.MigrationContext.configure(conn) heads = context.get_current_heads() def update_alembic_version(old, new): """Correct alembic head in order to upgrade DB using EMC method. :param:old: Actual alembic head :param:new: Expected alembic head to be updated """ meta = MetaData(engine) alembic_version = Table('alembic_version', meta, autoload=True) alembic_version.update().values( version_num=new).where( alembic_version.c.version_num == old).execute() if ("pike01" in heads): update_alembic_version("pike01", "pike_contract01") elif ("ocata01" in heads): update_alembic_version("ocata01", "ocata_contract01") heads = context.get_current_heads() return heads
def test_legacy_db_downgrade_version(self): migration.db_sync = mock.Mock() self._main_test_helper(['glance.cmd.manage', 'db_downgrade', '20'], migration.db_sync, db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, '20', sanity_check=True)
def test_legacy_db_version(self): migration.db_version = mock.Mock() self._main_test_helper(['glance.cmd.manage', 'db_version'], glance.openstack.common.db.sqlalchemy. migration.db_version, db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, 0)
def test_db_upgrade(self): migration.db_sync = mock.Mock() self._main_test_helper(['glance.cmd.manage', 'db', 'upgrade'], migration.db_sync, db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, None, sanity_check=True)
def test_legacy_db_sync(self): migration.db_sync = mock.Mock() self._main_test_helper(['glance.cmd.manage', 'db_sync'], glance.openstack.common.db.sqlalchemy. migration.db_sync, db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, None)
def load_metadefs(): """Read metadefinition files and insert data into the database""" return get_backend().db_load_metadefs(engine=db_api.get_engine(), metadata_path=None, merge=False, prefer_new=False, overwrite=False)
def test_db_metadefs_load_from_path_merge(self): db_metadata.db_load_metadefs = mock.Mock() self._main_test_helper(['glance.cmd.manage', 'db_load_metadefs', '/mock/', 'True'], db_metadata.db_load_metadefs, db_api.get_engine(), '/mock/', 'True', None, None)
def expand(self): """Run the expansion phase of a rolling upgrade procedure.""" engine = db_api.get_engine() if engine.engine.name != 'mysql': sys.exit( _('Rolling upgrades are currently supported only for ' 'MySQL')) expand_head = alembic_migrations.get_alembic_branch_head( db_migration.EXPAND_BRANCH) if not expand_head: sys.exit( _('Database expansion failed. Couldn\'t find head ' 'revision of expand branch.')) self.sync(version=expand_head) curr_heads = alembic_migrations.get_current_alembic_heads() if expand_head not in curr_heads: sys.exit( _('Database expansion failed. Database expansion should ' 'have brought the database version up to "%(e_rev)s" ' 'revision. But, current revisions are: %(curr_revs)s ') % { 'e_rev': expand_head, 'curr_revs': curr_heads })
def db_sync(version='heads', engine=None): """Migrate the database to `version` or the most recent version.""" if engine is None: engine = db_api.get_engine() alembic_config = alembic_migrations.get_alembic_config(engine=engine) alembic_command.upgrade(alembic_config, version)
def test_db_metadefs_load_prefer_new(self): db_metadata.db_load_metadefs = mock.Mock() self._main_test_helper([ 'glance.cmd.manage', 'db', 'load_metadefs', '--merge', '--prefer_new' ], db_metadata.db_load_metadefs, db_api.get_engine(), None, True, True, False)
def test_db_metadefs_load_prefer_new_and_overwrite_existing(self): db_metadata.db_load_metadefs = mock.Mock() self._main_test_helper([ 'glance.cmd.manage', 'db', 'load_metadefs', '--merge', '--prefer_new', '--overwrite' ], db_metadata.db_load_metadefs, db_api.get_engine(), None, True, True, True)
def downgrade(self, version=None): """Downgrade the database's migration level""" migration.db_sync( db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, version, sanity_check=self._need_sanity_check())
def test_legacy_db_downgrade_version(self, db_sync): self._main_test_helper(['glance.cmd.manage', 'db_downgrade', '20'], migration.db_sync, db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, '20', sanity_check=True)
def test_db_metadefs_load_from_path_overwrite_existing(self): db_metadata.db_load_metadefs = mock.Mock() self._main_test_helper([ 'glance.cmd.manage', 'db', 'load_metadefs', '--path', '/mock/', '--merge', '--overwrite' ], db_metadata.db_load_metadefs, db_api.get_engine(), '/mock/', True, False, True)
def test_db_metadefs_load_from_merge_and_prefer_new_and_overwrite(self): db_metadata.db_load_metadefs = mock.Mock() self._main_test_helper([ 'glance.cmd.manage', 'db_load_metadefs', '/mock/', 'True', 'True', 'True' ], db_metadata.db_load_metadefs, db_api.get_engine(), '/mock/', 'True', 'True', 'True')
def test_db_metadefs_load_from_path_overwrite_existing(self): db_metadata.db_load_metadefs = mock.Mock() self._main_test_helper(['glance.cmd.manage', 'db', 'load_metadefs', '--path', '/mock/', '--merge', '--overwrite'], db_metadata.db_load_metadefs, db_api.get_engine(), '/mock/', True, False, True)
def test_db_metadefs_export_with_specified_path(self): db_metadata.db_export_metadefs = mock.Mock() self._main_test_helper(['glance.cmd.manage', 'db', 'export_metadefs', '--path', '/mock/'], db_metadata.db_export_metadefs, db_api.get_engine(), '/mock/')
def test_db_metadefs_load_prefer_new(self): db_metadata.db_load_metadefs = mock.Mock() self._main_test_helper(['glance.cmd.manage', 'db', 'load_metadefs', '--merge', '--prefer_new'], db_metadata.db_load_metadefs, db_api.get_engine(), None, True, True, False)
def test_db_upgrade(self, db_sync): self._main_test_helper(['glance.cmd.manage', 'db', 'upgrade'], migration.db_sync, db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, None, sanity_check=True)
def test_db_metadefs_load_from_merge_and_prefer_new_and_overwrite(self): db_metadata.db_load_metadefs = mock.Mock() self._main_test_helper(['glance.cmd.manage', 'db_load_metadefs', '/mock/', 'True', 'True', 'True'], db_metadata.db_load_metadefs, db_api.get_engine(), '/mock/', 'True', 'True', 'True')
def test_db_metadefs_load_with_specified_path(self): db_metadata.db_load_metadefs = mock.Mock() self._main_test_helper(['glance.cmd.manage', 'db_load_metadefs', '/mock/'], db_metadata.db_load_metadefs, db_api.get_engine(), '/mock/', None, None, None)
def migrate(engine=None, release=db_migrations.CURRENT_RELEASE): if not engine: engine = db_api.get_engine() migrations = _find_migration_modules(release) rows_migrated = _run_migrations(engine, migrations) return rows_migrated
def test_db_metadefs_load_prefer_new_and_overwrite_existing(self): db_metadata.db_load_metadefs = mock.Mock() self._main_test_helper(['glance.cmd.manage', 'db', 'load_metadefs', '--merge', '--prefer_new', '--overwrite'], db_metadata.db_load_metadefs, db_api.get_engine(), None, True, True, True)
def test_db_downgrade_version_without_sanity_check(self, db_sync): manage.CONF.db_enforce_mysql_charset = False self._main_test_helper(['glance.cmd.manage', 'db', 'downgrade', '20'], migration.db_sync, db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, '20', sanity_check=False)
def expand(self, online_migration=True): """Run the expansion phase of a database migration.""" if online_migration: self._validate_engine(db_api.get_engine()) curr_heads = alembic_migrations.get_current_alembic_heads() expand_head = alembic_migrations.get_alembic_branch_head( db_migration.EXPAND_BRANCH) contract_head = alembic_migrations.get_alembic_branch_head( db_migration.CONTRACT_BRANCH) if not expand_head: sys.exit( _('Database expansion failed. Couldn\'t find head ' 'revision of expand branch.')) elif (contract_head in curr_heads): print(_('Database is up to date. No migrations needed.')) sys.exit() if expand_head not in curr_heads: self._sync(version=expand_head) curr_heads = alembic_migrations.get_current_alembic_heads() if expand_head not in curr_heads: sys.exit( _('Database expansion failed. Database expansion ' 'should have brought the database version up to ' '"%(e_rev)s" revision. But, current revisions are' ': %(curr_revs)s ') % { 'e_rev': expand_head, 'curr_revs': curr_heads }) else: print(_('Database expansion is up to date. No expansion needed.'))
def test_legacy_db_upgrade_version_without_sanity_check(self): migration.db_sync = mock.Mock() manage.CONF.db_enforce_mysql_charset = False self._main_test_helper(['glance.cmd.manage', 'db_upgrade', '40'], migration.db_sync, db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, '40', sanity_check=False)
def test_legacy_db_sync(self): migration.db_sync = mock.Mock() self._main_test_helper(['glance.cmd.manage', 'db_sync'], glance.openstack.common.db.sqlalchemy. migration.db_sync, db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, None, sanity_check=True)
def get_current_legacy_head(): try: legacy_head = sqla_migration.db_version(db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, db_migration.INIT_VERSION) except db_exception.DBMigrationError: legacy_head = None return legacy_head
def get_current_legacy_head(): try: legacy_head = sqla_migration.db_version(db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, db_migration.INIT_VERSION) except db_exception.DbMigrationError: legacy_head = None return legacy_head
def setUp(self): super(TestRegistryRPC, self).setUp() self.mapper = routes.Mapper() self.api = test_utils.FakeAuthMiddleware(rserver.API(self.mapper), is_admin=True) self.FIXTURES = [ {'id': UUID1, 'name': 'fake image #1', 'status': 'active', 'disk_format': 'ami', 'container_format': 'ami', 'is_public': False, 'created_at': timeutils.utcnow(), 'updated_at': timeutils.utcnow(), 'deleted_at': None, 'deleted': False, 'checksum': None, 'min_disk': 0, 'min_ram': 0, 'size': 13, 'locations': [{'url': "file:///%s/%s" % (self.test_dir, UUID1), 'metadata': {}}], 'properties': {'type': 'kernel'}}, {'id': UUID2, 'name': 'fake image #2', 'status': 'active', 'disk_format': 'vhd', 'container_format': 'ovf', 'is_public': True, 'created_at': timeutils.utcnow(), 'updated_at': timeutils.utcnow(), 'deleted_at': None, 'deleted': False, 'checksum': None, 'min_disk': 5, 'min_ram': 256, 'size': 19, 'locations': [{'url': "file:///%s/%s" % (self.test_dir, UUID2), 'metadata': {}}], 'properties': {}}] self.context = glance.context.RequestContext(is_admin=True) db_api.get_engine() self.destroy_fixtures() self.create_fixtures()
def setUp(self): """Establish a clean test environment""" super(TestRegistryV1Client, self).setUp() db_api.get_engine() self.context = context.RequestContext(is_admin=True) self.FIXTURES = [ self.get_fixture( id=UUID1, name='fake image #1', is_public=False, disk_format='ami', container_format='ami', size=13, location="swift://*****:*****@acct/container/obj.tar.0", properties={'type': 'kernel'}), self.get_fixture(id=UUID2, name='fake image #2', properties={}, size=19, location="file:///tmp/glance-tests/2")] self.destroy_fixtures() self.create_fixtures() self.client = rclient.RegistryClient("0.0.0.0")
def db_sync(version=None, init_version=0, engine=None): """Migrate the database to `version` or the most recent version.""" if engine is None: engine = db_api.get_engine() return IMPL.db_sync(engine=engine, abs_path=MIGRATE_REPO_PATH, version=version, init_version=init_version)
def run_sql_cmd(self, sql): """ Provides a crude mechanism to run manual SQL commands for backend DB verification within the functional tests. The raw result set is returned. """ engine = db_api.get_engine() return engine.execute(sql)
def test_legacy_db_sync_version(self): migration.db_sync = mock.Mock() self._main_test_helper(['glance.cmd.manage', 'db_sync', '20'], migration.db_sync, db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, '20', sanity_check=True)
def test_db_sync(self): migration.db_sync = mock.Mock() self._main_test_helper(['glance.cmd.manage', 'db', 'sync'], migration.db_sync, db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, None, sanity_check=True)
def has_pending_migrations(engine=None, release=db_migrations.CURRENT_RELEASE): if not engine: engine = db_api.get_engine() migrations = _find_migration_modules(release) if not migrations: return False return any([x.has_migrations(engine) for x in migrations])
def test_legacy_db_version(self, db_version): self._main_test_helper( ["glance.cmd.manage", "db_version"], migration.db_version, db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, 0, )
def load_metadefs(self, path=None, merge=False, prefer_new=False, overwrite=False): """Load metadefinition json files to database""" metadata.db_load_metadefs(db_api.get_engine(), path, merge, prefer_new, overwrite)
def test_db_metadefs_export_with_specified_path(self): db_metadata.db_export_metadefs = mock.Mock() self._main_test_helper( ["glance.cmd.manage", "db", "export_metadefs", "--path", "/mock/"], db_metadata.db_export_metadefs, db_api.get_engine(), "/mock/", )
def test_legacy_db_version_control(self, db_version_control): self._main_test_helper( ["glance.cmd.manage", "db_version_control"], migration.db_version_control, db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, None, )
def downgrade(self, version=None): """Downgrade the database's migration level""" print("Warning: DB downgrade is deprecated and will be removed in N " "release. Users should make a full database backup of the " "production data before attempting any upgrade.", file=sys.stderr) migration.db_sync(db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, version)