def test__initialize_pulp(self):
"""
_initialize_pulp() should raise an Exception if any of the packages aren't at their latest
version.
"""
# It is unusual to put an import in the middle of a test, but unfortunately this import will
# call start_logging() before our test super class can override the logging settings, and
# thus all the logging will be done to /var/log instead of to /tmp. Moving the import here
# from the top of the file solves the problem, though not elegantly.
from pulp.server.webservices.application import _initialize_pulp
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.get_collection().find({}).count(), 0)
# Make sure that our mock works. There are three valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
# Set all versions back to 0
for package in models.get_migration_packages():
package._migration_tracker.version = 0
package._migration_tracker.save()
# Let's make sure we raise the exception
try:
_initialize_pulp()
self.fail('_initialize_pulp() should have raised an Exception, but did not.')
except Exception, e:
self.assertEqual(str(e), ('There are unapplied migrations. Please '
'run the database management utility to apply them.'))
def test_current_version_too_high(self, mocked_file_config, mocked_logger,
mocked_stdout, mocked_stderr):
"""
Set the current package version higher than latest available version, then sit back and eat
popcorn.
"""
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.get_collection().find({}).count(),
0)
# Make sure that our mock works. There are four valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
# Set all versions to ridiculously high values
for package in models.get_migration_packages():
package._migration_tracker.version = 9999999
package._migration_tracker.save()
error_code = manage.main()
self.assertEqual(error_code, os.EX_DATAERR)
# There should have been a print to stderr about the Exception
expected_stderr_calls = [(
'The database for migration package unit.server.db.migration_packages.platform is at '
'version 9999999, which is larger than the latest version available, 1.'
), '\n']
stderr_calls = [
mock_call[1][0] for mock_call in mocked_stderr.mock_calls
]
self.assertEquals(stderr_calls, expected_stderr_calls)
def test_migrate_with_dry_run_flag(self, mock_file_config, initialize, mocked_apply_migration,
getLogger):
"""
Test that when a dry run is performed, no migrations actually occur.
"""
logger = MagicMock()
getLogger.return_value = logger
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.objects().count(), 0)
# Make sure that our mock works. There are three valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
# Set all versions back to 0
for package in models.get_migration_packages():
package._migration_tracker.version = 0
package._migration_tracker.save()
result = manage.main()
# Test that none of the mock objects were actually called
migration_modules_called = [
mock_call[1][1].name for mock_call in mocked_apply_migration.mock_calls]
self.assertEquals(0, len(migration_modules_called))
self.assertEquals(1, result)
for package in models.get_migration_packages():
self.assertEqual(package.current_version, 0)
initialize.assert_called_once_with(max_timeout=1)
def test_migrate_with_new_packages(self, initialize, start_logging_mock,
logger_mock, mocked_stdout,
mocked_stderr):
"""
Adding new packages to a system that doesn't have any trackers should advance
each package to the latest available version, applying all migrate() functions along the
way.
"""
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.objects().count(), 0)
# Make sure that our mock works. There are four valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
manage.main()
for package in models.get_migration_packages():
if 'raise_exception' in str(package):
# The Exception raising package should get to version 1, because version 2 raises
self.assertEqual(package.current_version, 1)
elif 'z' in str(package):
self.assertEquals(package.current_version, 0)
else:
# All other packages should reach their top versions
self.assertEqual(package.current_version,
package.latest_available_version)
initialize.assert_called_once_with(max_timeout=1)
def test_migrate(self, mock_file_config, initialize,
mocked_apply_migration, getLogger):
"""
Let's set all the packages to be at version 0, and then check that the migrations get
called in the correct order.
"""
logger = MagicMock()
getLogger.return_value = logger
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.objects().count(), 0)
# Make sure that our mock works. There are three valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
# Set all versions back to 0
for package in models.get_migration_packages():
package._migration_tracker.version = 0
package._migration_tracker.save()
manage.main()
# There should have been a critical log about the Exception
expected_messages = (
'Applying migration '
'unit.server.db.migration_packages.raise_exception.0002_oh_no failed.\n\n'
'Halting migrations due to a migration failure.',
"Bet you didn\'t see this coming.")
critical_messages = ''.join(
[mock_call[1][0] for mock_call in logger.critical.mock_calls])
for msg in expected_messages:
self.assertTrue(msg in critical_messages)
migration_modules_called = [
mock_call[1][1].name
for mock_call in mocked_apply_migration.mock_calls
]
# Note that none of the migrations that don't meet our criteria show up in this list. Also,
# Note that migration_packages.raise_exception.0003_shouldnt_run doesn't appear
# since migration_packages.raise_exception.0002_oh_no raised an Exception. Note
# also that even though the raise_exception package raised an Exception, we still run all
# the z migrations because we don't want one package to break another.
expected_migration_modules_called = [
'unit.server.db.migration_packages.platform.0001_stuff_and_junk',
'unit.server.db.migration_packages.raise_exception.0001_works_fine',
'unit.server.db.migration_packages.raise_exception.0002_oh_no'
]
self.assertEquals(migration_modules_called,
expected_migration_modules_called)
# Assert that our precious versions have been updated correctly
for package in models.get_migration_packages():
if package.name == 'unit.server.db.migration_packages.platform':
self.assertEqual(package.current_version,
package.latest_available_version)
elif package.name == 'unit.server.db.migration_packages.raise_exception':
# The raised Exception should have prevented us from getting past version 1
self.assertEquals(package.current_version, 1)
else:
# raise_exception should cause the migrations to stop
self.assertEqual(package.current_version, 0)
initialize.assert_called_once_with(max_timeout=1)
def test_current_version_too_high(self, mocked_file_config, initialize, getLogger):
"""
Set the current package version higher than latest available version, then sit back and eat
popcorn.
"""
logger = MagicMock()
getLogger.return_value = logger
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.objects().count(), 0)
# Make sure that our mock works. There are four valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
# Set all versions to ridiculously high values
for package in models.get_migration_packages():
package._migration_tracker.version = 9999999
package._migration_tracker.save()
error_code = manage.main()
self.assertEqual(error_code, os.EX_DATAERR)
# There should have been a critical log about the Exception
expected_messages = (
'The database for migration package unit.server.db.migration_packages.'
'platform is at version 9999999, which is larger than the latest version available, 1.')
critical_messages = ''.join([mock_call[1][0] for mock_call in logger.critical.mock_calls])
for msg in expected_messages:
self.assertTrue(msg in critical_messages)
initialize.assert_called_once_with(max_timeout=1)
def test_migrate(self, mock_file_config, initialize, mocked_apply_migration, getLogger):
"""
Let's set all the packages to be at version 0, and then check that the migrations get
called in the correct order.
"""
logger = MagicMock()
getLogger.return_value = logger
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.objects().count(), 0)
# Make sure that our mock works. There are three valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
# Set all versions back to 0
for package in models.get_migration_packages():
package._migration_tracker.version = 0
package._migration_tracker.save()
manage.main()
# There should have been a critical log about the Exception
expected_messages = (
'Applying migration '
'unit.server.db.migration_packages.raise_exception.0002_oh_no failed.\n\n'
'Halting migrations due to a migration failure.',
"Bet you didn\'t see this coming."
)
critical_messages = ''.join([mock_call[1][0] for mock_call in logger.critical.mock_calls])
for msg in expected_messages:
self.assertTrue(msg in critical_messages)
migration_modules_called = [
mock_call[1][1].name for mock_call in mocked_apply_migration.mock_calls]
# Note that none of the migrations that don't meet our criteria show up in this list. Also,
# Note that migration_packages.raise_exception.0003_shouldnt_run doesn't appear
# since migration_packages.raise_exception.0002_oh_no raised an Exception. Note
# also that even though the raise_exception package raised an Exception, we still run all
# the z migrations because we don't want one package to break another.
expected_migration_modules_called = [
'unit.server.db.migration_packages.platform.0001_stuff_and_junk',
'unit.server.db.migration_packages.raise_exception.0001_works_fine',
'unit.server.db.migration_packages.raise_exception.0002_oh_no']
self.assertEquals(migration_modules_called, expected_migration_modules_called)
# Assert that our precious versions have been updated correctly
for package in models.get_migration_packages():
if package.name == 'unit.server.db.migration_packages.platform':
self.assertEqual(package.current_version, package.latest_available_version)
elif package.name == 'unit.server.db.migration_packages.raise_exception':
# The raised Exception should have prevented us from getting past version 1
self.assertEquals(package.current_version, 1)
else:
# raise_exception should cause the migrations to stop
self.assertEqual(package.current_version, 0)
initialize.assert_called_once_with(max_timeout=1)
def test_migrate(self, file_config_mock, logger_mock,
mocked_apply_migration, mocked_stdout, mocked_stderr):
"""
Let's set all the packages to be at version 0, and then check that the migrations get
called in the correct order.
"""
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.get_collection().find({}).count(),
0)
# Make sure that our mock works. There are three valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
# Set all versions back to 0
for package in models.get_migration_packages():
package._migration_tracker.version = 0
package._migration_tracker.save()
manage.main()
# There should have been a print to stderr about the Exception
expected_stderr_calls = [(
'Applying migration unit.server.db.migration_packages.raise_exception.0002_oh_no '
'failed.'), ' ', ' See log for details.', '\n']
stderr_calls = [
mock_call[1][0] for mock_call in mocked_stderr.mock_calls
]
self.assertEquals(stderr_calls, expected_stderr_calls)
migration_modules_called = [
mock_call[1][1].name
for mock_call in mocked_apply_migration.mock_calls
]
# Note that none of the migrations that don't meet our criteria show up in this list. Also,
# Note that migration_packages.raise_exception.0003_shouldnt_run doesn't appear
# since migration_packages.raise_exception.0002_oh_no raised an Exception. Note
# also that even though the raise_exception package raised an Exception, we still run all
# the z migrations because we don't want one package to break another.
expected_migration_modules_called = [
'unit.server.db.migration_packages.platform.0001_stuff_and_junk',
'unit.server.db.migration_packages.raise_exception.0001_works_fine',
'unit.server.db.migration_packages.raise_exception.0002_oh_no',
'unit.server.db.migration_packages.z.0001_test',
'unit.server.db.migration_packages.z.0002_test',
'unit.server.db.migration_packages.z.0003_test'
]
self.assertEquals(migration_modules_called,
expected_migration_modules_called)
# Assert that our precious versions have been updated correctly
for package in models.get_migration_packages():
if package.name != 'unit.server.db.migration_packages.raise_exception':
self.assertEqual(package.current_version,
package.latest_available_version)
else:
# The raised Exception should have prevented us from getting past version 1
self.assertEqual(package.current_version, 1)
def test_migrate_with_test_flag(self, mock_file_config,
mocked_apply_migration, getLogger):
"""
Let's set all the packages to be at version 0, and then check that the migrations get called
in the correct order. We will also set the --test flag and ensure that the migration
versions do not get updated.
"""
logger = MagicMock()
getLogger.return_value = logger
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.get_collection().find({}).count(),
0)
# Make sure that our mock works. There are three valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
# Set all versions back to 0
for package in models.get_migration_packages():
package._migration_tracker.version = 0
package._migration_tracker.save()
manage.main()
# There should have been a critical log about the Exception
expected_messages = (
'Applying migration unit.server.db.migration_packages.raise_exception.0002_oh_no '
'failed.\n\nHalting migrations due to a migration failure.',
'Bet you didn\'t see this coming.')
critical_messages = [
mock_call[1][0] for mock_call in logger.critical.mock_calls
]
for msg in expected_messages:
self.assertTrue(msg in critical_messages)
migration_modules_called = [
mock_call[1][1].name
for mock_call in mocked_apply_migration.mock_calls
]
# Note that none of the migrations that don't meet our criteria show up in this list. Also,
# Note that migration_packages.raise_exception.0003_shouldnt_run doesn't appear
# since migration_packages.raise_exception.0002_oh_no raised an Exception. Note
# also that even though the raise_exception package raised an Exception, we still run all
# the z migrations because we don't want one package to break another.
expected_migration_modules_called = [
'unit.server.db.migration_packages.platform.0001_stuff_and_junk',
'unit.server.db.migration_packages.raise_exception.0001_works_fine',
'unit.server.db.migration_packages.raise_exception.0002_oh_no'
]
self.assertEquals(migration_modules_called,
expected_migration_modules_called)
# Assert that our precious versions have not been updated, since we have the --test flag
for package in models.get_migration_packages():
self.assertEqual(package.current_version, 0)
def test_get_migration_packages(self, log_mock):
"""
Ensure that pulp.server.db.migrate.models.get_migration_packages functions correctly.
"""
packages = models.get_migration_packages()
self.assertEquals(len(packages), 4)
self.assertTrue(
all([
isinstance(package, models.MigrationPackage)
for package in packages
]))
# Make sure that the packages are sorted correctly, with platform first
self.assertEquals(packages[0].name,
'unit.server.db.migration_packages.platform')
self.assertEquals(packages[1].name,
'unit.server.db.migration_packages.a')
self.assertEquals(packages[2].name,
'unit.server.db.migration_packages.raise_exception')
self.assertEquals(packages[3].name,
'unit.server.db.migration_packages.z')
# Assert that we logged the duplicate version exception and the version gap exception
expected_log_calls = [
call('There are two migration modules that share version 2 in '
'unit.server.db.migration_packages.duplicate_versions.'),
call('Migration version 2 is missing in '
'unit.server.db.migration_packages.version_gap.')
]
log_mock.assert_has_calls(expected_log_calls)
def migrate_database(options):
"""
Perform the migrations for each migration package found in pulp.server.db.migrations.
:param options: The command line parameters from the user
"""
migration_packages = models.get_migration_packages()
for migration_package in migration_packages:
if migration_package.current_version > migration_package.latest_available_version:
msg = _(
'The database for migration package %(p)s is at version %(v)s, which is larger '
'than the latest version available, %(a)s.')
msg = msg % ({
'p': migration_package.name,
'v': migration_package.current_version,
'a': migration_package.latest_available_version
})
raise DataError(msg)
if migration_package.current_version == migration_package.latest_available_version:
message = _(
'Migration package %(p)s is up to date at version %(v)s')
message = message % {
'p': migration_package.name,
'v': migration_package.latest_available_version
}
logger.info(message)
print message
continue
try:
for migration in migration_package.unapplied_migrations:
message = _('Applying %(p)s version %(v)s')
message = message % {
'p': migration_package.name,
'v': migration.version
}
print message
logger.info(message)
# We pass in !options.test to stop the apply_migration method from updating the
# package's current version when the --test flag is set
migration_package.apply_migration(
migration, update_current_version=not options.test)
message = _('Migration to %(p)s version %(v)s complete.')
message = message % {
'p': migration_package.name,
'v': migration_package.current_version
}
print message
logger.info(message)
except Exception, e:
# If an Exception is raised while applying the migrations, we should log and print it,
# and then continue with the other packages.
error_message = _('Applying migration %(m)s failed.')
error_message = error_message % {'m': migration.name}
print >> sys.stderr, str(error_message), _(' See log for details.')
logger.critical(error_message)
logger.critical(str(e))
logger.critical(''.join(
traceback.format_exception(*sys.exc_info())))
def test_migrate(self, file_config_mock, logger_mock, mocked_apply_migration, mocked_stdout,
mocked_stderr):
"""
Let's set all the packages to be at version 0, and then check that the migrations get
called in the correct order.
"""
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.get_collection().find({}).count(), 0)
# Make sure that our mock works. There are three valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
# Set all versions back to 0
for package in models.get_migration_packages():
package._migration_tracker.version = 0
package._migration_tracker.save()
manage.main()
# There should have been a print to stderr about the Exception
expected_stderr_calls = [
('Applying migration unit.server.db.migration_packages.raise_exception.0002_oh_no '
'failed.\n\nHalting migrations due to a migration failure.'), ' ', ' See log for details.', '\n',
'Bet you didn\'t see this coming.', '\n']
stderr_calls = [mock_call[1][0] for mock_call in mocked_stderr.mock_calls]
self.assertEquals(stderr_calls, expected_stderr_calls)
migration_modules_called = [
mock_call[1][1].name for mock_call in mocked_apply_migration.mock_calls]
# Note that none of the migrations that don't meet our criteria show up in this list. Also,
# Note that migration_packages.raise_exception.0003_shouldnt_run doesn't appear
# since migration_packages.raise_exception.0002_oh_no raised an Exception. Note
# also that even though the raise_exception package raised an Exception, we still run all
# the z migrations because we don't want one package to break another.
expected_migration_modules_called = [
'unit.server.db.migration_packages.platform.0001_stuff_and_junk',
'unit.server.db.migration_packages.raise_exception.0001_works_fine',
'unit.server.db.migration_packages.raise_exception.0002_oh_no']
self.assertEquals(migration_modules_called, expected_migration_modules_called)
# Assert that our precious versions have been updated correctly
for package in models.get_migration_packages():
if package.name == 'unit.server.db.migration_packages.platform':
self.assertEqual(package.current_version, package.latest_available_version)
elif package.name == 'unit.server.db.migration_packages.raise_exception':
# The raised Exception should have prevented us from getting past version 1
self.assertEquals(package.current_version, 1)
else:
# raise_exception should cause the migrations to stop
self.assertEqual(package.current_version, 0)
def test_migrate_with_test_flag(self, start_logging_mock,
mocked_apply_migration, mocked_stderr):
"""
Let's set all the packages to be at version 0, and then check that the migrations get called
in the correct order. We will also set the --test flag and ensure that the migration
versions do not get updated.
"""
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.get_collection().find({}).count(),
0)
# Make sure that our mock works. There are three valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
# Set all versions back to 0
for package in models.get_migration_packages():
package._migration_tracker.version = 0
package._migration_tracker.save()
manage.main()
# There should have been a print to stderr about the Exception
expected_stderr_calls = [
'Applying migration data.test_migration_packages.raise_exception.0002_oh_no failed.',
' ', ' See log for details.', '\n'
]
stderr_calls = [call[1][0] for call in mocked_stderr.mock_calls]
self.assertEquals(stderr_calls, expected_stderr_calls)
migration_modules_called = [
call[1][1].name for call in mocked_apply_migration.mock_calls
]
# Note that none of the migrations that don't meet our criteria show up in this list. Also,
# Note that data.test_migration_packages.raise_exception.0003_shouldnt_run doesn't appear
# since data.test_migration_packages.raise_exception.0002_oh_no raised an Exception. Note
# also that even though the raise_exception package raised an Exception, we still run all
# the z migrations because we don't want one package to break another.
expected_migration_modules_called = [
'data.test_migration_packages.platform.0001_stuff_and_junk',
'data.test_migration_packages.raise_exception.0001_works_fine',
'data.test_migration_packages.raise_exception.0002_oh_no',
'data.test_migration_packages.z.0001_test',
'data.test_migration_packages.z.0002_test',
'data.test_migration_packages.z.0003_test'
]
self.assertEquals(migration_modules_called,
expected_migration_modules_called)
# Assert that our precious versions have not been updated, since we have the --test flag
for package in models.get_migration_packages():
self.assertEqual(package.current_version, 0)
def test_migrate_with_test_flag(self, mock_file_config, mocked_apply_migration, getLogger):
"""
Let's set all the packages to be at version 0, and then check that the migrations get called
in the correct order. We will also set the --test flag and ensure that the migration
versions do not get updated.
"""
logger = MagicMock()
getLogger.return_value = logger
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.get_collection().find({}).count(), 0)
# Make sure that our mock works. There are three valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
# Set all versions back to 0
for package in models.get_migration_packages():
package._migration_tracker.version = 0
package._migration_tracker.save()
manage.main()
# There should have been a critical log about the Exception
expected_messages = (
'Applying migration unit.server.db.migration_packages.raise_exception.0002_oh_no '
'failed.\n\nHalting migrations due to a migration failure.',
'Bet you didn\'t see this coming.'
)
critical_messages = [mock_call[1][0] for mock_call in logger.critical.mock_calls]
for msg in expected_messages:
self.assertTrue(msg in critical_messages)
migration_modules_called = [
mock_call[1][1].name for mock_call in mocked_apply_migration.mock_calls]
# Note that none of the migrations that don't meet our criteria show up in this list. Also,
# Note that migration_packages.raise_exception.0003_shouldnt_run doesn't appear
# since migration_packages.raise_exception.0002_oh_no raised an Exception. Note
# also that even though the raise_exception package raised an Exception, we still run all
# the z migrations because we don't want one package to break another.
expected_migration_modules_called = [
'unit.server.db.migration_packages.platform.0001_stuff_and_junk',
'unit.server.db.migration_packages.raise_exception.0001_works_fine',
'unit.server.db.migration_packages.raise_exception.0002_oh_no']
self.assertEquals(migration_modules_called, expected_migration_modules_called)
# Assert that our precious versions have not been updated, since we have the --test flag
for package in models.get_migration_packages():
self.assertEqual(package.current_version, 0)
def test_migrate_with_new_packages(self, start_logging_mock, logger_mock, mocked_stderr):
"""
Adding new packages to a system that doesn't have any trackers should advance
each package to the latest available version, applying all migrate() functions along the way.
"""
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.get_collection().find({}).count(), 0)
# Make sure that our mock works. There are four valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
manage.main()
for package in models.get_migration_packages():
if 'raise_exception' in str(package):
# The Exception raising package should get to version 1, because version 2 raises
self.assertEqual(package.current_version, 1)
else:
# All other packages should reach their top versions
self.assertEqual(package.current_version, package.latest_available_version)
def migrate_database(options):
"""
Perform the migrations for each migration package found in pulp.server.db.migrations.
:param options: The command line parameters from the user
"""
migration_packages = models.get_migration_packages()
unperformed_migrations = False
for migration_package in migration_packages:
if migration_package.current_version > migration_package.latest_available_version:
msg = _(
"The database for migration package %(p)s is at version %(v)s, which is larger "
"than the latest version available, %(a)s."
)
msg = msg % (
{
"p": migration_package.name,
"v": migration_package.current_version,
"a": migration_package.latest_available_version,
}
)
raise DataError(msg)
if migration_package.current_version == migration_package.latest_available_version:
message = _("Migration package %(p)s is up to date at version %(v)s")
message = message % {"p": migration_package.name, "v": migration_package.latest_available_version}
_logger.info(message)
continue
try:
for migration in migration_package.unapplied_migrations:
message = _("Applying %(p)s version %(v)s")
message = message % {"p": migration_package.name, "v": migration.version}
_logger.info(message)
if options.dry_run:
unperformed_migrations = True
message = _("Would have applied migration to %(p)s version %(v)s")
message = message % {"p": migration_package.name, "v": migration.version}
else:
# We pass in !options.test to stop the apply_migration method from updating the
# package's current version when the --test flag is set
migration_package.apply_migration(migration, update_current_version=not options.test)
message = _("Migration to %(p)s version %(v)s complete.")
message = message % {"p": migration_package.name, "v": migration_package.current_version}
_logger.info(message)
except Exception:
# Log the error and what migration failed before allowing main() to handle the exception
error_message = _("Applying migration %(m)s failed.\n\nHalting migrations due to a " "migration failure.")
error_message = error_message % {"m": migration.name}
_logger.critical(error_message)
raise
if not options.dry_run:
ensure_database_indexes()
if options.dry_run and unperformed_migrations:
raise UnperformedMigrationException
def test_current_version_too_high(self, mocked_file_config, mocked_logger, mocked_stderr):
"""
Set the current package version higher than latest available version, then sit back and eat
popcorn.
"""
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.get_collection().find({}).count(), 0)
# Make sure that our mock works. There are four valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
# Set all versions to ridiculously high values
for package in models.get_migration_packages():
package._migration_tracker.version = 9999999
package._migration_tracker.save()
error_code = manage.main()
self.assertEqual(error_code, os.EX_DATAERR)
# There should have been a print to stderr about the Exception
expected_stderr_calls = [
'The database for migration package data.test_migration_packages.platform is at ' +\
'version 9999999, which is larger than the latest version available, 1.', '\n']
stderr_calls = [call[1][0] for call in mocked_stderr.mock_calls]
self.assertEquals(stderr_calls, expected_stderr_calls)
def test_migrate_with_new_packages(self, mocked_apply_migration):
"""
Adding new packages to a system that doesn't have any trackers should automatically advance
each package to the latest available version without calling any migrate() functions.
"""
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.get_collection().find({}).count(), 0)
# Make sure that our mock works. There are three valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
manage.main()
# No calls to apply_migration should have been made, and we should be at the latest package
# versions for each of the packages that have valid migrations.
self.assertFalse(mocked_apply_migration.called)
for package in models.get_migration_packages():
self.assertEqual(package.current_version, package.latest_available_version)
# Calling main() again should still not call apply_migration() or change the versions
manage.main()
self.assertFalse(mocked_apply_migration.called)
for package in models.get_migration_packages():
self.assertEqual(package.current_version, package.latest_available_version)
def test_migrate_with_test_flag(self, start_logging_mock, mocked_apply_migration, mocked_stdout,
mocked_stderr):
"""
Let's set all the packages to be at version 0, and then check that the migrations get called
in the correct order. We will also set the --test flag and ensure that the migration
versions do not get updated.
"""
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.get_collection().find({}).count(), 0)
# Make sure that our mock works. There are three valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
# Set all versions back to 0
for package in models.get_migration_packages():
package._migration_tracker.version = 0
package._migration_tracker.save()
manage.main()
# There should have been a print to stderr about the Exception
expected_stderr_calls = [
('Applying migration unit.server.db.migration_packages.raise_exception.0002_oh_no '
'failed.'), ' ', ' See log for details.', '\n']
stderr_calls = [mock_call[1][0] for mock_call in mocked_stderr.mock_calls]
self.assertEquals(stderr_calls, expected_stderr_calls)
migration_modules_called = [
mock_call[1][1].name for mock_call in mocked_apply_migration.mock_calls]
# Note that none of the migrations that don't meet our criteria show up in this list. Also,
# Note that migration_packages.raise_exception.0003_shouldnt_run doesn't appear
# since migration_packages.raise_exception.0002_oh_no raised an Exception. Note
# also that even though the raise_exception package raised an Exception, we still run all
# the z migrations because we don't want one package to break another.
expected_migration_modules_called = [
'unit.server.db.migration_packages.platform.0001_stuff_and_junk',
'unit.server.db.migration_packages.raise_exception.0001_works_fine',
'unit.server.db.migration_packages.raise_exception.0002_oh_no',
'unit.server.db.migration_packages.z.0001_test',
'unit.server.db.migration_packages.z.0002_test',
'unit.server.db.migration_packages.z.0003_test']
self.assertEquals(migration_modules_called, expected_migration_modules_called)
# Assert that our precious versions have not been updated, since we have the --test flag
for package in models.get_migration_packages():
self.assertEqual(package.current_version, 0)
def test_migrate_with_new_packages(self, initialize, start_logging_mock, logger_mock,
mocked_stdout, mocked_stderr):
"""
Adding new packages to a system that doesn't have any trackers should advance
each package to the latest available version, applying no migrate() functions along the
way.
"""
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.objects().count(), 0)
# Make sure that our mock works. There are five valid packages.
self.assertEquals(len(models.get_migration_packages()), 5)
manage.main()
for package in models.get_migration_packages():
if 'raise_exception' in str(package):
# The Exception raising package should get to version 3, despite the fact that
# version 2 raises an exception, because new trackers get fast-forwarded.
self.assertEqual(package.current_version, 3)
else:
# All other packages should reach their top versions
self.assertEqual(package.current_version, package.latest_available_version)
initialize.assert_called_once_with(max_timeout=1)
def migrate_database(options):
"""
Perform the migrations for each migration package found in pulp.server.db.migrations.
:param options: The command line parameters from the user
"""
migration_packages = models.get_migration_packages()
for migration_package in migration_packages:
if migration_package.current_version > migration_package.latest_available_version:
msg = _('The database for migration package %(p)s is at version %(v)s, which is larger '
'than the latest version available, %(a)s.')
msg = msg % ({'p': migration_package.name, 'v': migration_package.current_version,
'a': migration_package.latest_available_version})
raise DataError(msg)
if migration_package.current_version == migration_package.latest_available_version:
message = _('Migration package %(p)s is up to date at version %(v)s')
message = message % {'p': migration_package.name,
'v': migration_package.latest_available_version}
logger.info(message)
print message
continue
try:
for migration in migration_package.unapplied_migrations:
message = _('Applying %(p)s version %(v)s')
message = message % {'p': migration_package.name, 'v': migration.version}
print message
logger.info(message)
# We pass in !options.test to stop the apply_migration method from updating the
# package's current version when the --test flag is set
migration_package.apply_migration(migration,
update_current_version=not options.test)
message = _('Migration to %(p)s version %(v)s complete.')
message = message % {'p': migration_package.name,
'v': migration_package.current_version}
print message
logger.info(message)
except Exception, e:
# If an Exception is raised while applying the migrations, we should log and print it,
# and then continue with the other packages.
error_message = _('Applying migration %(m)s failed.')
error_message = error_message % {'m': migration.name}
print >> sys.stderr, str(error_message), _(' See log for details.')
logger.critical(error_message)
logger.critical(str(e))
logger.critical(''.join(traceback.format_exception(*sys.exc_info())))
def migrate_database(options):
"""
Perform the migrations for each migration package found in pulp.server.db.migrations.
:param options: The command line parameters from the user
"""
migration_packages = models.get_migration_packages()
for migration_package in migration_packages:
if migration_package.current_version > migration_package.latest_available_version:
msg = _('The database for migration package %(p)s is at version %(v)s, which is larger '
'than the latest version available, %(a)s.')
msg = msg % ({'p': migration_package.name, 'v': migration_package.current_version,
'a': migration_package.latest_available_version})
raise DataError(msg)
if migration_package.current_version == migration_package.latest_available_version:
message = _('Migration package %(p)s is up to date at version %(v)s')
message = message % {'p': migration_package.name,
'v': migration_package.latest_available_version}
logger.info(message)
print message
continue
try:
for migration in migration_package.unapplied_migrations:
message = _('Applying %(p)s version %(v)s')
message = message % {'p': migration_package.name, 'v': migration.version}
print message
logger.info(message)
# We pass in !options.test to stop the apply_migration method from updating the
# package's current version when the --test flag is set
migration_package.apply_migration(migration,
update_current_version=not options.test)
message = _('Migration to %(p)s version %(v)s complete.')
message = message % {'p': migration_package.name,
'v': migration_package.current_version}
print message
logger.info(message)
except Exception, e:
# Log and print the error and what migration failed before allowing main() to handle the exception
error_message = _('Applying migration %(m)s failed.\n\nHalting migrations due to a migration failure.')
error_message = error_message % {'m': migration.name}
print >> sys.stderr, str(error_message), _(' See log for details.')
logger.critical(error_message)
raise
def test_get_migration_packages(self, log_mock):
"""
Ensure that pulp.server.db.migrate.models.get_migration_packages functions correctly.
"""
packages = models.get_migration_packages()
self.assertEquals(len(packages), 4)
self.assertTrue(
all([isinstance(package, models.MigrationPackage) for package in packages]))
# Make sure that the packages are sorted correctly, with platform first
self.assertEquals(packages[0].name, 'unit.server.db.migration_packages.platform')
self.assertEquals(packages[1].name, 'unit.server.db.migration_packages.a')
self.assertEquals(packages[2].name, 'unit.server.db.migration_packages.raise_exception')
self.assertEquals(packages[3].name, 'unit.server.db.migration_packages.z')
# Assert that we logged the duplicate version exception and the version gap exception
expected_log_calls = [call('There are two migration modules that share version 2 in '
'unit.server.db.migration_packages.duplicate_versions.'),
call('Migration version 2 is missing in '
'unit.server.db.migration_packages.version_gap.')]
log_mock.assert_has_calls(expected_log_calls)
def migrate_database(options):
"""
Perform the migrations for each migration package found in pulp.server.db.migrations.
:param options: The command line parameters from the user
"""
migration_packages = models.get_migration_packages()
for migration_package in migration_packages:
if migration_package.current_version > migration_package.latest_available_version:
msg = _('The database for migration package %(p)s is at version %(v)s, which is larger '
'than the latest version available, %(a)s.')
msg = msg % ({'p': migration_package.name, 'v': migration_package.current_version,
'a': migration_package.latest_available_version})
raise DataError(msg)
if migration_package.current_version == migration_package.latest_available_version:
message = _('Migration package %(p)s is up to date at version %(v)s')
message = message % {'p': migration_package.name,
'v': migration_package.latest_available_version}
logger.info(message)
continue
try:
for migration in migration_package.unapplied_migrations:
message = _('Applying %(p)s version %(v)s')
message = message % {'p': migration_package.name, 'v': migration.version}
logger.info(message)
# We pass in !options.test to stop the apply_migration method from updating the
# package's current version when the --test flag is set
migration_package.apply_migration(migration,
update_current_version=not options.test)
message = _('Migration to %(p)s version %(v)s complete.')
message = message % {'p': migration_package.name,
'v': migration_package.current_version}
logger.info(message)
except Exception, e:
# Log the error and what migration failed before allowing main() to handle the exception
error_message = _('Applying migration %(m)s failed.\n\nHalting migrations due to a migration failure.')
error_message = error_message % {'m': migration.name}
logger.critical(error_message)
raise
def migrate_database(options):
"""
Perform the migrations for each migration package found in pulp.server.db.migrations.
:param options: The command line parameters from the user
"""
migration_packages = models.get_migration_packages()
unperformed_migrations = False
for migration_package in migration_packages:
if migration_package.current_version > migration_package.latest_available_version:
msg = _(
'The database for migration package %(p)s is at version %(v)s, which is larger '
'than the latest version available, %(a)s.')
msg = msg % ({
'p': migration_package.name,
'v': migration_package.current_version,
'a': migration_package.latest_available_version
})
raise DataError(msg)
if migration_package.current_version == migration_package.latest_available_version:
message = _(
'Migration package %(p)s is up to date at version %(v)s')
message = message % {
'p': migration_package.name,
'v': migration_package.latest_available_version
}
_logger.info(message)
continue
elif migration_package.current_version == -1 and migration_package.allow_fast_forward:
# -1 is the default for a brand-new tracker, so it indicates that no migrations
# previously existed. Thus we can skip the migrations and fast-forward to the latest
# version.
log_args = {
'v': migration_package.latest_available_version,
'p': migration_package.name
}
if options.dry_run:
unperformed_migrations = True
_logger.info(
_('Migration package %(p)s would have fast-forwarded '
'to version %(v)d' % log_args))
else:
# fast-forward if there is no pre-existing tracker
migration_package._migration_tracker.version = \
migration_package.latest_available_version
migration_package._migration_tracker.save()
_logger.info(
_('Migration package %(p)s fast-forwarded to '
'version %(v)d' % log_args))
continue
if migration_package.current_version == -1 and not migration_package.unapplied_migrations:
# for a new migration package with no migrations, go ahead and track it at version 0
log_args = {'n': migration_package.name}
if options.dry_run:
_logger.info(
_('Would have tracked migration %(n)s at version 0') %
log_args)
else:
_logger.info(
_('Tracking migration %(n)s at version 0') % log_args)
migration_package._migration_tracker.version = 0
migration_package._migration_tracker.save()
try:
for migration in migration_package.unapplied_migrations:
message = _('Applying %(p)s version %(v)s')
message = message % {
'p': migration_package.name,
'v': migration.version
}
_logger.info(message)
if options.dry_run:
unperformed_migrations = True
message = _(
'Would have applied migration to %(p)s version %(v)s')
message = message % {
'p': migration_package.name,
'v': migration.version
}
else:
# We pass in !options.test to stop the apply_migration method from updating the
# package's current version when the --test flag is set
migration_package.apply_migration(
migration, update_current_version=not options.test)
message = _('Migration to %(p)s version %(v)s complete.')
message = message % {
'p': migration_package.name,
'v': migration_package.current_version
}
_logger.info(message)
except models.MigrationRemovedError as e:
# keep the log message simpler than the generic message below.
_logger.critical(str(e))
raise
except Exception:
# Log the error and what migration failed before allowing main() to handle the exception
error_message = _(
'Applying migration %(m)s failed.\n\nHalting migrations due to a '
'migration failure.')
error_message = error_message % {'m': migration.name}
_logger.critical(error_message)
raise
if not options.dry_run:
ensure_database_indexes()
if options.dry_run and unperformed_migrations:
raise UnperformedMigrationException
def migrate_database(options):
"""
Perform the migrations for each migration package found in pulp.server.db.migrations.
:param options: The command line parameters from the user
"""
migration_packages = models.get_migration_packages()
unperformed_migrations = False
for migration_package in migration_packages:
if migration_package.current_version > migration_package.latest_available_version:
msg = _('The database for migration package %(p)s is at version %(v)s, which is larger '
'than the latest version available, %(a)s.')
msg = msg % ({'p': migration_package.name, 'v': migration_package.current_version,
'a': migration_package.latest_available_version})
raise DataError(msg)
if migration_package.current_version == migration_package.latest_available_version:
message = _('Migration package %(p)s is up to date at version %(v)s')
message = message % {'p': migration_package.name,
'v': migration_package.latest_available_version}
_logger.info(message)
continue
elif migration_package.current_version == -1 and migration_package.allow_fast_forward:
# -1 is the default for a brand-new tracker, so it indicates that no migrations
# previously existed. Thus we can skip the migrations and fast-forward to the latest
# version.
log_args = {
'v': migration_package.latest_available_version,
'p': migration_package.name
}
if options.dry_run:
unperformed_migrations = True
_logger.info(_('Migration package %(p)s would have fast-forwarded '
'to version %(v)d' % log_args))
else:
# fast-forward if there is no pre-existing tracker
migration_package._migration_tracker.version = \
migration_package.latest_available_version
migration_package._migration_tracker.save()
_logger.info(_('Migration package %(p)s fast-forwarded to '
'version %(v)d' % log_args))
continue
if migration_package.current_version == -1 and not migration_package.unapplied_migrations:
# for a new migration package with no migrations, go ahead and track it at version 0
log_args = {'n': migration_package.name}
if options.dry_run:
_logger.info(_('Would have tracked migration %(n)s at version 0') % log_args)
else:
_logger.info(_('Tracking migration %(n)s at version 0') % log_args)
migration_package._migration_tracker.version = 0
migration_package._migration_tracker.save()
try:
for migration in migration_package.unapplied_migrations:
message = _('Applying %(p)s version %(v)s')
message = message % {'p': migration_package.name, 'v': migration.version}
_logger.info(message)
if options.dry_run:
unperformed_migrations = True
message = _('Would have applied migration to %(p)s version %(v)s')
message = message % {'p': migration_package.name, 'v': migration.version}
else:
# We pass in !options.test to stop the apply_migration method from updating the
# package's current version when the --test flag is set
migration_package.apply_migration(migration,
update_current_version=not options.test)
message = _('Migration to %(p)s version %(v)s complete in %(t).3f seconds.')
message = message % {'p': migration_package.name,
't': migration_package.duration,
'v': migration_package.current_version}
_logger.info(message)
except models.MigrationRemovedError as e:
# keep the log message simpler than the generic message below.
_logger.critical(str(e))
raise
except Exception:
# Log the error and what migration failed before allowing main() to handle the exception
error_message = _('Applying migration %(m)s failed.\n\nHalting migrations due to a '
'migration failure.')
error_message = error_message % {'m': migration.name}
_logger.critical(error_message)
raise
if not options.dry_run:
ensure_database_indexes()
if options.dry_run and unperformed_migrations:
raise UnperformedMigrationException
def migrate_database(options):
"""
Perform the migrations for each migration package found in pulp.server.db.migrations.
Create indexes before running any migration to avoid duplicates, e.g. in case a new collection
is created.
WARNING: If any unapplied migrations include a "prepare_reindex_migration" function, these
functions will be run before indexes are applied. This is before ANY additional regular
migrations are applied, irrespective of what migration version the DB is currently in.
In addition, execution of these functions is not tracked independently of the regular migration
they are associated with. This means they may be reapplied (even if successfull) until the
associated regular migration has been successfully applied.
prepare_reindex_migration functions should only be used, to prepare database fields and indexes
that would otherwise prevent a successfull index change.
:param options: The command line parameters from the user
"""
migration_packages = models.get_migration_packages()
unperformed_migrations = False
for migration_package in migration_packages:
if migration_package.current_version > migration_package.latest_available_version:
msg = _(
'The database for migration package %(p)s is at version %(v)s, which is larger '
'than the latest version available, %(a)s.')
msg = msg % ({
'p': migration_package.name,
'v': migration_package.current_version,
'a': migration_package.latest_available_version
})
raise DataError(msg)
if migration_package.current_version == migration_package.latest_available_version:
message = _(
'Migration package %(p)s is up to date at version %(v)s')
message = message % {
'p': migration_package.name,
'v': migration_package.latest_available_version
}
_logger.info(message)
continue
elif migration_package.current_version == -1 and migration_package.allow_fast_forward:
# -1 is the default for a brand-new tracker, so it indicates that no migrations
# previously existed. Thus we can skip the migrations and fast-forward to the latest
# version.
log_args = {
'v': migration_package.latest_available_version,
'p': migration_package.name
}
if options.dry_run:
unperformed_migrations = True
_logger.info(
_('Migration package %(p)s would have fast-forwarded '
'to version %(v)d' % log_args))
else:
# fast-forward if there is no pre-existing tracker
migration_package._migration_tracker.version = \
migration_package.latest_available_version
migration_package._migration_tracker.save()
_logger.info(
_('Migration package %(p)s fast-forwarded to '
'version %(v)d' % log_args))
continue
if migration_package.current_version == -1 and not migration_package.unapplied_migrations:
# for a new migration package with no migrations, go ahead and track it at version 0
log_args = {'n': migration_package.name}
if options.dry_run:
_logger.info(
_('Would have tracked migration %(n)s at version 0') %
log_args)
else:
_logger.info(
_('Tracking migration %(n)s at version 0') % log_args)
migration_package._migration_tracker.version = 0
migration_package._migration_tracker.save()
try:
for migration in migration_package.unapplied_migrations:
if hasattr(migration, 'prepare_reindex_migration'):
message = _('Applying prepare part of %(p)s version %(v)s')
message = message % {
'p': migration_package.name,
'v': migration.version
}
_logger.info(message)
if options.dry_run:
unperformed_migrations = True
message = _(
'Would have applied prepare part of migration to %(p)s version %(v)s'
)
message = message % {
'p': migration_package.name,
'v': migration.version
}
else:
migration_package.apply_prepare_reindex_migration(
migration)
message = _(
'Prepare part of migration to %(p)s version %(v)s complete in %(t).3f seconds.'
) # noqa
message = message % {
'p': migration_package.name,
't': migration_package.duration,
'v': migration_package.current_version
}
_logger.info(message)
except models.MigrationRemovedError as e:
# keep the log message simpler than the generic message below.
_logger.critical(str(e))
raise
except Exception:
# Log the error and what migration failed before allowing main() to handle the exception
error_message = _(
'Applying migration %(m)s failed.\n\nHalting migrations due to a '
'migration failure.')
error_message = error_message % {'m': migration.name}
_logger.critical(error_message)
raise
if not options.dry_run:
ensure_database_indexes()
for migration_package in migration_packages:
try:
for migration in migration_package.unapplied_migrations:
message = _('Applying %(p)s version %(v)s')
message = message % {
'p': migration_package.name,
'v': migration.version
}
_logger.info(message)
if options.dry_run:
unperformed_migrations = True
message = _(
'Would have applied migration to %(p)s version %(v)s')
message = message % {
'p': migration_package.name,
'v': migration.version
}
else:
# We pass in !options.test to stop the apply_migration method from updating the
# package's current version when the --test flag is set
migration_package.apply_migration(
migration, update_current_version=not options.test)
message = _(
'Migration to %(p)s version %(v)s complete in %(t).3f seconds.'
)
message = message % {
'p': migration_package.name,
't': migration_package.duration,
'v': migration_package.current_version
}
_logger.info(message)
except models.MigrationRemovedError as e:
# keep the log message simpler than the generic message below.
_logger.critical(str(e))
raise
except Exception:
# Log the error and what migration failed before allowing main() to handle the exception
error_message = _(
'Applying migration %(m)s failed.\n\nHalting migrations due to a '
'migration failure.')
error_message = error_message % {'m': migration.name}
_logger.critical(error_message)
raise
if options.dry_run and unperformed_migrations:
raise UnperformedMigrationException