def __init__(self, python_package):
"""
Initialize the MigrationPackage to represent the Python migration package passed in.
:param python_package: The Python package this object should represent
:type python_package: package
"""
self._package = python_package
self.allow_fast_forward = getattr(python_package, 'allow_fast_forward',
False)
self.duration = None
try:
self._migration_tracker = MigrationTracker.objects().get(
name=self.name)
except DoesNotExist:
self._migration_tracker = MigrationTracker(name=self.name)
self._migration_tracker.save()
# Calculate the latest available version
available_versions = self.available_versions
if available_versions:
self.latest_available_version = available_versions[-1]
else:
self.latest_available_version = 0
def test_migrate_with_dry_run_flag(self, mock_file_config, initialize, mocked_apply_migration,
getLogger):
"""
Test that when a dry run is performed, no migrations actually occur.
"""
logger = MagicMock()
getLogger.return_value = logger
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.objects().count(), 0)
# Make sure that our mock works. There are three valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
# Set all versions back to 0
for package in models.get_migration_packages():
package._migration_tracker.version = 0
package._migration_tracker.save()
result = manage.main()
# Test that none of the mock objects were actually called
migration_modules_called = [
mock_call[1][1].name for mock_call in mocked_apply_migration.mock_calls]
self.assertEquals(0, len(migration_modules_called))
self.assertEquals(1, result)
for package in models.get_migration_packages():
self.assertEqual(package.current_version, 0)
initialize.assert_called_once_with(max_timeout=1)
def test_current_version_too_high(self, mocked_file_config, initialize, getLogger):
"""
Set the current package version higher than latest available version, then sit back and eat
popcorn.
"""
logger = MagicMock()
getLogger.return_value = logger
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.objects().count(), 0)
# Make sure that our mock works. There are four valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
# Set all versions to ridiculously high values
for package in models.get_migration_packages():
package._migration_tracker.version = 9999999
package._migration_tracker.save()
error_code = manage.main()
self.assertEqual(error_code, os.EX_DATAERR)
# There should have been a critical log about the Exception
expected_messages = (
'The database for migration package unit.server.db.migration_packages.'
'platform is at version 9999999, which is larger than the latest version available, 1.')
critical_messages = ''.join([mock_call[1][0] for mock_call in logger.critical.mock_calls])
for msg in expected_messages:
self.assertTrue(msg in critical_messages)
initialize.assert_called_once_with(max_timeout=1)
def test_migrate_with_new_packages(self, initialize, start_logging_mock,
logger_mock, mocked_stdout,
mocked_stderr):
"""
Adding new packages to a system that doesn't have any trackers should advance
each package to the latest available version, applying all migrate() functions along the
way.
"""
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.objects().count(), 0)
# Make sure that our mock works. There are four valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
manage.main()
for package in models.get_migration_packages():
if 'raise_exception' in str(package):
# The Exception raising package should get to version 1, because version 2 raises
self.assertEqual(package.current_version, 1)
elif 'z' in str(package):
self.assertEquals(package.current_version, 0)
else:
# All other packages should reach their top versions
self.assertEqual(package.current_version,
package.latest_available_version)
initialize.assert_called_once_with(max_timeout=1)
def test_migrate(self, mock_file_config, initialize,
mocked_apply_migration, getLogger):
"""
Let's set all the packages to be at version 0, and then check that the migrations get
called in the correct order.
"""
logger = MagicMock()
getLogger.return_value = logger
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.objects().count(), 0)
# Make sure that our mock works. There are three valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
# Set all versions back to 0
for package in models.get_migration_packages():
package._migration_tracker.version = 0
package._migration_tracker.save()
manage.main()
# There should have been a critical log about the Exception
expected_messages = (
'Applying migration '
'unit.server.db.migration_packages.raise_exception.0002_oh_no failed.\n\n'
'Halting migrations due to a migration failure.',
"Bet you didn\'t see this coming.")
critical_messages = ''.join(
[mock_call[1][0] for mock_call in logger.critical.mock_calls])
for msg in expected_messages:
self.assertTrue(msg in critical_messages)
migration_modules_called = [
mock_call[1][1].name
for mock_call in mocked_apply_migration.mock_calls
]
# Note that none of the migrations that don't meet our criteria show up in this list. Also,
# Note that migration_packages.raise_exception.0003_shouldnt_run doesn't appear
# since migration_packages.raise_exception.0002_oh_no raised an Exception. Note
# also that even though the raise_exception package raised an Exception, we still run all
# the z migrations because we don't want one package to break another.
expected_migration_modules_called = [
'unit.server.db.migration_packages.platform.0001_stuff_and_junk',
'unit.server.db.migration_packages.raise_exception.0001_works_fine',
'unit.server.db.migration_packages.raise_exception.0002_oh_no'
]
self.assertEquals(migration_modules_called,
expected_migration_modules_called)
# Assert that our precious versions have been updated correctly
for package in models.get_migration_packages():
if package.name == 'unit.server.db.migration_packages.platform':
self.assertEqual(package.current_version,
package.latest_available_version)
elif package.name == 'unit.server.db.migration_packages.raise_exception':
# The raised Exception should have prevented us from getting past version 1
self.assertEquals(package.current_version, 1)
else:
# raise_exception should cause the migrations to stop
self.assertEqual(package.current_version, 0)
initialize.assert_called_once_with(max_timeout=1)
def test_migrate(self, mock_file_config, initialize, mocked_apply_migration, getLogger):
"""
Let's set all the packages to be at version 0, and then check that the migrations get
called in the correct order.
"""
logger = MagicMock()
getLogger.return_value = logger
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.objects().count(), 0)
# Make sure that our mock works. There are three valid packages.
self.assertEquals(len(models.get_migration_packages()), 4)
# Set all versions back to 0
for package in models.get_migration_packages():
package._migration_tracker.version = 0
package._migration_tracker.save()
manage.main()
# There should have been a critical log about the Exception
expected_messages = (
'Applying migration '
'unit.server.db.migration_packages.raise_exception.0002_oh_no failed.\n\n'
'Halting migrations due to a migration failure.',
"Bet you didn\'t see this coming."
)
critical_messages = ''.join([mock_call[1][0] for mock_call in logger.critical.mock_calls])
for msg in expected_messages:
self.assertTrue(msg in critical_messages)
migration_modules_called = [
mock_call[1][1].name for mock_call in mocked_apply_migration.mock_calls]
# Note that none of the migrations that don't meet our criteria show up in this list. Also,
# Note that migration_packages.raise_exception.0003_shouldnt_run doesn't appear
# since migration_packages.raise_exception.0002_oh_no raised an Exception. Note
# also that even though the raise_exception package raised an Exception, we still run all
# the z migrations because we don't want one package to break another.
expected_migration_modules_called = [
'unit.server.db.migration_packages.platform.0001_stuff_and_junk',
'unit.server.db.migration_packages.raise_exception.0001_works_fine',
'unit.server.db.migration_packages.raise_exception.0002_oh_no']
self.assertEquals(migration_modules_called, expected_migration_modules_called)
# Assert that our precious versions have been updated correctly
for package in models.get_migration_packages():
if package.name == 'unit.server.db.migration_packages.platform':
self.assertEqual(package.current_version, package.latest_available_version)
elif package.name == 'unit.server.db.migration_packages.raise_exception':
# The raised Exception should have prevented us from getting past version 1
self.assertEquals(package.current_version, 1)
else:
# raise_exception should cause the migrations to stop
self.assertEqual(package.current_version, 0)
initialize.assert_called_once_with(max_timeout=1)
def __init__(self, python_package):
"""
Initialize the MigrationPackage to represent the Python migration package passed in.
:param python_package: The Python package this object should represent
:type python_package: package
"""
self._package = python_package
self.allow_fast_forward = getattr(python_package, 'allow_fast_forward', False)
try:
self._migration_tracker = MigrationTracker.objects().get(name=self.name)
except DoesNotExist:
self._migration_tracker = MigrationTracker(name=self.name)
self._migration_tracker.save()
# Calculate the latest available version
available_versions = self.available_versions
if available_versions:
self.latest_available_version = available_versions[-1]
else:
self.latest_available_version = 0
def __init__(self, python_package):
"""
Initialize the MigrationPackage to represent the Python migration package passed in.
:param python_package: The Python package this object should represent
:type python_package: package
"""
self._package = python_package
# This is an object representation of the DB object that keeps track of the migration
# version that has been applied
try:
self._migration_tracker = MigrationTracker.objects().get(name=self.name)
except DoesNotExist:
self._migration_tracker = MigrationTracker(name=self.name)
self._migration_tracker.save()
# Calculate the latest available version
available_versions = self.available_versions
if available_versions:
self.latest_available_version = available_versions[-1]
else:
self.latest_available_version = 0
def __init__(self, python_package):
"""
Initialize the MigrationPackage to represent the Python migration package passed in.
:param python_package: The Python package this object should represent
:type python_package: package
"""
self._package = python_package
# This is an object representation of the DB object that keeps track of the migration
# version that has been applied
try:
self._migration_tracker = MigrationTracker.objects().get(name=self.name)
except DoesNotExist:
self._migration_tracker = MigrationTracker(name=self.name)
self._migration_tracker.save()
# Calculate the latest available version
available_versions = self.available_versions
if available_versions:
self.latest_available_version = available_versions[-1]
else:
self.latest_available_version = 0
def test_migrate_with_new_packages(self, initialize, start_logging_mock, logger_mock,
mocked_stdout, mocked_stderr):
"""
Adding new packages to a system that doesn't have any trackers should advance
each package to the latest available version, applying no migrate() functions along the
way.
"""
# Make sure we start out with a clean slate
self.assertEquals(MigrationTracker.objects().count(), 0)
# Make sure that our mock works. There are five valid packages.
self.assertEquals(len(models.get_migration_packages()), 5)
manage.main()
for package in models.get_migration_packages():
if 'raise_exception' in str(package):
# The Exception raising package should get to version 3, despite the fact that
# version 2 raises an exception, because new trackers get fast-forwarded.
self.assertEqual(package.current_version, 3)
else:
# All other packages should reach their top versions
self.assertEqual(package.current_version, package.latest_available_version)
initialize.assert_called_once_with(max_timeout=1)
def clean(self):
super(MigrationTest, self).clean()
# Make sure each test doesn't have any lingering MigrationTrackers
MigrationTracker.objects().delete()
class MigrationPackage(object):
"""
A wrapper around the migration packages found in pulp.server.db.migrations. Has methods to
retrieve information about the migrations that are found there, and to apply the migrations.
"""
class DuplicateVersions(Exception):
"""
This is raised when a single migration package has two MigrationModules in it that have the
same version.
"""
pass
class MissingVersion(Exception):
"""
This is raised when a migration package has a gap in the MigrationModule versions.
"""
pass
def __init__(self, python_package):
"""
Initialize the MigrationPackage to represent the Python migration package passed in.
:param python_package: The Python package this object should represent
:type python_package: package
"""
self._package = python_package
# This is an object representation of the DB object that keeps track of the migration
# version that has been applied
try:
self._migration_tracker = MigrationTracker.objects().get(name=self.name)
except DoesNotExist:
self._migration_tracker = MigrationTracker(name=self.name)
self._migration_tracker.save()
# Calculate the latest available version
available_versions = self.available_versions
if available_versions:
self.latest_available_version = available_versions[-1]
else:
self.latest_available_version = 0
def apply_migration(self, migration, update_current_version=True):
"""
Apply the migration that is passed in, and update the DB to note the new version that this
migration represents.
:param migration: The migration to apply
:type migration: pulp.server.db.migrate.utils.MigrationModule
:param update_current_version: If True, update the package's current version after
successful application and enforce migration version order.
If False, don't enforce and don't update.
:type update_current_version: bool
"""
if update_current_version and migration.version != self.current_version + 1:
msg = _('Cannot apply migration %(name)s, because the next migration version is '
'%(version)s.')
msg = msg % {'name': migration.name, 'version': self.current_version + 1}
raise Exception(msg)
migration.migrate()
if update_current_version:
self._migration_tracker.version = migration.version
self._migration_tracker.save()
@property
def available_versions(self):
"""
Return a list of the migration versions that are available in this migration package.
:rtype: list
"""
migrations = self.migrations
versions = [migration.version for migration in migrations]
return versions
@property
def current_version(self):
"""
An integer that represents the migration version that the database is currently at.
None means that the migration package has never been run before.
:rtype: int
"""
return self._migration_tracker.version
@property
def migrations(self):
"""
Finds all available migration modules for the MigrationPackage,
and then sorts by the version. Return a list of MigrationModules.
:rtype: list
"""
# Generate a list of the names of the modules found inside this package
module_names = [name for module_loader, name, ispkg in
iter_modules([os.path.dirname(self._package.__file__)])]
migration_modules = []
for module_name in module_names:
try:
module_name = '%s.%s' % (self.name, module_name)
migration_modules.append(MigrationModule(module_name))
except MigrationModule.MissingMigrate:
msg = _("The module %(m)s doesn't have a migrate function. It will be ignored.")
msg = msg % {'m': module_name}
_logger.debug(msg)
except MigrationModule.MissingVersion:
msg = _("The module %(m)s doesn't conform to the migration package naming "
"conventions. It will be ignored.")
msg = msg % {'m': module_name}
_logger.debug(msg)
migration_modules.sort()
# We should have migrations starting at version 1, which each module version being exactly
# one larger than the migration preceeding it.
last_version = 0
for module in migration_modules:
if module.version == 0:
error_message = _('0 is a reserved migration version number, but the '
'module %(n)s has been assigned that version.')
error_message = error_message % {'n': module.name}
raise self.__class__.DuplicateVersions(error_message)
if module.version == last_version:
error_message = _('There are two migration modules that share version %(v)s in '
'%(n)s.')
error_message = error_message % {'v': module.version, 'n': self.name}
raise self.__class__.DuplicateVersions(error_message)
if module.version != last_version + 1:
msg = _('Migration version %(v)s is missing in %(n)s.')
msg = msg % ({'v': last_version + 1, 'n': self.name})
raise self.__class__.MissingVersion(msg)
last_version = module.version
return migration_modules
@property
def name(self):
"""
Returns the name of the Python package that this MigrationPackage represents.
:rtype: str
"""
return self._package.__name__
@property
def unapplied_migrations(self):
"""
Return a list of MigrationModules in this package that have not been applied yet.
:rtype: list
"""
return [migration for migration in self.migrations
if migration.version > self.current_version]
def __cmp__(self, other_package):
"""
This method returns a negative value if self.name < other_package.name, 0 if they are
equal, and a positive value if self.name > other_package.name. There is an exception to this
sorting rule, in that if self._package is pulp.server.db.migrations, this method will always
return -1, and if other_package is pulp.server.db.migrations, it will always return 1.
:rtype: int
"""
if self._package is pulp.server.db.migrations:
return -1
if other_package._package is pulp.server.db.migrations:
return 1
return cmp(self.name, other_package.name)
def __str__(self):
return self.name
def __repr__(self):
return str(self)
class MigrationPackage(object):
"""
A wrapper around the migration packages found in pulp.server.db.migrations. Has methods to
retrieve information about the migrations that are found there, and to apply the migrations.
"""
class DuplicateVersions(Exception):
"""
This is raised when a single migration package has two MigrationModules in it that have the
same version.
"""
pass
class MissingVersion(Exception):
"""
This is raised when a migration package has a gap in the MigrationModule versions.
"""
pass
def __init__(self, python_package):
"""
Initialize the MigrationPackage to represent the Python migration package passed in.
:param python_package: The Python package this object should represent
:type python_package: package
"""
self._package = python_package
# This is an object representation of the DB object that keeps track of the migration
# version that has been applied
try:
self._migration_tracker = MigrationTracker.objects().get(name=self.name)
except DoesNotExist:
self._migration_tracker = MigrationTracker(name=self.name)
self._migration_tracker.save()
# Calculate the latest available version
available_versions = self.available_versions
if available_versions:
self.latest_available_version = available_versions[-1]
else:
self.latest_available_version = 0
def apply_migration(self, migration, update_current_version=True):
"""
Apply the migration that is passed in, and update the DB to note the new version that this
migration represents.
:param migration: The migration to apply
:type migration: pulp.server.db.migrate.utils.MigrationModule
:param update_current_version: If True, update the package's current version after
successful application and enforce migration version order.
If False, don't enforce and don't update.
:type update_current_version: bool
"""
if update_current_version and migration.version != self.current_version + 1:
msg = _("Cannot apply migration %(name)s, because the next migration version is " "%(version)s.")
msg = msg % {"name": migration.name, "version": self.current_version + 1}
raise Exception(msg)
migration.migrate()
if update_current_version:
self._migration_tracker.version = migration.version
self._migration_tracker.save()
@property
def available_versions(self):
"""
Return a list of the migration versions that are available in this migration package.
:rtype: list
"""
migrations = self.migrations
versions = [migration.version for migration in migrations]
return versions
@property
def current_version(self):
"""
An integer that represents the migration version that the database is currently at.
None means that the migration package has never been run before.
:rtype: int
"""
return self._migration_tracker.version
@property
def migrations(self):
"""
Finds all available migration modules for the MigrationPackage,
and then sorts by the version. Return a list of MigrationModules.
:rtype: list
"""
# Generate a list of the names of the modules found inside this package
module_names = [name for module_loader, name, ispkg in iter_modules([os.path.dirname(self._package.__file__)])]
migration_modules = []
for module_name in module_names:
try:
module_name = "%s.%s" % (self.name, module_name)
migration_modules.append(MigrationModule(module_name))
except MigrationModule.MissingMigrate:
msg = _("The module %(m)s doesn't have a migrate function. It will be ignored.")
msg = msg % {"m": module_name}
_logger.debug(msg)
except MigrationModule.MissingVersion:
msg = _(
"The module %(m)s doesn't conform to the migration package naming "
"conventions. It will be ignored."
)
msg = msg % {"m": module_name}
_logger.debug(msg)
migration_modules.sort()
# We should have migrations starting at version 1, which each module version being exactly
# one larger than the migration preceeding it.
last_version = 0
for module in migration_modules:
if module.version == 0:
error_message = _(
"0 is a reserved migration version number, but the " "module %(n)s has been assigned that version."
)
error_message = error_message % {"n": module.name}
raise self.__class__.DuplicateVersions(error_message)
if module.version == last_version:
error_message = _("There are two migration modules that share version %(v)s in " "%(n)s.")
error_message = error_message % {"v": module.version, "n": self.name}
raise self.__class__.DuplicateVersions(error_message)
if module.version != last_version + 1:
msg = _("Migration version %(v)s is missing in %(n)s.")
msg = msg % ({"v": last_version + 1, "n": self.name})
raise self.__class__.MissingVersion(msg)
last_version = module.version
return migration_modules
@property
def name(self):
"""
Returns the name of the Python package that this MigrationPackage represents.
:rtype: str
"""
return self._package.__name__
@property
def unapplied_migrations(self):
"""
Return a list of MigrationModules in this package that have not been applied yet.
:rtype: list
"""
return [migration for migration in self.migrations if migration.version > self.current_version]
def __cmp__(self, other_package):
"""
This method returns a negative value if self.name < other_package.name, 0 if they are
equal, and a positive value if self.name > other_package.name. There is an exception to this
sorting rule, in that if self._package is pulp.server.db.migrations, this method will always
return -1, and if other_package is pulp.server.db.migrations, it will always return 1.
:rtype: int
"""
if self._package is pulp.server.db.migrations:
return -1
if other_package._package is pulp.server.db.migrations:
return 1
return cmp(self.name, other_package.name)
def __str__(self):
return self.name
def __repr__(self):
return str(self)
