async def migrate_repositories(plan):
"""
A coroutine to migrate pre-migrated repositories.
"""
repos_to_create = plan.get_pulp3_repository_setup().keys()
progress_data = dict(message='Creating repositories in Pulp 3',
code='creating.repositories',
total=0)
with ProgressReport(**progress_data) as pb:
pulp2repos_qs = Pulp2Repository.objects.filter(
pulp3_repository_version=None)
# no specific migration plan for repositories
if not repos_to_create:
pb.total += pulp2repos_qs.count()
pb.save()
for pulp2repo in pulp2repos_qs:
pulp3_repo_name = pulp2repo.pulp2_repo_id
repository_class = PLUGIN_MIGRATORS.get(
pulp2repo.type).pulp3_repository
repo, created = repository_class.objects.get_or_create(
name=pulp3_repo_name,
description=pulp2repo.pulp2_description)
if created:
pb.increment()
else:
pb.total -= 1
pb.save()
# specific migration plan for repositories
else:
pb.total += len(repos_to_create)
pb.save()
for pulp3_repo_name in repos_to_create:
try:
pulp2repo = pulp2repos_qs.get(
pulp2_repo_id=pulp3_repo_name)
except Pulp2Repository.DoesNotExist:
description = pulp3_repo_name
else:
description = pulp2repo.pulp2_description
repository_class = PLUGIN_MIGRATORS.get(
pulp2repo.type).pulp3_repository
repo, created = repository_class.objects.get_or_create(
name=pulp3_repo_name, description=description)
if created:
pb.increment()
else:
pb.total -= 1
pb.save()
async def migrate_content(plan):
"""
A coroutine to initiate content migration for each plugin.
Args:
plan (MigrationPlan): Migration Plan to use
"""
content_migration_coros = []
plugins_to_migrate = plan.get_plugins()
progress_data = dict(message='Migrating content to Pulp 3',
code='migrating.content',
total=0)
with ProgressReport(**progress_data) as pb:
# schedule content migration into Pulp 3 using pre-migrated Pulp 2 content
for plugin in plugins_to_migrate:
plugin_migrator = PLUGIN_MIGRATORS.get(plugin)
content_migration_coros.append(
plugin_migrator.migrate_content_to_pulp3())
# only used for progress bar counters
content_types = plugin_migrator.content_models.keys()
pulp2content_qs = Pulp2Content.objects.filter(
pulp2_content_type_id__in=content_types, pulp3_content=None)
pb.total += pulp2content_qs.count()
pb.save()
await asyncio.wait(content_migration_coros)
pb.done = pb.total
def _parse_plugin_plan(self, repository_data):
# Circular import avoidance
from pulp_2to3_migration.app.plugin import PLUGIN_MIGRATORS
self.type = repository_data['type']
self.migrator = PLUGIN_MIGRATORS.get(self.type)
repositories = repository_data.get('repositories')
if repositories:
self.empty = False
for repository in repositories:
name = repository['name']
_find_importer_repo = repository[
'pulp2_importer_repository_id']
self.repositories_importers_to_migrate.append(
_find_importer_repo)
repository_versions = []
for repository_version in repository.get(
'repository_versions', []):
pulp2_repository_id = repository_version[
'pulp2_repository_id']
self.repositories_to_migrate.append(pulp2_repository_id)
repository_versions.append(pulp2_repository_id)
distributor_ids = repository_version.get(
'distributor_ids', [])
self.repositories_distributors_to_migrate.extend(
distributor_ids)
self.repositories_to_create[name] = {
"pulp2_importer_repository_id": _find_importer_repo,
"versions": repository_versions
}
def create_repo_version(pulp3_repo_name, pulp2_repo):
"""
Create a repo version based on a pulp2 repository
Args:
pulp3_repo_name(str): repository name in Pulp 3
pulp2_repo(Pulp2Repository): a pre-migrated repository to create a repo version for
"""
repository_class = PLUGIN_MIGRATORS.get(
pulp2_repo.type).pulp3_repository
pulp3_repo = repository_class.objects.get(name=pulp3_repo_name)
unit_ids = Pulp2RepoContent.objects.filter(
pulp2_repository=pulp2_repo).values_list('pulp2_unit_id',
flat=True)
incoming_content = set(
Pulp2Content.objects.filter(
pulp2_id__in=unit_ids).only('pulp3_content').values_list(
'pulp3_content__pk', flat=True))
with pulp3_repo.new_version() as new_version:
repo_content = set(new_version.content.values_list('pk',
flat=True))
to_add = incoming_content - repo_content
to_delete = repo_content - incoming_content
new_version.add_content(Content.objects.filter(pk__in=to_add))
new_version.remove_content(
Content.objects.filter(pk__in=to_delete))
return new_version
async def pre_migrate_all_content(plan):
"""
Pre-migrate all content for the specified plugins.
Args:
plan (MigrationPlan): Migration Plan to use for migration.
"""
plugins_to_migrate = plan.get_plugins()
pre_migrators = []
# get all the content models for the migrating plugins
for plugin, plugin_migrator in PLUGIN_MIGRATORS.items():
if plugin not in plugins_to_migrate:
continue
for content_type in plugin_migrator.pulp2_content_models:
# mongodb model
pulp2_content_model = plugin_migrator.pulp2_content_models[content_type]
# postgresql model
pulp_2to3_detail_model = plugin_migrator.content_models[content_type]
content_model = ContentModel(pulp2=pulp2_content_model,
pulp_2to3_detail=pulp_2to3_detail_model)
pre_migrators.append(pre_migrate_content(content_model))
_logger.debug('Pre-migrating Pulp 2 content')
await asyncio.wait(pre_migrators)
def validate(self, data):
"""
Validate that the Serializer contains valid data.
Validates JSON structure of migration_plan.
Checks pulp2 and pulp3 plugins are installed.
"""
schema = json.loads(SCHEMA)
validator = Draft7Validator(schema)
if isinstance(data['plan'], str):
loaded_plan = json.loads(data['plan'])
elif isinstance(data['plan'], dict):
loaded_plan = data['plan']
else:
raise serializers.ValidationError(
_("Must provide a (JSON-encoded) string or dict for 'plan', not list")
)
err = []
for error in sorted(validator.iter_errors(loaded_plan), key=str):
err.append(error.message)
if err:
raise serializers.ValidationError(
_("Provided Migration Plan format is invalid:'{}'".format(err))
)
plugins_to_migrate = set()
for plugin_type in loaded_plan['plugins']:
plugins_to_migrate.add(plugin_type['type'])
if len(loaded_plan['plugins']) != len(plugins_to_migrate):
raise serializers.ValidationError(
_("Provided Migration Plan contains same plugin type specified more that once.")
)
# MongoDB connection initialization
connection.initialize()
db = connection.get_database()
for plugin in plugins_to_migrate:
plugin_migrator = PLUGIN_MIGRATORS.get(plugin)
if not plugin_migrator:
raise serializers.ValidationError(
_("Migration of {} plugin is not supported.".format(plugin))
)
if plugin_migrator.pulp3_plugin not in INSTALLED_PULP_PLUGINS:
raise serializers.ValidationError(
_("Plugin {} is not installed in pulp3.".format(plugin))
)
try:
db.command("collstats", plugin_migrator.pulp2_collection)
except OperationFailure:
raise serializers.ValidationError(
_("Plugin {} is not installed in pulp2.".format(plugin))
)
data['plan'] = loaded_plan
return data
def _parse_plugin_plan(self, repository_data):
# Circular import avoidance
from pulp_2to3_migration.app.plugin import PLUGIN_MIGRATORS
self.type = repository_data['type']
self.migrator = PLUGIN_MIGRATORS.get(self.type)
repositories = repository_data.get('repositories')
if repositories:
self.empty = False
for repository in repositories:
name = repository['name']
importer_repo_id = repository.get(
'pulp2_importer_repository_id')
if importer_repo_id:
self.repositories_importers_to_migrate.append(
importer_repo_id)
repository_versions = []
for repository_version in repository.get(
'repository_versions', []):
pulp2_repository_id = repository_version[
'pulp2_repository_id']
self.repositories_to_migrate.append(pulp2_repository_id)
distributor_repo_ids = repository_version.get(
'pulp2_distributor_repository_ids', [])
self.repositories_distributors_to_migrate.extend(
distributor_repo_ids)
repository_versions.append({
'repo_id':
pulp2_repository_id,
'dist_repo_ids':
distributor_repo_ids
})
signing_service = repository.get("signing_service")
RepoSetup.set_importer(pulp2_repository_id, self.type,
importer_repo_id)
RepoSetup.set_distributors(pulp2_repository_id, self.type,
distributor_repo_ids)
self.repositories_to_create[name] = {
"pulp2_importer_repository_id": importer_repo_id,
"repository_versions": repository_versions,
"signing_service": signing_service,
}
async def migrate_importers(plan):
"""
A coroutine to migrate pre-migrated importers.
Args:
plan (MigrationPlan): Migration Plan to use.
"""
# gather all needed plugin importer migrators
importer_migrators = {}
plugins_to_migrate = plan.get_plugins()
for plugin, plugin_migrator in PLUGIN_MIGRATORS.items():
if plugin not in plugins_to_migrate:
continue
importer_migrators.update(**plugin_migrator.importer_migrators)
progress_data = dict(message='Migrating importers to Pulp 3',
code='migrating.importers',
total=0)
with ProgressReport(**progress_data) as pb:
# Temp fix until https://pulp.plan.io/issues/5485 is done
pulp2importers_qs = Pulp2Importer.objects.filter(
pulp2_type_id__in=importer_migrators.keys(), pulp3_remote=None)
pb.total += pulp2importers_qs.count()
pb.save()
for pulp2importer in pulp2importers_qs:
importer_migrator = importer_migrators.get(
pulp2importer.pulp2_type_id)
remote, created = await importer_migrator.migrate_to_pulp3(
pulp2importer)
pulp2importer.pulp3_remote = remote
pulp2importer.is_migrated = True
pulp2importer.save()
if created:
pb.increment()
else:
pb.total -= 1
pb.save()
def complex_repo_migration(plugin_type, pulp3_repo_setup, repo_name):
"""Perform a complex migration for a particular repo using the repo setup config.
Create all repository versions, publications, distributions.
Args:
plugin_type(str): Plugin type
pulp3_repo_setup: Pulp 3 repo setup config for a plugin
repo_name: Name of the repo to be migrated
"""
from pulp_2to3_migration.app.plugin import PLUGIN_MIGRATORS
migrator = PLUGIN_MIGRATORS.get(plugin_type)
distributor_migrators = migrator.distributor_migrators
distributor_types = list(distributor_migrators.keys())
repo_versions_setup = pulp3_repo_setup[repo_name]['repository_versions']
signing_service = None
signing_service_name = pulp3_repo_setup[repo_name].get("signing_service")
if signing_service_name:
_logger.info(
"Signing Service %r requested for %r", signing_service_name, repo_name
)
try:
signing_service = SigningService.objects.get(name=signing_service_name)
except SigningService.DoesNotExist:
_logger.warning(
"Could not find signing-service named %r", signing_service_name
)
raise
# importer might not be migrated, e.g. config is empty or it's not specified in a MP
pulp3_remote = None
pulp2_importer_repo_id = pulp3_repo_setup[repo_name].get('pulp2_importer_repository_id')
if pulp2_importer_repo_id:
try:
pulp2_importer = Pulp2Importer.objects.get(
pulp2_repo_id=pulp2_importer_repo_id,
not_in_plan=False
)
pulp3_remote = pulp2_importer.pulp3_remote
except Pulp2Importer.DoesNotExist:
pass
task_group = TaskGroup.current()
# find appropriate group_progress_reports that later will be updated
progress_dist = task_group.group_progress_reports.filter(
code='create.distribution'
)
progress_rv = task_group.group_progress_reports.filter(
code='create.repo_version'
)
for pulp2_repo_info in repo_versions_setup:
try:
pulp2_repo = Pulp2Repository.objects.get(
pulp2_repo_id=pulp2_repo_info['repo_id'],
not_in_plan=False
)
except Pulp2Repository.DoesNotExist:
# not in Pulp 2 anymore
continue
else:
# it's possible to have a random order of the repo versions (after migration
# re-run, a repo can be changed in pulp 2 and it might not be for the last
# repo version)
create_repo_version(progress_rv, pulp2_repo, pulp3_remote)
for pulp2_repo_info in repo_versions_setup:
# find pulp2repo by id
repo_id = pulp2_repo_info['repo_id']
dist_repositories = pulp2_repo_info['dist_repo_ids']
try:
migrated_repo = Pulp2Repository.objects.get(pulp2_repo_id=repo_id,
not_in_plan=False,
is_migrated=True)
except Pulp2Repository.DoesNotExist:
# not in Pulp 2 anymore
continue
else:
pulp2dist = Pulp2Distributor.objects.filter(
is_migrated=False,
not_in_plan=False,
pulp2_repo_id__in=dist_repositories,
pulp2_type_id__in=distributor_types,
)
# decrease the number of total because some dists have already been migrated
decrease_total = len(dist_repositories) - len(pulp2dist)
if decrease_total:
progress_dist.update(total=F('total') - decrease_total)
for dist in pulp2dist:
dist_migrator = distributor_migrators.get(dist.pulp2_type_id)
migrate_repo_distributor(
dist_migrator, progress_dist, dist,
migrated_repo.pulp3_repository_version, signing_service
)
# add distirbutors specified in the complex plan
# these can be native and not native distributors
migrated_repo.pulp2_dists.add(dist)
