def pre_migrate_repocontent(repo):
"""
Pre-migrate a relation between repositories and content in pulp 2.
Args:
repo(Pulp2Repository): A pre-migrated pulp 2 repository which importer should be migrated
"""
if repo.is_migrated:
return
# At this stage the pre-migrated repo is either new or changed since the last run.
# For the case when something changed, old repo-content relations should be removed.
Pulp2RepoContent.objects.filter(pulp2_repository=repo).delete()
mongo_repocontent_q = mongo_Q(repo_id=repo.pulp2_repo_id)
mongo_repocontent_qs = RepositoryContentUnit.objects(mongo_repocontent_q)
repocontent = []
for repocontent_data in mongo_repocontent_qs.exclude('repo_id').as_pymongo().no_cache():
item = Pulp2RepoContent(
pulp2_unit_id=repocontent_data['unit_id'],
pulp2_content_type_id=repocontent_data['unit_type_id'],
pulp2_repository=repo,
pulp2_created=repocontent_data['created'],
pulp2_updated=repocontent_data['updated']
)
repocontent.append(item)
if not repocontent:
# Either the repo no longer exists in Pulp 2, or the repo is empty.
return
Pulp2RepoContent.objects.bulk_create(repocontent, batch_size=DEFAULT_BATCH_SIZE)
async def pre_migrate_repocontent(repo):
"""
Pre-migrate a relation between repositories and content in pulp 2.
Args:
repo(Pulp2Repository): A pre-migrated pulp 2 repository which importer should be migrated
"""
if repo.is_migrated:
return
# At this stage the pre-migrated repo is either new or changed since the last run.
# For the case when something changed, old repo-content relations should be removed.
Pulp2RepoContent.objects.filter(pulp2_repository=repo).delete()
mongo_repocontent_q = mongo_Q(repo_id=repo.pulp2_repo_id)
mongo_repocontent_qs = RepositoryContentUnit.objects(mongo_repocontent_q)
if not mongo_repocontent_qs:
# Either the repo no longer exists in Pulp 2,
# or the repo is empty.
return
repocontent = []
for repocontent_data in mongo_repocontent_qs.only('unit_id',
'unit_type_id'):
item = Pulp2RepoContent(
pulp2_unit_id=repocontent_data.unit_id,
pulp2_content_type_id=repocontent_data.unit_type_id,
pulp2_repository=repo)
repocontent.append(item)
Pulp2RepoContent.objects.bulk_create(repocontent)
def get_repo_types(plan):
"""
Create mappings for pulp 2 repository types.
Identify type by inspecting content of a repo.
One mapping is repo_id -> repo_type, the other is repo_type -> list of repo_ids.
It's used later during pre-migration and identification of removed repos from pulp 2
Args:
plan(MigrationPlan): A Migration Plan
Returns:
repo_id_to_type(dict): mapping from a pulp 2 repo_id to a plugin/repo type
type_to_repo_ids(dict): mapping from a plugin/repo type to the list of repo_ids
"""
repo_id_to_type = {}
type_to_repo_ids = defaultdict(set)
# mapping content type -> plugin/repo type, e.g. 'docker_blob' -> 'docker'
content_type_to_plugin = {}
for plugin in plan.get_plugin_plans():
for content_type in plugin.migrator.pulp2_content_models:
content_type_to_plugin[
content_type] = plugin.migrator.pulp2_plugin
repos = set(plugin.get_repositories())
repos |= set(plugin.get_importers_repos())
repos |= set(plugin.get_distributors_repos())
for repo in repos:
repo_id_to_type[repo] = plugin.type
type_to_repo_ids[plugin.type].update(repos)
# TODO: optimizations.
# It looks at each content at the moment. Potential optimizations:
# - This is a big query, paginate?
# - Filter by repos from the plan
# - Query any but one record for a repo
for rec in RepositoryContentUnit.objects().\
only('repo_id', 'unit_type_id').as_pymongo().no_cache():
repo_id = rec['repo_id']
unit_type_id = rec['unit_type_id']
# a type for a repo is already known or this content/repo type is not supported
if repo_id in repo_id_to_type or unit_type_id not in content_type_to_plugin:
continue
plugin_name = content_type_to_plugin[unit_type_id]
repo_id_to_type[repo_id] = plugin_name
type_to_repo_ids[plugin_name].add(repo_id)
return repo_id_to_type, type_to_repo_ids
def get_repo_types(plan):
"""
Create mappings for pulp 2 repository types.
Identify type by inspecting content of a repo.
One mapping is repo_id -> repo_type, the other is repo_type -> list of repo_ids.
It's used later during pre-migration and identification of removed repos from pulp 2
Args:
plan(MigrationPlan): A Migration Plan
Returns:
repo_id_to_type(dict): mapping from a pulp 2 repo_id to a plugin/repo type
type_to_repo_ids(dict): mapping from a plugin/repo type to the list of repo_ids
"""
repo_id_to_type = {}
type_to_repo_ids = defaultdict(set)
is_simple_plan = False
# mapping content type -> plugin/repo type, e.g. 'docker_blob' -> 'docker'
content_type_to_plugin = {}
for plugin in plan.get_plugin_plans():
for content_type in plugin.migrator.pulp2_content_models:
content_type_to_plugin[content_type] = plugin.migrator.pulp2_plugin
# if any of the plugin plans is empty, we'd need to go through all repo content relations
# to determine repo types correctly.
if plugin.empty:
is_simple_plan = True
continue
repos = set(plugin.get_repositories())
repos |= set(plugin.get_importers_repos())
repos |= set(plugin.get_distributors_repos())
for repo in repos:
repo_id_to_type[repo] = plugin.type
type_to_repo_ids[plugin.type].update(repos)
# Go through repo content relations only when at least one of the plans is not complex,
# otherwise the type is determined by the plan in a much more efficient way.
if is_simple_plan:
# TODO: optimizations.
# It looks at each content at the moment. Potential optimizations:
# - This is a big query, paginate?
# - Filter by repos from the plan
# - Query any but one record for a repo
for rec in (RepositoryContentUnit.objects().only(
"repo_id", "unit_type_id").as_pymongo().no_cache()):
repo_id = rec["repo_id"]
unit_type_id = rec["unit_type_id"]
# a type for a repo is already known or this content/repo type is not supported
if repo_id in repo_id_to_type or unit_type_id not in content_type_to_plugin:
continue
plugin_name = content_type_to_plugin[unit_type_id]
repo_id_to_type[repo_id] = plugin_name
type_to_repo_ids[plugin_name].add(repo_id)
return repo_id_to_type, type_to_repo_ids