def pre_migrate_repocontent(repo): """ Pre-migrate a relation between repositories and content in pulp 2. Args: repo(Pulp2Repository): A pre-migrated pulp 2 repository which importer should be migrated """ if repo.is_migrated: return # At this stage the pre-migrated repo is either new or changed since the last run. # For the case when something changed, old repo-content relations should be removed. Pulp2RepoContent.objects.filter(pulp2_repository=repo).delete() mongo_repocontent_q = mongo_Q(repo_id=repo.pulp2_repo_id) mongo_repocontent_qs = RepositoryContentUnit.objects(mongo_repocontent_q) repocontent = [] for repocontent_data in mongo_repocontent_qs.exclude('repo_id').as_pymongo().no_cache(): item = Pulp2RepoContent( pulp2_unit_id=repocontent_data['unit_id'], pulp2_content_type_id=repocontent_data['unit_type_id'], pulp2_repository=repo, pulp2_created=repocontent_data['created'], pulp2_updated=repocontent_data['updated'] ) repocontent.append(item) if not repocontent: # Either the repo no longer exists in Pulp 2, or the repo is empty. return Pulp2RepoContent.objects.bulk_create(repocontent, batch_size=DEFAULT_BATCH_SIZE)
async def pre_migrate_repocontent(repo): """ Pre-migrate a relation between repositories and content in pulp 2. Args: repo(Pulp2Repository): A pre-migrated pulp 2 repository which importer should be migrated """ if repo.is_migrated: return # At this stage the pre-migrated repo is either new or changed since the last run. # For the case when something changed, old repo-content relations should be removed. Pulp2RepoContent.objects.filter(pulp2_repository=repo).delete() mongo_repocontent_q = mongo_Q(repo_id=repo.pulp2_repo_id) mongo_repocontent_qs = RepositoryContentUnit.objects(mongo_repocontent_q) if not mongo_repocontent_qs: # Either the repo no longer exists in Pulp 2, # or the repo is empty. return repocontent = [] for repocontent_data in mongo_repocontent_qs.only('unit_id', 'unit_type_id'): item = Pulp2RepoContent( pulp2_unit_id=repocontent_data.unit_id, pulp2_content_type_id=repocontent_data.unit_type_id, pulp2_repository=repo) repocontent.append(item) Pulp2RepoContent.objects.bulk_create(repocontent)
def get_repo_types(plan): """ Create mappings for pulp 2 repository types. Identify type by inspecting content of a repo. One mapping is repo_id -> repo_type, the other is repo_type -> list of repo_ids. It's used later during pre-migration and identification of removed repos from pulp 2 Args: plan(MigrationPlan): A Migration Plan Returns: repo_id_to_type(dict): mapping from a pulp 2 repo_id to a plugin/repo type type_to_repo_ids(dict): mapping from a plugin/repo type to the list of repo_ids """ repo_id_to_type = {} type_to_repo_ids = defaultdict(set) # mapping content type -> plugin/repo type, e.g. 'docker_blob' -> 'docker' content_type_to_plugin = {} for plugin in plan.get_plugin_plans(): for content_type in plugin.migrator.pulp2_content_models: content_type_to_plugin[ content_type] = plugin.migrator.pulp2_plugin repos = set(plugin.get_repositories()) repos |= set(plugin.get_importers_repos()) repos |= set(plugin.get_distributors_repos()) for repo in repos: repo_id_to_type[repo] = plugin.type type_to_repo_ids[plugin.type].update(repos) # TODO: optimizations. # It looks at each content at the moment. Potential optimizations: # - This is a big query, paginate? # - Filter by repos from the plan # - Query any but one record for a repo for rec in RepositoryContentUnit.objects().\ only('repo_id', 'unit_type_id').as_pymongo().no_cache(): repo_id = rec['repo_id'] unit_type_id = rec['unit_type_id'] # a type for a repo is already known or this content/repo type is not supported if repo_id in repo_id_to_type or unit_type_id not in content_type_to_plugin: continue plugin_name = content_type_to_plugin[unit_type_id] repo_id_to_type[repo_id] = plugin_name type_to_repo_ids[plugin_name].add(repo_id) return repo_id_to_type, type_to_repo_ids
def get_repo_types(plan): """ Create mappings for pulp 2 repository types. Identify type by inspecting content of a repo. One mapping is repo_id -> repo_type, the other is repo_type -> list of repo_ids. It's used later during pre-migration and identification of removed repos from pulp 2 Args: plan(MigrationPlan): A Migration Plan Returns: repo_id_to_type(dict): mapping from a pulp 2 repo_id to a plugin/repo type type_to_repo_ids(dict): mapping from a plugin/repo type to the list of repo_ids """ repo_id_to_type = {} type_to_repo_ids = defaultdict(set) is_simple_plan = False # mapping content type -> plugin/repo type, e.g. 'docker_blob' -> 'docker' content_type_to_plugin = {} for plugin in plan.get_plugin_plans(): for content_type in plugin.migrator.pulp2_content_models: content_type_to_plugin[content_type] = plugin.migrator.pulp2_plugin # if any of the plugin plans is empty, we'd need to go through all repo content relations # to determine repo types correctly. if plugin.empty: is_simple_plan = True continue repos = set(plugin.get_repositories()) repos |= set(plugin.get_importers_repos()) repos |= set(plugin.get_distributors_repos()) for repo in repos: repo_id_to_type[repo] = plugin.type type_to_repo_ids[plugin.type].update(repos) # Go through repo content relations only when at least one of the plans is not complex, # otherwise the type is determined by the plan in a much more efficient way. if is_simple_plan: # TODO: optimizations. # It looks at each content at the moment. Potential optimizations: # - This is a big query, paginate? # - Filter by repos from the plan # - Query any but one record for a repo for rec in (RepositoryContentUnit.objects().only( "repo_id", "unit_type_id").as_pymongo().no_cache()): repo_id = rec["repo_id"] unit_type_id = rec["unit_type_id"] # a type for a repo is already known or this content/repo type is not supported if repo_id in repo_id_to_type or unit_type_id not in content_type_to_plugin: continue plugin_name = content_type_to_plugin[unit_type_id] repo_id_to_type[repo_id] = plugin_name type_to_repo_ids[plugin_name].add(repo_id) return repo_id_to_type, type_to_repo_ids