async def migrate_repositories(plan): """ A coroutine to migrate pre-migrated repositories. """ repos_to_create = plan.get_pulp3_repository_setup().keys() progress_data = dict(message='Creating repositories in Pulp 3', code='creating.repositories', total=0) with ProgressReport(**progress_data) as pb: pulp2repos_qs = Pulp2Repository.objects.filter( pulp3_repository_version=None) # no specific migration plan for repositories if not repos_to_create: pb.total += pulp2repos_qs.count() pb.save() for pulp2repo in pulp2repos_qs: pulp3_repo_name = pulp2repo.pulp2_repo_id repository_class = PLUGIN_MIGRATORS.get( pulp2repo.type).pulp3_repository repo, created = repository_class.objects.get_or_create( name=pulp3_repo_name, description=pulp2repo.pulp2_description) if created: pb.increment() else: pb.total -= 1 pb.save() # specific migration plan for repositories else: pb.total += len(repos_to_create) pb.save() for pulp3_repo_name in repos_to_create: try: pulp2repo = pulp2repos_qs.get( pulp2_repo_id=pulp3_repo_name) except Pulp2Repository.DoesNotExist: description = pulp3_repo_name else: description = pulp2repo.pulp2_description repository_class = PLUGIN_MIGRATORS.get( pulp2repo.type).pulp3_repository repo, created = repository_class.objects.get_or_create( name=pulp3_repo_name, description=description) if created: pb.increment() else: pb.total -= 1 pb.save()
async def migrate_content(plan): """ A coroutine to initiate content migration for each plugin. Args: plan (MigrationPlan): Migration Plan to use """ content_migration_coros = [] plugins_to_migrate = plan.get_plugins() progress_data = dict(message='Migrating content to Pulp 3', code='migrating.content', total=0) with ProgressReport(**progress_data) as pb: # schedule content migration into Pulp 3 using pre-migrated Pulp 2 content for plugin in plugins_to_migrate: plugin_migrator = PLUGIN_MIGRATORS.get(plugin) content_migration_coros.append( plugin_migrator.migrate_content_to_pulp3()) # only used for progress bar counters content_types = plugin_migrator.content_models.keys() pulp2content_qs = Pulp2Content.objects.filter( pulp2_content_type_id__in=content_types, pulp3_content=None) pb.total += pulp2content_qs.count() pb.save() await asyncio.wait(content_migration_coros) pb.done = pb.total
def create_repo_version(pulp3_repo_name, pulp2_repo): """ Create a repo version based on a pulp2 repository Args: pulp3_repo_name(str): repository name in Pulp 3 pulp2_repo(Pulp2Repository): a pre-migrated repository to create a repo version for """ repository_class = PLUGIN_MIGRATORS.get( pulp2_repo.type).pulp3_repository pulp3_repo = repository_class.objects.get(name=pulp3_repo_name) unit_ids = Pulp2RepoContent.objects.filter( pulp2_repository=pulp2_repo).values_list('pulp2_unit_id', flat=True) incoming_content = set( Pulp2Content.objects.filter( pulp2_id__in=unit_ids).only('pulp3_content').values_list( 'pulp3_content__pk', flat=True)) with pulp3_repo.new_version() as new_version: repo_content = set(new_version.content.values_list('pk', flat=True)) to_add = incoming_content - repo_content to_delete = repo_content - incoming_content new_version.add_content(Content.objects.filter(pk__in=to_add)) new_version.remove_content( Content.objects.filter(pk__in=to_delete)) return new_version
def _parse_plugin_plan(self, repository_data): # Circular import avoidance from pulp_2to3_migration.app.plugin import PLUGIN_MIGRATORS self.type = repository_data['type'] self.migrator = PLUGIN_MIGRATORS.get(self.type) repositories = repository_data.get('repositories') if repositories: self.empty = False for repository in repositories: name = repository['name'] _find_importer_repo = repository[ 'pulp2_importer_repository_id'] self.repositories_importers_to_migrate.append( _find_importer_repo) repository_versions = [] for repository_version in repository.get( 'repository_versions', []): pulp2_repository_id = repository_version[ 'pulp2_repository_id'] self.repositories_to_migrate.append(pulp2_repository_id) repository_versions.append(pulp2_repository_id) distributor_ids = repository_version.get( 'distributor_ids', []) self.repositories_distributors_to_migrate.extend( distributor_ids) self.repositories_to_create[name] = { "pulp2_importer_repository_id": _find_importer_repo, "versions": repository_versions }
def validate(self, data): """ Validate that the Serializer contains valid data. Validates JSON structure of migration_plan. Checks pulp2 and pulp3 plugins are installed. """ schema = json.loads(SCHEMA) validator = Draft7Validator(schema) if isinstance(data['plan'], str): loaded_plan = json.loads(data['plan']) elif isinstance(data['plan'], dict): loaded_plan = data['plan'] else: raise serializers.ValidationError( _("Must provide a (JSON-encoded) string or dict for 'plan', not list") ) err = [] for error in sorted(validator.iter_errors(loaded_plan), key=str): err.append(error.message) if err: raise serializers.ValidationError( _("Provided Migration Plan format is invalid:'{}'".format(err)) ) plugins_to_migrate = set() for plugin_type in loaded_plan['plugins']: plugins_to_migrate.add(plugin_type['type']) if len(loaded_plan['plugins']) != len(plugins_to_migrate): raise serializers.ValidationError( _("Provided Migration Plan contains same plugin type specified more that once.") ) # MongoDB connection initialization connection.initialize() db = connection.get_database() for plugin in plugins_to_migrate: plugin_migrator = PLUGIN_MIGRATORS.get(plugin) if not plugin_migrator: raise serializers.ValidationError( _("Migration of {} plugin is not supported.".format(plugin)) ) if plugin_migrator.pulp3_plugin not in INSTALLED_PULP_PLUGINS: raise serializers.ValidationError( _("Plugin {} is not installed in pulp3.".format(plugin)) ) try: db.command("collstats", plugin_migrator.pulp2_collection) except OperationFailure: raise serializers.ValidationError( _("Plugin {} is not installed in pulp2.".format(plugin)) ) data['plan'] = loaded_plan return data
def _parse_plugin_plan(self, repository_data): # Circular import avoidance from pulp_2to3_migration.app.plugin import PLUGIN_MIGRATORS self.type = repository_data['type'] self.migrator = PLUGIN_MIGRATORS.get(self.type) repositories = repository_data.get('repositories') if repositories: self.empty = False for repository in repositories: name = repository['name'] importer_repo_id = repository.get( 'pulp2_importer_repository_id') if importer_repo_id: self.repositories_importers_to_migrate.append( importer_repo_id) repository_versions = [] for repository_version in repository.get( 'repository_versions', []): pulp2_repository_id = repository_version[ 'pulp2_repository_id'] self.repositories_to_migrate.append(pulp2_repository_id) distributor_repo_ids = repository_version.get( 'pulp2_distributor_repository_ids', []) self.repositories_distributors_to_migrate.extend( distributor_repo_ids) repository_versions.append({ 'repo_id': pulp2_repository_id, 'dist_repo_ids': distributor_repo_ids }) signing_service = repository.get("signing_service") RepoSetup.set_importer(pulp2_repository_id, self.type, importer_repo_id) RepoSetup.set_distributors(pulp2_repository_id, self.type, distributor_repo_ids) self.repositories_to_create[name] = { "pulp2_importer_repository_id": importer_repo_id, "repository_versions": repository_versions, "signing_service": signing_service, }
def complex_repo_migration(plugin_type, pulp3_repo_setup, repo_name): """Perform a complex migration for a particular repo using the repo setup config. Create all repository versions, publications, distributions. Args: plugin_type(str): Plugin type pulp3_repo_setup: Pulp 3 repo setup config for a plugin repo_name: Name of the repo to be migrated """ from pulp_2to3_migration.app.plugin import PLUGIN_MIGRATORS migrator = PLUGIN_MIGRATORS.get(plugin_type) distributor_migrators = migrator.distributor_migrators distributor_types = list(distributor_migrators.keys()) repo_versions_setup = pulp3_repo_setup[repo_name]['repository_versions'] signing_service = None signing_service_name = pulp3_repo_setup[repo_name].get("signing_service") if signing_service_name: _logger.info( "Signing Service %r requested for %r", signing_service_name, repo_name ) try: signing_service = SigningService.objects.get(name=signing_service_name) except SigningService.DoesNotExist: _logger.warning( "Could not find signing-service named %r", signing_service_name ) raise # importer might not be migrated, e.g. config is empty or it's not specified in a MP pulp3_remote = None pulp2_importer_repo_id = pulp3_repo_setup[repo_name].get('pulp2_importer_repository_id') if pulp2_importer_repo_id: try: pulp2_importer = Pulp2Importer.objects.get( pulp2_repo_id=pulp2_importer_repo_id, not_in_plan=False ) pulp3_remote = pulp2_importer.pulp3_remote except Pulp2Importer.DoesNotExist: pass task_group = TaskGroup.current() # find appropriate group_progress_reports that later will be updated progress_dist = task_group.group_progress_reports.filter( code='create.distribution' ) progress_rv = task_group.group_progress_reports.filter( code='create.repo_version' ) for pulp2_repo_info in repo_versions_setup: try: pulp2_repo = Pulp2Repository.objects.get( pulp2_repo_id=pulp2_repo_info['repo_id'], not_in_plan=False ) except Pulp2Repository.DoesNotExist: # not in Pulp 2 anymore continue else: # it's possible to have a random order of the repo versions (after migration # re-run, a repo can be changed in pulp 2 and it might not be for the last # repo version) create_repo_version(progress_rv, pulp2_repo, pulp3_remote) for pulp2_repo_info in repo_versions_setup: # find pulp2repo by id repo_id = pulp2_repo_info['repo_id'] dist_repositories = pulp2_repo_info['dist_repo_ids'] try: migrated_repo = Pulp2Repository.objects.get(pulp2_repo_id=repo_id, not_in_plan=False, is_migrated=True) except Pulp2Repository.DoesNotExist: # not in Pulp 2 anymore continue else: pulp2dist = Pulp2Distributor.objects.filter( is_migrated=False, not_in_plan=False, pulp2_repo_id__in=dist_repositories, pulp2_type_id__in=distributor_types, ) # decrease the number of total because some dists have already been migrated decrease_total = len(dist_repositories) - len(pulp2dist) if decrease_total: progress_dist.update(total=F('total') - decrease_total) for dist in pulp2dist: dist_migrator = distributor_migrators.get(dist.pulp2_type_id) migrate_repo_distributor( dist_migrator, progress_dist, dist, migrated_repo.pulp3_repository_version, signing_service ) # add distirbutors specified in the complex plan # these can be native and not native distributors migrated_repo.pulp2_dists.add(dist)