async def run(self): """ Build and emit `DeclarativeContent` from the Manifest data. """ deferred_download = self.remote.policy != Remote.IMMEDIATE # Interpret download policy with ProgressReport(message="Downloading Metadata", code="downloading.metadata") as pb: parsed_url = urlparse(self.remote.url) root_dir = os.path.dirname(parsed_url.path) downloader = self.remote.get_downloader(url=self.remote.url) result = await downloader.run() pb.increment() with ProgressReport(message="Parsing Metadata Lines", code="parsing.metadata") as pb: manifest = Manifest(result.path) entries = list(manifest.read()) pb.total = len(entries) pb.save() for entry in entries: path = os.path.join(root_dir, entry.relative_path) url = urlunparse(parsed_url._replace(path=path)) file = FileContent(relative_path=entry.relative_path, digest=entry.digest) artifact = Artifact(size=entry.size, sha256=entry.digest) da = DeclarativeArtifact( artifact=artifact, url=url, relative_path=entry.relative_path, remote=self.remote, deferred_download=deferred_download, ) dc = DeclarativeContent(content=file, d_artifacts=[da]) pb.increment() await self.put(dc)
def __init__(self, remote, repository, optimize): """ The first stage of a pulp_ansible sync pipeline. Args: remote (CollectionRemote): The remote data to be used when syncing repository (AnsibleRepository): The repository being syncedself. optimize (boolean): Whether to optimize sync or not. """ super().__init__() msg = _("Parsing CollectionVersion Metadata") self.parsing_metadata_progress_bar = ProgressReport( message=msg, code="parsing.metadata") self.remote = remote self.repository = repository self.optimize = optimize self.collection_info = parse_collections_requirements_file( remote.requirements_file) self.deprecations = Q() self.add_dependents = self.collection_info and self.remote.sync_dependencies self.already_synced = set() self._unpaginated_collection_metadata = None self._unpaginated_collection_version_metadata = None self.last_synced_metadata_time = None # Interpret download policy self.deferred_download = self.remote.policy != Remote.IMMEDIATE
async def run(self): """ The coroutine for this stage. Returns: The coroutine for this stage. """ with ProgressReport(message="Associating Content", code="associating.content") as pb: to_delete = set( self.new_version.content.values_list("pk", flat=True)) async for batch in self.batches(): to_add = set() for d_content in batch: try: to_delete.remove(d_content.content.pk) except KeyError: to_add.add(d_content.content.pk) await self.put(d_content) if to_add: self.new_version.add_content( Content.objects.filter(pk__in=to_add)) pb.increase_by(len(to_add)) if self.allow_delete: with ProgressReport(message="Un-Associating Content", code="unassociating.content") as pb: if to_delete: self.new_version.remove_content( Content.objects.filter(pk__in=to_delete)) pb.increase_by(len(to_delete))
async def run(self): """Signs collections if they have not been signed with key.""" tasks = [] # Filter out any content that already has a signature with pubkey_fingerprint current_signatures = CollectionVersionSignature.objects.filter( pubkey_fingerprint=self.signing_service.pubkey_fingerprint) new_content = self.content.exclude(signatures__in=current_signatures) ntotal = await sync_to_async(new_content.count)() nmsg = _("Signing new CollectionVersions") async with ProgressReport(message=nmsg, code="sign.new.signature", total=ntotal) as p: self.progress_report = p async for collection_version in sync_to_async_iterable( new_content.iterator()): tasks.append( asyncio.create_task( self.sign_collection_version(collection_version))) await asyncio.gather(*tasks) # Add any signatures already present in Pulp if part of content list present_content = current_signatures.filter( signed_collection__in=self.content).exclude( pk__in=self.repos_current_signatures) ptotal = await sync_to_async(present_content.count)() pmsg = _("Adding present CollectionVersionSignatures") async with ProgressReport(message=pmsg, code="sign.present.signature", total=ptotal) as np: async for signature in sync_to_async_iterable( present_content.iterator()): await np.aincrement() await self.put(DeclarativeContent(content=signature))
async def pre_migrate_all_without_content(plan): """ Pre-migrate repositories, relations to their contents, importers and distributors. NOTE: MongoDB and Django handle datetime fields differently. MongoDB doesn't care about timezones and provides "naive" time, while Django is complaining about time without a timezone. The problem is that naive time != time with specified timezone, that's why all the time for MongoDB comparisons should be naive and all the time for Django/PostgreSQL should be timezone aware. Args: plan(MigrationPlan): A Migration Plan """ repos = plan.get_repositories() importers = plan.get_importers() distributors = plan.get_distributors() _logger.debug('Pre-migrating Pulp 2 repositories') # the latest time we have in the migration tool in Pulp2Repository table zero_datetime = timezone.make_aware(datetime(1970, 1, 1), timezone.utc) last_added = Pulp2Repository.objects.aggregate( Max('pulp2_last_unit_added' ))['pulp2_last_unit_added__max'] or zero_datetime last_removed = Pulp2Repository.objects.aggregate( Max('pulp2_last_unit_removed') )['pulp2_last_unit_removed__max'] or zero_datetime last_updated = max(last_added, last_removed) last_updated_naive = timezone.make_naive(last_updated, timezone=timezone.utc) with ProgressReport( message= 'Pre-migrating Pulp 2 repositories, importers, distributors', code='premigrating.repositories') as pb: # we pre-migrate: # - empty repos (last_unit_added is not set) # - repos which were updated since last migration (last_unit_added/removed >= last_updated) mongo_repo_q = (mongo_Q(last_unit_added__exists=False) | mongo_Q(last_unit_added__gte=last_updated_naive) | mongo_Q(last_unit_removed__gte=last_updated_naive)) # in case only certain repositories are specified in the migration plan if repos: mongo_repo_q &= mongo_Q(repo_id__in=repos) mongo_repo_qs = Repository.objects(mongo_repo_q) pb.total = mongo_repo_qs.count() pb.save() for repo_data in mongo_repo_qs.only('id', 'repo_id', 'last_unit_added', 'last_unit_removed', 'description'): with transaction.atomic(): repo = await pre_migrate_repo(repo_data) await pre_migrate_importer(repo, importers) await pre_migrate_distributor(repo, distributors) await pre_migrate_repocontent(repo) pb.increment()
async def run(self): """ The coroutine for this stage. Returns: The coroutine for this stage. """ with ProgressReport(message='Associating Content', code='associating.content') as pb: to_delete = set( self.new_version.content.values_list('pk', flat=True)) async for batch in self.batches(): to_add = set() for d_content in batch: try: to_delete.remove(d_content.content.pk) except KeyError: to_add.add(d_content.content.pk) if to_add: self.new_version.add_content( Content.objects.filter(pk__in=to_add)) pb.done = pb.done + len(to_add) pb.save() if to_delete: await self.put(Content.objects.filter(pk__in=to_delete))
def _parse_modulemd_default_names(self, modulemd_index): modulemd_default_names = parse_defaults(modulemd_index) # Parsing module-defaults happens all at one time, and from here on no useful # work happens. So just report that it finished this stage. modulemd_defaults_pb_data = { "message": "Parsed Modulemd-defaults", "code": "parsing.modulemd_defaults", } with ProgressReport( **modulemd_defaults_pb_data) as modulemd_defaults_pb: modulemd_defaults_total = len(modulemd_default_names) modulemd_defaults_pb.total = modulemd_defaults_total modulemd_defaults_pb.done = modulemd_defaults_total for default in modulemd_default_names: artifact = default.pop("artifact") relative_path = "{}{}snippet".format( default[PULP_MODULEDEFAULTS_ATTR.MODULE], default[PULP_MODULEDEFAULTS_ATTR.STREAM]) da = DeclarativeArtifact(artifact=artifact, relative_path=relative_path, url=self.data.modules_url) default_content = ModulemdDefaults(**default) self.default_content_dcs.append( DeclarativeContent(content=default_content, d_artifacts=[da])) # delete list now that we're done with it for memory savings del modulemd_default_names
async def run(self): """Build `DeclarativeContent` from the repodata.""" self.data.remote_url = self.new_url or self.remote.url progress_data = dict(message="Downloading Metadata Files", code="downloading.metadata") with ProgressReport(**progress_data) as metadata_pb: self.data.metadata_pb = metadata_pb downloader = self.remote.get_downloader( url=urljoin(self.data.remote_url, "repodata/repomd.xml")) result = await downloader.run() metadata_pb.increment() repomd_path = result.path self.data.repomd = cr.Repomd(repomd_path) self.repository.last_sync_revision_number = self.data.repomd.revision self.repository.last_sync_repomd_checksum = get_sha256(repomd_path) await self.parse_distribution_tree() await self.parse_repository_metadata() await self.parse_modules_metadata() await self.parse_packages_components() await self.parse_content() # now send modules down the pipeline since all relations have been set up for modulemd in self.data.modulemd_list: await self.put(modulemd) for dc_group in self.data.dc_groups: await self.put(dc_group)
async def run(self): """ Build and emit `DeclarativeContent` from the ansible metadata. """ if self.optimize: should_we_sync = await self._should_we_sync() if should_we_sync is False: log.debug(_("no-op: remote wasn't updated since last sync.")) return tasks = [] loop = asyncio.get_event_loop() msg = _("Parsing CollectionVersion Metadata") async with ProgressReport(message=msg, code="sync.parsing.metadata") as pb: self.parsing_metadata_progress_bar = pb await self._download_unpaginated_metadata() if self.collection_info: for requirement_entry in self.collection_info: tasks.append( loop.create_task( self._fetch_collection_metadata( requirement_entry))) else: tasks.append(loop.create_task(self._find_all_collections())) await asyncio.gather(*tasks)
async def run(self): """ Schedules multiple coroutines to migrate pre-migrated content to Pulp 3 It migrates content type by type. If a plugin needs to have more control over the order of content migration, it should override this method. """ content_types = self.migrator.content_models.keys() for ctype in content_types: # we need to go through all content in case any of Remotes changed pulp2content_qs = Pulp2Content.objects.filter( pulp2_content_type_id=ctype) total_pulp2content = pulp2content_qs.count() with ProgressReport( message='Migrating {} content to Pulp 3 {}'.format( self.migrator.pulp2_plugin, ctype), code='migrating.{}.content'.format( self.migrator.pulp2_plugin), total=total_pulp2content) as pb: # We are waiting on the coroutine to finish, because the order of the processed # content for plugins like Container and RPM is important because of the relations # between the content types. await asyncio.wait( [self.migrate_to_pulp3(pulp2content_qs, pb=pb)])
async def _fetch_galaxy_pages(self): """ Fetch the roles in a remote repository. Returns: async generator: dicts that represent pages from galaxy api """ page_count = 0 remote = self.remote progress_data = dict(message="Parsing Pages from Galaxy Roles API", code="parsing.roles") with ProgressReport(**progress_data) as progress_bar: api_version = get_api_version(remote.url) downloader = remote.get_downloader(url=get_page_url(remote.url, api_version)) metadata = parse_metadata(await downloader.run()) page_count = math.ceil(float(metadata["count"]) / float(PAGE_SIZE)) progress_bar.total = page_count progress_bar.save() yield metadata progress_bar.increment() # Concurrent downloads are limited by aiohttp... not_done = set( remote.get_downloader(url=get_page_url(remote.url, api_version, page)).run() for page in range(2, page_count + 1) ) while not_done: done, not_done = await asyncio.wait(not_done, return_when=FIRST_COMPLETED) for item in done: yield parse_metadata(item.result()) progress_bar.increment()
def publish(repository_version_pk): """ Create a Publication based on a RepositoryVersion. Args: repository_version_pk (str): Create a publication from this repository version. """ repository_version = RepositoryVersion.objects.get( pk=repository_version_pk) log.info( _("Publishing: repository=%(repository)s, version=%(version)d"), { "repository": repository_version.repository.name, "version": repository_version.number }, ) with tempfile.TemporaryDirectory("."): with ProgressReport(message="Publishing Content", code="publishing.content") as progress_report: with CookbookPublication.create(repository_version) as publication: universe = Universe("__universe__") universe.write( populate(publication, progress_report=progress_report)) PublishedMetadata.create_from_file( relative_path=os.path.basename(universe.relative_path), publication=publication, file=File(open(universe.relative_path, "rb")), ) log.info(_("Publication: %(publication)s created"), {"publication": publication.pk})
async def run(self): """ Parse Release content units. Update release content with information obtained from its artifact. """ with ProgressReport(message="Update Release units", code="update.release") as pb: async for d_content in self.items(): if isinstance(d_content.content, Release): release = d_content.content release_artifact = d_content.d_artifacts[0].artifact release.sha256 = release_artifact.sha256 release_dict = deb822.Release(release_artifact.file) release.codename = release_dict["Codename"] release.suite = release_dict["Suite"] # TODO split of extra stuff e.g. : 'updates/main' -> 'main' release.components = _filter_ssl( release_dict["Components"], self.components) release.architectures = _filter_ssl( release_dict["Architectures"], self.architectures) log.debug("Codename: {}".format(release.codename)) log.debug("Components: {}".format(release.components)) log.debug("Architectures: {}".format( release.architectures)) pb.increment() await self.put(d_content)
async def run(self): """ Parse PackageIndex content units. Ensure, that an uncompressed artifact is available. """ with ProgressReport(message="Update PackageIndex units", code="update.packageindex") as pb: async for d_content in self.items(): if isinstance(d_content.content, PackageIndex): if not d_content.d_artifacts: raise NoPackageIndexFile() content = d_content.content if not [ da for da in d_content.d_artifacts if da.artifact.sha256 == content.sha256 ]: # No main_artifact found uncompress one filename = _uncompress_artifact(d_content.d_artifacts) da = DeclarativeArtifact( Artifact(sha256=content.sha256), filename, content.relative_path, d_content.d_artifacts[0].remote, ) d_content.d_artifacts.append(da) await da.download() da.artifact.save() log.info( "*** Expected: {} *** Uncompressed: {} ***".format( content.sha256, da.artifact.sha256)) pb.increment() await self.put(d_content)
async def migrate_content(plan): """ A coroutine to initiate content migration for each plugin. Args: plan (MigrationPlan): Migration Plan to use """ content_migration_coros = [] plugins_to_migrate = plan.get_plugins() progress_data = dict(message='Migrating content to Pulp 3', code='migrating.content', total=0) with ProgressReport(**progress_data) as pb: # schedule content migration into Pulp 3 using pre-migrated Pulp 2 content for plugin in plugins_to_migrate: plugin_migrator = PLUGIN_MIGRATORS.get(plugin) content_migration_coros.append( plugin_migrator.migrate_content_to_pulp3()) # only used for progress bar counters content_types = plugin_migrator.content_models.keys() pulp2content_qs = Pulp2Content.objects.filter( pulp2_content_type_id__in=content_types, pulp3_content=None) pb.total += pulp2content_qs.count() pb.save() await asyncio.wait(content_migration_coros) pb.done = pb.total
async def run(self): """ Build and emit `DeclarativeContent` from the ansible metadata. """ with ProgressReport(message="Parsing Collection Metadata", code="parsing.metadata") as pb: async for metadata in self._fetch_collections(): url = metadata["download_url"] collection_version = CollectionVersion( namespace=metadata["namespace"]["name"], name=metadata["collection"]["name"], version=metadata["version"], ) artifact = metadata["artifact"] d_artifact = DeclarativeArtifact( artifact=Artifact(sha256=artifact["sha256"], size=artifact["size"]), url=url, relative_path=collection_version.relative_path, remote=self.remote, deferred_download=self.deferred_download, ) d_content = DeclarativeContent(content=collection_version, d_artifacts=[d_artifact]) pb.increment() await self.put(d_content)
def migrate_content(plan, skip_corrupted=False): """ A coroutine to initiate content migration for each plugin. Args: plan (MigrationPlan): Migration Plan to use skip_corrupted (bool): If True, corrupted content is skipped during migration, no task failure. """ progress_data = dict(message='Migrating content to Pulp 3', code='migrating.content', total=0) with ProgressReport(**progress_data) as pb: # schedule content migration into Pulp 3 using pre-migrated Pulp 2 content for plugin in plan.get_plugin_plans(): # only used for progress bar counters content_types = plugin.migrator.content_models.keys() num_to_migrate = Pulp2Content.objects.filter( pulp2_content_type_id__in=content_types, pulp3_content=None).count() pb.total += num_to_migrate pb.save() # migrate plugin.migrator.migrate_content_to_pulp3( skip_corrupted=skip_corrupted) pb.done = pb.total pb.save()
async def run(self): """ Build and emit `DeclarativeContent` from the ansible metadata. """ async with ProgressReport( message="Parsing Role Metadata", code="sync.parsing.metadata" ) as pb: async for metadata in self._fetch_roles(): for version in metadata["summary_fields"]["versions"]: url = GITHUB_URL % ( metadata["github_user"], metadata["github_repo"], version["name"], ) role = Role( version=version["name"], name=metadata["name"], namespace=metadata["namespace"], ) relative_path = "%s/%s/%s.tar.gz" % ( metadata["namespace"], metadata["name"], version["name"], ) d_artifact = DeclarativeArtifact( artifact=Artifact(), url=url, relative_path=relative_path, remote=self.remote, deferred_download=self.deferred_download, ) d_content = DeclarativeContent(content=role, d_artifacts=[d_artifact]) await pb.aincrement() await self.put(d_content)
async def migrate_repositories(): """ A coroutine to migrate pre-migrated repositories. """ progress_data = dict(message='Creating repositories in Pulp 3', code='creating.repositories', total=0) with ProgressReport(**progress_data) as pb: pulp2repos_qs = Pulp2Repository.objects.filter( pulp3_repository_version=None) pb.total += pulp2repos_qs.count() pb.save() for pulp2repo in pulp2repos_qs: # if pulp2 repo_id is too long, its hash is included in pulp3 repo name pulp3_repo_name = pulp2repo.pulp2_repo_id if len(pulp3_repo_name) > 255: repo_name_hash = hashlib.sha256( pulp3_repo_name.encode()).hexdigest() pulp3_repo_name = '{}-{}'.format(pulp3_repo_name[:190], repo_name_hash) repo, created = Repository.objects.get_or_create( name=pulp3_repo_name, description=pulp2repo.pulp2_description) if created: pb.increment() else: pb.total -= 1 pb.save()
async def run(self): """ If includes is specified, then only sync those,else try to sync all other packages """ # TODO Change Bandersnatch internal API to take proxy settings in from config parameters if self.remote.proxy_url: environ['http_proxy'] = self.remote.proxy_url # local & global timeouts defaults to 10secs and 5 hours async with Master(self.remote.url) as master: if self.remote.proxy_url: environ.pop('http_proxy') deferred_download = self.remote.policy != Remote.IMMEDIATE workers = self.remote.download_concurrency or self.remote.DEFAULT_DOWNLOAD_CONCURRENCY with ProgressReport( message="Fetching Project Metadata", code="sync.fetching.project" ) as p: pmirror = PulpMirror( serial=0, # Serial currently isn't supported by Pulp master=master, workers=workers, deferred_download=deferred_download, python_stage=self, progress_report=p, ) packages_to_sync = None if self.remote.includes: packages_to_sync = [ Requirement(pkg).name for pkg in self.remote.includes ] await pmirror.synchronize(packages_to_sync)
def migrate_importers(plan): """ A coroutine to migrate pre-migrated importers. Args: plan (MigrationPlan): Migration Plan to use. """ # gather all needed plugin importer migrators importer_migrators = {} for plugin in plan.get_plugin_plans(): importer_migrators.update(**plugin.migrator.importer_migrators) progress_data = dict(message='Migrating importers to Pulp 3', code='migrating.importers', total=0) with ProgressReport(**progress_data) as pb: pulp2importers_qs = Pulp2Importer.objects.filter(is_migrated=False, not_in_plan=False) pb.total += pulp2importers_qs.count() pb.save() for pulp2importer in pulp2importers_qs: importer_migrator = importer_migrators.get( pulp2importer.pulp2_type_id) remote, created = importer_migrator.migrate_to_pulp3(pulp2importer) pulp2importer.pulp3_remote = remote pulp2importer.is_migrated = True pulp2importer.save() if created: pb.increment() else: pb.total -= 1 pb.save()
async def run(self): """ Build and emit `DeclarativeContent` from the Spec data. """ # Interpret policy to download Artifacts or not deferred_download = self.remote.policy != Remote.IMMEDIATE with ProgressReport(message="Downloading Metadata") as progress: parsed_url = urlparse(self.remote.url) root_dir = parsed_url.path specs_path = os.path.join(root_dir, "specs.4.8.gz") specs_url = urlunparse(parsed_url._replace(path=specs_path)) downloader = self.remote.get_downloader(url=specs_url) result = await downloader.run() progress.increment() with ProgressReport(message="Parsing Metadata") as progress: for key in read_specs(result.path): relative_path = os.path.join( "gems", key.name + "-" + key.version + ".gem") path = os.path.join(root_dir, relative_path) url = urlunparse(parsed_url._replace(path=path)) spec_relative_path = os.path.join( "quick/Marshal.4.8", key.name + "-" + key.version + ".gemspec.rz") spec_path = os.path.join(root_dir, spec_relative_path) spec_url = urlunparse(parsed_url._replace(path=spec_path)) gem = GemContent(name=key.name, version=key.version) da_gem = DeclarativeArtifact( artifact=Artifact(), url=url, relative_path=relative_path, remote=self.remote, deferred_download=deferred_download, ) da_spec = DeclarativeArtifact( artifact=Artifact(), url=spec_url, relative_path=spec_relative_path, remote=self.remote, deferred_download=deferred_download, ) dc = DeclarativeContent(content=gem, d_artifacts=[da_gem, da_spec]) progress.increment() await self.put(dc)
async def _fetch_collections(self): """ Fetch the collections in a remote repository. Returns: async generator: dicts that represent collections from galaxy api """ page_count = 1 remote = self.remote collection_info = self.collection_info def _get_url(page): if collection_info: name, version, source = collection_info[page - 1] namespace, name = name.split(".") root = source or remote.url url = f"{root}/api/v2/collections/{namespace}/{name}" return url return get_page_url(remote.url, page) progress_data = dict(message="Parsing Galaxy Collections API", code="parsing.collections") with ProgressReport(**progress_data) as progress_bar: url = _get_url(page_count) downloader = remote.get_downloader(url=url) initial_data = parse_metadata(await downloader.run()) count = len(self.collection_info) or initial_data.get("count", 1) page_count = math.ceil(float(count) / float(PAGE_SIZE)) progress_bar.total = count progress_bar.save() # Concurrent downloads are limited by aiohttp... not_done = set() for page in range(1, page_count + 1): downloader = remote.get_downloader(url=_get_url(page)) not_done.add(downloader.run()) while not_done: done, not_done = await asyncio.wait(not_done, return_when=asyncio.FIRST_COMPLETED) for item in done: data = parse_metadata(item.result()) for result in data.get("results", [data]): download_url = result.get("download_url") if result.get("versions_url"): not_done.update( [remote.get_downloader(url=result["versions_url"]).run()] ) if result.get("version") and not download_url: not_done.update([remote.get_downloader(url=result["href"]).run()]) if download_url: yield data progress_bar.increment()
async def run(self): """ Build and emit `DeclarativeContent` from the Manifest data. Args: in_q (asyncio.Queue): Unused because the first stage doesn't read from an input queue. out_q (asyncio.Queue): The out_q to send `DeclarativeContent` objects to """ remote_url = self.remote.url if not remote_url.endswith('/index.yaml'): remote_url += '/index.yaml' # TODO Skip reading generator to list? index_yaml = [] with ProgressReport(message="Downloading Index", code="downloading.metadata") as pb: downloader = self.remote.get_downloader(url=remote_url) result = await downloader.run() index_yaml = list(self.read_index_yaml(result.path)) pb.increment() with ProgressReport(message="Parsing Entries", code="parsing.metadata") as pb: pb.total = len(index_yaml) pb.save() for entry in index_yaml: content_entry = dict( filter(lambda e: e[0] not in ('url'), entry.items())) unit = ChartContent(**content_entry) artifact = Artifact(sha256=entry['digest']) da = DeclarativeArtifact( artifact, urljoin(remote_url, entry['url']), "{}-{}.tgz".format(entry['name'], entry['version']), self.remote, deferred_download=self.deferred_download, ) dc = DeclarativeContent(content=unit, d_artifacts=[da]) pb.increment() await self.put(dc)
async def migrate_repositories(plan): """ A coroutine to migrate pre-migrated repositories. """ progress_data = dict(message='Creating repositories in Pulp 3', code='creating.repositories', total=0) with ProgressReport(**progress_data) as pb: for plugin in plan.get_plugin_plans(): pulp2repos_qs = Pulp2Repository.objects.filter( pulp3_repository_version=None, not_in_plan=False, pulp2_repo_type=plugin.type, ) repos_to_create = plugin.get_repo_creation_setup() # no specific migration plan for repositories if not repos_to_create: pb.total += pulp2repos_qs.count() pb.save() for pulp2repo in pulp2repos_qs: pulp3_repo_name = pulp2repo.pulp2_repo_id repository_class = plugin.migrator.pulp3_repository repo, created = repository_class.objects.get_or_create( name=pulp3_repo_name, defaults={'description': pulp2repo.pulp2_description}) if created: pb.increment() else: pb.total -= 1 pb.save() # specific migration plan for repositories else: pb.total += len(repos_to_create) pb.save() for pulp3_repo_name in repos_to_create: try: pulp2repo = pulp2repos_qs.get( pulp2_repo_id=pulp3_repo_name) except Pulp2Repository.DoesNotExist: description = pulp3_repo_name else: description = pulp2repo.pulp2_description repository_class = plugin.migrator.pulp3_repository repo, created = repository_class.objects.get_or_create( name=pulp3_repo_name, defaults={'description': description}) if created: pb.increment() else: pb.total -= 1 pb.save()
def migrate_repositories(plan): """ A coroutine to migrate pre-migrated repositories. """ progress_data = dict(message='Creating repositories in Pulp 3', code='creating.repositories', total=0) with ProgressReport(**progress_data) as pb: for plugin in plan.get_plugin_plans(): # all pulp2 repos in current plan were already migrated, no need to proceed not_migrated_repos = Pulp2Repository.objects.filter( is_migrated=False, not_in_plan=False, pulp2_repo_type=plugin.type) if not not_migrated_repos.exists(): continue pulp2repos_qs = Pulp2Repository.objects.filter( pulp3_repository_version=None, not_in_plan=False, pulp2_repo_type=plugin.type, ) repos_to_create = plugin.get_repo_creation_setup() pb.total += len(repos_to_create) pb.save() for pulp3_repo_name in repos_to_create: try: pulp2repo = pulp2repos_qs.get( pulp2_repo_id=pulp3_repo_name) except Pulp2Repository.DoesNotExist: description = pulp3_repo_name else: description = pulp2repo.pulp2_description repository_class = plugin.migrator.pulp3_repository repo, created = repository_class.objects.get_or_create( name=pulp3_repo_name, defaults={'description': description}) pulp2_repo_ids = [] repo_version_setup = repos_to_create[pulp3_repo_name].get( 'repository_versions') for repo_version in repo_version_setup: pulp2_repo_ids.append(repo_version['repo_id']) pulp2repos_qs = Pulp2Repository.objects.filter( pulp2_repo_id__in=pulp2_repo_ids, pulp3_repository__isnull=True) pulp2repos_qs.update(pulp3_repository=repo) if created: pb.increment() else: pb.total -= 1 pb.save()
async def migrate_repositories(plan): """ A coroutine to migrate pre-migrated repositories. """ repos_to_create = plan.get_pulp3_repository_setup().keys() progress_data = dict( message='Creating repositories in Pulp 3', code='creating.repositories', total=0 ) with ProgressReport(**progress_data) as pb: pulp2repos_qs = Pulp2Repository.objects.filter(pulp3_repository_version=None) # no specific migration plan for repositories if not repos_to_create: pb.total += pulp2repos_qs.count() pb.save() for pulp2repo in pulp2repos_qs: # if pulp2 repo_id is too long, its hash is included in pulp3 repo name pulp3_repo_name = pulp2repo.pulp2_repo_id if len(pulp3_repo_name) > 255: repo_name_hash = hashlib.sha256(pulp3_repo_name.encode()).hexdigest() pulp3_repo_name = '{}-{}'.format(pulp3_repo_name[:190], repo_name_hash) repo, created = Repository.objects.get_or_create( name=pulp3_repo_name, description=pulp2repo.pulp2_description) if created: pb.increment() else: pb.total -= 1 pb.save() # specific migration plan for repositories else: pb.total += len(repos_to_create) pb.save() for pulp3_repo_name in repos_to_create: try: pulp2repo = pulp2repos_qs.get(pulp2_repo_id=pulp3_repo_name) except Pulp2Repository.DoesNotExist: description = pulp3_repo_name else: description = pulp2repo.pulp2_description repo, created = Repository.objects.get_or_create( name=pulp3_repo_name[:255], description=description) if created: pb.increment() else: pb.total -= 1 pb.save()
async def migrate_repositories(plan): """ A coroutine to migrate pre-migrated repositories. """ repos_to_create = plan.get_pulp3_repository_setup().keys() progress_data = dict(message='Creating repositories in Pulp 3', code='creating.repositories', total=0) with ProgressReport(**progress_data) as pb: pulp2repos_qs = Pulp2Repository.objects.filter( pulp3_repository_version=None) # no specific migration plan for repositories if not repos_to_create: pb.total += pulp2repos_qs.count() pb.save() for pulp2repo in pulp2repos_qs: pulp3_repo_name = pulp2repo.pulp2_repo_id repository_class = PLUGIN_MIGRATORS.get( pulp2repo.type).pulp3_repository repo, created = repository_class.objects.get_or_create( name=pulp3_repo_name, description=pulp2repo.pulp2_description) if created: pb.increment() else: pb.total -= 1 pb.save() # specific migration plan for repositories else: pb.total += len(repos_to_create) pb.save() for pulp3_repo_name in repos_to_create: try: pulp2repo = pulp2repos_qs.get( pulp2_repo_id=pulp3_repo_name) except Pulp2Repository.DoesNotExist: description = pulp3_repo_name else: description = pulp2repo.pulp2_description repository_class = PLUGIN_MIGRATORS.get( pulp2repo.type).pulp3_repository repo, created = repository_class.objects.get_or_create( name=pulp3_repo_name, description=description) if created: pb.increment() else: pb.total -= 1 pb.save()
async def run(self): """ Build and emit `DeclarativeContent` from the ansible metadata. """ async with ProgressReport( message="Cloning Git repository for Collection", code="sync.git.clone") as pb: d_content = await declarative_content_from_git_repo( self.remote, self.remote.url, self.remote.git_ref, self.metadata_only) await self.put(d_content) await pb.aincrement()
async def run(self): """ Build and emit `DeclarativeContent` from the Manifest data. If a cookbook specifier is set in the remote, cookbooks are filtered using this specifier. """ async with ProgressReport(message="Downloading Metadata", code="downloading.metadata", total=1) as pb: downloader = self.remote.get_downloader( url=urljoin(self.remote.url + "/", "universe")) result = await downloader.run() await pb.aincrement() cookbook_names = self.remote.specifier_cookbook_names() async with ProgressReport(message="Parsing Metadata", code="parsing.metadata") as pb: universe = Universe(result.path) for entry in universe.read(): if cookbook_names and entry.name not in cookbook_names: continue cookbook = CookbookPackageContent( name=entry.name, version=entry.version, dependencies=entry.dependencies) artifact = Artifact() da = DeclarativeArtifact( artifact=artifact, url=entry.download_url, relative_path=cookbook.relative_path(), remote=self.remote, deferred_download=not self.download_artifacts, ) dc = DeclarativeContent(content=cookbook, d_artifacts=[da]) await pb.aincrement() await self.put(dc)