async def run(self):
"""
Build and emit `DeclarativeContent` from the Manifest data.
"""
deferred_download = self.remote.policy != Remote.IMMEDIATE # Interpret download policy
with ProgressReport(message="Downloading Metadata", code="downloading.metadata") as pb:
parsed_url = urlparse(self.remote.url)
root_dir = os.path.dirname(parsed_url.path)
downloader = self.remote.get_downloader(url=self.remote.url)
result = await downloader.run()
pb.increment()
with ProgressReport(message="Parsing Metadata Lines", code="parsing.metadata") as pb:
manifest = Manifest(result.path)
entries = list(manifest.read())
pb.total = len(entries)
pb.save()
for entry in entries:
path = os.path.join(root_dir, entry.relative_path)
url = urlunparse(parsed_url._replace(path=path))
file = FileContent(relative_path=entry.relative_path, digest=entry.digest)
artifact = Artifact(size=entry.size, sha256=entry.digest)
da = DeclarativeArtifact(
artifact=artifact,
url=url,
relative_path=entry.relative_path,
remote=self.remote,
deferred_download=deferred_download,
)
dc = DeclarativeContent(content=file, d_artifacts=[da])
pb.increment()
await self.put(dc)
def __init__(self, remote, repository, optimize):
"""
The first stage of a pulp_ansible sync pipeline.
Args:
remote (CollectionRemote): The remote data to be used when syncing
repository (AnsibleRepository): The repository being syncedself.
optimize (boolean): Whether to optimize sync or not.
"""
super().__init__()
msg = _("Parsing CollectionVersion Metadata")
self.parsing_metadata_progress_bar = ProgressReport(
message=msg, code="parsing.metadata")
self.remote = remote
self.repository = repository
self.optimize = optimize
self.collection_info = parse_collections_requirements_file(
remote.requirements_file)
self.deprecations = Q()
self.add_dependents = self.collection_info and self.remote.sync_dependencies
self.already_synced = set()
self._unpaginated_collection_metadata = None
self._unpaginated_collection_version_metadata = None
self.last_synced_metadata_time = None
# Interpret download policy
self.deferred_download = self.remote.policy != Remote.IMMEDIATE
async def run(self):
"""
The coroutine for this stage.
Returns:
The coroutine for this stage.
"""
with ProgressReport(message="Associating Content",
code="associating.content") as pb:
to_delete = set(
self.new_version.content.values_list("pk", flat=True))
async for batch in self.batches():
to_add = set()
for d_content in batch:
try:
to_delete.remove(d_content.content.pk)
except KeyError:
to_add.add(d_content.content.pk)
await self.put(d_content)
if to_add:
self.new_version.add_content(
Content.objects.filter(pk__in=to_add))
pb.increase_by(len(to_add))
if self.allow_delete:
with ProgressReport(message="Un-Associating Content",
code="unassociating.content") as pb:
if to_delete:
self.new_version.remove_content(
Content.objects.filter(pk__in=to_delete))
pb.increase_by(len(to_delete))
async def run(self):
"""Signs collections if they have not been signed with key."""
tasks = []
# Filter out any content that already has a signature with pubkey_fingerprint
current_signatures = CollectionVersionSignature.objects.filter(
pubkey_fingerprint=self.signing_service.pubkey_fingerprint)
new_content = self.content.exclude(signatures__in=current_signatures)
ntotal = await sync_to_async(new_content.count)()
nmsg = _("Signing new CollectionVersions")
async with ProgressReport(message=nmsg,
code="sign.new.signature",
total=ntotal) as p:
self.progress_report = p
async for collection_version in sync_to_async_iterable(
new_content.iterator()):
tasks.append(
asyncio.create_task(
self.sign_collection_version(collection_version)))
await asyncio.gather(*tasks)
# Add any signatures already present in Pulp if part of content list
present_content = current_signatures.filter(
signed_collection__in=self.content).exclude(
pk__in=self.repos_current_signatures)
ptotal = await sync_to_async(present_content.count)()
pmsg = _("Adding present CollectionVersionSignatures")
async with ProgressReport(message=pmsg,
code="sign.present.signature",
total=ptotal) as np:
async for signature in sync_to_async_iterable(
present_content.iterator()):
await np.aincrement()
await self.put(DeclarativeContent(content=signature))
async def pre_migrate_all_without_content(plan):
"""
Pre-migrate repositories, relations to their contents, importers and distributors.
NOTE: MongoDB and Django handle datetime fields differently. MongoDB doesn't care about
timezones and provides "naive" time, while Django is complaining about time without a timezone.
The problem is that naive time != time with specified timezone, that's why all the time for
MongoDB comparisons should be naive and all the time for Django/PostgreSQL should be timezone
aware.
Args:
plan(MigrationPlan): A Migration Plan
"""
repos = plan.get_repositories()
importers = plan.get_importers()
distributors = plan.get_distributors()
_logger.debug('Pre-migrating Pulp 2 repositories')
# the latest time we have in the migration tool in Pulp2Repository table
zero_datetime = timezone.make_aware(datetime(1970, 1, 1), timezone.utc)
last_added = Pulp2Repository.objects.aggregate(
Max('pulp2_last_unit_added'
))['pulp2_last_unit_added__max'] or zero_datetime
last_removed = Pulp2Repository.objects.aggregate(
Max('pulp2_last_unit_removed')
)['pulp2_last_unit_removed__max'] or zero_datetime
last_updated = max(last_added, last_removed)
last_updated_naive = timezone.make_naive(last_updated,
timezone=timezone.utc)
with ProgressReport(
message=
'Pre-migrating Pulp 2 repositories, importers, distributors',
code='premigrating.repositories') as pb:
# we pre-migrate:
# - empty repos (last_unit_added is not set)
# - repos which were updated since last migration (last_unit_added/removed >= last_updated)
mongo_repo_q = (mongo_Q(last_unit_added__exists=False)
| mongo_Q(last_unit_added__gte=last_updated_naive)
| mongo_Q(last_unit_removed__gte=last_updated_naive))
# in case only certain repositories are specified in the migration plan
if repos:
mongo_repo_q &= mongo_Q(repo_id__in=repos)
mongo_repo_qs = Repository.objects(mongo_repo_q)
pb.total = mongo_repo_qs.count()
pb.save()
for repo_data in mongo_repo_qs.only('id', 'repo_id', 'last_unit_added',
'last_unit_removed',
'description'):
with transaction.atomic():
repo = await pre_migrate_repo(repo_data)
await pre_migrate_importer(repo, importers)
await pre_migrate_distributor(repo, distributors)
await pre_migrate_repocontent(repo)
pb.increment()
async def run(self):
"""
The coroutine for this stage.
Returns:
The coroutine for this stage.
"""
with ProgressReport(message='Associating Content',
code='associating.content') as pb:
to_delete = set(
self.new_version.content.values_list('pk', flat=True))
async for batch in self.batches():
to_add = set()
for d_content in batch:
try:
to_delete.remove(d_content.content.pk)
except KeyError:
to_add.add(d_content.content.pk)
if to_add:
self.new_version.add_content(
Content.objects.filter(pk__in=to_add))
pb.done = pb.done + len(to_add)
pb.save()
if to_delete:
await self.put(Content.objects.filter(pk__in=to_delete))
def _parse_modulemd_default_names(self, modulemd_index):
modulemd_default_names = parse_defaults(modulemd_index)
# Parsing module-defaults happens all at one time, and from here on no useful
# work happens. So just report that it finished this stage.
modulemd_defaults_pb_data = {
"message": "Parsed Modulemd-defaults",
"code": "parsing.modulemd_defaults",
}
with ProgressReport(
**modulemd_defaults_pb_data) as modulemd_defaults_pb:
modulemd_defaults_total = len(modulemd_default_names)
modulemd_defaults_pb.total = modulemd_defaults_total
modulemd_defaults_pb.done = modulemd_defaults_total
for default in modulemd_default_names:
artifact = default.pop("artifact")
relative_path = "{}{}snippet".format(
default[PULP_MODULEDEFAULTS_ATTR.MODULE],
default[PULP_MODULEDEFAULTS_ATTR.STREAM])
da = DeclarativeArtifact(artifact=artifact,
relative_path=relative_path,
url=self.data.modules_url)
default_content = ModulemdDefaults(**default)
self.default_content_dcs.append(
DeclarativeContent(content=default_content, d_artifacts=[da]))
# delete list now that we're done with it for memory savings
del modulemd_default_names
async def run(self):
"""Build `DeclarativeContent` from the repodata."""
self.data.remote_url = self.new_url or self.remote.url
progress_data = dict(message="Downloading Metadata Files",
code="downloading.metadata")
with ProgressReport(**progress_data) as metadata_pb:
self.data.metadata_pb = metadata_pb
downloader = self.remote.get_downloader(
url=urljoin(self.data.remote_url, "repodata/repomd.xml"))
result = await downloader.run()
metadata_pb.increment()
repomd_path = result.path
self.data.repomd = cr.Repomd(repomd_path)
self.repository.last_sync_revision_number = self.data.repomd.revision
self.repository.last_sync_repomd_checksum = get_sha256(repomd_path)
await self.parse_distribution_tree()
await self.parse_repository_metadata()
await self.parse_modules_metadata()
await self.parse_packages_components()
await self.parse_content()
# now send modules down the pipeline since all relations have been set up
for modulemd in self.data.modulemd_list:
await self.put(modulemd)
for dc_group in self.data.dc_groups:
await self.put(dc_group)
async def run(self):
"""
Build and emit `DeclarativeContent` from the ansible metadata.
"""
if self.optimize:
should_we_sync = await self._should_we_sync()
if should_we_sync is False:
log.debug(_("no-op: remote wasn't updated since last sync."))
return
tasks = []
loop = asyncio.get_event_loop()
msg = _("Parsing CollectionVersion Metadata")
async with ProgressReport(message=msg,
code="sync.parsing.metadata") as pb:
self.parsing_metadata_progress_bar = pb
await self._download_unpaginated_metadata()
if self.collection_info:
for requirement_entry in self.collection_info:
tasks.append(
loop.create_task(
self._fetch_collection_metadata(
requirement_entry)))
else:
tasks.append(loop.create_task(self._find_all_collections()))
await asyncio.gather(*tasks)
async def run(self):
"""
Schedules multiple coroutines to migrate pre-migrated content to Pulp 3
It migrates content type by type.
If a plugin needs to have more control over the order of content migration, it should
override this method.
"""
content_types = self.migrator.content_models.keys()
for ctype in content_types:
# we need to go through all content in case any of Remotes changed
pulp2content_qs = Pulp2Content.objects.filter(
pulp2_content_type_id=ctype)
total_pulp2content = pulp2content_qs.count()
with ProgressReport(
message='Migrating {} content to Pulp 3 {}'.format(
self.migrator.pulp2_plugin, ctype),
code='migrating.{}.content'.format(
self.migrator.pulp2_plugin),
total=total_pulp2content) as pb:
# We are waiting on the coroutine to finish, because the order of the processed
# content for plugins like Container and RPM is important because of the relations
# between the content types.
await asyncio.wait(
[self.migrate_to_pulp3(pulp2content_qs, pb=pb)])
async def _fetch_galaxy_pages(self):
"""
Fetch the roles in a remote repository.
Returns:
async generator: dicts that represent pages from galaxy api
"""
page_count = 0
remote = self.remote
progress_data = dict(message="Parsing Pages from Galaxy Roles API", code="parsing.roles")
with ProgressReport(**progress_data) as progress_bar:
api_version = get_api_version(remote.url)
downloader = remote.get_downloader(url=get_page_url(remote.url, api_version))
metadata = parse_metadata(await downloader.run())
page_count = math.ceil(float(metadata["count"]) / float(PAGE_SIZE))
progress_bar.total = page_count
progress_bar.save()
yield metadata
progress_bar.increment()
# Concurrent downloads are limited by aiohttp...
not_done = set(
remote.get_downloader(url=get_page_url(remote.url, api_version, page)).run()
for page in range(2, page_count + 1)
)
while not_done:
done, not_done = await asyncio.wait(not_done, return_when=FIRST_COMPLETED)
for item in done:
yield parse_metadata(item.result())
progress_bar.increment()
def publish(repository_version_pk):
"""
Create a Publication based on a RepositoryVersion.
Args:
repository_version_pk (str): Create a publication from this repository version.
"""
repository_version = RepositoryVersion.objects.get(
pk=repository_version_pk)
log.info(
_("Publishing: repository=%(repository)s, version=%(version)d"),
{
"repository": repository_version.repository.name,
"version": repository_version.number
},
)
with tempfile.TemporaryDirectory("."):
with ProgressReport(message="Publishing Content",
code="publishing.content") as progress_report:
with CookbookPublication.create(repository_version) as publication:
universe = Universe("__universe__")
universe.write(
populate(publication, progress_report=progress_report))
PublishedMetadata.create_from_file(
relative_path=os.path.basename(universe.relative_path),
publication=publication,
file=File(open(universe.relative_path, "rb")),
)
log.info(_("Publication: %(publication)s created"),
{"publication": publication.pk})
async def run(self):
"""
Parse Release content units.
Update release content with information obtained from its artifact.
"""
with ProgressReport(message="Update Release units",
code="update.release") as pb:
async for d_content in self.items():
if isinstance(d_content.content, Release):
release = d_content.content
release_artifact = d_content.d_artifacts[0].artifact
release.sha256 = release_artifact.sha256
release_dict = deb822.Release(release_artifact.file)
release.codename = release_dict["Codename"]
release.suite = release_dict["Suite"]
# TODO split of extra stuff e.g. : 'updates/main' -> 'main'
release.components = _filter_ssl(
release_dict["Components"], self.components)
release.architectures = _filter_ssl(
release_dict["Architectures"], self.architectures)
log.debug("Codename: {}".format(release.codename))
log.debug("Components: {}".format(release.components))
log.debug("Architectures: {}".format(
release.architectures))
pb.increment()
await self.put(d_content)
async def run(self):
"""
Parse PackageIndex content units.
Ensure, that an uncompressed artifact is available.
"""
with ProgressReport(message="Update PackageIndex units",
code="update.packageindex") as pb:
async for d_content in self.items():
if isinstance(d_content.content, PackageIndex):
if not d_content.d_artifacts:
raise NoPackageIndexFile()
content = d_content.content
if not [
da for da in d_content.d_artifacts
if da.artifact.sha256 == content.sha256
]:
# No main_artifact found uncompress one
filename = _uncompress_artifact(d_content.d_artifacts)
da = DeclarativeArtifact(
Artifact(sha256=content.sha256),
filename,
content.relative_path,
d_content.d_artifacts[0].remote,
)
d_content.d_artifacts.append(da)
await da.download()
da.artifact.save()
log.info(
"*** Expected: {} *** Uncompressed: {} ***".format(
content.sha256, da.artifact.sha256))
pb.increment()
await self.put(d_content)
async def migrate_content(plan):
"""
A coroutine to initiate content migration for each plugin.
Args:
plan (MigrationPlan): Migration Plan to use
"""
content_migration_coros = []
plugins_to_migrate = plan.get_plugins()
progress_data = dict(message='Migrating content to Pulp 3',
code='migrating.content',
total=0)
with ProgressReport(**progress_data) as pb:
# schedule content migration into Pulp 3 using pre-migrated Pulp 2 content
for plugin in plugins_to_migrate:
plugin_migrator = PLUGIN_MIGRATORS.get(plugin)
content_migration_coros.append(
plugin_migrator.migrate_content_to_pulp3())
# only used for progress bar counters
content_types = plugin_migrator.content_models.keys()
pulp2content_qs = Pulp2Content.objects.filter(
pulp2_content_type_id__in=content_types, pulp3_content=None)
pb.total += pulp2content_qs.count()
pb.save()
await asyncio.wait(content_migration_coros)
pb.done = pb.total
async def run(self):
"""
Build and emit `DeclarativeContent` from the ansible metadata.
"""
with ProgressReport(message="Parsing Collection Metadata",
code="parsing.metadata") as pb:
async for metadata in self._fetch_collections():
url = metadata["download_url"]
collection_version = CollectionVersion(
namespace=metadata["namespace"]["name"],
name=metadata["collection"]["name"],
version=metadata["version"],
)
artifact = metadata["artifact"]
d_artifact = DeclarativeArtifact(
artifact=Artifact(sha256=artifact["sha256"],
size=artifact["size"]),
url=url,
relative_path=collection_version.relative_path,
remote=self.remote,
deferred_download=self.deferred_download,
)
d_content = DeclarativeContent(content=collection_version,
d_artifacts=[d_artifact])
pb.increment()
await self.put(d_content)
def migrate_content(plan, skip_corrupted=False):
"""
A coroutine to initiate content migration for each plugin.
Args:
plan (MigrationPlan): Migration Plan to use
skip_corrupted (bool): If True, corrupted content is skipped during migration,
no task failure.
"""
progress_data = dict(message='Migrating content to Pulp 3',
code='migrating.content',
total=0)
with ProgressReport(**progress_data) as pb:
# schedule content migration into Pulp 3 using pre-migrated Pulp 2 content
for plugin in plan.get_plugin_plans():
# only used for progress bar counters
content_types = plugin.migrator.content_models.keys()
num_to_migrate = Pulp2Content.objects.filter(
pulp2_content_type_id__in=content_types,
pulp3_content=None).count()
pb.total += num_to_migrate
pb.save()
# migrate
plugin.migrator.migrate_content_to_pulp3(
skip_corrupted=skip_corrupted)
pb.done = pb.total
pb.save()
async def run(self):
"""
Build and emit `DeclarativeContent` from the ansible metadata.
"""
async with ProgressReport(
message="Parsing Role Metadata", code="sync.parsing.metadata"
) as pb:
async for metadata in self._fetch_roles():
for version in metadata["summary_fields"]["versions"]:
url = GITHUB_URL % (
metadata["github_user"],
metadata["github_repo"],
version["name"],
)
role = Role(
version=version["name"],
name=metadata["name"],
namespace=metadata["namespace"],
)
relative_path = "%s/%s/%s.tar.gz" % (
metadata["namespace"],
metadata["name"],
version["name"],
)
d_artifact = DeclarativeArtifact(
artifact=Artifact(),
url=url,
relative_path=relative_path,
remote=self.remote,
deferred_download=self.deferred_download,
)
d_content = DeclarativeContent(content=role, d_artifacts=[d_artifact])
await pb.aincrement()
await self.put(d_content)
async def migrate_repositories():
"""
A coroutine to migrate pre-migrated repositories.
"""
progress_data = dict(message='Creating repositories in Pulp 3',
code='creating.repositories',
total=0)
with ProgressReport(**progress_data) as pb:
pulp2repos_qs = Pulp2Repository.objects.filter(
pulp3_repository_version=None)
pb.total += pulp2repos_qs.count()
pb.save()
for pulp2repo in pulp2repos_qs:
# if pulp2 repo_id is too long, its hash is included in pulp3 repo name
pulp3_repo_name = pulp2repo.pulp2_repo_id
if len(pulp3_repo_name) > 255:
repo_name_hash = hashlib.sha256(
pulp3_repo_name.encode()).hexdigest()
pulp3_repo_name = '{}-{}'.format(pulp3_repo_name[:190],
repo_name_hash)
repo, created = Repository.objects.get_or_create(
name=pulp3_repo_name, description=pulp2repo.pulp2_description)
if created:
pb.increment()
else:
pb.total -= 1
pb.save()
async def run(self):
"""
If includes is specified, then only sync those,else try to sync all other packages
"""
# TODO Change Bandersnatch internal API to take proxy settings in from config parameters
if self.remote.proxy_url:
environ['http_proxy'] = self.remote.proxy_url
# local & global timeouts defaults to 10secs and 5 hours
async with Master(self.remote.url) as master:
if self.remote.proxy_url:
environ.pop('http_proxy')
deferred_download = self.remote.policy != Remote.IMMEDIATE
workers = self.remote.download_concurrency or self.remote.DEFAULT_DOWNLOAD_CONCURRENCY
with ProgressReport(
message="Fetching Project Metadata", code="sync.fetching.project"
) as p:
pmirror = PulpMirror(
serial=0, # Serial currently isn't supported by Pulp
master=master,
workers=workers,
deferred_download=deferred_download,
python_stage=self,
progress_report=p,
)
packages_to_sync = None
if self.remote.includes:
packages_to_sync = [
Requirement(pkg).name for pkg in self.remote.includes
]
await pmirror.synchronize(packages_to_sync)
def migrate_importers(plan):
"""
A coroutine to migrate pre-migrated importers.
Args:
plan (MigrationPlan): Migration Plan to use.
"""
# gather all needed plugin importer migrators
importer_migrators = {}
for plugin in plan.get_plugin_plans():
importer_migrators.update(**plugin.migrator.importer_migrators)
progress_data = dict(message='Migrating importers to Pulp 3',
code='migrating.importers',
total=0)
with ProgressReport(**progress_data) as pb:
pulp2importers_qs = Pulp2Importer.objects.filter(is_migrated=False,
not_in_plan=False)
pb.total += pulp2importers_qs.count()
pb.save()
for pulp2importer in pulp2importers_qs:
importer_migrator = importer_migrators.get(
pulp2importer.pulp2_type_id)
remote, created = importer_migrator.migrate_to_pulp3(pulp2importer)
pulp2importer.pulp3_remote = remote
pulp2importer.is_migrated = True
pulp2importer.save()
if created:
pb.increment()
else:
pb.total -= 1
pb.save()
async def run(self):
"""
Build and emit `DeclarativeContent` from the Spec data.
"""
# Interpret policy to download Artifacts or not
deferred_download = self.remote.policy != Remote.IMMEDIATE
with ProgressReport(message="Downloading Metadata") as progress:
parsed_url = urlparse(self.remote.url)
root_dir = parsed_url.path
specs_path = os.path.join(root_dir, "specs.4.8.gz")
specs_url = urlunparse(parsed_url._replace(path=specs_path))
downloader = self.remote.get_downloader(url=specs_url)
result = await downloader.run()
progress.increment()
with ProgressReport(message="Parsing Metadata") as progress:
for key in read_specs(result.path):
relative_path = os.path.join(
"gems", key.name + "-" + key.version + ".gem")
path = os.path.join(root_dir, relative_path)
url = urlunparse(parsed_url._replace(path=path))
spec_relative_path = os.path.join(
"quick/Marshal.4.8",
key.name + "-" + key.version + ".gemspec.rz")
spec_path = os.path.join(root_dir, spec_relative_path)
spec_url = urlunparse(parsed_url._replace(path=spec_path))
gem = GemContent(name=key.name, version=key.version)
da_gem = DeclarativeArtifact(
artifact=Artifact(),
url=url,
relative_path=relative_path,
remote=self.remote,
deferred_download=deferred_download,
)
da_spec = DeclarativeArtifact(
artifact=Artifact(),
url=spec_url,
relative_path=spec_relative_path,
remote=self.remote,
deferred_download=deferred_download,
)
dc = DeclarativeContent(content=gem,
d_artifacts=[da_gem, da_spec])
progress.increment()
await self.put(dc)
async def _fetch_collections(self):
"""
Fetch the collections in a remote repository.
Returns:
async generator: dicts that represent collections from galaxy api
"""
page_count = 1
remote = self.remote
collection_info = self.collection_info
def _get_url(page):
if collection_info:
name, version, source = collection_info[page - 1]
namespace, name = name.split(".")
root = source or remote.url
url = f"{root}/api/v2/collections/{namespace}/{name}"
return url
return get_page_url(remote.url, page)
progress_data = dict(message="Parsing Galaxy Collections API", code="parsing.collections")
with ProgressReport(**progress_data) as progress_bar:
url = _get_url(page_count)
downloader = remote.get_downloader(url=url)
initial_data = parse_metadata(await downloader.run())
count = len(self.collection_info) or initial_data.get("count", 1)
page_count = math.ceil(float(count) / float(PAGE_SIZE))
progress_bar.total = count
progress_bar.save()
# Concurrent downloads are limited by aiohttp...
not_done = set()
for page in range(1, page_count + 1):
downloader = remote.get_downloader(url=_get_url(page))
not_done.add(downloader.run())
while not_done:
done, not_done = await asyncio.wait(not_done, return_when=asyncio.FIRST_COMPLETED)
for item in done:
data = parse_metadata(item.result())
for result in data.get("results", [data]):
download_url = result.get("download_url")
if result.get("versions_url"):
not_done.update(
[remote.get_downloader(url=result["versions_url"]).run()]
)
if result.get("version") and not download_url:
not_done.update([remote.get_downloader(url=result["href"]).run()])
if download_url:
yield data
progress_bar.increment()
async def run(self):
"""
Build and emit `DeclarativeContent` from the Manifest data.
Args:
in_q (asyncio.Queue): Unused because the first stage doesn't read from an input queue.
out_q (asyncio.Queue): The out_q to send `DeclarativeContent` objects to
"""
remote_url = self.remote.url
if not remote_url.endswith('/index.yaml'):
remote_url += '/index.yaml'
# TODO Skip reading generator to list?
index_yaml = []
with ProgressReport(message="Downloading Index",
code="downloading.metadata") as pb:
downloader = self.remote.get_downloader(url=remote_url)
result = await downloader.run()
index_yaml = list(self.read_index_yaml(result.path))
pb.increment()
with ProgressReport(message="Parsing Entries",
code="parsing.metadata") as pb:
pb.total = len(index_yaml)
pb.save()
for entry in index_yaml:
content_entry = dict(
filter(lambda e: e[0] not in ('url'), entry.items()))
unit = ChartContent(**content_entry)
artifact = Artifact(sha256=entry['digest'])
da = DeclarativeArtifact(
artifact,
urljoin(remote_url, entry['url']),
"{}-{}.tgz".format(entry['name'], entry['version']),
self.remote,
deferred_download=self.deferred_download,
)
dc = DeclarativeContent(content=unit, d_artifacts=[da])
pb.increment()
await self.put(dc)
async def migrate_repositories(plan):
"""
A coroutine to migrate pre-migrated repositories.
"""
progress_data = dict(message='Creating repositories in Pulp 3',
code='creating.repositories',
total=0)
with ProgressReport(**progress_data) as pb:
for plugin in plan.get_plugin_plans():
pulp2repos_qs = Pulp2Repository.objects.filter(
pulp3_repository_version=None,
not_in_plan=False,
pulp2_repo_type=plugin.type,
)
repos_to_create = plugin.get_repo_creation_setup()
# no specific migration plan for repositories
if not repos_to_create:
pb.total += pulp2repos_qs.count()
pb.save()
for pulp2repo in pulp2repos_qs:
pulp3_repo_name = pulp2repo.pulp2_repo_id
repository_class = plugin.migrator.pulp3_repository
repo, created = repository_class.objects.get_or_create(
name=pulp3_repo_name,
defaults={'description': pulp2repo.pulp2_description})
if created:
pb.increment()
else:
pb.total -= 1
pb.save()
# specific migration plan for repositories
else:
pb.total += len(repos_to_create)
pb.save()
for pulp3_repo_name in repos_to_create:
try:
pulp2repo = pulp2repos_qs.get(
pulp2_repo_id=pulp3_repo_name)
except Pulp2Repository.DoesNotExist:
description = pulp3_repo_name
else:
description = pulp2repo.pulp2_description
repository_class = plugin.migrator.pulp3_repository
repo, created = repository_class.objects.get_or_create(
name=pulp3_repo_name,
defaults={'description': description})
if created:
pb.increment()
else:
pb.total -= 1
pb.save()
def migrate_repositories(plan):
"""
A coroutine to migrate pre-migrated repositories.
"""
progress_data = dict(message='Creating repositories in Pulp 3',
code='creating.repositories',
total=0)
with ProgressReport(**progress_data) as pb:
for plugin in plan.get_plugin_plans():
# all pulp2 repos in current plan were already migrated, no need to proceed
not_migrated_repos = Pulp2Repository.objects.filter(
is_migrated=False,
not_in_plan=False,
pulp2_repo_type=plugin.type)
if not not_migrated_repos.exists():
continue
pulp2repos_qs = Pulp2Repository.objects.filter(
pulp3_repository_version=None,
not_in_plan=False,
pulp2_repo_type=plugin.type,
)
repos_to_create = plugin.get_repo_creation_setup()
pb.total += len(repos_to_create)
pb.save()
for pulp3_repo_name in repos_to_create:
try:
pulp2repo = pulp2repos_qs.get(
pulp2_repo_id=pulp3_repo_name)
except Pulp2Repository.DoesNotExist:
description = pulp3_repo_name
else:
description = pulp2repo.pulp2_description
repository_class = plugin.migrator.pulp3_repository
repo, created = repository_class.objects.get_or_create(
name=pulp3_repo_name,
defaults={'description': description})
pulp2_repo_ids = []
repo_version_setup = repos_to_create[pulp3_repo_name].get(
'repository_versions')
for repo_version in repo_version_setup:
pulp2_repo_ids.append(repo_version['repo_id'])
pulp2repos_qs = Pulp2Repository.objects.filter(
pulp2_repo_id__in=pulp2_repo_ids,
pulp3_repository__isnull=True)
pulp2repos_qs.update(pulp3_repository=repo)
if created:
pb.increment()
else:
pb.total -= 1
pb.save()
async def migrate_repositories(plan):
"""
A coroutine to migrate pre-migrated repositories.
"""
repos_to_create = plan.get_pulp3_repository_setup().keys()
progress_data = dict(
message='Creating repositories in Pulp 3', code='creating.repositories', total=0
)
with ProgressReport(**progress_data) as pb:
pulp2repos_qs = Pulp2Repository.objects.filter(pulp3_repository_version=None)
# no specific migration plan for repositories
if not repos_to_create:
pb.total += pulp2repos_qs.count()
pb.save()
for pulp2repo in pulp2repos_qs:
# if pulp2 repo_id is too long, its hash is included in pulp3 repo name
pulp3_repo_name = pulp2repo.pulp2_repo_id
if len(pulp3_repo_name) > 255:
repo_name_hash = hashlib.sha256(pulp3_repo_name.encode()).hexdigest()
pulp3_repo_name = '{}-{}'.format(pulp3_repo_name[:190], repo_name_hash)
repo, created = Repository.objects.get_or_create(
name=pulp3_repo_name,
description=pulp2repo.pulp2_description)
if created:
pb.increment()
else:
pb.total -= 1
pb.save()
# specific migration plan for repositories
else:
pb.total += len(repos_to_create)
pb.save()
for pulp3_repo_name in repos_to_create:
try:
pulp2repo = pulp2repos_qs.get(pulp2_repo_id=pulp3_repo_name)
except Pulp2Repository.DoesNotExist:
description = pulp3_repo_name
else:
description = pulp2repo.pulp2_description
repo, created = Repository.objects.get_or_create(
name=pulp3_repo_name[:255],
description=description)
if created:
pb.increment()
else:
pb.total -= 1
pb.save()
async def migrate_repositories(plan):
"""
A coroutine to migrate pre-migrated repositories.
"""
repos_to_create = plan.get_pulp3_repository_setup().keys()
progress_data = dict(message='Creating repositories in Pulp 3',
code='creating.repositories',
total=0)
with ProgressReport(**progress_data) as pb:
pulp2repos_qs = Pulp2Repository.objects.filter(
pulp3_repository_version=None)
# no specific migration plan for repositories
if not repos_to_create:
pb.total += pulp2repos_qs.count()
pb.save()
for pulp2repo in pulp2repos_qs:
pulp3_repo_name = pulp2repo.pulp2_repo_id
repository_class = PLUGIN_MIGRATORS.get(
pulp2repo.type).pulp3_repository
repo, created = repository_class.objects.get_or_create(
name=pulp3_repo_name,
description=pulp2repo.pulp2_description)
if created:
pb.increment()
else:
pb.total -= 1
pb.save()
# specific migration plan for repositories
else:
pb.total += len(repos_to_create)
pb.save()
for pulp3_repo_name in repos_to_create:
try:
pulp2repo = pulp2repos_qs.get(
pulp2_repo_id=pulp3_repo_name)
except Pulp2Repository.DoesNotExist:
description = pulp3_repo_name
else:
description = pulp2repo.pulp2_description
repository_class = PLUGIN_MIGRATORS.get(
pulp2repo.type).pulp3_repository
repo, created = repository_class.objects.get_or_create(
name=pulp3_repo_name, description=description)
if created:
pb.increment()
else:
pb.total -= 1
pb.save()
async def run(self):
"""
Build and emit `DeclarativeContent` from the ansible metadata.
"""
async with ProgressReport(
message="Cloning Git repository for Collection",
code="sync.git.clone") as pb:
d_content = await declarative_content_from_git_repo(
self.remote, self.remote.url, self.remote.git_ref,
self.metadata_only)
await self.put(d_content)
await pb.aincrement()
async def run(self):
"""
Build and emit `DeclarativeContent` from the Manifest data.
If a cookbook specifier is set in the remote, cookbooks are filtered
using this specifier.
"""
async with ProgressReport(message="Downloading Metadata",
code="downloading.metadata",
total=1) as pb:
downloader = self.remote.get_downloader(
url=urljoin(self.remote.url + "/", "universe"))
result = await downloader.run()
await pb.aincrement()
cookbook_names = self.remote.specifier_cookbook_names()
async with ProgressReport(message="Parsing Metadata",
code="parsing.metadata") as pb:
universe = Universe(result.path)
for entry in universe.read():
if cookbook_names and entry.name not in cookbook_names:
continue
cookbook = CookbookPackageContent(
name=entry.name,
version=entry.version,
dependencies=entry.dependencies)
artifact = Artifact()
da = DeclarativeArtifact(
artifact=artifact,
url=entry.download_url,
relative_path=cookbook.relative_path(),
remote=self.remote,
deferred_download=not self.download_artifacts,
)
dc = DeclarativeContent(content=cookbook, d_artifacts=[da])
await pb.aincrement()
await self.put(dc)
