def pipeline_stages(self, new_version):
"""
Build a list of stages feeding into the ContentUnitAssociation stage.
This defines the "architecture" of the entire sync.
Args:
new_version (:class:`~pulpcore.plugin.models.RepositoryVersion`): The
new repository version that is going to be built.
Returns:
list: List of :class:`~pulpcore.plugin.stages.Stage` instances
"""
return [
TagListStage(self.remote),
# In: Pending Tags (not downloaded yet)
ArtifactDownloader(),
ArtifactSaver(),
ProcessContentStage(self.remote),
SerialContentSave(),
RemoteArtifactSaver(),
# Out: Finished Tags, Finished ManifestLists, Finished ImageManifests,
# Pending ImageManifests, Pending ManifestBlobs
# In: Pending ImageManifests, Pending Blobs
# In: Finished content (no-op)
ArtifactDownloader(),
ArtifactSaver(),
ProcessContentStage(self.remote),
SerialContentSave(),
RemoteArtifactSaver(),
# Out: No-op (Finished Tags, ManifestLists, ImageManifests)
# Out: Finished ImageManifests, Finished ManifestBlobs, Pending ManifestBlobs
# In: Pending Blobs
# In: Finished content (no-op)
ArtifactDownloader(),
ArtifactSaver(),
SerialContentSave(),
RemoteArtifactSaver(),
# Out: Finished content, Tags, ManifestLists, ImageManifests, ManifestBlobs
# In: Tags, ManifestLists, ImageManifests, ManifestBlobs (downloaded, processed, and
# saved)
# Requires that all content (and related content in dc.extra_data) is already saved.
InterrelateContent(),
# Out: Content that has been related to other Content.
]
def pipeline_stages(self, new_version):
"""
Build the list of pipeline stages feeding into the ContentAssociation stage.
Args:
new_version (:class:`~pulpcore.plugin.models.RepositoryVersion`): The
new repository version that is going to be built.
Returns:
list: List of :class:`~pulpcore.plugin.stages.Stage` instances
"""
pipeline = [
self.first_stage,
QueryExistingArtifacts(),
ArtifactDownloader(),
DebDropEmptyContent(),
ArtifactSaver(),
# This is dependent on
# https://salsa.debian.org/python-debian-team/python-debian/merge_requests/11
# DebUpdatePackageAttributes(),
DebUpdateReleaseAttributes(self.first_stage.components,
self.first_stage.architectures),
DebUpdatePackageIndexAttributes(),
QueryExistingContents(),
ContentSaver(),
RemoteArtifactSaver(),
ResolveContentFutures(),
]
return pipeline
def pipeline_stages(self, new_version):
"""
Build a list of stages feeding into the ContentUnitAssociation stage.
This defines the "architecture" of the entire sync.
Args:
new_version (:class:`~pulpcore.plugin.models.RepositoryVersion`): The
new repository version that is going to be built.
Returns:
list: List of :class:`~pulpcore.plugin.stages.Stage` instances
"""
pipeline = [
self.first_stage,
QueryExistingArtifacts(),
ArtifactDownloader(),
ArtifactSaver(),
QueryExistingContents(),
ContentSaver(),
RemoteArtifactSaver(),
ResolveContentFutures(),
InterrelateContent(),
]
return pipeline
def pipeline_stages(self, new_version):
"""
Build the list of pipeline stages feeding into the ContentAssociation stage.
Args:
new_version (:class:`~pulpcore.plugin.models.RepositoryVersion`): The
new repository version that is going to be built.
Returns:
list: List of :class:`~pulpcore.plugin.stages.Stage` instances
"""
pipeline = [
self.first_stage,
QueryExistingArtifacts(),
ArtifactDownloader(),
DebDropFailedArtifacts(),
ArtifactSaver(),
DebUpdateReleaseFileAttributes(remote=self.first_stage.remote),
DebUpdatePackageIndexAttributes(),
QueryExistingContents(),
ContentSaver(),
RemoteArtifactSaver(),
ResolveContentFutures(),
]
return pipeline
def pipeline_stages(self, new_version):
"""
Build a list of stages feeding into the ContentUnitAssociation stage.
This defines the "architecture" of the entire sync.
Args:
new_version (:class:`~pulpcore.plugin.models.RepositoryVersion`): The
new repository version that is going to be built.
Returns:
list: List of :class:`~pulpcore.plugin.stages.Stage` instances
"""
pipeline = [
self.first_stage,
QueryExistingArtifacts(),
ArtifactDownloader(),
ArtifactSaver(),
QueryExistingContents(),
RpmContentSaver(),
RemoteArtifactSaver(),
]
for dupe_query_dict in self.remove_duplicates:
pipeline.append(RemoveDuplicates(new_version, **dupe_query_dict))
return pipeline
def pipeline_stages(self, new_version):
return [
self.first_stage,
QueryExistingContentUnits(),
ExistingContentNeedsNoArtifacts(),
ArtifactDownloader(),
ArtifactSaver(),
ContentUnitSaver()
]
def pipeline_stages(self, new_version):
"""
Build a list of stages feeding into the ContentUnitAssociation stage.
This defines the "architecture" of the entire sync.
Args:
new_version (:class:`~pulpcore.plugin.models.RepositoryVersion`): The
new repository version that is going to be built.
Returns:
list: List of :class:`~pulpcore.plugin.stages.Stage` instances
"""
# We only want to create a single instance of each stage. Each call to the stage is
# encapsulated, so it isn't necessary to create a new instance.
downloader = ArtifactDownloader()
serial_artifact_save = SerialArtifactSave()
serial_content_save = SerialContentSave()
process_content = ProcessContentStage(self.remote)
return [
TagListStage(self.remote),
# In: Pending Tags (not downloaded yet)
downloader,
serial_artifact_save,
process_content,
serial_content_save,
# Out: Finished Tags, Finished ManifestLists, Finished ImageManifests,
# Pending ImageManifests, Pending ManifestBlobs
# In: Pending ImageManifests, Pending Blobs
# In: Finished content (no-op)
downloader,
serial_artifact_save,
process_content,
serial_content_save,
# Out: No-op (Finished Tags, ManifestLists, ImageManifests)
# Out: Finished ImageManifests, Finished ManifestBlobs, Pending ManifestBlobs
# In: Pending Blobs
# In: Finished content (no-op)
downloader,
serial_artifact_save,
serial_content_save,
# Out: Finished content, Tags, ManifestLists, ImageManifests, ManifestBlobs
# In: Tags, ManifestLists, ImageManifests, ManifestBlobs (downloaded, processed, and
# saved)
# Requires that all content (and related content in dc.extra_data) is already saved.
InterrelateContent(),
# Out: Content that has been related to other Content.
]
def pipeline_stages(self, new_version):
pipeline = [self.first_stage, QueryExistingRepoContentAndArtifacts(new_version=new_version)]
if self.download_artifacts:
pipeline.extend(
[
ArtifactDownloader(),
ArtifactSaver(),
UpdateContentWithDownloadResult(),
QueryExistingContents(), # share content with known digest
]
)
pipeline.append(ContentSaver())
pipeline.append(RemoteArtifactSaver())
return pipeline
def synchronize(remote_pk, repository_pk):
"""
Sync content from the remote repository.
Create a new version of the repository that is synchronized with the remote.
Args:
remote_pk (str): The remote PK.
repository_pk (str): The repository PK.
Raises:
ValueError: If the remote does not specify a url to sync.
"""
remote = RpmRemote.objects.get(pk=remote_pk)
repository = Repository.objects.get(pk=repository_pk)
dupe_criteria = {'model': Package,
'field_names': ['name', 'epoch', 'version', 'release', 'arch']}
if not remote.url:
raise ValueError(_('A remote must have a url specified to synchronize.'))
log.info(_('Synchronizing: repository={r} remote={p}').format(
r=repository.name, p=remote.name))
download_artifacts = (remote.policy == Remote.IMMEDIATE)
first_stage = RpmFirstStage(remote)
with WorkingDirectory():
with RepositoryVersion.create(repository) as new_version:
loop = asyncio.get_event_loop()
remove_duplicates_stage = RemoveDuplicates(new_version, **dupe_criteria)
stages = [first_stage]
if download_artifacts:
stages.extend([QueryExistingArtifacts(), ArtifactDownloader(), ArtifactSaver()])
stages.extend([
QueryExistingContents(), ErratumContentSaver(), remove_duplicates_stage,
ContentAssociation(new_version), EndStage()
])
pipeline = create_pipeline(stages)
loop.run_until_complete(pipeline)
def synchronize(remote_pk, repository_pk):
"""
Sync content from the remote repository.
Create a new version of the repository that is synchronized with the remote.
Args:
remote_pk (str): The remote PK.
repository_pk (str): The repository PK.
Raises:
ValueError: If the remote does not specify a url to sync.
"""
remote = RpmRemote.objects.get(pk=remote_pk)
repository = Repository.objects.get(pk=repository_pk)
if not remote.url:
raise ValueError(
_('A remote must have a url specified to synchronize.'))
log.info(
_('Synchronizing: repository={r} remote={p}').format(r=repository.name,
p=remote.name))
first_stage = RpmFirstStage(remote)
with WorkingDirectory():
with RepositoryVersion.create(repository) as new_version:
loop = asyncio.get_event_loop()
stages = [
first_stage,
QueryExistingArtifacts(),
ArtifactDownloader(),
ArtifactSaver(),
QueryExistingContentUnits(),
ErratumContentUnitSaver(),
ContentUnitAssociation(new_version),
EndStage()
]
pipeline = create_pipeline(stages)
loop.run_until_complete(pipeline)
def create(self):
"""
Perform the work. This is the long-blocking call where all syncing occurs.
"""
with WorkingDirectory():
with RepositoryVersion.create(self.repository) as new_version:
loop = asyncio.get_event_loop()
stages = [
self.first_stage,
QueryExistingContentUnits(),
ExistingContentNeedsNoArtifacts(),
ArtifactDownloader(),
ArtifactSaver(),
ContentUnitSaver(),
ContentUnitAssociation(new_version)
]
if self.mirror:
stages.append(ContentUnitUnassociation(new_version))
stages.append(EndStage())
pipeline = create_pipeline(stages)
loop.run_until_complete(pipeline)
def pipeline_stages(self, new_version):
"""
Build the list of pipeline stages feeding into the ContentUnitAssociation stage.
This is overwritten to create a custom pipeline.
Args:
new_version (:class:`~pulpcore.plugin.models.RepositoryVersion`): The
new repository version that is going to be built.
Returns:
list: List of :class:`~pulpcore.plugin.stages.Stage` instances
"""
pipeline = [
self.first_stage,
QueryExistingContents(),
UpdateExistingContentArtifacts(),
ArtifactDownloader(),
ArtifactSaver(),
ContentSaver(),
RemoteArtifactSaver(),
]
return pipeline