def pipeline_stages(self, new_version): """ Build a list of stages feeding into the ContentUnitAssociation stage. This defines the "architecture" of the entire sync. Args: new_version (:class:`~pulpcore.plugin.models.RepositoryVersion`): The new repository version that is going to be built. Returns: list: List of :class:`~pulpcore.plugin.stages.Stage` instances """ return [ TagListStage(self.remote), # In: Pending Tags (not downloaded yet) ArtifactDownloader(), ArtifactSaver(), ProcessContentStage(self.remote), SerialContentSave(), RemoteArtifactSaver(), # Out: Finished Tags, Finished ManifestLists, Finished ImageManifests, # Pending ImageManifests, Pending ManifestBlobs # In: Pending ImageManifests, Pending Blobs # In: Finished content (no-op) ArtifactDownloader(), ArtifactSaver(), ProcessContentStage(self.remote), SerialContentSave(), RemoteArtifactSaver(), # Out: No-op (Finished Tags, ManifestLists, ImageManifests) # Out: Finished ImageManifests, Finished ManifestBlobs, Pending ManifestBlobs # In: Pending Blobs # In: Finished content (no-op) ArtifactDownloader(), ArtifactSaver(), SerialContentSave(), RemoteArtifactSaver(), # Out: Finished content, Tags, ManifestLists, ImageManifests, ManifestBlobs # In: Tags, ManifestLists, ImageManifests, ManifestBlobs (downloaded, processed, and # saved) # Requires that all content (and related content in dc.extra_data) is already saved. InterrelateContent(), # Out: Content that has been related to other Content. ]
def pipeline_stages(self, new_version): """ Build the list of pipeline stages feeding into the ContentAssociation stage. Args: new_version (:class:`~pulpcore.plugin.models.RepositoryVersion`): The new repository version that is going to be built. Returns: list: List of :class:`~pulpcore.plugin.stages.Stage` instances """ pipeline = [ self.first_stage, QueryExistingArtifacts(), ArtifactDownloader(), DebDropEmptyContent(), ArtifactSaver(), # This is dependent on # https://salsa.debian.org/python-debian-team/python-debian/merge_requests/11 # DebUpdatePackageAttributes(), DebUpdateReleaseAttributes(self.first_stage.components, self.first_stage.architectures), DebUpdatePackageIndexAttributes(), QueryExistingContents(), ContentSaver(), RemoteArtifactSaver(), ResolveContentFutures(), ] return pipeline
def pipeline_stages(self, new_version): """ Build a list of stages feeding into the ContentUnitAssociation stage. This defines the "architecture" of the entire sync. Args: new_version (:class:`~pulpcore.plugin.models.RepositoryVersion`): The new repository version that is going to be built. Returns: list: List of :class:`~pulpcore.plugin.stages.Stage` instances """ pipeline = [ self.first_stage, QueryExistingArtifacts(), ArtifactDownloader(), ArtifactSaver(), QueryExistingContents(), ContentSaver(), RemoteArtifactSaver(), ResolveContentFutures(), InterrelateContent(), ] return pipeline
def pipeline_stages(self, new_version): """ Build the list of pipeline stages feeding into the ContentAssociation stage. Args: new_version (:class:`~pulpcore.plugin.models.RepositoryVersion`): The new repository version that is going to be built. Returns: list: List of :class:`~pulpcore.plugin.stages.Stage` instances """ pipeline = [ self.first_stage, QueryExistingArtifacts(), ArtifactDownloader(), DebDropFailedArtifacts(), ArtifactSaver(), DebUpdateReleaseFileAttributes(remote=self.first_stage.remote), DebUpdatePackageIndexAttributes(), QueryExistingContents(), ContentSaver(), RemoteArtifactSaver(), ResolveContentFutures(), ] return pipeline
def pipeline_stages(self, new_version): """ Build a list of stages feeding into the ContentUnitAssociation stage. This defines the "architecture" of the entire sync. Args: new_version (:class:`~pulpcore.plugin.models.RepositoryVersion`): The new repository version that is going to be built. Returns: list: List of :class:`~pulpcore.plugin.stages.Stage` instances """ pipeline = [ self.first_stage, QueryExistingArtifacts(), ArtifactDownloader(), ArtifactSaver(), QueryExistingContents(), RpmContentSaver(), RemoteArtifactSaver(), ] for dupe_query_dict in self.remove_duplicates: pipeline.append(RemoveDuplicates(new_version, **dupe_query_dict)) return pipeline
def pipeline_stages(self, new_version): return [ self.first_stage, QueryExistingContentUnits(), ExistingContentNeedsNoArtifacts(), ArtifactDownloader(), ArtifactSaver(), ContentUnitSaver() ]
def pipeline_stages(self, new_version): """ Build a list of stages feeding into the ContentUnitAssociation stage. This defines the "architecture" of the entire sync. Args: new_version (:class:`~pulpcore.plugin.models.RepositoryVersion`): The new repository version that is going to be built. Returns: list: List of :class:`~pulpcore.plugin.stages.Stage` instances """ # We only want to create a single instance of each stage. Each call to the stage is # encapsulated, so it isn't necessary to create a new instance. downloader = ArtifactDownloader() serial_artifact_save = SerialArtifactSave() serial_content_save = SerialContentSave() process_content = ProcessContentStage(self.remote) return [ TagListStage(self.remote), # In: Pending Tags (not downloaded yet) downloader, serial_artifact_save, process_content, serial_content_save, # Out: Finished Tags, Finished ManifestLists, Finished ImageManifests, # Pending ImageManifests, Pending ManifestBlobs # In: Pending ImageManifests, Pending Blobs # In: Finished content (no-op) downloader, serial_artifact_save, process_content, serial_content_save, # Out: No-op (Finished Tags, ManifestLists, ImageManifests) # Out: Finished ImageManifests, Finished ManifestBlobs, Pending ManifestBlobs # In: Pending Blobs # In: Finished content (no-op) downloader, serial_artifact_save, serial_content_save, # Out: Finished content, Tags, ManifestLists, ImageManifests, ManifestBlobs # In: Tags, ManifestLists, ImageManifests, ManifestBlobs (downloaded, processed, and # saved) # Requires that all content (and related content in dc.extra_data) is already saved. InterrelateContent(), # Out: Content that has been related to other Content. ]
def pipeline_stages(self, new_version): pipeline = [self.first_stage, QueryExistingRepoContentAndArtifacts(new_version=new_version)] if self.download_artifacts: pipeline.extend( [ ArtifactDownloader(), ArtifactSaver(), UpdateContentWithDownloadResult(), QueryExistingContents(), # share content with known digest ] ) pipeline.append(ContentSaver()) pipeline.append(RemoteArtifactSaver()) return pipeline
def synchronize(remote_pk, repository_pk): """ Sync content from the remote repository. Create a new version of the repository that is synchronized with the remote. Args: remote_pk (str): The remote PK. repository_pk (str): The repository PK. Raises: ValueError: If the remote does not specify a url to sync. """ remote = RpmRemote.objects.get(pk=remote_pk) repository = Repository.objects.get(pk=repository_pk) dupe_criteria = {'model': Package, 'field_names': ['name', 'epoch', 'version', 'release', 'arch']} if not remote.url: raise ValueError(_('A remote must have a url specified to synchronize.')) log.info(_('Synchronizing: repository={r} remote={p}').format( r=repository.name, p=remote.name)) download_artifacts = (remote.policy == Remote.IMMEDIATE) first_stage = RpmFirstStage(remote) with WorkingDirectory(): with RepositoryVersion.create(repository) as new_version: loop = asyncio.get_event_loop() remove_duplicates_stage = RemoveDuplicates(new_version, **dupe_criteria) stages = [first_stage] if download_artifacts: stages.extend([QueryExistingArtifacts(), ArtifactDownloader(), ArtifactSaver()]) stages.extend([ QueryExistingContents(), ErratumContentSaver(), remove_duplicates_stage, ContentAssociation(new_version), EndStage() ]) pipeline = create_pipeline(stages) loop.run_until_complete(pipeline)
def synchronize(remote_pk, repository_pk): """ Sync content from the remote repository. Create a new version of the repository that is synchronized with the remote. Args: remote_pk (str): The remote PK. repository_pk (str): The repository PK. Raises: ValueError: If the remote does not specify a url to sync. """ remote = RpmRemote.objects.get(pk=remote_pk) repository = Repository.objects.get(pk=repository_pk) if not remote.url: raise ValueError( _('A remote must have a url specified to synchronize.')) log.info( _('Synchronizing: repository={r} remote={p}').format(r=repository.name, p=remote.name)) first_stage = RpmFirstStage(remote) with WorkingDirectory(): with RepositoryVersion.create(repository) as new_version: loop = asyncio.get_event_loop() stages = [ first_stage, QueryExistingArtifacts(), ArtifactDownloader(), ArtifactSaver(), QueryExistingContentUnits(), ErratumContentUnitSaver(), ContentUnitAssociation(new_version), EndStage() ] pipeline = create_pipeline(stages) loop.run_until_complete(pipeline)
def create(self): """ Perform the work. This is the long-blocking call where all syncing occurs. """ with WorkingDirectory(): with RepositoryVersion.create(self.repository) as new_version: loop = asyncio.get_event_loop() stages = [ self.first_stage, QueryExistingContentUnits(), ExistingContentNeedsNoArtifacts(), ArtifactDownloader(), ArtifactSaver(), ContentUnitSaver(), ContentUnitAssociation(new_version) ] if self.mirror: stages.append(ContentUnitUnassociation(new_version)) stages.append(EndStage()) pipeline = create_pipeline(stages) loop.run_until_complete(pipeline)
def pipeline_stages(self, new_version): """ Build the list of pipeline stages feeding into the ContentUnitAssociation stage. This is overwritten to create a custom pipeline. Args: new_version (:class:`~pulpcore.plugin.models.RepositoryVersion`): The new repository version that is going to be built. Returns: list: List of :class:`~pulpcore.plugin.stages.Stage` instances """ pipeline = [ self.first_stage, QueryExistingContents(), UpdateExistingContentArtifacts(), ArtifactDownloader(), ArtifactSaver(), ContentSaver(), RemoteArtifactSaver(), ] return pipeline