コード例 #1
0
    def __init__(self, working_dir, conduit, config, repo):
        """
        :param working_dir: working directory to create the filelists.xml.gz in
        :type  working_dir: str
        :param conduit: The conduit to get api calls
        :type conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
        :param config: Pulp configuration for the distributor
        :type  config: pulp.plugins.config.PluginCallConfiguration
        :param repo: Pulp managed repository
        :type  repo: pulp.plugins.model.Repository
        """

        self.repo_id = repo.id
        metadata_file_path = os.path.join(working_dir,
                                          configuration.get_redirect_file_name(repo))
        super(RedirectFileContext, self).__init__(metadata_file_path)
        scratchpad = conduit.get_repo_scratchpad()

        tag_list = scratchpad.get(u'tags', [])
        self.tags = self.convert_tag_list_to_dict(tag_list)

        self.registry = configuration.get_repo_registry_id(repo, config)

        self.redirect_url = configuration.get_redirect_url(config, repo, 'v1')
        if config.get('protected', False):
            self.protected = "true"
        else:
            self.protected = "false"
コード例 #2
0
ファイル: publish_steps.py プロジェクト: vrutkovs/pulp_docker
    def process_main(self):
        """
        Publish the JSON file for Crane.
        """
        registry = configuration.get_repo_registry_id(self.get_repo(),
                                                      self.get_config())
        redirect_url = configuration.get_redirect_url(self.get_config(),
                                                      self.get_repo(), 'v2')
        schema2_data = self.redirect_data[2]
        manifest_list_data = self.redirect_data['list']
        manifest_list_amd64 = self.redirect_data['amd64']

        redirect_data = {
            'type': 'pulp-docker-redirect',
            'version': 4,
            'repository': self.get_repo().id,
            'repo-registry-id': registry,
            'url': redirect_url,
            'protected': self.get_config().get('protected', False),
            'schema2_data': list(schema2_data),
            'manifest_list_data': list(manifest_list_data),
            'manifest_list_amd64_tags': manifest_list_amd64
        }

        misc.mkdir(os.path.dirname(self.app_publish_location))
        with open(self.app_publish_location, 'w') as app_file:
            app_file.write(json.dumps(redirect_data))
コード例 #3
0
 def finalize(self):
     """
     Write the Tag list file so that clients can retrieve the list of available Tags.
     """
     tags_path = os.path.join(self.parent.get_working_dir(), 'tags')
     misc.mkdir(tags_path)
     with open(os.path.join(tags_path, 'list'), 'w') as list_file:
         tag_data = {
             'name': configuration.get_repo_registry_id(self.get_repo(), self.get_config()),
             'tags': list(self._tag_names)}
         list_file.write(json.dumps(tag_data))
     # We don't need the tag names anymore
     del self._tag_names
コード例 #4
0
ファイル: publish_steps.py プロジェクト: snmpboy/pulp_docker
 def finalize(self):
     """
     Write the Tag list file so that clients can retrieve the list of available Tags.
     """
     tags_path = os.path.join(self.parent.get_working_dir(), 'tags')
     misc.mkdir(tags_path)
     with open(os.path.join(tags_path, 'list'), 'w') as list_file:
         tag_data = {
             'name': configuration.get_repo_registry_id(self.get_repo(), self.get_config()),
             'tags': list(self._tag_names)}
         list_file.write(json.dumps(tag_data))
     # We don't need the tag names anymore
     del self._tag_names
コード例 #5
0
 def process_main(self, item=False):
     repo_registry_id = configuration.get_repo_registry_id(
         self.get_repo(), self.get_config())
     for img_name, tags in self.parent.imgname_to_tags.items():
         dest_name = os.path.join(self.get_working_dir(), img_name, "tags",
                                  constants.MANIFEST_LIST_TYPE)
         repo_name = os.path.join(repo_registry_id, img_name)
         tag_data = {
             'name': repo_name,
             'tags': sorted(tags),
         }
         misc.mkdir(os.path.dirname(dest_name))
         with open(dest_name, 'w') as list_file:
             list_file.write(json.dumps(tag_data))
コード例 #6
0
ファイル: publish_steps.py プロジェクト: twaugh/pulp_docker
    def process_main(self):
        """
        Publish the JSON file for Crane.
        """
        registry = configuration.get_repo_registry_id(self.get_repo(), self.get_config())
        redirect_url = configuration.get_redirect_url(self.get_config(), self.get_repo(), 'v2')

        redirect_data = {
            'type': 'pulp-docker-redirect', 'version': 2, 'repository': self.get_repo().id,
            'repo-registry-id': registry, 'url': redirect_url,
            'protected': self.get_config().get('protected', False)}

        misc.mkdir(os.path.dirname(self.app_publish_location))
        with open(self.app_publish_location, 'w') as app_file:
            app_file.write(json.dumps(redirect_data))
コード例 #7
0
    def process_main(self):
        """
        Publish the JSON file for Crane.
        """
        repo = self.get_repo()
        config = self.get_config()
        registry_prefix = configuration.get_repo_registry_id(repo, config)
        global_app_publish_dir = os.path.join(
            configuration.get_app_publish_dir(config,
                                              self.parent.docker_api_version),
            registry_prefix)
        self.parent.atomic_publish_step.publish_locations.append(
            ("app", global_app_publish_dir))
        # Compute image references: tags and manifest IDs
        img_refs = dict(
            (img_name, set(tags))
            for img_name, tags in self.parent.imgname_to_tags.items())
        for img_name, refs in self.parent.imgname_to_mfid.items():
            img_refs.setdefault(img_name, set()).update(refs)

        redirect_url = configuration.get_redirect_url(
            config, repo, self.parent.docker_api_version)

        # Defaults
        redirect_tmpl = {
            'type': 'pulp-docker-redirect',
            'version': 4,
            'repository': self.get_repo().id,
            'protected': config.get('protected', False),
            'manifest_list_data': [],
            'manifest_list_amd64_tags': {}
        }
        working_dir = self.get_working_dir()

        for img_name, refs in img_refs.items():
            app_publish_path = os.path.join("app", "%s.json" % img_name)
            app_publish_location = os.path.join(working_dir, app_publish_path)
            misc.mkdir(os.path.dirname(app_publish_location))
            redirect_data = dict(redirect_tmpl)
            redirect_data['repo-registry-id'] = os.path.join(
                registry_prefix, img_name)
            redirect_data['url'] = os.path.join(redirect_url, img_name)
            redirect_data['schema2_data'] = sorted(refs)
            with open(app_publish_location, 'w') as app_file:
                app_file.write(json.dumps(redirect_data))
コード例 #8
0
 def process_main(self):
     repo = self.get_repo()
     config = self.get_config()
     registry_prefix = configuration.get_repo_registry_id(repo, config)
     dest_name = os.path.join(self.get_working_dir(), 'image_list.json')
     name_to_mfdata = dict()
     for manifest_id, name_tags in sorted(
             self.parent.manifest_to_imgname_to_tags.items()):
         for imgname, tags in sorted(name_tags.items()):
             name_to_mfdata.setdefault("%s/%s" % (registry_prefix, imgname),
                                       []).append(
                                           dict(manifest_id=manifest_id,
                                                tags=sorted(x[0]
                                                            for x in tags)))
     images = []
     for name, mfdata in sorted(name_to_mfdata.items()):
         images.append(dict(name=name, manifests=mfdata))
     with open(dest_name, "w") as fobj:
         json.dump(dict(images=images), fobj)
コード例 #9
0
ファイル: publish_steps.py プロジェクト: twaugh/pulp_docker
    def process_main(self):
        """
        Create the list file and add the manifest tag links.

        :param unit: The unit to process
        :type unit:  pulp_docker.common.models.Tag
        """
        tags_path = os.path.join(self.parent.get_working_dir(), 'tags')
        misc.mkdir(tags_path)
        with open(os.path.join(tags_path, 'list'), 'w') as list_file:
            tag_data = {
                'name': configuration.get_repo_registry_id(self.get_repo(), self.get_config()),
                'tags': list(self.parent.tags)}
            list_file.write(json.dumps(tag_data))

        # Add the links to make Manifests accessible by tags as well
        for tag, unit in self.parent.tags.items():
            self.parent.publish_manifests_step._create_symlink(
                unit.storage_path,
                os.path.join(self.parent.publish_manifests_step.get_manifests_directory(), tag))
コード例 #10
0
    def process_main(self):
        """
        Publish the JSON file for Crane.
        """
        registry = configuration.get_repo_registry_id(self.get_repo(),
                                                      self.get_config())
        redirect_url = configuration.get_redirect_url(self.get_config(),
                                                      self.get_repo(), 'v2')

        redirect_data = {
            'type': 'pulp-docker-redirect',
            'version': 2,
            'repository': self.get_repo().id,
            'repo-registry-id': registry,
            'url': redirect_url,
            'protected': self.get_config().get('protected', False)
        }

        misc.mkdir(os.path.dirname(self.app_publish_location))
        with open(self.app_publish_location, 'w') as app_file:
            app_file.write(json.dumps(redirect_data))
コード例 #11
0
ファイル: publish_steps.py プロジェクト: vrutkovs/pulp_docker
    def _add_necesary_steps(self, date_filter=None, config=None):
        """
        This method adds all the steps that are needed to accomplish an RPM rsync publish. This
        includes:

        Unit Query Step - selects units associated with the repo based on the date_filter and
                          creates relative symlinks
        Tag Generation Step  - creates relative symlinks for for all tags and writes out list file
        Rsync Step (content units) - rsyncs content units from /var/lib/pulp/content to remote
                                     server
        Rsync Step (symlinks) - rsyncs symlinks from working directory to remote server

        :param date_filter: Q object with start and/or end dates, or None if start and end dates
                             are not provided
        :type date_filter: mongoengine.Q or types.NoneType
        :param config: distributor configuration
        :type config: pulp.plugins.config.PluginCallConfiguration

        :return: None
        """
        postdistributor = self._get_postdistributor()
        repo_registry_id = configuration.get_repo_registry_id(
            self.repo, postdistributor.config)
        remote_repo_path = configuration.get_remote_repo_relative_path(
            self.repo, self.config)

        unit_models = {
            constants.IMAGE_TYPE_ID: models.Image,
            constants.MANIFEST_TYPE_ID: models.Manifest,
            constants.MANIFEST_LIST_TYPE_ID: models.ManifestList,
            constants.BLOB_TYPE_ID: models.Blob
        }

        for unit_type in DockerRsyncPublisher.REPO_CONTENT_TYPES:
            gen_step = RSyncFastForwardUnitPublishStep(
                "Unit query step (things)", [unit_models[unit_type]],
                repo=self.repo,
                repo_content_unit_q=date_filter,
                remote_repo_path=remote_repo_path)
            self.add_child(gen_step)

        self.add_child(
            PublishTagsForRsyncStep("Generate tags step",
                                    repo=self.repo,
                                    repo_content_unit_q=date_filter,
                                    remote_repo_path=remote_repo_path,
                                    repo_registry_id=repo_registry_id))

        origin_dest_prefix = self.get_units_directory_dest_path()
        origin_src_prefix = self.get_units_src_path()

        self.add_child(
            RSyncPublishStep("Rsync step (content units)",
                             self.content_unit_file_list,
                             origin_src_prefix,
                             origin_dest_prefix,
                             config=config))

        # Stop here if distributor is only supposed to publish actual content
        if self.get_config().flatten().get("content_units_only"):
            return

        self.add_child(
            RSyncPublishStep("Rsync step (symlinks)",
                             self.symlink_list,
                             self.symlink_src,
                             remote_repo_path,
                             config=config,
                             links=True,
                             delete=self.config.get("delete")))

        if constants.TAG_TYPE_ID in self.repo.content_unit_counts:
            self.add_child(
                RSyncPublishStep("Rsync step (tags list)", ["tags/list"],
                                 os.path.join(self.get_working_dir(),
                                              '.relative'),
                                 remote_repo_path,
                                 config=config,
                                 links=True))
コード例 #12
0
    def _add_necesary_steps(self, date_filter=None, config=None):
        """
        This method adds all the steps that are needed to accomplish an RPM rsync publish. This
        includes:

        Unit Query Step - selects units associated with the repo based on the date_filter and
                          creates relative symlinks
        Tag Generation Step  - creates relative symlinks for for all tags and writes out list file
        Rsync Step (content units) - rsyncs content units from /var/lib/pulp/content to remote
                                     server
        Rsync Step (symlinks) - rsyncs symlinks from working directory to remote server

        :param date_filter: Q object with start and/or end dates, or None if start and end dates
                             are not provided
        :type date_filter: mongoengine.Q or types.NoneType
        :param config: distributor configuration
        :type config: pulp.plugins.config.PluginCallConfiguration

        :return: None
        """
        postdistributor = self._get_postdistributor()
        repo_registry_id = configuration.get_repo_registry_id(self.repo, postdistributor.config)
        remote_repo_path = configuration.get_remote_repo_relative_path(self.repo, self.config)

        unit_info = {constants.IMAGE_TYPE_ID: {'extra_path': [''], 'model': models.Image},
                     constants.MANIFEST_TYPE_ID: {'extra_path': ['manifests'],
                                                  'model': models.Manifest},
                     constants.BLOB_TYPE_ID: {'extra_path': ['blobs'], 'model': models.Blob}}

        for unit_type in DockerRsyncPublisher.REPO_CONTENT_TYPES:
            unit_path = unit_info[unit_type]['extra_path']
            gen_step = RSyncFastForwardUnitPublishStep("Unit query step (things)",
                                                       [unit_info[unit_type]['model']],
                                                       repo=self.repo,
                                                       repo_content_unit_q=date_filter,
                                                       remote_repo_path=remote_repo_path,
                                                       published_unit_path=unit_path)
            self.add_child(gen_step)

        self.add_child(PublishTagsForRsyncStep("Generate tags step",
                                               repo=self.repo,
                                               repo_content_unit_q=date_filter,
                                               remote_repo_path=remote_repo_path,
                                               repo_registry_id=repo_registry_id))

        origin_dest_prefix = self.get_units_directory_dest_path()
        origin_src_prefix = self.get_units_src_path()

        self.add_child(RSyncPublishStep("Rsync step (content units)", self.content_unit_file_list,
                                        origin_src_prefix, origin_dest_prefix,
                                        config=config))

        # Stop here if distributor is only supposed to publish actual content
        if self.get_config().flatten().get("content_units_only"):
            return

        self.add_child(RSyncPublishStep("Rsync step (symlinks)",
                                        self.symlink_list, self.symlink_src, remote_repo_path,
                                        config=config, links=True,
                                        delete=self.config.get("delete")))

        if constants.TAG_TYPE_ID in self.repo.content_unit_counts:
            self.add_child(RSyncPublishStep("Rsync step (tags list)", ["tags/list"],
                                            os.path.join(self.get_working_dir(), '.relative'),
                                            remote_repo_path, config=config, links=True))