def run(self): pulp = dockpulp.Pulp(env=self.pulp_registry_name) self.set_auth(pulp) # We only want the hostname[:port] pulp_registry = re.sub(r'^https?://([^/]*)/?.*', lambda m: m.groups()[0], pulp.registry) # Store the registry URI in the push configuration self.workflow.push_conf.add_pulp_registry(self.pulp_registry_name, pulp_registry) self.log.info("syncing from docker V2 registry %s", self.docker_registry) images = [] repos = {} # pulp repo -> repo id with dockpulp_config(docker_registry=self.docker_registry) as config: for image in self.workflow.tag_conf.primary_images: if image.pulp_repo not in repos: self.log.info("syncing %s", image.pulp_repo) repoinfo = pulp.syncRepo(config.env, image.pulp_repo, config_file=config.name) repos[image.pulp_repo] = repoinfo[0]['id'] images.append(ImageName(registry=pulp_registry, repo=image.repo)) self.log.info("publishing to crane") pulp.crane(list(repos.values()), wait=True) # Return the set of qualitifed repo names for this image return images
def run(self): pulp = dockpulp.Pulp(env=self.pulp_registry_name) self.set_auth(pulp) # We only want the hostname[:port] hostname_and_port = re.compile(r'^https?://([^/]*)/?.*') pulp_registry = hostname_and_port.sub(lambda m: m.groups()[0], pulp.registry) # Store the registry URI in the push configuration self.workflow.push_conf.add_pulp_registry(self.pulp_registry_name, pulp_registry, server_side_sync=True) self.log.info("syncing from docker V2 registry %s", self.docker_registry) docker_registry = hostname_and_port.sub(lambda m: m.groups()[0], self.docker_registry) kwargs = self.get_dockercfg_credentials(docker_registry) if self.insecure_registry is not None: kwargs['ssl_validation'] = not self.insecure_registry images = [] repos = {} # pulp repo -> repo id for image in self.workflow.tag_conf.images: if image.pulp_repo not in repos: repo_id = self.create_repo_if_missing( pulp, image.pulp_repo, image.to_str(registry=False, tag=False)) self.log.info("syncing %s", repo_id) pulp.syncRepo(repo=repo_id, feed=self.docker_registry, **kwargs) repos[image.pulp_repo] = repo_id images.append( ImageName(registry=pulp_registry, repo=image.repo, namespace=image.namespace, tag=image.tag)) if self.publish: self.log.info("publishing to crane") pulp.crane(list(repos.values()), wait=True) for image_name in images: self.log.info("image available at %s", image_name.to_str()) # Return the set of qualified repo names for this image return images
def pulp_login(bopts): p = dockpulp.Pulp(env=bopts.server, config_file=bopts.config_file) if bopts.debug: p.setDebug() if bopts.cert and bopts.key: p.certificate = bopts.cert p.key = bopts.key elif not os.path.exists( os.path.join(os.path.expanduser('~/.pulp'), 'pulp.cer')): log.error('You need to log in with a user/password first.') sys.exit(1) else: creddir = os.path.expanduser('~/.pulp') p.certificate = os.path.join(creddir, 'pulp.cer') p.key = os.path.join(creddir, 'pulp.key') return p
def push_tarball_to_pulp(self, image_names): self.log.info("checking image before upload") self._check_file() p = dockpulp.Pulp(env=self.pulp_instance) self._set_auth(p) # { # "repo-id": { # "registry-id": "", # "tags": [], # }, # ... # } repos_tags_mapping = {} for image in image_names: repo = image.pulp_repo repos_tags_mapping.setdefault(repo, {}) repos_tags_mapping[repo]["registry-id"] = image.to_str( registry=False, tag=False) repos_tags_mapping[repo].setdefault("tags", []) repos_tags_mapping[repo]["tags"].append(image.tag) self.log.info("repo_tags_mapping = %s", repos_tags_mapping) task_ids = p.push_tar_to_pulp(repos_tags_mapping, self.filename) self.log.info("waiting for repos to be published to crane") p.watch_tasks(task_ids) # Store the registry URI in the push configuration # We only want the hostname[:port] pulp_registry = re.sub(r'^https?://([^/]*)/?.*', lambda m: m.groups()[0], p.registry) self.workflow.push_conf.add_pulp_registry(self.pulp_instance, pulp_registry) # Return the set of qualified repo names for this image return [ ImageName(registry=pulp_registry, repo=repodata["registry-id"], tag=tag) for repo, repodata in repos_tags_mapping.items() for tag in repodata['tags'] ]
def do_login(bopts, bargs, parser): """Login into pulp and get a session certificate. dock-pulp login [options] """ parser.add_option('-p', '--password', help='specify an account password') parser.add_option('-u', '--username', default='admin', help='pick username') opts, args = parser.parse_args(bargs) if not opts.password: parser.error('You should provide a password too') p = dockpulp.Pulp(env=bopts.server, config_file=bopts.config_file) p.login(opts.username, opts.password) creddir = os.path.expanduser('~/.pulp') if not os.path.exists(creddir): os.makedirs(creddir) shutil.copy(p.certificate, creddir) shutil.copy(p.key, creddir) log.info('Credentials stored in %s.' % creddir) log.info('You may run commands without a user/password now.')
def pulp_login(bopts): p = dockpulp.Pulp(env=bopts.server, config_file=bopts.config_file) if bopts.debug: p.setDebug() if bopts.cert: p.certificate = bopts.cert if bopts.key: p.key = bopts.key default_creddir = os.path.expanduser('~/.pulp') if (not p.certificate or not p.key) and\ (not os.path.exists(os.path.join(default_creddir, p.AUTH_CER_FILE)) or not os.path.exists(os.path.join(default_creddir, p.AUTH_KEY_FILE))): log.error('You need to log in with a user/password first.') sys.exit(1) else: creddir = os.path.expanduser('~/.pulp') p.set_certs(os.path.join(creddir, p.AUTH_CER_FILE), os.path.join(creddir, p.AUTH_KEY_FILE)) return p
def pulp_login(bopts): p = dockpulp.Pulp(env=bopts.server, config_file=bopts.config_file) if bopts.debug: p.setDebug() if bopts.cert: p.certificate = bopts.cert if bopts.key: p.key = bopts.key default_creddir = os.path.expanduser('~/.pulp') default_cerfile = os.path.join(default_creddir, p.AUTH_CER_FILE % bopts.server) default_keyfile = os.path.join(default_creddir, p.AUTH_KEY_FILE % bopts.server) if not (p.certificate and p.key) and not \ (os.path.exists(default_cerfile) and os.path.exists(default_keyfile)): log.error('You need to log in with a user/password first.') sys.exit(1) else: p.set_certs(default_cerfile, default_keyfile) return p
def push_tarball_to_pulp(self, image_names, repo_prefix="redhat-"): self.log.info("checking image before upload") self._check_file() p = dockpulp.Pulp(env=self.pulp_instance) self._set_auth(p) # pulp_repos is mapping from repo-ids to registry-ids and tags # which should be applied to those repos, expected structure: # { # "my-image": PulpRepo(registry_id="nick/my-image", tags=["v1", "latest"]) # ... # } pulp_repos = {} for image in image_names: repo_id = image.pulp_repo tag = image.tag if image.tag else 'latest' if repo_prefix: repo_id = repo_prefix + repo_id if repo_id in pulp_repos: pulp_repos[repo_id].tags.append(tag) else: pulp_repos[repo_id] = PulpRepo(registry_id=image.to_str( registry=False, tag=False), tags=[tag]) self.log.info("pulp_repos = %s", pulp_repos) self._create_missing_repos(p, pulp_repos, repo_prefix) _, file_extension = os.path.splitext(self.filename) try: top_layer, layers = self._get_tar_metadata(self.filename) # getImageIdsExist was introduced in rh-dockpulp 0.6+ existing_imageids = p.getImageIdsExist(layers) self.log.debug("existing layers: %s", existing_imageids) # Strip existing layers from the tar and repack it remove_layers = [ str(os.path.join(x, 'layer.tar')) for x in existing_imageids ] commands = { '.xz': 'xzcat', '.gz': 'zcat', '.bz2': 'bzcat', '.tar': 'cat' } unpacker = commands.get(file_extension, None) self.log.debug("using unpacker %s for extension %s", unpacker, file_extension) if unpacker is None: raise Exception("Unknown tarball format: %s" % self.filename) with NamedTemporaryFile(prefix='strip_tar_', suffix='.gz') as outfile: cmd = "set -o pipefail; {0} {1} | tar --delete {2} | gzip - > {3}".format( unpacker, self.filename, ' '.join(remove_layers), outfile.name) self.log.debug("running %s", cmd) subprocess.check_call(cmd, shell=True) self.log.debug("uploading %s", outfile.name) p.upload(outfile.name) except: self.log.debug("Error on creating deduplicated layers tar", exc_info=True) try: if file_extension != '.tar': raise RuntimeError("tar is already compressed") with NamedTemporaryFile(prefix='full_tar_', suffix='.gz') as outfile: cmd = "set -o pipefail; cat {0} | gzip - > {1}".format( self.filename, outfile.name) self.log.debug("Compressing tar using '%s' command", cmd) subprocess.check_call(cmd, shell=True) self.log.debug("uploading %s", outfile.name) p.upload(outfile.name) except: self.log.info("Falling back to full tar upload") p.upload(self.filename) for repo_id, pulp_repo in pulp_repos.items(): for layer in layers: p.copy(repo_id, layer) p.updateRepo( repo_id, {"tag": "%s:%s" % (",".join(pulp_repo.tags), top_layer)}) # Only publish if we don't the pulp_sync plugin also configured if self.publish: task_ids = p.crane(pulp_repos.keys(), wait=True) self.log.info( "waiting for repos to be published to crane, tasks: %s", ", ".join(map(str, task_ids))) p.watch_tasks(task_ids) else: self.log.info("publishing deferred until %s plugin runs", PLUGIN_PULP_SYNC_KEY) # Store the registry URI in the push configuration # We only want the hostname[:port] pulp_registry = re.sub(r'^https?://([^/]*)/?.*', lambda m: m.groups()[0], p.registry) self.workflow.push_conf.add_pulp_registry(self.pulp_instance, pulp_registry) # Return the set of qualified repo names for this image return [ ImageName(registry=pulp_registry, repo=repodata.registry_id, tag=tag) for dummy_repo, repodata in pulp_repos.items() for tag in repodata.tags ] # noqa
def run(self): pulp = dockpulp.Pulp(env=self.pulp_registry_name) self.set_auth(pulp) # We only want the hostname[:port] hostname_and_port = re.compile(r'^https?://([^/]*)/?.*') pulp_registry = hostname_and_port.sub( lambda m: m.groups()[0], pulp.registry # pylint: disable=no-member ) # Store the registry URI in the push configuration self.workflow.push_conf.add_pulp_registry(self.pulp_registry_name, pulp_registry, server_side_sync=True) self.log.info("syncing from docker V2 registry %s", self.docker_registry) docker_registry = hostname_and_port.sub(lambda m: m.groups()[0], self.docker_registry) kwargs = self.get_dockercfg_credentials(docker_registry) if self.insecure_registry is not None: kwargs['ssl_validation'] = not self.insecure_registry images = [] repos = {} # pulp repo -> repo id for image in self.workflow.tag_conf.images: if image.pulp_repo not in repos: repo_id = self.create_repo_if_missing( pulp, image.pulp_repo, image.to_str(registry=False, tag=False)) self.log.info("syncing %s", repo_id) pulp.syncRepo(repo=repo_id, feed=self.docker_registry, **kwargs) repos[image.pulp_repo] = repo_id images.append( ImageName(registry=pulp_registry, repo=image.repo, namespace=image.namespace, tag=image.tag)) if self.publish: self.log.info("publishing to crane") pulp.crane(list(repos.values()), wait=True) for image_name in images: self.log.info("image available at %s", image_name.to_str()) # Fetch the repository content so we can remove v2 schema 2 # manifests from Koji metadata if they are not present # (i.e. if Pulp does not have v2 schema 2 support). self.log.info("fetching repository content") manifest_refs = set() for content in pulp.listRepos(list(repos.values()), content=True): manifest_refs |= set(content['manifests'].keys()) self.workflow.plugin_workspace[PulpSyncPlugin.key] = list( manifest_refs) # Return the set of qualified repo names for this image return images
def push_tarball_to_pulp(self, image_names, repo_prefix="redhat-"): self.log.info("checking image before upload") self._check_file() p = dockpulp.Pulp(env=self.pulp_instance) self._set_auth(p) # pulp_repos is mapping from repo-ids to registry-ids and tags # which should be applied to those repos, expected structure: # { # "my-image": PulpRepo(registry_id="nick/my-image", tags=["v1", "latest"]) # ... # } pulp_repos = {} for image in image_names: repo_id = image.pulp_repo tag = image.tag if image.tag else 'latest' if repo_prefix: repo_id = repo_prefix + repo_id if repo_id in pulp_repos: pulp_repos[repo_id].tags.append(tag) else: pulp_repos[repo_id] = PulpRepo(registry_id=image.to_str( registry=False, tag=False), tags=[tag]) top_layer, layers = self._get_tar_metadata(self.filename) self.log.info("pulp_repos = %s", pulp_repos) self._create_missing_repos(p, pulp_repos, repo_prefix) p.upload(self.filename) for repo_id, pulp_repo in pulp_repos.items(): for layer in layers: p.copy(repo_id, layer) p.updateRepo( repo_id, {"tag": "%s:%s" % (",".join(pulp_repo.tags), top_layer)}) task_ids = p.crane(pulp_repos.keys(), wait=True) self.log.info("waiting for repos to be published to crane, tasks: %s", ", ".join(map(str, task_ids))) p.watch_tasks(task_ids) # Store the registry URI in the push configuration # We only want the hostname[:port] pulp_registry = re.sub(r'^https?://([^/]*)/?.*', lambda m: m.groups()[0], p.registry) self.workflow.push_conf.add_pulp_registry(self.pulp_instance, pulp_registry) # Return the set of qualified repo names for this image return [ ImageName(registry=pulp_registry, repo=repodata.registry_id, tag=tag) for dummy_repo, repodata in pulp_repos.items() for tag in repodata.tags ]
parser.add_option('-p', '--password', help='specify the account password') parser.add_option('-u', '--username', default='admin', help='provide an account besides "admin"') opts, args = parser.parse_args() if len(args) != 1: parser.error( 'You must provide an environment to create the repository') return opts, args def create_everything(dpo): """create the everything repo with the given dockpulp object""" dpo.createRepo( dockpulp.HIDDEN, '/content/this/does/not/matter', desc='hidden repository for RCM use that contains everything', title='RCM Hidden repository', distributors=False) # remove distributors from the hidden repository so it is never published if __name__ == '__main__': opts, args = get_opts() p = dockpulp.Pulp(env=args[0]) p.login(opts.username, opts.password) if opts.debug: p.setDebug() create_everything(p)
def create_dockpulp(self): self.p = dockpulp.Pulp(env=self.pulp_instance) self._set_auth()