def run(self):
pulp = dockpulp.Pulp(env=self.pulp_registry_name)
self.set_auth(pulp)
# We only want the hostname[:port]
pulp_registry = re.sub(r'^https?://([^/]*)/?.*',
lambda m: m.groups()[0],
pulp.registry)
# Store the registry URI in the push configuration
self.workflow.push_conf.add_pulp_registry(self.pulp_registry_name,
pulp_registry)
self.log.info("syncing from docker V2 registry %s",
self.docker_registry)
images = []
repos = {} # pulp repo -> repo id
with dockpulp_config(docker_registry=self.docker_registry) as config:
for image in self.workflow.tag_conf.primary_images:
if image.pulp_repo not in repos:
self.log.info("syncing %s", image.pulp_repo)
repoinfo = pulp.syncRepo(config.env, image.pulp_repo,
config_file=config.name)
repos[image.pulp_repo] = repoinfo[0]['id']
images.append(ImageName(registry=pulp_registry,
repo=image.repo))
self.log.info("publishing to crane")
pulp.crane(list(repos.values()), wait=True)
# Return the set of qualitifed repo names for this image
return images
def run(self):
pulp = dockpulp.Pulp(env=self.pulp_registry_name)
self.set_auth(pulp)
# We only want the hostname[:port]
hostname_and_port = re.compile(r'^https?://([^/]*)/?.*')
pulp_registry = hostname_and_port.sub(lambda m: m.groups()[0],
pulp.registry)
# Store the registry URI in the push configuration
self.workflow.push_conf.add_pulp_registry(self.pulp_registry_name,
pulp_registry,
server_side_sync=True)
self.log.info("syncing from docker V2 registry %s",
self.docker_registry)
docker_registry = hostname_and_port.sub(lambda m: m.groups()[0],
self.docker_registry)
kwargs = self.get_dockercfg_credentials(docker_registry)
if self.insecure_registry is not None:
kwargs['ssl_validation'] = not self.insecure_registry
images = []
repos = {} # pulp repo -> repo id
for image in self.workflow.tag_conf.images:
if image.pulp_repo not in repos:
repo_id = self.create_repo_if_missing(
pulp, image.pulp_repo,
image.to_str(registry=False, tag=False))
self.log.info("syncing %s", repo_id)
pulp.syncRepo(repo=repo_id,
feed=self.docker_registry,
**kwargs)
repos[image.pulp_repo] = repo_id
images.append(
ImageName(registry=pulp_registry,
repo=image.repo,
namespace=image.namespace,
tag=image.tag))
if self.publish:
self.log.info("publishing to crane")
pulp.crane(list(repos.values()), wait=True)
for image_name in images:
self.log.info("image available at %s", image_name.to_str())
# Return the set of qualified repo names for this image
return images
def pulp_login(bopts):
p = dockpulp.Pulp(env=bopts.server, config_file=bopts.config_file)
if bopts.debug:
p.setDebug()
if bopts.cert and bopts.key:
p.certificate = bopts.cert
p.key = bopts.key
elif not os.path.exists(
os.path.join(os.path.expanduser('~/.pulp'), 'pulp.cer')):
log.error('You need to log in with a user/password first.')
sys.exit(1)
else:
creddir = os.path.expanduser('~/.pulp')
p.certificate = os.path.join(creddir, 'pulp.cer')
p.key = os.path.join(creddir, 'pulp.key')
return p
def push_tarball_to_pulp(self, image_names):
self.log.info("checking image before upload")
self._check_file()
p = dockpulp.Pulp(env=self.pulp_instance)
self._set_auth(p)
# {
# "repo-id": {
# "registry-id": "",
# "tags": [],
# },
# ...
# }
repos_tags_mapping = {}
for image in image_names:
repo = image.pulp_repo
repos_tags_mapping.setdefault(repo, {})
repos_tags_mapping[repo]["registry-id"] = image.to_str(
registry=False, tag=False)
repos_tags_mapping[repo].setdefault("tags", [])
repos_tags_mapping[repo]["tags"].append(image.tag)
self.log.info("repo_tags_mapping = %s", repos_tags_mapping)
task_ids = p.push_tar_to_pulp(repos_tags_mapping, self.filename)
self.log.info("waiting for repos to be published to crane")
p.watch_tasks(task_ids)
# Store the registry URI in the push configuration
# We only want the hostname[:port]
pulp_registry = re.sub(r'^https?://([^/]*)/?.*',
lambda m: m.groups()[0], p.registry)
self.workflow.push_conf.add_pulp_registry(self.pulp_instance,
pulp_registry)
# Return the set of qualified repo names for this image
return [
ImageName(registry=pulp_registry,
repo=repodata["registry-id"],
tag=tag)
for repo, repodata in repos_tags_mapping.items()
for tag in repodata['tags']
]
def do_login(bopts, bargs, parser):
"""Login into pulp and get a session certificate.
dock-pulp login [options]
"""
parser.add_option('-p', '--password', help='specify an account password')
parser.add_option('-u', '--username', default='admin', help='pick username')
opts, args = parser.parse_args(bargs)
if not opts.password:
parser.error('You should provide a password too')
p = dockpulp.Pulp(env=bopts.server, config_file=bopts.config_file)
p.login(opts.username, opts.password)
creddir = os.path.expanduser('~/.pulp')
if not os.path.exists(creddir):
os.makedirs(creddir)
shutil.copy(p.certificate, creddir)
shutil.copy(p.key, creddir)
log.info('Credentials stored in %s.' % creddir)
log.info('You may run commands without a user/password now.')
def pulp_login(bopts):
p = dockpulp.Pulp(env=bopts.server, config_file=bopts.config_file)
if bopts.debug:
p.setDebug()
if bopts.cert:
p.certificate = bopts.cert
if bopts.key:
p.key = bopts.key
default_creddir = os.path.expanduser('~/.pulp')
if (not p.certificate or not p.key) and\
(not os.path.exists(os.path.join(default_creddir, p.AUTH_CER_FILE)) or
not os.path.exists(os.path.join(default_creddir, p.AUTH_KEY_FILE))):
log.error('You need to log in with a user/password first.')
sys.exit(1)
else:
creddir = os.path.expanduser('~/.pulp')
p.set_certs(os.path.join(creddir, p.AUTH_CER_FILE),
os.path.join(creddir, p.AUTH_KEY_FILE))
return p
def pulp_login(bopts):
p = dockpulp.Pulp(env=bopts.server, config_file=bopts.config_file)
if bopts.debug:
p.setDebug()
if bopts.cert:
p.certificate = bopts.cert
if bopts.key:
p.key = bopts.key
default_creddir = os.path.expanduser('~/.pulp')
default_cerfile = os.path.join(default_creddir,
p.AUTH_CER_FILE % bopts.server)
default_keyfile = os.path.join(default_creddir,
p.AUTH_KEY_FILE % bopts.server)
if not (p.certificate and p.key) and not \
(os.path.exists(default_cerfile) and os.path.exists(default_keyfile)):
log.error('You need to log in with a user/password first.')
sys.exit(1)
else:
p.set_certs(default_cerfile, default_keyfile)
return p
def push_tarball_to_pulp(self, image_names, repo_prefix="redhat-"):
self.log.info("checking image before upload")
self._check_file()
p = dockpulp.Pulp(env=self.pulp_instance)
self._set_auth(p)
# pulp_repos is mapping from repo-ids to registry-ids and tags
# which should be applied to those repos, expected structure:
# {
# "my-image": PulpRepo(registry_id="nick/my-image", tags=["v1", "latest"])
# ...
# }
pulp_repos = {}
for image in image_names:
repo_id = image.pulp_repo
tag = image.tag if image.tag else 'latest'
if repo_prefix:
repo_id = repo_prefix + repo_id
if repo_id in pulp_repos:
pulp_repos[repo_id].tags.append(tag)
else:
pulp_repos[repo_id] = PulpRepo(registry_id=image.to_str(
registry=False, tag=False),
tags=[tag])
self.log.info("pulp_repos = %s", pulp_repos)
self._create_missing_repos(p, pulp_repos, repo_prefix)
_, file_extension = os.path.splitext(self.filename)
try:
top_layer, layers = self._get_tar_metadata(self.filename)
# getImageIdsExist was introduced in rh-dockpulp 0.6+
existing_imageids = p.getImageIdsExist(layers)
self.log.debug("existing layers: %s", existing_imageids)
# Strip existing layers from the tar and repack it
remove_layers = [
str(os.path.join(x, 'layer.tar')) for x in existing_imageids
]
commands = {
'.xz': 'xzcat',
'.gz': 'zcat',
'.bz2': 'bzcat',
'.tar': 'cat'
}
unpacker = commands.get(file_extension, None)
self.log.debug("using unpacker %s for extension %s", unpacker,
file_extension)
if unpacker is None:
raise Exception("Unknown tarball format: %s" % self.filename)
with NamedTemporaryFile(prefix='strip_tar_',
suffix='.gz') as outfile:
cmd = "set -o pipefail; {0} {1} | tar --delete {2} | gzip - > {3}".format(
unpacker, self.filename, ' '.join(remove_layers),
outfile.name)
self.log.debug("running %s", cmd)
subprocess.check_call(cmd, shell=True)
self.log.debug("uploading %s", outfile.name)
p.upload(outfile.name)
except:
self.log.debug("Error on creating deduplicated layers tar",
exc_info=True)
try:
if file_extension != '.tar':
raise RuntimeError("tar is already compressed")
with NamedTemporaryFile(prefix='full_tar_',
suffix='.gz') as outfile:
cmd = "set -o pipefail; cat {0} | gzip - > {1}".format(
self.filename, outfile.name)
self.log.debug("Compressing tar using '%s' command", cmd)
subprocess.check_call(cmd, shell=True)
self.log.debug("uploading %s", outfile.name)
p.upload(outfile.name)
except:
self.log.info("Falling back to full tar upload")
p.upload(self.filename)
for repo_id, pulp_repo in pulp_repos.items():
for layer in layers:
p.copy(repo_id, layer)
p.updateRepo(
repo_id,
{"tag": "%s:%s" % (",".join(pulp_repo.tags), top_layer)})
# Only publish if we don't the pulp_sync plugin also configured
if self.publish:
task_ids = p.crane(pulp_repos.keys(), wait=True)
self.log.info(
"waiting for repos to be published to crane, tasks: %s",
", ".join(map(str, task_ids)))
p.watch_tasks(task_ids)
else:
self.log.info("publishing deferred until %s plugin runs",
PLUGIN_PULP_SYNC_KEY)
# Store the registry URI in the push configuration
# We only want the hostname[:port]
pulp_registry = re.sub(r'^https?://([^/]*)/?.*',
lambda m: m.groups()[0], p.registry)
self.workflow.push_conf.add_pulp_registry(self.pulp_instance,
pulp_registry)
# Return the set of qualified repo names for this image
return [
ImageName(registry=pulp_registry,
repo=repodata.registry_id,
tag=tag) for dummy_repo, repodata in pulp_repos.items()
for tag in repodata.tags
] # noqa
def run(self):
pulp = dockpulp.Pulp(env=self.pulp_registry_name)
self.set_auth(pulp)
# We only want the hostname[:port]
hostname_and_port = re.compile(r'^https?://([^/]*)/?.*')
pulp_registry = hostname_and_port.sub(
lambda m: m.groups()[0],
pulp.registry # pylint: disable=no-member
)
# Store the registry URI in the push configuration
self.workflow.push_conf.add_pulp_registry(self.pulp_registry_name,
pulp_registry,
server_side_sync=True)
self.log.info("syncing from docker V2 registry %s",
self.docker_registry)
docker_registry = hostname_and_port.sub(lambda m: m.groups()[0],
self.docker_registry)
kwargs = self.get_dockercfg_credentials(docker_registry)
if self.insecure_registry is not None:
kwargs['ssl_validation'] = not self.insecure_registry
images = []
repos = {} # pulp repo -> repo id
for image in self.workflow.tag_conf.images:
if image.pulp_repo not in repos:
repo_id = self.create_repo_if_missing(
pulp, image.pulp_repo,
image.to_str(registry=False, tag=False))
self.log.info("syncing %s", repo_id)
pulp.syncRepo(repo=repo_id,
feed=self.docker_registry,
**kwargs)
repos[image.pulp_repo] = repo_id
images.append(
ImageName(registry=pulp_registry,
repo=image.repo,
namespace=image.namespace,
tag=image.tag))
if self.publish:
self.log.info("publishing to crane")
pulp.crane(list(repos.values()), wait=True)
for image_name in images:
self.log.info("image available at %s", image_name.to_str())
# Fetch the repository content so we can remove v2 schema 2
# manifests from Koji metadata if they are not present
# (i.e. if Pulp does not have v2 schema 2 support).
self.log.info("fetching repository content")
manifest_refs = set()
for content in pulp.listRepos(list(repos.values()), content=True):
manifest_refs |= set(content['manifests'].keys())
self.workflow.plugin_workspace[PulpSyncPlugin.key] = list(
manifest_refs)
# Return the set of qualified repo names for this image
return images
def push_tarball_to_pulp(self, image_names, repo_prefix="redhat-"):
self.log.info("checking image before upload")
self._check_file()
p = dockpulp.Pulp(env=self.pulp_instance)
self._set_auth(p)
# pulp_repos is mapping from repo-ids to registry-ids and tags
# which should be applied to those repos, expected structure:
# {
# "my-image": PulpRepo(registry_id="nick/my-image", tags=["v1", "latest"])
# ...
# }
pulp_repos = {}
for image in image_names:
repo_id = image.pulp_repo
tag = image.tag if image.tag else 'latest'
if repo_prefix:
repo_id = repo_prefix + repo_id
if repo_id in pulp_repos:
pulp_repos[repo_id].tags.append(tag)
else:
pulp_repos[repo_id] = PulpRepo(registry_id=image.to_str(
registry=False, tag=False),
tags=[tag])
top_layer, layers = self._get_tar_metadata(self.filename)
self.log.info("pulp_repos = %s", pulp_repos)
self._create_missing_repos(p, pulp_repos, repo_prefix)
p.upload(self.filename)
for repo_id, pulp_repo in pulp_repos.items():
for layer in layers:
p.copy(repo_id, layer)
p.updateRepo(
repo_id,
{"tag": "%s:%s" % (",".join(pulp_repo.tags), top_layer)})
task_ids = p.crane(pulp_repos.keys(), wait=True)
self.log.info("waiting for repos to be published to crane, tasks: %s",
", ".join(map(str, task_ids)))
p.watch_tasks(task_ids)
# Store the registry URI in the push configuration
# We only want the hostname[:port]
pulp_registry = re.sub(r'^https?://([^/]*)/?.*',
lambda m: m.groups()[0], p.registry)
self.workflow.push_conf.add_pulp_registry(self.pulp_instance,
pulp_registry)
# Return the set of qualified repo names for this image
return [
ImageName(registry=pulp_registry,
repo=repodata.registry_id,
tag=tag) for dummy_repo, repodata in pulp_repos.items()
for tag in repodata.tags
]
parser.add_option('-p', '--password', help='specify the account password')
parser.add_option('-u',
'--username',
default='admin',
help='provide an account besides "admin"')
opts, args = parser.parse_args()
if len(args) != 1:
parser.error(
'You must provide an environment to create the repository')
return opts, args
def create_everything(dpo):
"""create the everything repo with the given dockpulp object"""
dpo.createRepo(
dockpulp.HIDDEN,
'/content/this/does/not/matter',
desc='hidden repository for RCM use that contains everything',
title='RCM Hidden repository',
distributors=False)
# remove distributors from the hidden repository so it is never published
if __name__ == '__main__':
opts, args = get_opts()
p = dockpulp.Pulp(env=args[0])
p.login(opts.username, opts.password)
if opts.debug:
p.setDebug()
create_everything(p)
def create_dockpulp(self):
self.p = dockpulp.Pulp(env=self.pulp_instance)
self._set_auth()
