def test_check_file(tmpdir, check_repo_retval, should_raise):
log = logging.getLogger("tests.test_pulp_util")
pulp_registry_name = 'registry.example.com'
testfile = 'foo'
_, workflow = prepare(testfile, check_repo_retval)
handler = PulpHandler(workflow, pulp_registry_name, log)
if should_raise:
with pytest.raises(Exception):
handler.check_file(testfile)
return
handler.check_file(testfile)
def test_check_file(tmpdir, check_repo_retval, should_raise):
log = logging.getLogger("tests.test_pulp_util")
pulp_registry_name = 'registry.example.com'
testfile = 'foo'
_, workflow = prepare(testfile, check_repo_retval)
handler = PulpHandler(workflow, pulp_registry_name, log)
if should_raise:
with pytest.raises(Exception):
handler.check_file(testfile)
return
handler.check_file(testfile)
class PulpPushPlugin(PostBuildPlugin):
key = PLUGIN_PULP_PUSH_KEY
is_allowed_to_fail = False
def __init__(self, tasker, workflow, pulp_registry_name, load_squashed_image=None,
load_exported_image=None, image_names=None, pulp_secret_path=None,
username=None, password=None, dockpulp_loglevel=None, publish=True):
"""
constructor
:param tasker: DockerTasker instance
:param workflow: DockerBuildWorkflow instance
:param pulp_registry_name: str, name of pulp registry to use, specified in /etc/
dockpulp.conf
:param load_squashed_image: obsolete name for load_exported_image, please don't use
:param load_exported_image: bool, use exported tar instead of image from Docker
:param image_names: list of additional image names
:param pulp_secret_path: path to pulp.cer and pulp.key; $SOURCE_SECRET_PATH otherwise
:param username: pulp username, used in preference to certificate and key
:param password: pulp password, used in preference to certificate and key
:param publish: Bool, whether to publish to crane or not
"""
# call parent constructor
super(PulpPushPlugin, self).__init__(tasker, workflow)
self.pulp_registry_name = pulp_registry_name
self.image_names = image_names
if load_squashed_image is not None and load_exported_image is not None and \
(load_squashed_image != load_exported_image):
raise RuntimeError("Can\'t use load_squashed_image and "
"load_exported_image with different values")
if load_squashed_image is not None:
self.log.warning('load_squashed_image argument is obsolete and will be '
'removed in a future version; please use load_exported_image instead')
self.load_exported_image = load_exported_image or load_squashed_image or False
self.pulp_secret_path = pulp_secret_path
self.username = username
self.password = password
self.publish = publish and not are_plugins_in_order(self.workflow.postbuild_plugins_conf,
self.key, PLUGIN_PULP_SYNC_KEY)
self.dockpulp_loglevel = dockpulp_loglevel
self.pulp_handler = PulpHandler(self.workflow, self.pulp_registry_name, self.log,
pulp_secret_path=self.pulp_secret_path,
username=self.username, password=self.password,
dockpulp_loglevel=self.dockpulp_loglevel)
def push_tar(self, filename, image_names=None, repo_prefix="redhat-"):
# Find out how to tag this image.
self.log.info("image names: %s", [str(image_name) for image_name in image_names])
self.log.info("checking image before upload %s", filename)
self.pulp_handler.check_file(filename)
pulp_repos = self.pulp_handler.create_dockpulp_and_repos(image_names, repo_prefix)
_, file_extension = os.path.splitext(filename)
try:
top_layer, layers = self.pulp_handler.get_tar_metadata(filename)
# getImageIdsExist was introduced in rh-dockpulp 0.6+
existing_imageids = self.pulp_handler.get_image_ids_existing(layers)
self.log.debug("existing layers: %s", existing_imageids)
# Strip existing layers from the tar and repack it
remove_layers = [str(os.path.join(x, 'layer.tar')) for x in existing_imageids]
commands = {'.xz': 'xzcat', '.gz': 'zcat', '.bz2': 'bzcat', '.tar': 'cat'}
unpacker = commands.get(file_extension, None)
self.log.debug("using unpacker %s for extension %s", unpacker, file_extension)
if unpacker is None:
raise Exception("Unknown tarball format: %s" % filename)
with NamedTemporaryFile(prefix='strip_tar_', suffix='.gz') as outfile:
cmd = "set -o pipefail; {0} {1} | tar --delete {2} | gzip - > {3}".format(
unpacker, filename, ' '.join(remove_layers), outfile.name)
self.log.debug("running %s", cmd)
subprocess.check_call(cmd, shell=True)
self.log.debug("uploading %s", outfile.name)
self.pulp_handler.upload(outfile.name)
except:
self.log.debug("Error on creating deduplicated layers tar", exc_info=True)
try:
if file_extension != '.tar':
raise RuntimeError("tar is already compressed")
with NamedTemporaryFile(prefix='full_tar_', suffix='.gz') as outfile:
cmd = "set -o pipefail; cat {0} | gzip - > {1}".format(
filename, outfile.name)
self.log.debug("Compressing tar using '%s' command", cmd)
subprocess.check_call(cmd, shell=True)
self.log.debug("uploading %s", outfile.name)
self.pulp_handler.upload(outfile.name)
except:
self.log.info("Falling back to full tar upload")
self.pulp_handler.upload(filename)
for repo_id, pulp_repo in pulp_repos.items():
for layer in layers:
self.pulp_handler.copy(repo_id, layer)
self.pulp_handler.update_repo(repo_id, {"tag": "%s:%s" % (",".join(pulp_repo.tags),
top_layer)})
# Only publish if we don't the pulp_sync plugin also configured
if self.publish:
self.pulp_handler.publish(pulp_repos.keys())
else:
self.log.info("publishing deferred until %s plugin runs", PLUGIN_PULP_SYNC_KEY)
# Store the registry URI in the push configuration
# We only want the hostname[:port]
pulp_registry = self.pulp_handler.get_registry_hostname()
self.workflow.push_conf.add_pulp_registry(self.pulp_handler.get_pulp_instance(),
pulp_registry,
server_side_sync=False)
# Return the set of qualified repo names for this image
return top_layer, [ImageName(registry=pulp_registry, repo=repodata.registry_id, tag=tag)
for dummy_repo, repodata in pulp_repos.items()
for tag in repodata.tags] # noqa
def run(self):
image_names = self.workflow.tag_conf.images[:]
# Add in additional image names, if any
if self.image_names:
self.log.info("extending image names: %s", self.image_names)
image_names += [ImageName.parse(x) for x in self.image_names]
if self.load_exported_image:
if len(self.workflow.exported_image_sequence) == 0:
raise RuntimeError('no exported image to push to pulp')
export_path = self.workflow.exported_image_sequence[-1].get("path")
top_layer, crane_repos = self.push_tar(export_path, image_names)
else:
# Work out image ID
image = self.workflow.image
self.log.info("fetching image %s from docker", image)
with tempfile.NamedTemporaryFile(prefix='docker-image-', suffix='.tar') as image_file:
image_file.write(self.tasker.d.get_image(image).data)
# This file will be referenced by its filename, not file
# descriptor - must ensure contents are written to disk
image_file.flush()
top_layer, crane_repos = self.push_tar(image_file.name, image_names)
if self.publish:
for image_name in crane_repos:
self.log.info("image available at %s", str(image_name))
return top_layer, crane_repos
class PulpPushPlugin(PostBuildPlugin):
key = PLUGIN_PULP_PUSH_KEY
is_allowed_to_fail = False
def __init__(self,
tasker,
workflow,
pulp_registry_name,
load_squashed_image=None,
load_exported_image=None,
image_names=None,
pulp_secret_path=None,
username=None,
password=None,
dockpulp_loglevel=None,
publish=True):
"""
constructor
:param tasker: DockerTasker instance
:param workflow: DockerBuildWorkflow instance
:param pulp_registry_name: str, name of pulp registry to use, specified in /etc/
dockpulp.conf
:param load_squashed_image: obsolete name for load_exported_image, please don't use
:param load_exported_image: bool, use exported tar instead of image from Docker
:param image_names: list of additional image names
:param pulp_secret_path: path to pulp.cer and pulp.key; $SOURCE_SECRET_PATH otherwise
:param username: pulp username, used in preference to certificate and key
:param password: pulp password, used in preference to certificate and key
:param publish: Bool, whether to publish to crane or not
"""
# call parent constructor
super(PulpPushPlugin, self).__init__(tasker, workflow)
self.pulp_registry_name = pulp_registry_name
self.image_names = image_names
if load_squashed_image is not None and load_exported_image is not None and \
(load_squashed_image != load_exported_image):
raise RuntimeError("Can\'t use load_squashed_image and "
"load_exported_image with different values")
if load_squashed_image is not None:
self.log.warning(
'load_squashed_image argument is obsolete and will be '
'removed in a future version; please use load_exported_image instead'
)
self.load_exported_image = load_exported_image or load_squashed_image or False
self.pulp_secret_path = pulp_secret_path
self.username = username
self.password = password
self.publish = publish and not are_plugins_in_order(
self.workflow.postbuild_plugins_conf, self.key,
PLUGIN_PULP_SYNC_KEY)
self.dockpulp_loglevel = dockpulp_loglevel
self.pulp_handler = PulpHandler(
self.workflow,
self.pulp_registry_name,
self.log,
pulp_secret_path=self.pulp_secret_path,
username=self.username,
password=self.password,
dockpulp_loglevel=self.dockpulp_loglevel)
def push_tar(self, filename, image_names=None, repo_prefix="redhat-"):
# Find out how to tag this image.
self.log.info("image names: %s",
[str(image_name) for image_name in image_names])
self.log.info("checking image before upload %s", filename)
self.pulp_handler.check_file(filename)
pulp_repos = self.pulp_handler.create_dockpulp_and_repos(
image_names, repo_prefix)
_, file_extension = os.path.splitext(filename)
try:
top_layer, layers = self.pulp_handler.get_tar_metadata(filename)
# getImageIdsExist was introduced in rh-dockpulp 0.6+
existing_imageids = self.pulp_handler.get_image_ids_existing(
layers)
self.log.debug("existing layers: %s", existing_imageids)
# Strip existing layers from the tar and repack it
remove_layers = [
str(os.path.join(x, 'layer.tar')) for x in existing_imageids
]
commands = {
'.xz': 'xzcat',
'.gz': 'zcat',
'.bz2': 'bzcat',
'.tar': 'cat'
}
unpacker = commands.get(file_extension, None)
self.log.debug("using unpacker %s for extension %s", unpacker,
file_extension)
if unpacker is None:
raise Exception("Unknown tarball format: %s" % filename)
with NamedTemporaryFile(prefix='strip_tar_',
suffix='.gz') as outfile:
cmd = "set -o pipefail; {0} {1} | tar --delete {2} | gzip - > {3}".format(
unpacker, filename, ' '.join(remove_layers), outfile.name)
self.log.debug("running %s", cmd)
subprocess.check_call(cmd, shell=True)
self.log.debug("uploading %s", outfile.name)
self.pulp_handler.upload(outfile.name)
except:
self.log.debug("Error on creating deduplicated layers tar",
exc_info=True)
try:
if file_extension != '.tar':
raise RuntimeError("tar is already compressed")
with NamedTemporaryFile(prefix='full_tar_',
suffix='.gz') as outfile:
cmd = "set -o pipefail; cat {0} | gzip - > {1}".format(
filename, outfile.name)
self.log.debug("Compressing tar using '%s' command", cmd)
subprocess.check_call(cmd, shell=True)
self.log.debug("uploading %s", outfile.name)
self.pulp_handler.upload(outfile.name)
except:
self.log.info("Falling back to full tar upload")
self.pulp_handler.upload(filename)
for repo_id, pulp_repo in pulp_repos.items():
for layer in layers:
self.pulp_handler.copy(repo_id, layer)
self.pulp_handler.update_repo(
repo_id,
{"tag": "%s:%s" % (",".join(pulp_repo.tags), top_layer)})
# Only publish if we don't the pulp_sync plugin also configured
if self.publish:
self.pulp_handler.publish(pulp_repos.keys())
else:
self.log.info("publishing deferred until %s plugin runs",
PLUGIN_PULP_SYNC_KEY)
# Store the registry URI in the push configuration
# We only want the hostname[:port]
pulp_registry = self.pulp_handler.get_registry_hostname()
self.workflow.push_conf.add_pulp_registry(
self.pulp_handler.get_pulp_instance(),
pulp_registry,
server_side_sync=False)
# Return the set of qualified repo names for this image
return top_layer, [
ImageName(registry=pulp_registry,
repo=repodata.registry_id,
tag=tag) for dummy_repo, repodata in pulp_repos.items()
for tag in repodata.tags
] # noqa
def run(self):
image_names = self.workflow.tag_conf.images[:]
# Add in additional image names, if any
if self.image_names:
self.log.info("extending image names: %s", self.image_names)
image_names += [ImageName.parse(x) for x in self.image_names]
if self.load_exported_image:
if len(self.workflow.exported_image_sequence) == 0:
raise RuntimeError('no exported image to push to pulp')
export_path = self.workflow.exported_image_sequence[-1].get("path")
top_layer, crane_repos = self.push_tar(export_path, image_names)
else:
# Work out image ID
image = self.workflow.image
self.log.info("fetching image %s from docker", image)
with tempfile.NamedTemporaryFile(prefix='docker-image-',
suffix='.tar') as image_file:
image_file.write(self.tasker.d.get_image(image).data)
# This file will be referenced by its filename, not file
# descriptor - must ensure contents are written to disk
image_file.flush()
top_layer, crane_repos = self.push_tar(image_file.name,
image_names)
if self.publish:
for image_name in crane_repos:
self.log.info("image available at %s", str(image_name))
return top_layer, crane_repos