def test_create_dockpulp_and_repos(tmpdir, user, secret_path, secret_env, invalid,
monkeypatch):
log = logging.getLogger("tests.test_pulp_util")
pulp_registry_name = 'registry.example.com'
testfile = 'foo'
if secret_env:
monkeypatch.setenv('SOURCE_SECRET_PATH', str(tmpdir))
if secret_path:
secret_path = str(tmpdir)
if not invalid:
with open(os.path.join(str(tmpdir), "pulp.cer"), "wt") as cer:
cer.write("pulp certificate\n")
with open(os.path.join(str(tmpdir), "pulp.key"), "wt") as key:
key.write("pulp key\n")
_, workflow = prepare(testfile)
image_names = workflow.tag_conf.images[:]
handler = PulpHandler(workflow, pulp_registry_name, log, pulp_secret_path=secret_path,
username=user, password=user)
if invalid:
with pytest.raises(Exception):
handler.create_dockpulp_and_repos(image_names)
return
expected = {
'redhat-image-name1': PulpRepo(registry_id='image-name1', tags=['latest']),
'redhat-image-name3': PulpRepo(registry_id='image-name3', tags=['asd']),
'redhat-prefix-image-name2': PulpRepo(registry_id='prefix/image-name2', tags=['latest'])
}
assert handler.create_dockpulp_and_repos(image_names) == expected
def __init__(self,
tasker,
workflow,
pulp_registry_name,
pulp_secret_path=None,
username=None,
password=None,
dockpulp_loglevel=None):
"""
constructor
:param tasker: DockerTasker instance
:param workflow: DockerBuildWorkflow instance
:param pulp_registry_name: str, name of pulp registry to use, specified in /etc/
dockpulp.conf
:param pulp_secret_path: path to pulp.cer and pulp.key; $SOURCE_SECRET_PATH otherwise
:param username: pulp username, used in preference to certificate and key
:param password: pulp password, used in preference to certificate and key
"""
# call parent constructor
super(PulpPublishPlugin, self).__init__(tasker, workflow)
self.workflow = workflow
self.pulp_handler = PulpHandler(self.workflow,
pulp_registry_name,
self.log,
pulp_secret_path=pulp_secret_path,
username=username,
password=password,
dockpulp_loglevel=dockpulp_loglevel)
def test_get_pulp_instance():
log = logging.getLogger("tests.test_pulp_util")
pulp_registry_name = 'registry.example.com'
testfile = 'foo'
_, workflow = prepare(testfile)
handler = PulpHandler(workflow, pulp_registry_name, log)
assert handler.get_pulp_instance() == pulp_registry_name
def test_get_pulp_instance():
log = logging.getLogger("tests.test_pulp_util")
pulp_registry_name = 'registry.example.com'
testfile = 'foo'
_, workflow = prepare(testfile)
handler = PulpHandler(workflow, pulp_registry_name, log)
assert handler.get_pulp_instance() == pulp_registry_name
def __init__(self,
tasker,
workflow,
pulp_registry_name,
load_squashed_image=None,
load_exported_image=None,
image_names=None,
pulp_secret_path=None,
username=None,
password=None,
dockpulp_loglevel=None,
publish=True):
"""
constructor
:param tasker: DockerTasker instance
:param workflow: DockerBuildWorkflow instance
:param pulp_registry_name: str, name of pulp registry to use, specified in /etc/
dockpulp.conf
:param load_squashed_image: obsolete name for load_exported_image, please don't use
:param load_exported_image: bool, use exported tar instead of image from Docker
:param image_names: list of additional image names
:param pulp_secret_path: path to pulp.cer and pulp.key; $SOURCE_SECRET_PATH otherwise
:param username: pulp username, used in preference to certificate and key
:param password: pulp password, used in preference to certificate and key
:param publish: Bool, whether to publish to crane or not
"""
# call parent constructor
super(PulpPushPlugin, self).__init__(tasker, workflow)
self.pulp_registry_name = pulp_registry_name
self.image_names = image_names
if load_squashed_image is not None and load_exported_image is not None and \
(load_squashed_image != load_exported_image):
raise RuntimeError("Can\'t use load_squashed_image and "
"load_exported_image with different values")
if load_squashed_image is not None:
self.log.warning(
'load_squashed_image argument is obsolete and will be '
'removed in a future version; please use load_exported_image instead'
)
self.load_exported_image = load_exported_image or load_squashed_image or False
self.pulp_secret_path = pulp_secret_path
self.username = username
self.password = password
self.publish = publish and not are_plugins_in_order(
self.workflow.postbuild_plugins_conf, self.key,
PLUGIN_PULP_SYNC_KEY)
self.dockpulp_loglevel = dockpulp_loglevel
self.pulp_handler = PulpHandler(
self.workflow,
self.pulp_registry_name,
self.log,
pulp_secret_path=self.pulp_secret_path,
username=self.username,
password=self.password,
dockpulp_loglevel=self.dockpulp_loglevel)
def test_get_registry_hostname():
log = logging.getLogger("tests.test_pulp_util")
pulp_registry_name = 'registry.example.com'
testfile = 'foo'
_, workflow = prepare(testfile)
handler = PulpHandler(workflow, pulp_registry_name, log)
handler.create_dockpulp_and_repos([])
assert handler.get_registry_hostname() == pulp_registry_name
def test_get_registry_hostname():
log = logging.getLogger("tests.test_pulp_util")
pulp_registry_name = 'registry.example.com'
testfile = 'foo'
_, workflow = prepare(testfile)
handler = PulpHandler(workflow, pulp_registry_name, log)
handler.create_dockpulp_and_repos([])
assert handler.get_registry_hostname() == pulp_registry_name
def test_get_tar_metadata():
log = logging.getLogger("tests.test_pulp_util")
pulp_registry_name = 'registry.example.com'
testfile = 'foo'
tasker, workflow = prepare(testfile)
handler = PulpHandler(workflow, pulp_registry_name, log)
expected = ("foo", ["foo"])
assert handler.get_tar_metadata(testfile) == expected
def test_get_tar_metadata():
log = logging.getLogger("tests.test_pulp_util")
pulp_registry_name = 'registry.example.com'
testfile = 'foo'
tasker, workflow = prepare(testfile)
handler = PulpHandler(workflow, pulp_registry_name, log)
expected = ("foo", ["foo"])
assert handler.get_tar_metadata(testfile) == expected
def set_v1_tags(self, v1_image_id):
image_names = self.workflow.tag_conf.images[:]
handler = PulpHandler(self.workflow, self.pulp_registry_name, self.log,
pulp_secret_path=self.pulp_secret_path, username=self.username,
password=self.password, dockpulp_loglevel=self.dockpulp_loglevel)
pulp_repos = handler.create_dockpulp_and_repos(image_names)
repo_tags = {}
for repo_id, pulp_repo in pulp_repos.items():
repo_tags[repo_id] = {"tag": "%s:%s" % (",".join(pulp_repo.tags), v1_image_id)}
handler.update_repo(repo_id, repo_tags[repo_id])
return repo_tags
def test_check_file(tmpdir, check_repo_retval, should_raise):
log = logging.getLogger("tests.test_pulp_util")
pulp_registry_name = 'registry.example.com'
testfile = 'foo'
_, workflow = prepare(testfile, check_repo_retval)
handler = PulpHandler(workflow, pulp_registry_name, log)
if should_raise:
with pytest.raises(Exception):
handler.check_file(testfile)
return
handler.check_file(testfile)
def test_ensure_repos(auto_publish, unsupported):
dist_data = [
{
"repo_id": "redhat-myproject-hello-world",
"auto_publish": auto_publish,
}
]
mock_get_data = [{"id": "redhat-myproject-hello-world"}]
data_with_dist = copy.deepcopy(mock_get_data)
data_with_dist[0]["distributors"] = dist_data
log = logging.getLogger("tests.test_pulp_util")
pulp_registry_name = 'registry.example.com'
testfile = 'foo'
_, workflow = prepare(testfile)
if unsupported:
(flexmock(dockpulp.Pulp)
.should_receive('getRepos')
.with_args(['redhat-myproject-hello-world'], fields=['id'], distributors=True)
.and_raise(TypeError)
.once())
else:
(flexmock(dockpulp.Pulp)
.should_receive('getRepos')
.with_args(['redhat-myproject-hello-world'], fields=['id'], distributors=True)
.and_return(data_with_dist)
.once())
(flexmock(dockpulp.Pulp)
.should_receive('getRepos')
.with_args(['redhat-myproject-hello-world'], fields=['id'])
.and_return(mock_get_data)
.times(1 if unsupported else 0))
(flexmock(dockpulp.Pulp)
.should_receive('updateRepo')
.with_args(data_with_dist[0]["id"], {'auto_publish': False})
.times(1 if auto_publish and not unsupported else 0))
image_names = [ImageName(repo="myproject-hello-world")]
handler = PulpHandler(workflow, pulp_registry_name, log)
handler.create_dockpulp_and_repos(image_names)
def get_pulp_session(workflow, logger, fallback):
config = get_pulp(workflow, fallback)
from atomic_reactor.pulp_util import PulpHandler
return PulpHandler(workflow,
config['name'],
logger,
pulp_secret_path=config['auth'].get('ssl_certs_dir'),
username=config['auth'].get('username'),
password=config['auth'].get('password'),
dockpulp_loglevel=config.get('loglevel'))
def test_upload(unsupported, caplog):
log = logging.getLogger("tests.test_pulp_util")
caplog.set_level(logging.DEBUG, logger='tests.test_pulp_util')
pulp_registry_name = 'registry.example.com'
testfile = 'foo'
upload_file = 'test_file'
repo_id = 'redhat-myproject-hello-world'
_, workflow = prepare(testfile)
image_names = [ImageName(repo="myproject-hello-world")]
handler = PulpHandler(workflow, pulp_registry_name, log)
handler.create_dockpulp_and_repos(image_names)
if unsupported:
(flexmock(dockpulp.Pulp)
.should_receive('upload')
.with_args(upload_file, repo_id)
.and_raise(TypeError)
.once()
.ordered())
(flexmock(dockpulp.Pulp)
.should_receive('upload')
.with_args(upload_file)
.and_return(True)
.once()
.ordered())
else:
(flexmock(dockpulp.Pulp)
.should_receive('upload')
.with_args(upload_file, repo_id)
.and_return(False)
.once())
handler.upload(upload_file, repo_id)
assert "Uploading %s to %s" % (upload_file, repo_id) in caplog.text
if unsupported:
assert "Falling back to uploading %s to redhat-everything repo" %\
upload_file in caplog.text
def test_create_dockpulp_and_repos(tmpdir, user, secret_path, secret_env,
invalid, monkeypatch):
log = logging.getLogger("tests.test_pulp_util")
pulp_registry_name = 'registry.example.com'
testfile = 'foo'
if secret_env:
monkeypatch.setenv('SOURCE_SECRET_PATH', str(tmpdir))
if secret_path:
secret_path = str(tmpdir)
if not invalid:
with open(os.path.join(str(tmpdir), "pulp.cer"), "wt") as cer:
cer.write("pulp certificate\n")
with open(os.path.join(str(tmpdir), "pulp.key"), "wt") as key:
key.write("pulp key\n")
_, workflow = prepare(testfile)
image_names = workflow.tag_conf.images[:]
handler = PulpHandler(workflow,
pulp_registry_name,
log,
pulp_secret_path=secret_path,
username=user,
password=user)
if invalid:
with pytest.raises(Exception):
handler.create_dockpulp_and_repos(image_names)
return
expected = {
'redhat-image-name1':
PulpRepo(registry_id='image-name1', tags=['latest']),
'redhat-image-name3':
PulpRepo(registry_id='image-name3', tags=['asd']),
'redhat-prefix-image-name2':
PulpRepo(registry_id='prefix/image-name2', tags=['latest'])
}
assert handler.create_dockpulp_and_repos(image_names) == expected
def test_upload(unsupported, caplog):
log = logging.getLogger("tests.test_pulp_util")
pulp_registry_name = 'registry.example.com'
testfile = 'foo'
upload_file = 'test_file'
repo_id = 'redhat-myproject-hello-world'
_, workflow = prepare(testfile)
image_names = [ImageName(repo="myproject-hello-world")]
handler = PulpHandler(workflow, pulp_registry_name, log)
handler.create_dockpulp_and_repos(image_names)
if unsupported:
(flexmock(dockpulp.Pulp).should_receive('upload').with_args(
upload_file, repo_id).and_raise(TypeError).once().ordered())
(flexmock(dockpulp.Pulp).should_receive('upload').with_args(
upload_file).and_return(True).once().ordered())
else:
(flexmock(dockpulp.Pulp).should_receive('upload').with_args(
upload_file, repo_id).and_return(False).once())
handler.upload(upload_file, repo_id)
assert "Uploading %s to %s" % (upload_file, repo_id) in caplog.text()
if unsupported:
assert "Falling back to uploading %s to redhat-everything repo" %\
upload_file in caplog.text()
def test_ensure_repos(auto_publish, unsupported):
dist_data = [{
"repo_id": "redhat-myproject-hello-world",
"auto_publish": auto_publish,
}]
mock_get_data = [{"id": "redhat-myproject-hello-world"}]
data_with_dist = copy.deepcopy(mock_get_data)
data_with_dist[0]["distributors"] = dist_data
log = logging.getLogger("tests.test_pulp_util")
pulp_registry_name = 'registry.example.com'
testfile = 'foo'
_, workflow = prepare(testfile)
if unsupported:
(flexmock(dockpulp.Pulp).should_receive('getRepos').with_args(
['redhat-myproject-hello-world'], fields=['id'],
distributors=True).and_raise(TypeError).once())
else:
(flexmock(dockpulp.Pulp).should_receive('getRepos').with_args(
['redhat-myproject-hello-world'], fields=['id'],
distributors=True).and_return(data_with_dist).once())
(flexmock(dockpulp.Pulp).should_receive('getRepos').with_args(
['redhat-myproject-hello-world'],
fields=['id'
]).and_return(mock_get_data).times(1 if unsupported else 0))
(flexmock(dockpulp.Pulp).should_receive('updateRepo').with_args(
data_with_dist[0]["id"], {
'auto_publish': False
}).times(1 if auto_publish and not unsupported else 0))
image_names = [ImageName(repo="myproject-hello-world")]
handler = PulpHandler(workflow, pulp_registry_name, log)
handler.create_dockpulp_and_repos(image_names)
def test_check_file(tmpdir, check_repo_retval, should_raise):
log = logging.getLogger("tests.test_pulp_util")
pulp_registry_name = 'registry.example.com'
testfile = 'foo'
_, workflow = prepare(testfile, check_repo_retval)
handler = PulpHandler(workflow, pulp_registry_name, log)
if should_raise:
with pytest.raises(Exception):
handler.check_file(testfile)
return
handler.check_file(testfile)
def __init__(self, tasker, workflow, pulp_registry_name, load_squashed_image=None,
load_exported_image=None, image_names=None, pulp_secret_path=None,
username=None, password=None, dockpulp_loglevel=None, publish=True):
"""
constructor
:param tasker: DockerTasker instance
:param workflow: DockerBuildWorkflow instance
:param pulp_registry_name: str, name of pulp registry to use, specified in /etc/
dockpulp.conf
:param load_squashed_image: obsolete name for load_exported_image, please don't use
:param load_exported_image: bool, use exported tar instead of image from Docker
:param image_names: list of additional image names
:param pulp_secret_path: path to pulp.cer and pulp.key; $SOURCE_SECRET_PATH otherwise
:param username: pulp username, used in preference to certificate and key
:param password: pulp password, used in preference to certificate and key
:param publish: Bool, whether to publish to crane or not
"""
# call parent constructor
super(PulpPushPlugin, self).__init__(tasker, workflow)
self.pulp_registry_name = pulp_registry_name
self.image_names = image_names
if load_squashed_image is not None and load_exported_image is not None and \
(load_squashed_image != load_exported_image):
raise RuntimeError("Can\'t use load_squashed_image and "
"load_exported_image with different values")
if load_squashed_image is not None:
self.log.warning('load_squashed_image argument is obsolete and will be '
'removed in a future version; please use load_exported_image instead')
self.load_exported_image = load_exported_image or load_squashed_image or False
self.pulp_secret_path = pulp_secret_path
self.username = username
self.password = password
self.publish = publish and not are_plugins_in_order(self.workflow.postbuild_plugins_conf,
self.key, PLUGIN_PULP_SYNC_KEY)
self.dockpulp_loglevel = dockpulp_loglevel
self.pulp_handler = PulpHandler(self.workflow, self.pulp_registry_name, self.log,
pulp_secret_path=self.pulp_secret_path,
username=self.username, password=self.password,
dockpulp_loglevel=self.dockpulp_loglevel)
def __init__(self, tasker, workflow, pulp_registry_name,
pulp_secret_path=None, username=None, password=None,
dockpulp_loglevel=None):
"""
constructor
:param tasker: DockerTasker instance
:param workflow: DockerBuildWorkflow instance
:param pulp_registry_name: str, name of pulp registry to use, specified in /etc/
dockpulp.conf
:param pulp_secret_path: path to pulp.cer and pulp.key; $SOURCE_SECRET_PATH otherwise
:param username: pulp username, used in preference to certificate and key
:param password: pulp password, used in preference to certificate and key
"""
# call parent constructor
super(PulpPublishPlugin, self).__init__(tasker, workflow)
self.workflow = workflow
self.pulp_handler = PulpHandler(self.workflow, pulp_registry_name, self.log,
pulp_secret_path=pulp_secret_path,
username=username, password=password,
dockpulp_loglevel=dockpulp_loglevel)
class PulpPublishPlugin(ExitPlugin):
key = PLUGIN_PULP_PUBLISH_KEY
is_allowed_to_fail = False
def __init__(self,
tasker,
workflow,
pulp_registry_name,
pulp_secret_path=None,
username=None,
password=None,
dockpulp_loglevel=None):
"""
constructor
:param tasker: DockerTasker instance
:param workflow: DockerBuildWorkflow instance
:param pulp_registry_name: str, name of pulp registry to use, specified in /etc/
dockpulp.conf
:param pulp_secret_path: path to pulp.cer and pulp.key; $SOURCE_SECRET_PATH otherwise
:param username: pulp username, used in preference to certificate and key
:param password: pulp password, used in preference to certificate and key
"""
# call parent constructor
super(PulpPublishPlugin, self).__init__(tasker, workflow)
self.workflow = workflow
self.pulp_handler = PulpHandler(self.workflow,
pulp_registry_name,
self.log,
pulp_secret_path=pulp_secret_path,
username=username,
password=password,
dockpulp_loglevel=dockpulp_loglevel)
def publish_to_crane(self, repo_prefix="redhat-"):
image_names = self.workflow.tag_conf.images[:]
# Find out how to publish this image.
self.log.info("image names: %s",
[str(image_name) for image_name in image_names])
self.pulp_handler.create_dockpulp()
if not repo_prefix:
repo_prefix = ''
pulp_repos = set(
['%s%s' % (repo_prefix, image.pulp_repo) for image in image_names])
self.pulp_handler.publish(pulp_repos)
pulp_registry = self.pulp_handler.get_registry_hostname()
crane_repos = [
ImageName(registry=pulp_registry,
repo=image.to_str(registry=False, tag=False),
tag=image.tag or 'latest') for image in image_names
]
for image_name in crane_repos:
self.log.info("image available at %s", str(image_name))
return crane_repos
def delete_v1_layers(self, repo_prefix="redhat-"):
annotations = self.workflow.build_result.annotations
if not annotations:
# No worker builds created
return
worker_builds = annotations['worker-builds']
for platform in worker_builds:
build_info = get_worker_build_info(self.workflow, platform)
annotations = build_info.build.get_annotations()
v1_image_id = annotations.get('v1-image-id')
if v1_image_id:
image_names = self.workflow.tag_conf.images
self.pulp_handler.create_dockpulp()
if not repo_prefix:
repo_prefix = ''
pulp_repos = set([
'%s%s' % (repo_prefix, image.pulp_repo)
for image in image_names
])
for repo_id in pulp_repos:
self.log.info("removing %s from repo %s", v1_image_id,
repo_id)
self.pulp_handler.remove_image(repo_id, v1_image_id)
def run(self):
if self.workflow.build_process_failed:
self.delete_v1_layers()
return []
else:
return self.publish_to_crane()
class PulpPushPlugin(PostBuildPlugin):
key = PLUGIN_PULP_PUSH_KEY
is_allowed_to_fail = False
def __init__(self,
tasker,
workflow,
pulp_registry_name,
load_squashed_image=None,
load_exported_image=None,
image_names=None,
pulp_secret_path=None,
username=None,
password=None,
dockpulp_loglevel=None,
publish=True):
"""
constructor
:param tasker: DockerTasker instance
:param workflow: DockerBuildWorkflow instance
:param pulp_registry_name: str, name of pulp registry to use, specified in /etc/
dockpulp.conf
:param load_squashed_image: obsolete name for load_exported_image, please don't use
:param load_exported_image: bool, use exported tar instead of image from Docker
:param image_names: list of additional image names
:param pulp_secret_path: path to pulp.cer and pulp.key; $SOURCE_SECRET_PATH otherwise
:param username: pulp username, used in preference to certificate and key
:param password: pulp password, used in preference to certificate and key
:param publish: Bool, whether to publish to crane or not
"""
# call parent constructor
super(PulpPushPlugin, self).__init__(tasker, workflow)
self.pulp_registry_name = pulp_registry_name
self.image_names = image_names
if load_squashed_image is not None and load_exported_image is not None and \
(load_squashed_image != load_exported_image):
raise RuntimeError("Can\'t use load_squashed_image and "
"load_exported_image with different values")
if load_squashed_image is not None:
self.log.warning(
'load_squashed_image argument is obsolete and will be '
'removed in a future version; please use load_exported_image instead'
)
self.load_exported_image = load_exported_image or load_squashed_image or False
self.pulp_secret_path = pulp_secret_path
self.username = username
self.password = password
self.publish = publish and not are_plugins_in_order(
self.workflow.postbuild_plugins_conf, self.key,
PLUGIN_PULP_SYNC_KEY)
self.dockpulp_loglevel = dockpulp_loglevel
self.pulp_handler = PulpHandler(
self.workflow,
self.pulp_registry_name,
self.log,
pulp_secret_path=self.pulp_secret_path,
username=self.username,
password=self.password,
dockpulp_loglevel=self.dockpulp_loglevel)
def push_tar(self, filename, image_names=None, repo_prefix="redhat-"):
# Find out how to tag this image.
self.log.info("image names: %s",
[str(image_name) for image_name in image_names])
self.log.info("checking image before upload %s", filename)
self.pulp_handler.check_file(filename)
pulp_repos = self.pulp_handler.create_dockpulp_and_repos(
image_names, repo_prefix)
_, file_extension = os.path.splitext(filename)
try:
top_layer, layers = self.pulp_handler.get_tar_metadata(filename)
# getImageIdsExist was introduced in rh-dockpulp 0.6+
existing_imageids = self.pulp_handler.get_image_ids_existing(
layers)
self.log.debug("existing layers: %s", existing_imageids)
# Strip existing layers from the tar and repack it
remove_layers = [
str(os.path.join(x, 'layer.tar')) for x in existing_imageids
]
commands = {
'.xz': 'xzcat',
'.gz': 'zcat',
'.bz2': 'bzcat',
'.tar': 'cat'
}
unpacker = commands.get(file_extension, None)
self.log.debug("using unpacker %s for extension %s", unpacker,
file_extension)
if unpacker is None:
raise Exception("Unknown tarball format: %s" % filename)
with NamedTemporaryFile(prefix='strip_tar_',
suffix='.gz') as outfile:
cmd = "set -o pipefail; {0} {1} | tar --delete {2} | gzip - > {3}".format(
unpacker, filename, ' '.join(remove_layers), outfile.name)
self.log.debug("running %s", cmd)
subprocess.check_call(cmd, shell=True)
self.log.debug("uploading %s", outfile.name)
self.pulp_handler.upload(outfile.name)
except:
self.log.debug("Error on creating deduplicated layers tar",
exc_info=True)
try:
if file_extension != '.tar':
raise RuntimeError("tar is already compressed")
with NamedTemporaryFile(prefix='full_tar_',
suffix='.gz') as outfile:
cmd = "set -o pipefail; cat {0} | gzip - > {1}".format(
filename, outfile.name)
self.log.debug("Compressing tar using '%s' command", cmd)
subprocess.check_call(cmd, shell=True)
self.log.debug("uploading %s", outfile.name)
self.pulp_handler.upload(outfile.name)
except:
self.log.info("Falling back to full tar upload")
self.pulp_handler.upload(filename)
for repo_id, pulp_repo in pulp_repos.items():
for layer in layers:
self.pulp_handler.copy(repo_id, layer)
self.pulp_handler.update_repo(
repo_id,
{"tag": "%s:%s" % (",".join(pulp_repo.tags), top_layer)})
# Only publish if we don't the pulp_sync plugin also configured
if self.publish:
self.pulp_handler.publish(pulp_repos.keys())
else:
self.log.info("publishing deferred until %s plugin runs",
PLUGIN_PULP_SYNC_KEY)
# Store the registry URI in the push configuration
# We only want the hostname[:port]
pulp_registry = self.pulp_handler.get_registry_hostname()
self.workflow.push_conf.add_pulp_registry(
self.pulp_handler.get_pulp_instance(),
pulp_registry,
server_side_sync=False)
# Return the set of qualified repo names for this image
return top_layer, [
ImageName(registry=pulp_registry,
repo=repodata.registry_id,
tag=tag) for dummy_repo, repodata in pulp_repos.items()
for tag in repodata.tags
] # noqa
def run(self):
image_names = self.workflow.tag_conf.images[:]
# Add in additional image names, if any
if self.image_names:
self.log.info("extending image names: %s", self.image_names)
image_names += [ImageName.parse(x) for x in self.image_names]
if self.load_exported_image:
if len(self.workflow.exported_image_sequence) == 0:
raise RuntimeError('no exported image to push to pulp')
export_path = self.workflow.exported_image_sequence[-1].get("path")
top_layer, crane_repos = self.push_tar(export_path, image_names)
else:
# Work out image ID
image = self.workflow.image
self.log.info("fetching image %s from docker", image)
with tempfile.NamedTemporaryFile(prefix='docker-image-',
suffix='.tar') as image_file:
image_file.write(self.tasker.d.get_image(image).data)
# This file will be referenced by its filename, not file
# descriptor - must ensure contents are written to disk
image_file.flush()
top_layer, crane_repos = self.push_tar(image_file.name,
image_names)
if self.publish:
for image_name in crane_repos:
self.log.info("image available at %s", str(image_name))
return top_layer, crane_repos
class PulpPublishPlugin(ExitPlugin):
key = PLUGIN_PULP_PUBLISH_KEY
is_allowed_to_fail = False
def __init__(self, tasker, workflow, pulp_registry_name,
pulp_secret_path=None, username=None, password=None,
dockpulp_loglevel=None):
"""
constructor
:param tasker: DockerTasker instance
:param workflow: DockerBuildWorkflow instance
:param pulp_registry_name: str, name of pulp registry to use, specified in /etc/
dockpulp.conf
:param pulp_secret_path: path to pulp.cer and pulp.key; $SOURCE_SECRET_PATH otherwise
:param username: pulp username, used in preference to certificate and key
:param password: pulp password, used in preference to certificate and key
"""
# call parent constructor
super(PulpPublishPlugin, self).__init__(tasker, workflow)
self.workflow = workflow
self.pulp_handler = PulpHandler(self.workflow, pulp_registry_name, self.log,
pulp_secret_path=pulp_secret_path,
username=username, password=password,
dockpulp_loglevel=dockpulp_loglevel)
def publish_to_crane(self, repo_prefix="redhat-"):
image_names = self.workflow.tag_conf.images[:]
# Find out how to publish this image.
self.log.info("image names: %s", [str(image_name) for image_name in image_names])
self.pulp_handler.create_dockpulp()
if not repo_prefix:
repo_prefix = ''
pulp_repos = set(['%s%s' % (repo_prefix, image.pulp_repo)
for image in image_names])
self.pulp_handler.publish(pulp_repos)
pulp_registry = self.pulp_handler.get_registry_hostname()
crane_repos = [ImageName(registry=pulp_registry,
repo=image.to_str(registry=False, tag=False),
tag=image.tag or 'latest') for image in image_names]
for image_name in crane_repos:
self.log.info("image available at %s", str(image_name))
return crane_repos
def delete_v1_layers(self, repo_prefix="redhat-"):
annotations = self.workflow.build_result.annotations
if not annotations:
# No worker builds created
return
worker_builds = annotations['worker-builds']
for platform in worker_builds:
build_info = get_worker_build_info(self.workflow, platform)
annotations = build_info.build.get_annotations()
v1_image_id = annotations.get('v1-image-id')
if v1_image_id:
image_names = self.workflow.tag_conf.images
self.pulp_handler.create_dockpulp()
if not repo_prefix:
repo_prefix = ''
pulp_repos = set(['%s%s' % (repo_prefix, image.pulp_repo) for image in image_names])
for repo_id in pulp_repos:
self.log.info("removing %s from repo %s", v1_image_id, repo_id)
self.pulp_handler.remove_image(repo_id, v1_image_id)
def run(self):
if self.workflow.build_process_failed:
self.delete_v1_layers()
return []
else:
return self.publish_to_crane()
class PulpPushPlugin(PostBuildPlugin):
key = PLUGIN_PULP_PUSH_KEY
is_allowed_to_fail = False
def __init__(self, tasker, workflow, pulp_registry_name, load_squashed_image=None,
load_exported_image=None, image_names=None, pulp_secret_path=None,
username=None, password=None, dockpulp_loglevel=None, publish=True):
"""
constructor
:param tasker: DockerTasker instance
:param workflow: DockerBuildWorkflow instance
:param pulp_registry_name: str, name of pulp registry to use, specified in /etc/
dockpulp.conf
:param load_squashed_image: obsolete name for load_exported_image, please don't use
:param load_exported_image: bool, use exported tar instead of image from Docker
:param image_names: list of additional image names
:param pulp_secret_path: path to pulp.cer and pulp.key; $SOURCE_SECRET_PATH otherwise
:param username: pulp username, used in preference to certificate and key
:param password: pulp password, used in preference to certificate and key
:param publish: Bool, whether to publish to crane or not
"""
# call parent constructor
super(PulpPushPlugin, self).__init__(tasker, workflow)
self.pulp_registry_name = pulp_registry_name
self.image_names = image_names
if load_squashed_image is not None and load_exported_image is not None and \
(load_squashed_image != load_exported_image):
raise RuntimeError("Can\'t use load_squashed_image and "
"load_exported_image with different values")
if load_squashed_image is not None:
self.log.warning('load_squashed_image argument is obsolete and will be '
'removed in a future version; please use load_exported_image instead')
self.load_exported_image = load_exported_image or load_squashed_image or False
self.pulp_secret_path = pulp_secret_path
self.username = username
self.password = password
self.publish = publish and not are_plugins_in_order(self.workflow.postbuild_plugins_conf,
self.key, PLUGIN_PULP_SYNC_KEY)
self.dockpulp_loglevel = dockpulp_loglevel
self.pulp_handler = PulpHandler(self.workflow, self.pulp_registry_name, self.log,
pulp_secret_path=self.pulp_secret_path,
username=self.username, password=self.password,
dockpulp_loglevel=self.dockpulp_loglevel)
def push_tar(self, filename, image_names=None, repo_prefix="redhat-"):
# Find out how to tag this image.
self.log.info("image names: %s", [str(image_name) for image_name in image_names])
self.log.info("checking image before upload %s", filename)
self.pulp_handler.check_file(filename)
pulp_repos = self.pulp_handler.create_dockpulp_and_repos(image_names, repo_prefix)
_, file_extension = os.path.splitext(filename)
try:
top_layer, layers = self.pulp_handler.get_tar_metadata(filename)
# getImageIdsExist was introduced in rh-dockpulp 0.6+
existing_imageids = self.pulp_handler.get_image_ids_existing(layers)
self.log.debug("existing layers: %s", existing_imageids)
# Strip existing layers from the tar and repack it
remove_layers = [str(os.path.join(x, 'layer.tar')) for x in existing_imageids]
commands = {'.xz': 'xzcat', '.gz': 'zcat', '.bz2': 'bzcat', '.tar': 'cat'}
unpacker = commands.get(file_extension, None)
self.log.debug("using unpacker %s for extension %s", unpacker, file_extension)
if unpacker is None:
raise Exception("Unknown tarball format: %s" % filename)
with NamedTemporaryFile(prefix='strip_tar_', suffix='.gz') as outfile:
cmd = "set -o pipefail; {0} {1} | tar --delete {2} | gzip - > {3}".format(
unpacker, filename, ' '.join(remove_layers), outfile.name)
self.log.debug("running %s", cmd)
subprocess.check_call(cmd, shell=True)
self.log.debug("uploading %s", outfile.name)
self.pulp_handler.upload(outfile.name)
except:
self.log.debug("Error on creating deduplicated layers tar", exc_info=True)
try:
if file_extension != '.tar':
raise RuntimeError("tar is already compressed")
with NamedTemporaryFile(prefix='full_tar_', suffix='.gz') as outfile:
cmd = "set -o pipefail; cat {0} | gzip - > {1}".format(
filename, outfile.name)
self.log.debug("Compressing tar using '%s' command", cmd)
subprocess.check_call(cmd, shell=True)
self.log.debug("uploading %s", outfile.name)
self.pulp_handler.upload(outfile.name)
except:
self.log.info("Falling back to full tar upload")
self.pulp_handler.upload(filename)
for repo_id, pulp_repo in pulp_repos.items():
for layer in layers:
self.pulp_handler.copy(repo_id, layer)
self.pulp_handler.update_repo(repo_id, {"tag": "%s:%s" % (",".join(pulp_repo.tags),
top_layer)})
# Only publish if we don't the pulp_sync plugin also configured
if self.publish:
self.pulp_handler.publish(pulp_repos.keys())
else:
self.log.info("publishing deferred until %s plugin runs", PLUGIN_PULP_SYNC_KEY)
# Store the registry URI in the push configuration
# We only want the hostname[:port]
pulp_registry = self.pulp_handler.get_registry_hostname()
self.workflow.push_conf.add_pulp_registry(self.pulp_handler.get_pulp_instance(),
pulp_registry,
server_side_sync=False)
# Return the set of qualified repo names for this image
return top_layer, [ImageName(registry=pulp_registry, repo=repodata.registry_id, tag=tag)
for dummy_repo, repodata in pulp_repos.items()
for tag in repodata.tags] # noqa
def run(self):
image_names = self.workflow.tag_conf.images[:]
# Add in additional image names, if any
if self.image_names:
self.log.info("extending image names: %s", self.image_names)
image_names += [ImageName.parse(x) for x in self.image_names]
if self.load_exported_image:
if len(self.workflow.exported_image_sequence) == 0:
raise RuntimeError('no exported image to push to pulp')
export_path = self.workflow.exported_image_sequence[-1].get("path")
top_layer, crane_repos = self.push_tar(export_path, image_names)
else:
# Work out image ID
image = self.workflow.image
self.log.info("fetching image %s from docker", image)
with tempfile.NamedTemporaryFile(prefix='docker-image-', suffix='.tar') as image_file:
image_file.write(self.tasker.d.get_image(image).data)
# This file will be referenced by its filename, not file
# descriptor - must ensure contents are written to disk
image_file.flush()
top_layer, crane_repos = self.push_tar(image_file.name, image_names)
if self.publish:
for image_name in crane_repos:
self.log.info("image available at %s", str(image_name))
return top_layer, crane_repos