def tag_manifest_into_registry(self, session, worker_digest):
"""
Tags the manifest identified by worker_digest into session.registry with all the
configured tags found in workflow.tag_conf.
"""
self.log.info("%s: Tagging manifest", session.registry)
digest = worker_digest['digest']
source_repo = worker_digest['repository']
image_manifest, _, media_type, _ = self.get_manifest(
session, source_repo, digest)
if media_type == MEDIA_TYPE_DOCKER_V2_SCHEMA2:
digests = ManifestDigest(v1=digest)
elif media_type == MEDIA_TYPE_OCI_V1:
digests = ManifestDigest(oci=digest)
else:
raise RuntimeError(
"Unexpected media type found in worker repository: {}".format(
media_type))
push_conf_registry = self.workflow.push_conf.add_docker_registry(
session.registry, insecure=session.insecure)
for image in self.workflow.tag_conf.images:
target_repo = image.to_str(registry=False, tag=False)
self.store_manifest_in_repository(session,
image_manifest,
media_type,
source_repo,
target_repo,
tag=image.tag)
# add a tag for any plugins running later that expect it
push_conf_registry.digests[image.tag] = digests
def prepare(pulp_registries=None, docker_registries=None):
if pulp_registries is None:
pulp_registries = (
("test", LOCALHOST_REGISTRY),
)
if docker_registries is None:
docker_registries = (DOCKER0_REGISTRY,)
def set_annotations_on_build(build_id, annotations):
pass
def update_labels_on_build(build_id, labels):
pass
new_environ = deepcopy(os.environ)
new_environ["BUILD"] = dedent('''\
{
"metadata": {
"name": "asd",
"namespace": "namespace"
}
}
''')
flexmock(OSBS, set_annotations_on_build=set_annotations_on_build)
flexmock(OSBS, update_labels_on_build=update_labels_on_build)
(flexmock(osbs.conf)
.should_call("Configuration")
.with_args(namespace="namespace", conf_file=None, verify_ssl=True,
openshift_url="http://example.com/", openshift_uri="http://example.com/",
use_auth=True))
flexmock(os)
os.should_receive("environ").and_return(new_environ)
workflow = DockerBuildWorkflow({"provider": "git", "uri": "asd"}, "test-image")
for name, crane_uri in pulp_registries:
workflow.push_conf.add_pulp_registry(name, crane_uri)
workflow.tag_conf.add_primary_image(TEST_IMAGE)
workflow.tag_conf.add_unique_image("namespace/image:asd123")
for docker_registry in docker_registries:
r = workflow.push_conf.add_docker_registry(docker_registry)
r.digests[TEST_IMAGE] = ManifestDigest(v1='not-used', v2=DIGEST1)
r.digests["namespace/image:asd123"] = ManifestDigest(v1='not-used',
v2=DIGEST2)
setattr(workflow, 'builder', X)
setattr(workflow, '_base_image_inspect', {'Id': '01234567'})
workflow.build_logs = [
"a", "b",
]
workflow.source.lg = LazyGit(None, commit="commit")
flexmock(workflow.source.lg)
workflow.source.lg.should_receive("_commit_id").and_return("commit")
return workflow
def test_delete_from_registry_plugin(saved_digests, req_registries, tmpdir):
if MOCK:
mock_docker()
tasker = DockerTasker()
workflow = DockerBuildWorkflow({"provider": "git", "uri": "asd"}, TEST_IMAGE)
setattr(workflow, 'builder', X)
args_registries = {}
for reg, use_secret in req_registries.items():
if use_secret:
temp_dir = mkdtemp(dir=str(tmpdir))
with open(os.path.join(temp_dir, ".dockercfg"), "w+") as dockerconfig:
dockerconfig_contents = {
reg: {
"username": "******", "password": reg
}
}
dockerconfig.write(json.dumps(dockerconfig_contents))
dockerconfig.flush()
args_registries[reg] = { 'secret': temp_dir }
else:
args_registries[reg] = {}
for reg, digests in saved_digests.items():
r = DockerRegistry(reg)
for tag, dig in digests.items():
r.digests[tag] = ManifestDigest(v1='not-used', v2=dig)
workflow.push_conf._registries['docker'].append(r)
runner = ExitPluginsRunner(
tasker,
workflow,
[{
'name': DeleteFromRegistryPlugin.key,
'args': {
'registries': args_registries
},
}]
)
deleted_digests = set()
for reg, digests in saved_digests.items():
if reg not in req_registries:
continue
for tag, dig in digests.items():
if dig in deleted_digests:
continue
url = "https://" + reg + "/v2/" + tag.split(":")[0] + "/manifests/" + dig
auth_type = requests.auth.HTTPBasicAuth if req_registries[reg] else None
(flexmock(requests)
.should_receive('delete')
.with_args(url, verify=bool, auth=auth_type)
.once()
.and_return(flexmock(status_code=202)))
deleted_digests.add(dig)
result = runner.run()
assert result[DeleteFromRegistryPlugin.key] == deleted_digests
def test_manifest_digest(v1, v2, v2_list, oci, oci_index, default):
md = ManifestDigest(v1=v1, v2=v2, v2_list=v2_list, oci=oci, oci_index=oci_index)
assert md.v1 == v1
assert md.v2 == v2
assert md.v2_list == v2_list
assert md.oci == oci
assert md.default == default
with pytest.raises(AttributeError):
assert md.no_such_version
def tag_manifest_into_registry(self, session, digest, source_repo, configured_tags):
"""
Tags the manifest identified by worker_digest into session.registry with all the
configured_tags
"""
self.log.info("%s: Tagging manifest", session.registry)
image_manifest, _, media_type, _ = self.get_manifest(session, source_repo, digest)
if media_type == MEDIA_TYPE_DOCKER_V2_SCHEMA2:
digests = ManifestDigest(v1=digest)
elif media_type == MEDIA_TYPE_OCI_V1:
digests = ManifestDigest(oci=digest)
else:
raise RuntimeError("Unexpected media type {} found in source_repo: {}"
.format(media_type, source_repo))
self.add_tag_and_manifest(session, image_manifest, media_type, digests, source_repo,
configured_tags)
return image_manifest, media_type, digests
def test_exceed_binary_image_size(image_size_limit, workflow):
config = {
'version': 1,
'registries': [
{'url': LOCALHOST_REGISTRY}
],
}
if image_size_limit is not None:
config['image_size_limit'] = image_size_limit
# workflow = DockerBuildWorkflow(source=MOCK_SOURCE)
workflow.plugin_workspace[ReactorConfigPlugin.key] = {
WORKSPACE_CONF_KEY: ReactorConfig(config)
}
workflow.builder = StubInsideBuilder()
workflow.builder.image_id = INPUT_IMAGE
# fake layer sizes of the test image
workflow.layer_sizes = [
{'diff_id': '12345', 'size': 1000},
{'diff_id': '23456', 'size': 2000},
{'diff_id': '34567', 'size': 3000},
]
mock_docker()
plugin = TagAndPushPlugin(DockerTasker(), workflow)
if image_size_limit is None or image_size_limit['binary_image'] == 0:
# The plugin should skip the check on image size
(flexmock(atomic_reactor.plugins.post_tag_and_push)
.should_receive('get_manifest_digests')
.and_return(ManifestDigest({
'v2': 'application/vnd.docker.distribution.manifest.list.v2+json',
})))
(flexmock(atomic_reactor.plugins.post_tag_and_push)
.should_receive('get_config_from_registry'))
assert workflow.image == plugin.run()[0].repo
else:
with pytest.raises(ExceedsImageSizeError):
plugin.run()
def test_get_built_images_multiple_manifest_types(workflow):
MockEnv(workflow).set_check_platforms_result(["x86_64"])
workflow.data.tag_conf.add_unique_image(UNIQUE_IMAGE)
flexmock(ManifestUtil).should_receive("__init__") # and do nothing, this test doesn't use it
(
flexmock(RegistryClient)
.should_receive("get_manifest_digests")
.with_args(ImageName.parse(f"{UNIQUE_IMAGE}-x86_64"), versions=("v2", "oci"))
.and_return(ManifestDigest({"v2": make_digest("foo"), "oci": make_digest("bar")}))
)
plugin = GroupManifestsPlugin(workflow)
session = RegistrySession(REGISTRY_V2)
expect_error = (
f"Expected to find a single manifest digest for {UNIQUE_IMAGE}-x86_64, "
"but found multiple: {'v2': .*, 'oci': .*}"
)
with pytest.raises(RuntimeError, match=expect_error):
plugin.get_built_images(session)
def test_manifest_digest(v1, v2, default):
md = ManifestDigest(v1=v1, v2=v2)
assert md.v1 == v1
assert md.v2 == v2
assert md.default == default
def prepare(workflow, registry=None, no_dockerfile=True):
if not registry:
registry = LOCALHOST_REGISTRY
def update_annotations_on_build(build_id, annotations):
pass
flexmock(OSBS, update_annotations_on_build=update_annotations_on_build)
if no_dockerfile:
os.remove(os.path.join(workflow.source.path, 'Dockerfile'))
workflow.build_dir.init_build_dirs(["x86_64"], workflow.source)
config_kwargs = {
'namespace': workflow.namespace,
'verify_ssl': True,
'openshift_url': 'http://example.com/',
'use_auth': True,
'conf_file': None,
}
(flexmock(osbs.conf.Configuration)
.should_call("__init__")
.with_args(**config_kwargs))
openshift_map = {
'url': 'http://example.com/',
'insecure': False,
'auth': {'enable': True},
}
registries_conf = [{'url': registry,
'insecure': True}]
rcm = {'version': 1, 'openshift': openshift_map, 'registries': registries_conf}
workflow.conf.conf = rcm
add_koji_map_in_workflow(workflow, hub_url='/', root_url='')
tag_conf = workflow.data.tag_conf
tag_conf.add_floating_image(f'{registry}/{TEST_IMAGE}')
tag_conf.add_primary_image(f'{registry}/namespace/image:version-release')
tag_conf.add_unique_image(f'{registry}/namespace/image:asd123')
(flexmock(RegistryClient)
.should_receive('get_manifest_digests')
.with_args(image=ImageName.parse(f'{registry}/{TEST_IMAGE_NAME}'),
versions=('v1', 'v2', 'v2_list', 'oci', 'oci_index'), require_digest=True)
.and_return(ManifestDigest(v1=DIGEST_NOT_USED, v2=DIGEST1)))
(flexmock(RegistryClient)
.should_receive('get_manifest_digests')
.with_args(image=ImageName.parse(f'{registry}/namespace/image:version-release'),
versions=('v1', 'v2', 'v2_list', 'oci', 'oci_index'), require_digest=True)
.and_return(None))
(flexmock(RegistryClient)
.should_receive('get_manifest_digests')
.with_args(image=ImageName.parse(f'{registry}/namespace/image:asd123'),
versions=('v1', 'v2', 'v2_list', 'oci', 'oci_index'), require_digest=True)
.and_return(ManifestDigest(v1=DIGEST_NOT_USED, v2=DIGEST2)))
flexmock(workflow.imageutil).should_receive('base_image_inspect').and_return({'Id': '01234567'})
workflow.build_logs = [
"a", "b",
]
workflow.source = GitSource('git', 'git://fake-url.com/repo')
flexmock(workflow.source).should_receive('commit_id').and_return('commit')
def test_delete_from_registry_plugin(saved_digests, req_registries, tmpdir, orchestrator,
manifest_list_digests, reactor_config_map):
if MOCK:
mock_docker()
mock_get_retry_session()
buildstep_plugin = None
if orchestrator:
ann_digests = []
buildstep_plugin = [{
'name': OrchestrateBuildPlugin.key,
'args': {
'platforms': "x86_64"
},
}]
tasker = DockerTasker()
workflow = DockerBuildWorkflow({"provider": "git", "uri": "asd"}, TEST_IMAGE,
buildstep_plugins=buildstep_plugin, )
setattr(workflow, 'builder', X)
args_registries = {}
config_map_regiestries = []
for reg, use_secret in req_registries.items():
cm_reg = {'url': reg}
if use_secret:
temp_dir = mkdtemp(dir=str(tmpdir))
with open(os.path.join(temp_dir, ".dockercfg"), "w+") as dockerconfig:
dockerconfig_contents = {
reg: {
"username": "******", "password": reg
}
}
dockerconfig.write(json.dumps(dockerconfig_contents))
dockerconfig.flush()
args_registries[reg] = {'secret': temp_dir}
cm_reg['auth'] = {'cfg_path': temp_dir}
else:
args_registries[reg] = {}
config_map_regiestries.append(cm_reg)
for reg, digests in saved_digests.items():
if orchestrator:
for tag, dig in digests.items():
repo = tag.split(':')[0]
t = tag.split(':')[1]
ann_digests.append({
'digest': dig,
'tag': t,
'repository': repo,
'registry': reg,
})
else:
r = DockerRegistry(reg)
for tag, dig in digests.items():
r.digests[tag] = ManifestDigest(v1='not-used', v2=dig)
workflow.push_conf._registries['docker'].append(r)
group_manifest_digests = {}
if orchestrator:
build_annotations = {'digests': ann_digests}
annotations = {'worker-builds': {'x86_64': build_annotations}}
setattr(workflow, 'build_result', Y)
setattr(workflow.build_result, 'annotations', annotations)
# group_manifest digest should be added only
# if there are worker builds and images are pushed to one registry
if len(req_registries) == 1 and len(saved_digests.keys()) == 1 and \
all(saved_digests.values()):
workflow.postbuild_results[PLUGIN_GROUP_MANIFESTS_KEY] = manifest_list_digests
for ml_repo, ml_digest in manifest_list_digests.items():
for reg in req_registries:
if reg in saved_digests:
group_manifest_digests.setdefault(reg, {})
group_manifest_digests[reg] = saved_digests[reg].copy()
group_manifest_digests[reg][ml_repo] = ml_digest.default
result_digests = saved_digests.copy()
result_digests.update(group_manifest_digests)
if reactor_config_map:
workflow.plugin_workspace[ReactorConfigPlugin.key] = {}
workflow.plugin_workspace[ReactorConfigPlugin.key][WORKSPACE_CONF_KEY] =\
ReactorConfig({'version': 1, 'registries': config_map_regiestries})
runner = ExitPluginsRunner(
tasker,
workflow,
[{
'name': DeleteFromRegistryPlugin.key,
'args': {
'registries': args_registries
},
}]
)
deleted_digests = set()
for reg, digests in result_digests.items():
if reg not in req_registries:
continue
for tag, dig in digests.items():
if dig in deleted_digests:
continue
url = "https://" + reg + "/v2/" + tag.split(":")[0] + "/manifests/" + dig
auth_type = HTTPRegistryAuth
(flexmock(requests.Session)
.should_receive('delete')
.with_args(url, verify=bool, auth=auth_type)
.once()
.and_return(flexmock(status_code=202, ok=True, raise_for_status=lambda: None)))
deleted_digests.add(dig)
result = runner.run()
assert result[DeleteFromRegistryPlugin.key] == deleted_digests
def test_delete_from_registry_failures(tmpdir, status_code, reactor_config_map):
if MOCK:
mock_docker()
mock_get_retry_session()
req_registries = {DOCKER0_REGISTRY: True}
saved_digests = {DOCKER0_REGISTRY: {'foo/bar:latest': DIGEST1}}
tasker = DockerTasker()
workflow = DockerBuildWorkflow({"provider": "git", "uri": "asd"}, TEST_IMAGE)
setattr(workflow, 'builder', X)
args_registries = {}
config_map_regiestries = []
for reg, use_secret in req_registries.items():
cm_reg = {'url': reg}
if use_secret:
temp_dir = mkdtemp(dir=str(tmpdir))
with open(os.path.join(temp_dir, ".dockercfg"), "w+") as dockerconfig:
dockerconfig_contents = {
reg: {
"username": "******", "password": reg
}
}
dockerconfig.write(json.dumps(dockerconfig_contents))
dockerconfig.flush()
args_registries[reg] = {'secret': temp_dir}
cm_reg['auth'] = {'cfg_path': temp_dir}
else:
args_registries[reg] = {}
config_map_regiestries.append(cm_reg)
for reg, digests in saved_digests.items():
r = DockerRegistry(reg)
for tag, dig in digests.items():
r.digests[tag] = ManifestDigest(v1='not-used', v2=dig)
workflow.push_conf._registries['docker'].append(r)
if reactor_config_map:
workflow.plugin_workspace[ReactorConfigPlugin.key] = {}
workflow.plugin_workspace[ReactorConfigPlugin.key][WORKSPACE_CONF_KEY] =\
ReactorConfig({'version': 1, 'registries': config_map_regiestries})
runner = ExitPluginsRunner(
tasker,
workflow,
[{
'name': DeleteFromRegistryPlugin.key,
'args': {
'registries': args_registries
},
}]
)
deleted_digests = set()
for reg, digests in saved_digests.items():
if reg not in req_registries:
continue
for tag, dig in digests.items():
if dig in deleted_digests:
continue
url = "https://" + reg + "/v2/" + tag.split(":")[0] + "/manifests/" + dig
auth_type = HTTPRegistryAuth
response = requests.Response()
response.status_code = status_code
(flexmock(requests.Session)
.should_receive('delete')
.with_args(url, verify=bool, auth=auth_type)
.and_return(response))
deleted_digests.add(dig)
if status_code == 520:
with pytest.raises(PluginFailedException):
result = runner.run()
assert result[DeleteFromRegistryPlugin.key] == set([])
else:
result = runner.run()
if status_code == requests.codes.ACCEPTED:
assert result[DeleteFromRegistryPlugin.key] == deleted_digests
else:
assert result[DeleteFromRegistryPlugin.key] == set([])
def test_tag_and_push_plugin(
tmpdir, monkeypatch, image_name, logs, should_raise, has_config, use_secret):
if MOCK:
mock_docker()
flexmock(docker.Client, push=lambda iid, **kwargs: iter(logs),
login=lambda username, registry, dockercfg_path: {'Status': 'Login Succeeded'})
tasker = DockerTasker()
workflow = DockerBuildWorkflow({"provider": "git", "uri": "asd"}, TEST_IMAGE)
workflow.tag_conf.add_primary_image(image_name)
setattr(workflow, 'builder', X)
secret_path = None
if use_secret:
temp_dir = mkdtemp()
with open(os.path.join(temp_dir, ".dockercfg"), "w+") as dockerconfig:
dockerconfig_contents = {
LOCALHOST_REGISTRY: {
"username": "******", "email": "*****@*****.**", "password": "******"}}
dockerconfig.write(json.dumps(dockerconfig_contents))
dockerconfig.flush()
secret_path = temp_dir
CONFIG_DIGEST = 'sha256:2c782e3a93d34d89ea4cf54052768be117caed54803263dd1f3798ce42aac14e'
media_type = 'application/vnd.docker.distribution.manifest.v2+json'
response_config_json = {
'config': {
'digest': CONFIG_DIGEST,
'mediaType': 'application/octet-stream',
'size': 4132
},
'layers': [
{
'digest': 'sha256:16dc1f96e3a1bb628be2e00518fec2bb97bd5933859de592a00e2eb7774b6ecf',
'mediaType': 'application/vnd.docker.image.rootfs.diff.tar.gzip',
'size': 71907148
},
{
'digest': 'sha256:cebc0565e1f096016765f55fde87a6f60fdb1208c0b5017e35a856ff578f5ccb',
'mediaType': 'application/vnd.docker.image.rootfs.diff.tar.gzip',
'size': 3945724
}
],
'mediaType': media_type,
'schemaVersion': 2
}
response_json = {
'config': {
'Size': 12509448,
'architecture': 'amd64',
'author': 'Red Hat, Inc.',
'config': {
'Cmd': ['/bin/rsyslog.sh'],
'Entrypoint': None,
'Image': 'c3fb36aafd5692d2a45115d32bb120edb6edf6c0c3c783ed6592a8dab969fb88',
'Labels': {
'Architecture': 'x86_64',
'Authoritative_Registry': 'registry.access.redhat.com',
'BZComponent': 'rsyslog-docker',
'Name': 'rhel7/rsyslog',
'Release': '28.vrutkovs.31',
'Vendor': 'Red Hat, Inc.',
'Version': '7.2',
},
},
'created': '2016-10-07T10:20:05.38595Z',
'docker_version': '1.9.1',
'id': '1ca220fbc2aed7c141b236c8729fe59db5771b32bf2da74e4a663407f32ec2a2',
'os': 'linux',
'parent': '47eed7a8ea3f131e9281ae09fcbfb0041872fd8b74a048f1c739302c8148505d'
},
'container_config': {
'foo': 'bar',
'spam': 'maps'
},
'id': '1ca220fbc2aed7c141b236c8729fe59db5771b32bf2da74e4a663407f32ec2a2',
'parent_id': 'c3fb36aafd5692d2a45115d32bb120edb6edf6c0c3c783ed6592a8dab969fb88'
}
if not has_config:
response_json = None
config_latest_url = "https://{}/v2/{}/manifests/latest".format(LOCALHOST_REGISTRY, TEST_IMAGE,)
config_url = "https://{}/v2/{}/manifests/{}".format(LOCALHOST_REGISTRY, TEST_IMAGE, DIGEST_V2)
blob_url = "https://{}/v2/{}/blobs/{}".format(
LOCALHOST_REGISTRY, TEST_IMAGE, CONFIG_DIGEST)
config_response_config_v1 = requests.Response()
(flexmock(config_response_config_v1,
raise_for_status=lambda: None,
json=response_config_json,
headers={
'Content-Type': 'application/vnd.docker.distribution.manifest.v1+json',
'Docker-Content-Digest': DIGEST_V1
}))
config_response_config_v2 = requests.Response()
(flexmock(config_response_config_v2,
raise_for_status=lambda: None,
json=response_config_json,
headers={
'Content-Type': 'application/vnd.docker.distribution.manifest.v2+json',
'Docker-Content-Digest': DIGEST_V2
}))
blob_config = requests.Response()
(flexmock(blob_config, raise_for_status=lambda: None, json=response_json))
def custom_get(url, headers, **kwargs):
if url == config_latest_url:
if headers['Accept'] == 'application/vnd.docker.distribution.manifest.v1+json':
return config_response_config_v1
if headers['Accept'] == 'application/vnd.docker.distribution.manifest.v2+json':
return config_response_config_v2
if url == config_url:
return config_response_config_v2
if url == blob_url:
return blob_config
(flexmock(requests)
.should_receive('get')
.replace_with(custom_get))
runner = PostBuildPluginsRunner(
tasker,
workflow,
[{
'name': TagAndPushPlugin.key,
'args': {
'registries': {
LOCALHOST_REGISTRY: {
'insecure': True,
'secret': secret_path
}
}
},
}]
)
if should_raise:
with pytest.raises(Exception):
runner.run()
else:
output = runner.run()
image = output[TagAndPushPlugin.key][0]
tasker.remove_image(image)
assert len(workflow.push_conf.docker_registries) > 0
if MOCK:
# we only test this when mocking docker because we don't expect
# running actual docker against v2 registry
expected_digest = ManifestDigest(v1=DIGEST_V1, v2=DIGEST_V2)
assert workflow.push_conf.docker_registries[0].digests[image_name].v1 == \
expected_digest.v1
assert workflow.push_conf.docker_registries[0].digests[image_name].v2 == \
expected_digest.v2
if has_config:
assert isinstance(workflow.push_conf.docker_registries[0].config, dict)
else:
assert workflow.push_conf.docker_registries[0].config is None
def test_binary_build_get_output(no_v2_digest: bool, from_scratch: bool,
is_flatpak: bool,
workflow: DockerBuildWorkflow, tmpdir):
platform = "x86_64"
buildroot_id = f'{platform}-1'
image_pullspec = ImageName.parse("ns/image:latest")
expected_components = []
if not from_scratch:
package_list = [
'python-docker-py;1.3.1;1.fc24;noarch;(none);191456;'
'7c1f60d8cde73e97a45e0c489f4a3b26;1438058212;(none);(none);(none);(none)',
'fedora-repos-rawhide;24;0.1;noarch;(none);2149;'
'd41df1e059544d906363605d47477e60;1436940126;(none);(none);(none);(none)',
'gpg-pubkey-doc;1.0;1;noarch;(none);1000;'
'00000000000000000000000000000000;1436940126;(none);(none);(none);(none)'
]
expected_components = parse_rpm_output(package_list)
if is_flatpak:
workflow.user_params['flatpak'] = True
workflow.data.image_components = {platform: expected_components}
if from_scratch:
workflow.data.dockerfile_images = DockerfileImages(['scratch'])
parent_id = None
else:
workflow.data.dockerfile_images = DockerfileImages(['fedora:35'])
parent_id = 'parent-id'
(flexmock(workflow.imageutil).should_receive(
'base_image_inspect').with_args(platform).and_return(
{'Id': parent_id}))
# For verifying the tags in final metadata
workflow.data.tag_conf.add_unique_image("ns/image:1")
workflow.data.tag_conf.add_unique_image("ns/image:2")
# This primary image is noise. For binary build, this should not be
# included in the metadata.
workflow.data.tag_conf.add_primary_image("ns/image:1-2")
# Mock for ImageUtil.get_uncompressed_layer_sizes
layer_sizes = [
{
"diff_id": 1,
"size": 100
},
{
"diff_id": 2,
"size": 200
},
]
workflow.build_dir.init_build_dirs([platform], workflow.source)
platform_dir = workflow.build_dir.platform_dir(platform)
(flexmock(workflow.imageutil).should_receive(
'get_uncompressed_image_layer_sizes').with_args(
str(platform_dir.exported_squashed_image)).and_return(layer_sizes))
workflow.conf.conf = {
'registries': [
{
'url': 'https://registry.host/',
'insecure': False
},
],
}
# Mock get_inspect_for_image
image_id = 'image-id-1234'
(flexmock(
workflow.imageutil).should_receive('get_inspect_for_image').with_args(
image_pullspec, platform=platform).and_return({'Id': image_id}))
# Mock get manifest digests
image_manifest_digest = ManifestDigest(
{'oci': 'oci-1234'} if no_v2_digest else {'v2': '1234'})
(flexmock(RegistryClient).should_receive(
'get_manifest_digests').and_return(image_manifest_digest))
# Mock getting image config
blob_config = {'oci': 'oci-1234'} if no_v2_digest else {'v2': '1234'}
(flexmock(RegistryClient).should_receive(
'get_config_and_id_from_registry').and_return((blob_config, None)))
# Assume FetchDockerArchivePlugin has run and metadata of the
# platform-specific built image archive has been saved.
workflow.data.plugins_results[FetchDockerArchivePlugin.key] = {
platform: {
'type': IMAGE_TYPE_DOCKER_ARCHIVE
}
}
output, output_file = get_output(workflow,
buildroot_id,
image_pullspec,
platform,
source_build=False)
# Prepare expected metadata
expected_repositories = sorted([
# Pull image with a specific tag
image_pullspec.to_str(),
# Pull image with a specific digest
f'{image_pullspec.to_str(tag=False)}@{image_manifest_digest.oci}'
if no_v2_digest else
f'{image_pullspec.to_str(tag=False)}@{image_manifest_digest.v2}',
])
per_platform_image_tags = sorted(
image.tag
for image in workflow.data.tag_conf.get_unique_images_with_platform(
platform))
expected_metadata: Dict[str, Any] = {
'buildroot_id': buildroot_id,
'checksum_type': 'md5',
'arch': platform,
'type': 'docker-image',
'components': expected_components,
'extra': {
'image': {
'arch': platform
},
'docker': {
'id': image_id,
'repositories': expected_repositories,
'layer_sizes': layer_sizes,
'tags': per_platform_image_tags,
'config': blob_config,
'digests': None, # Set later below
},
},
}
extra_docker = expected_metadata['extra']['docker']
if not from_scratch:
extra_docker['parent_id'] = parent_id
extra_docker['digests'] = (
{
ManifestDigest.content_type['oci']: image_manifest_digest.oci
} if no_v2_digest else {
ManifestDigest.content_type['v2']: image_manifest_digest.v2
})
# Start assertions
assert output_file is None
assert len(output) == 1
image_metadata = output[0].metadata
# Assert these image metadata firstly, then remove them and assert the
# rest. So, no need to mock anything for get_image_output.
assert f'docker-image-{image_id}.x86_64.tar.gz' == image_metadata.pop(
'filename')
assert image_metadata.pop('filesize') > 0
assert re.match(r'^[0-9a-f]+$', image_metadata.pop('checksum'))
# Make it easier for comparison below
extra_docker = image_metadata['extra']['docker']
extra_docker['repositories'] = sorted(extra_docker['repositories'])
assert expected_metadata == image_metadata
def prepare(docker_registries=None, before_dockerfile=False):
if docker_registries is None:
docker_registries = (
LOCALHOST_REGISTRY,
DOCKER0_REGISTRY,
)
def update_annotations_on_build(build_id, annotations):
pass
def update_labels_on_build(build_id, labels):
pass
new_environ = deepcopy(os.environ)
new_environ["BUILD"] = dedent('''\
{
"metadata": {
"name": "asd",
"namespace": "namespace"
}
}
''')
flexmock(OSBS, update_annotations_on_build=update_annotations_on_build)
flexmock(OSBS, update_labels_on_build=update_labels_on_build)
config_kwargs = {
'namespace': 'namespace',
'verify_ssl': True,
'openshift_url': 'http://example.com/',
'use_auth': True,
'conf_file': None,
'build_json_dir': None
}
(flexmock(osbs.conf.Configuration).should_call("__init__").with_args(
**config_kwargs))
flexmock(os)
os.should_receive("environ").and_return(new_environ) # pylint: disable=no-member
workflow = DockerBuildWorkflow(source=MOCK_SOURCE)
openshift_map = {
'url': 'http://example.com/',
'insecure': False,
'auth': {
'enable': True
},
}
workflow.plugin_workspace[ReactorConfigPlugin.key] = {}
workflow.plugin_workspace[ReactorConfigPlugin.key][WORKSPACE_CONF_KEY] =\
ReactorConfig({'version': 1, 'openshift': openshift_map})
add_koji_map_in_workflow(workflow, hub_url='/', root_url='')
workflow.tag_conf.add_floating_image(TEST_IMAGE)
workflow.tag_conf.add_primary_image("namespace/image:version-release")
workflow.tag_conf.add_unique_image("namespace/image:asd123")
for docker_registry in docker_registries:
r = workflow.push_conf.add_docker_registry(docker_registry)
r.digests[TEST_IMAGE_NAME] = ManifestDigest(v1=DIGEST_NOT_USED,
v2=DIGEST1)
r.digests["namespace/image:asd123"] = ManifestDigest(
v1=DIGEST_NOT_USED, v2=DIGEST2)
if before_dockerfile:
setattr(workflow, 'builder', XBeforeDockerfile())
setattr(workflow.builder, 'base_image_inspect', {})
else:
setattr(workflow, 'builder', X())
setattr(workflow.builder, 'base_image_inspect', {'Id': '01234567'})
workflow.build_logs = [
"a",
"b",
]
workflow.source.lg = LazyGit(None, commit="commit")
flexmock(workflow.source.lg)
# pylint: disable=no-member
workflow.source.lg.should_receive("_commit_id").and_return("commit")
# pylint: enable=no-member
return workflow
def mock_environment(tmpdir,
session=None,
name=None,
component=None,
version=None,
release=None,
source=None,
build_process_failed=False,
docker_registry=True,
pulp_registries=0,
blocksize=None,
task_states=None,
additional_tags=None,
has_config=None,
prefer_schema1_digest=True):
if session is None:
session = MockedClientSession('', task_states=None)
if source is None:
source = GitSource('git', 'git://hostname/path')
if MOCK:
mock_docker()
tasker = DockerTasker()
workflow = DockerBuildWorkflow(SOURCE, "test-image")
base_image_id = '123456parent-id'
workflow.source = StubSource()
workflow.builder = StubInsideBuilder().for_workflow(workflow)
workflow.builder.image_id = '123456imageid'
workflow.builder.set_inspection_data({'Id': base_image_id})
setattr(workflow, 'tag_conf', TagConf())
with open(os.path.join(str(tmpdir), 'Dockerfile'), 'wt') as df:
df.write(
'FROM base\n'
'LABEL BZComponent={component} com.redhat.component={component}\n'
'LABEL Version={version} version={version}\n'
'LABEL Release={release} release={release}\n'.format(
component=component, version=version, release=release))
workflow.builder.set_df_path(df.name)
if name and version:
workflow.tag_conf.add_unique_image(
'user/test-image:{v}-timestamp'.format(v=version))
if name and version and release:
workflow.tag_conf.add_primary_images([
"{0}:{1}-{2}".format(name, version, release),
"{0}:{1}".format(name, version), "{0}:latest".format(name)
])
if additional_tags:
workflow.tag_conf.add_primary_images(
["{0}:{1}".format(name, tag) for tag in additional_tags])
flexmock(subprocess, Popen=fake_Popen)
flexmock(koji, ClientSession=lambda hub, opts: session)
flexmock(GitSource)
setattr(workflow, 'source', source)
setattr(workflow.source, 'lg', X())
setattr(workflow.source.lg, 'commit_id', '123456')
setattr(workflow, 'push_conf', PushConf())
if docker_registry:
docker_reg = workflow.push_conf.add_docker_registry(
'docker.example.com')
for image in workflow.tag_conf.images:
tag = image.to_str(registry=False)
if pulp_registries and prefer_schema1_digest:
docker_reg.digests[tag] = ManifestDigest(v1=fake_digest(image),
v2='sha256:not-used')
else:
docker_reg.digests[tag] = ManifestDigest(v1='sha256:not-used',
v2=fake_digest(image))
if has_config:
docker_reg.config = {
'config': {
'architecture': LOCAL_ARCH
},
'container_config': {}
}
for pulp_registry in range(pulp_registries):
workflow.push_conf.add_pulp_registry('env', 'pulp.example.com')
with open(os.path.join(str(tmpdir), 'image.tar.xz'), 'wt') as fp:
fp.write('x' * 2**12)
setattr(workflow, 'exported_image_sequence',
[{
'path': fp.name,
'type': IMAGE_TYPE_DOCKER_ARCHIVE
}])
if build_process_failed:
workflow.build_result = BuildResult(
logs=["docker build log - \u2018 \u2017 \u2019 \n'"],
fail_reason="not built")
else:
workflow.build_result = BuildResult(
logs=["docker build log - \u2018 \u2017 \u2019 \n'"],
image_id="id1234")
workflow.prebuild_plugins_conf = {}
workflow.image_components = parse_rpm_output([
"name1;1.0;1;" + LOCAL_ARCH + ";0;2000;" + FAKE_SIGMD5.decode() +
";23000;"
"RSA/SHA256, Tue 30 Aug 2016 00:00:00, Key ID 01234567890abc;(none)",
"name2;2.0;1;" + LOCAL_ARCH + ";0;3000;" + FAKE_SIGMD5.decode() +
";24000"
"RSA/SHA256, Tue 30 Aug 2016 00:00:00, Key ID 01234567890abd;(none)",
])
return tasker, workflow
wf_data.tag_conf.add_primary_image(image)
wf_data.tag_conf.add_unique_image(primary_images[0])
if floating_images:
image: str
for image in floating_images:
wf_data.tag_conf.add_floating_image(image)
return env
REGISTRY_V2 = 'registry_v2.example.com'
GROUPED_V2_RESULTS = {
"manifest_digest": ManifestDigest(v2_list="sha256:11c3ecdbfa"),
"media_type": "application/vnd.docker.distribution.manifest.list.v2+json",
"manifest": json.dumps({
"manifests": [
{
"digest": "sha256:9dc3bbcd6c",
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"platform": {
"architecture": "amd64",
"os": "linux"
},
"size": 306
},
{
"digest": "sha256:cd619643ae",
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
def mock_environment(tmpdir, session=None, name=None,
component=None, version=None, release=None,
source=None, build_process_failed=False,
is_rebuild=True, docker_registry=True,
pulp_registries=0, blocksize=None,
task_states=None, additional_tags=None,
has_config=None):
if session is None:
session = MockedClientSession('', task_states=None)
if source is None:
source = GitSource('git', 'git://hostname/path')
if MOCK:
mock_docker()
tasker = DockerTasker()
workflow = DockerBuildWorkflow(SOURCE, "test-image")
base_image_id = '123456parent-id'
setattr(workflow, '_base_image_inspect', {'Id': base_image_id})
setattr(workflow, 'builder', X())
setattr(workflow.builder, 'image_id', '123456imageid')
setattr(workflow.builder, 'base_image', ImageName(repo='Fedora', tag='22'))
setattr(workflow.builder, 'source', X())
setattr(workflow.builder, 'built_image_info', {'ParentId': base_image_id})
setattr(workflow.builder.source, 'dockerfile_path', None)
setattr(workflow.builder.source, 'path', None)
setattr(workflow, 'tag_conf', TagConf())
with open(os.path.join(str(tmpdir), 'Dockerfile'), 'wt') as df:
df.write('FROM base\n'
'LABEL BZComponent={component} com.redhat.component={component}\n'
'LABEL Version={version} version={version}\n'
'LABEL Release={release} release={release}\n'
.format(component=component, version=version, release=release))
setattr(workflow.builder, 'df_path', df.name)
if name and version:
workflow.tag_conf.add_unique_image('user/test-image:{v}-timestamp'
.format(v=version))
if name and version and release:
workflow.tag_conf.add_primary_images(["{0}:{1}-{2}".format(name,
version,
release),
"{0}:{1}".format(name, version),
"{0}:latest".format(name)])
if additional_tags:
workflow.tag_conf.add_primary_images(["{0}:{1}".format(name, tag)
for tag in additional_tags])
flexmock(subprocess, Popen=fake_Popen)
flexmock(koji, ClientSession=lambda hub, opts: session)
flexmock(GitSource)
(flexmock(OSBS)
.should_receive('get_build_logs')
.with_args(BUILD_ID)
.and_return('build logs - \u2018 \u2017 \u2019'))
(flexmock(OSBS)
.should_receive('get_pod_for_build')
.with_args(BUILD_ID)
.and_return(MockedPodResponse()))
setattr(workflow, 'source', source)
setattr(workflow.source, 'lg', X())
setattr(workflow.source.lg, 'commit_id', '123456')
setattr(workflow, 'push_conf', PushConf())
if docker_registry:
docker_reg = workflow.push_conf.add_docker_registry('docker.example.com')
for image in workflow.tag_conf.images:
tag = image.to_str(registry=False)
if pulp_registries:
docker_reg.digests[tag] = ManifestDigest(v1=fake_digest(image),
v2='sha256:not-used')
else:
docker_reg.digests[tag] = ManifestDigest(v1='sha256:not-used',
v2=fake_digest(image))
if has_config:
docker_reg.config = {
'config': {'architecture': 'x86_64'},
'container_config': {}
}
for pulp_registry in range(pulp_registries):
workflow.push_conf.add_pulp_registry('env', 'pulp.example.com')
with open(os.path.join(str(tmpdir), 'image.tar.xz'), 'wt') as fp:
fp.write('x' * 2**12)
setattr(workflow, 'exported_image_sequence', [{'path': fp.name}])
if build_process_failed:
workflow.build_result = BuildResult(logs=["docker build log - \u2018 \u2017 \u2019 \n'"],
fail_reason="not built")
else:
workflow.build_result = BuildResult(logs=["docker build log - \u2018 \u2017 \u2019 \n'"],
image_id="id1234")
workflow.prebuild_plugins_conf = {}
workflow.prebuild_results[CheckAndSetRebuildPlugin.key] = is_rebuild
workflow.postbuild_results[PostBuildRPMqaPlugin.key] = [
"name1;1.0;1;x86_64;0;2000;" + FAKE_SIGMD5.decode() + ";23000;"
"RSA/SHA256, Tue 30 Aug 2016 00:00:00, Key ID 01234567890abc;(none)",
"name2;2.0;1;x86_64;0;3000;" + FAKE_SIGMD5.decode() + ";24000"
"RSA/SHA256, Tue 30 Aug 2016 00:00:00, Key ID 01234567890abd;(none)",
]
return tasker, workflow
def prepare(
pulp_registries=None,
docker_registries=None,
before_dockerfile=False, # noqa
reactor_config_map=False):
if pulp_registries is None:
pulp_registries = (("test", LOCALHOST_REGISTRY), )
if docker_registries is None:
docker_registries = (DOCKER0_REGISTRY, )
def update_annotations_on_build(build_id, annotations):
pass
def update_labels_on_build(build_id, labels):
pass
new_environ = deepcopy(os.environ)
new_environ["BUILD"] = dedent('''\
{
"metadata": {
"name": "asd",
"namespace": "namespace"
}
}
''')
flexmock(OSBS, update_annotations_on_build=update_annotations_on_build)
flexmock(OSBS, update_labels_on_build=update_labels_on_build)
config_kwargs = {
'namespace': 'namespace',
'verify_ssl': True,
'openshift_url': 'http://example.com/',
'use_auth': True,
'conf_file': None,
'build_json_dir': None
}
(flexmock(osbs.conf.Configuration).should_call("__init__").with_args(
**config_kwargs))
flexmock(os)
os.should_receive("environ").and_return(new_environ)
workflow = DockerBuildWorkflow({
"provider": "git",
"uri": "asd"
}, "test-image")
if reactor_config_map:
openshift_map = {
'url': 'http://example.com/',
'insecure': False,
'auth': {
'enable': True
},
}
workflow.plugin_workspace[ReactorConfigPlugin.key] = {}
workflow.plugin_workspace[ReactorConfigPlugin.key][WORKSPACE_CONF_KEY] =\
ReactorConfig({'version': 1, 'openshift': openshift_map})
for name, crane_uri in pulp_registries:
workflow.push_conf.add_pulp_registry(name, crane_uri)
workflow.tag_conf.add_primary_image(TEST_IMAGE)
workflow.tag_conf.add_unique_image("namespace/image:asd123")
for docker_registry in docker_registries:
r = workflow.push_conf.add_docker_registry(docker_registry)
r.digests[TEST_IMAGE] = ManifestDigest(v1=DIGEST_NOT_USED, v2=DIGEST1)
r.digests["namespace/image:asd123"] = ManifestDigest(
v1=DIGEST_NOT_USED, v2=DIGEST2)
if before_dockerfile:
setattr(workflow, 'builder', XBeforeDockerfile())
else:
setattr(workflow, 'builder', X)
setattr(workflow, '_base_image_inspect', {'Id': '01234567'})
workflow.build_logs = [
"a",
"b",
]
workflow.source.lg = LazyGit(None, commit="commit")
flexmock(workflow.source.lg)
workflow.source.lg.should_receive("_commit_id").and_return("commit")
return workflow
def test_get_manifest_media_version_unknown():
with pytest.raises(RuntimeError):
assert get_manifest_media_version(ManifestDigest())
def test_tag_and_push_plugin(tmpdir, monkeypatch, image_name, logs,
should_raise, has_config, use_secret,
reactor_config_map, file_name,
dockerconfig_contents):
if MOCK:
mock_docker()
flexmock(docker.APIClient,
push=lambda iid, **kwargs: iter(logs),
login=lambda username, registry, dockercfg_path:
{'Status': 'Login Succeeded'})
tasker = DockerTasker(retry_times=0)
workflow = DockerBuildWorkflow({
"provider": "git",
"uri": "asd"
}, TEST_IMAGE)
workflow.tag_conf.add_primary_image(image_name)
setattr(workflow, 'builder', X)
secret_path = None
if use_secret:
temp_dir = mkdtemp()
with open(os.path.join(temp_dir, file_name), "w+") as dockerconfig:
dockerconfig.write(json.dumps(dockerconfig_contents))
dockerconfig.flush()
secret_path = temp_dir
CONFIG_DIGEST = 'sha256:2c782e3a93d34d89ea4cf54052768be117caed54803263dd1f3798ce42aac14e'
media_type = 'application/vnd.docker.distribution.manifest.v2+json'
manifest_json = {
'config': {
'digest': CONFIG_DIGEST,
'mediaType': 'application/octet-stream',
'size': 4132
},
'layers': [{
'digest':
'sha256:16dc1f96e3a1bb628be2e00518fec2bb97bd5933859de592a00e2eb7774b6ecf',
'mediaType': 'application/vnd.docker.image.rootfs.diff.tar.gzip',
'size': 71907148
}, {
'digest':
'sha256:cebc0565e1f096016765f55fde87a6f60fdb1208c0b5017e35a856ff578f5ccb',
'mediaType': 'application/vnd.docker.image.rootfs.diff.tar.gzip',
'size': 3945724
}],
'mediaType':
media_type,
'schemaVersion':
2
}
config_json = {
'config': {
'Size':
12509448,
'architecture':
'amd64',
'author':
'Red Hat, Inc.',
'config': {
'Cmd': ['/bin/rsyslog.sh'],
'Entrypoint': None,
'Image':
'c3fb36aafd5692d2a45115d32bb120edb6edf6c0c3c783ed6592a8dab969fb88',
'Labels': {
'Architecture': 'x86_64',
'Authoritative_Registry': 'registry.access.redhat.com',
'BZComponent': 'rsyslog-docker',
'Name': 'rhel7/rsyslog',
'Release': '28.vrutkovs.31',
'Vendor': 'Red Hat, Inc.',
'Version': '7.2',
},
},
'created':
'2016-10-07T10:20:05.38595Z',
'docker_version':
'1.9.1',
'id':
'1ca220fbc2aed7c141b236c8729fe59db5771b32bf2da74e4a663407f32ec2a2',
'os':
'linux',
'parent':
'47eed7a8ea3f131e9281ae09fcbfb0041872fd8b74a048f1c739302c8148505d'
},
'container_config': {
'foo': 'bar',
'spam': 'maps'
},
'id':
'1ca220fbc2aed7c141b236c8729fe59db5771b32bf2da74e4a663407f32ec2a2',
'parent_id':
'c3fb36aafd5692d2a45115d32bb120edb6edf6c0c3c783ed6592a8dab969fb88'
}
# To test out the lack of a config, we really should be testing what happens
# when we only return a v1 response and not a v2 response at all; what are
# doing now is simply testing that if we return a None instead of json for the
# config blob, that None is stored rather than json
if not has_config:
config_json = None
manifest_latest_url = "https://{}/v2/{}/manifests/latest".format(
LOCALHOST_REGISTRY, TEST_IMAGE)
manifest_url = "https://{}/v2/{}/manifests/{}".format(
LOCALHOST_REGISTRY, TEST_IMAGE, DIGEST_V2)
config_blob_url = "https://{}/v2/{}/blobs/{}".format(
LOCALHOST_REGISTRY, TEST_IMAGE, CONFIG_DIGEST)
# We return our v2 manifest in the mocked v1 response as a placeholder - only the
# digest matters anyways
manifest_response_v1 = requests.Response()
(flexmock(manifest_response_v1,
status_code=200,
json=manifest_json,
headers={
'Content-Type':
'application/vnd.docker.distribution.manifest.v1+json',
'Docker-Content-Digest': DIGEST_V1
}))
manifest_response_v2 = requests.Response()
(flexmock(manifest_response_v2,
status_code=200,
json=manifest_json,
headers={
'Content-Type':
'application/vnd.docker.distribution.manifest.v2+json',
'Docker-Content-Digest': DIGEST_V2
}))
manifest_response_v2_list = requests.Response()
(flexmock(manifest_response_v2_list,
raise_for_status=lambda: None,
json=manifest_json,
headers={
'Content-Type':
'application/vnd.docker.distribution.manifest.list.v2+json',
}))
config_blob_response = requests.Response()
(flexmock(config_blob_response, status_code=200, json=config_json))
def custom_get(method, url, headers, **kwargs):
if url == manifest_latest_url:
# For a manifest stored as v2 or v1, the docker registry defaults to
# returning a v1 manifest if a v2 manifest is not explicitly requested
if headers[
'Accept'] == 'application/vnd.docker.distribution.manifest.v2+json':
return manifest_response_v2
else:
return manifest_response_v1
if headers[
'Accept'] == 'application/vnd.docker.distribution.manifest.list.v2+json':
return manifest_response_v2_list
if url == manifest_url:
return manifest_response_v2
if url == config_blob_url:
return config_blob_response
mock_get_retry_session()
(flexmock(
requests.Session).should_receive('request').replace_with(custom_get))
if reactor_config_map:
workflow.plugin_workspace[ReactorConfigPlugin.key] = {}
workflow.plugin_workspace[ReactorConfigPlugin.key][WORKSPACE_CONF_KEY] =\
ReactorConfig({'version': 1, 'registries': [{
'url': LOCALHOST_REGISTRY,
'insecure': True,
'auth': {'cfg_path': secret_path},
}]})
runner = PostBuildPluginsRunner(tasker, workflow,
[{
'name': TagAndPushPlugin.key,
'args': {
'registries': {
LOCALHOST_REGISTRY: {
'insecure': True,
'secret': secret_path
}
}
},
}])
if should_raise:
with pytest.raises(PluginFailedException):
runner.run()
else:
output = runner.run()
image = output[TagAndPushPlugin.key][0]
tasker.remove_image(image)
assert len(workflow.push_conf.docker_registries) > 0
if MOCK:
# we only test this when mocking docker because we don't expect
# running actual docker against v2 registry
expected_digest = ManifestDigest(v1=DIGEST_V1,
v2=DIGEST_V2,
oci=None)
assert workflow.push_conf.docker_registries[0].digests[image_name].v1 == \
expected_digest.v1
assert workflow.push_conf.docker_registries[0].digests[image_name].v2 == \
expected_digest.v2
assert workflow.push_conf.docker_registries[0].digests[image_name].oci == \
expected_digest.oci
if has_config:
assert isinstance(
workflow.push_conf.docker_registries[0].config, dict)
else:
assert workflow.push_conf.docker_registries[0].config is None
{DOCKER0_REGISTRY: {'foo/bar:latest': DIGEST1}},
{DOCKER0_REGISTRY: {'foo/bar:latest': DIGEST1, 'foo/bar:1.0-1': DIGEST1}},
{DOCKER0_REGISTRY: {'foo/bar:1.0-1': DIGEST1, 'foo/bar:1.0': DIGEST2}},
{DOCKER0_REGISTRY: {'foo/bar:1.0-1': DIGEST1}, LOCALHOST_REGISTRY: {'foo/bar:1.0-1': DIGEST2}},
])
@pytest.mark.parametrize("req_registries", [
{},
{LOCALHOST_REGISTRY: True},
{DOCKER0_REGISTRY: False},
{DOCKER0_REGISTRY: True, LOCALHOST_REGISTRY: True},
{DOCKER0_REGISTRY: False, LOCALHOST_REGISTRY: True},
])
@pytest.mark.parametrize("orchestrator", [True, False])
@pytest.mark.parametrize("manifest_list_digests", [
{},
{'foo/bar': ManifestDigest(v2_list=DIGEST_LIST)}
])
def test_delete_from_registry_plugin(saved_digests, req_registries, tmpdir, orchestrator,
manifest_list_digests, reactor_config_map):
if MOCK:
mock_docker()
mock_get_retry_session()
buildstep_plugin = None
if orchestrator:
ann_digests = []
buildstep_plugin = [{
'name': OrchestrateBuildPlugin.key,
'args': {
'platforms': "x86_64"
},
def test_delete_from_registry_plugin(saved_digests, req_registries, tmpdir, orchestrator):
if MOCK:
mock_docker()
mock_get_retry_session()
buildstep_plugin = None
if orchestrator:
ann_digests = []
buildstep_plugin = [{
'name': OrchestrateBuildPlugin.key,
'args': {
'platforms': "x86_64"
},
}]
tasker = DockerTasker()
workflow = DockerBuildWorkflow({"provider": "git", "uri": "asd"}, TEST_IMAGE,
buildstep_plugins=buildstep_plugin, )
setattr(workflow, 'builder', X)
args_registries = {}
for reg, use_secret in req_registries.items():
if use_secret:
temp_dir = mkdtemp(dir=str(tmpdir))
with open(os.path.join(temp_dir, ".dockercfg"), "w+") as dockerconfig:
dockerconfig_contents = {
reg: {
"username": "******", "password": reg
}
}
dockerconfig.write(json.dumps(dockerconfig_contents))
dockerconfig.flush()
args_registries[reg] = {'secret': temp_dir}
else:
args_registries[reg] = {}
for reg, digests in saved_digests.items():
if orchestrator:
for tag, dig in digests.items():
repo = tag.split(':')[0]
t = tag.split(':')[1]
ann_digests.append({
'digest': dig,
'tag': t,
'repository': repo,
'registry': reg,
})
else:
r = DockerRegistry(reg)
for tag, dig in digests.items():
r.digests[tag] = ManifestDigest(v1='not-used', v2=dig)
workflow.push_conf._registries['docker'].append(r)
if orchestrator:
build_annotations = {'digests': ann_digests}
annotations = {'worker-builds': {'x86_64': build_annotations}}
setattr(workflow, 'build_result', Y)
setattr(workflow.build_result, 'annotations', annotations)
runner = ExitPluginsRunner(
tasker,
workflow,
[{
'name': DeleteFromRegistryPlugin.key,
'args': {
'registries': args_registries
},
}]
)
deleted_digests = set()
for reg, digests in saved_digests.items():
if reg not in req_registries:
continue
for tag, dig in digests.items():
if dig in deleted_digests:
continue
url = "https://" + reg + "/v2/" + tag.split(":")[0] + "/manifests/" + dig
auth_type = requests.auth.HTTPBasicAuth if req_registries[reg] else None
(flexmock(requests.Session)
.should_receive('delete')
.with_args(url, verify=bool, auth=auth_type)
.once()
.and_return(flexmock(status_code=202, ok=True, raise_for_status=lambda: None)))
deleted_digests.add(dig)
result = runner.run()
assert result[DeleteFromRegistryPlugin.key] == deleted_digests
def group_manifests_and_tag(self, session, worker_digests):
"""
Creates a manifest list or OCI image index that groups the different manifests
in worker_digests, then tags the result with with all the configured tags found
in workflow.tag_conf.
"""
self.log.info("%s: Creating manifest list", session.registry)
# Extract information about the manifests that we will group - we get the
# size and content type of the manifest by querying the registry
manifests = []
for platform, worker_image in worker_digests.items():
repository = worker_image['repository']
digest = worker_image['digest']
media_type = get_manifest_media_type(worker_image['version'])
if media_type not in self.manifest_util.manifest_media_types:
continue
content, _, media_type, size = self.manifest_util.get_manifest(
session, repository, digest)
manifests.append({
'content':
content,
'repository':
repository,
'digest':
digest,
'size':
size,
'media_type':
media_type,
'architecture':
self.goarch.get(platform, platform),
})
list_type, list_json = self.manifest_util.build_list(manifests)
self.log.info("%s: Created manifest, Content-Type=%s\n%s",
session.registry, list_type, list_json)
# Now push the manifest list to the registry once per each tag
self.log.info("%s: Tagging manifest list", session.registry)
for image in self.non_floating_images:
target_repo = image.to_str(registry=False, tag=False)
# We have to call store_manifest_in_repository directly for each
# referenced manifest, since they potentially come from different repos
for manifest in manifests:
self.manifest_util.store_manifest_in_repository(
session,
manifest['content'],
manifest['media_type'],
manifest['repository'],
target_repo,
ref=manifest['digest'])
self.manifest_util.store_manifest_in_repository(session,
list_json,
list_type,
target_repo,
target_repo,
ref=image.tag)
# Get the digest of the manifest list using one of the tags
registry_image = get_unique_images(self.workflow)[0]
_, digest_str, _, _ = self.manifest_util.get_manifest(
session, registry_image.to_str(registry=False, tag=False),
registry_image.tag)
if list_type == MEDIA_TYPE_OCI_V1_INDEX:
digest = ManifestDigest(oci_index=digest_str)
else:
digest = ManifestDigest(v2_list=digest_str)
# And store the manifest list in the push_conf
push_conf_registry = self.workflow.push_conf.add_docker_registry(
session.registry, insecure=session.insecure)
tags = []
for image in self.non_floating_images:
push_conf_registry.digests[image.tag] = digest
tags.append(image.tag)
self.log.info("%s: Manifest list digest is %s", session.registry,
digest_str)
self.log.debug("tags: %s digest: %s", tags, digest)
return {
'manifest': list_json,
'media_type': list_type,
'manifest_digest': digest
}
def mock_environment(tmpdir, session=None, name=None,
component=None, version=None, release=None,
source=None, build_process_failed=False,
is_rebuild=True, docker_registry=True,
pulp_registries=0, blocksize=None,
task_states=None, additional_tags=None,
has_config=None,
logs_return_bytes=True):
if session is None:
session = MockedClientSession('', task_states=None)
if source is None:
source = GitSource('git', 'git://hostname/path')
if MOCK:
mock_docker()
tasker = DockerTasker()
workflow = DockerBuildWorkflow(SOURCE, "test-image")
base_image_id = '123456parent-id'
setattr(workflow, '_base_image_inspect', {'Id': base_image_id})
setattr(workflow, 'builder', X())
setattr(workflow.builder, 'image_id', '123456imageid')
setattr(workflow.builder, 'base_image', ImageName(repo='Fedora', tag='22'))
setattr(workflow.builder, 'source', X())
setattr(workflow.builder, 'built_image_info', {'ParentId': base_image_id})
setattr(workflow.builder.source, 'dockerfile_path', None)
setattr(workflow.builder.source, 'path', None)
setattr(workflow, 'tag_conf', TagConf())
with open(os.path.join(str(tmpdir), 'Dockerfile'), 'wt') as df:
df.write('FROM base\n'
'LABEL BZComponent={component} com.redhat.component={component}\n'
'LABEL Version={version} version={version}\n'
'LABEL Release={release} release={release}\n'
.format(component=component, version=version, release=release))
setattr(workflow.builder, 'df_path', df.name)
if name and version:
workflow.tag_conf.add_unique_image('user/test-image:{v}-timestamp'
.format(v=version))
if name and version and release:
workflow.tag_conf.add_primary_images(["{0}:{1}-{2}".format(name,
version,
release),
"{0}:{1}".format(name, version),
"{0}:latest".format(name)])
if additional_tags:
workflow.tag_conf.add_primary_images(["{0}:{1}".format(name, tag)
for tag in additional_tags])
flexmock(subprocess, Popen=fake_Popen)
flexmock(koji, ClientSession=lambda hub, opts: session)
flexmock(GitSource)
if logs_return_bytes:
logs = b'build logs - \xe2\x80\x98 \xe2\x80\x97 \xe2\x80\x99'
else:
logs = 'build logs - \u2018 \u2017 \u2019'
(flexmock(OSBS)
.should_receive('get_build_logs')
.with_args(BUILD_ID)
.and_return(logs))
(flexmock(OSBS)
.should_receive('get_pod_for_build')
.with_args(BUILD_ID)
.and_return(MockedPodResponse()))
setattr(workflow, 'source', source)
setattr(workflow.source, 'lg', X())
setattr(workflow.source.lg, 'commit_id', '123456')
setattr(workflow, 'push_conf', PushConf())
if docker_registry:
docker_reg = workflow.push_conf.add_docker_registry('docker.example.com')
for image in workflow.tag_conf.images:
tag = image.to_str(registry=False)
if pulp_registries:
docker_reg.digests[tag] = ManifestDigest(v1=fake_digest(image),
v2='sha256:not-used')
else:
docker_reg.digests[tag] = ManifestDigest(v1='sha256:not-used',
v2=fake_digest(image))
if has_config:
docker_reg.config = {
'config': {'architecture': 'x86_64'},
'container_config': {}
}
for pulp_registry in range(pulp_registries):
workflow.push_conf.add_pulp_registry('env', 'pulp.example.com')
with open(os.path.join(str(tmpdir), 'image.tar.xz'), 'wt') as fp:
fp.write('x' * 2**12)
setattr(workflow, 'exported_image_sequence', [{'path': fp.name}])
annotations = {
'worker-builds': {
'x86_64': {
'build': {
'build-name': 'build-1-x64_64',
},
'metadata_fragment': 'configmap/build-1-x86_64-md',
'metadata_fragment_key': 'metadata.json',
},
'ppc64le': {
'build': {
'build-name': 'build-1-ppc64le',
},
'metadata_fragment': 'configmap/build-1-ppc64le-md',
'metadata_fragment_key': 'metadata.json',
}
}
}
if build_process_failed:
workflow.build_result = BuildResult(logs=["docker build log - \u2018 \u2017 \u2019 \n'"],
fail_reason="not built")
else:
workflow.build_result = BuildResult(logs=["docker build log - \u2018 \u2017 \u2019 \n'"],
image_id="id1234",
annotations=annotations)
workflow.prebuild_plugins_conf = {}
workflow.prebuild_results[CheckAndSetRebuildPlugin.key] = is_rebuild
workflow.postbuild_results[PostBuildRPMqaPlugin.key] = [
"name1;1.0;1;x86_64;0;2000;" + FAKE_SIGMD5.decode() + ";23000;"
"RSA/SHA256, Tue 30 Aug 2016 00:00:00, Key ID 01234567890abc;(none)",
"name2;2.0;1;x86_64;0;3000;" + FAKE_SIGMD5.decode() + ";24000"
"RSA/SHA256, Tue 30 Aug 2016 00:00:00, Key ID 01234567890abd;(none)",
]
workflow.postbuild_results[FetchWorkerMetadataPlugin.key] = {
'x86_64': {
'buildroots': [
{
'container': {
'type': 'docker',
'arch': 'x86_64'
},
'extra': {
'osbs': {
'build_id': '12345',
'builder_image_id': '67890'
}
},
'content_generator': {
'version': '1.6.23',
'name': 'atomic-reactor'
},
'host': {
'os': 'Red Hat Enterprise Linux Server 7.3 (Maipo)',
'arch': 'x86_64'
},
'components': [
{
'name': 'perl-Net-LibIDN',
'sigmd5': '1dba38d073ea8f3e6c99cfe32785c81e',
'arch': 'x86_64',
'epoch': None,
'version': '0.12',
'signature': '199e2f91fd431d51',
'release': '15.el7',
'type': 'rpm'
},
{
'name': 'tzdata',
'sigmd5': '2255a5807ca7e4d7274995db18e52bea',
'arch': 'noarch',
'epoch': None,
'version': '2017b',
'signature': '199e2f91fd431d51',
'release': '1.el7',
'type': 'rpm'
},
],
'tools': [
{
'version': '1.12.6',
'name': 'docker'
}
],
'id': 1
}
],
'metadata_version': 0,
'output': [
{
'type': 'log',
'arch': 'noarch',
'filename': 'openshift-final.log',
'filesize': 106690,
'checksum': '2efa754467c0d2ea1a98fb8bfe435955',
'checksum_type': 'md5',
'buildroot_id': 1
},
{
'type': 'log',
'arch': 'noarch',
'filename': 'build.log',
'filesize': 1660,
'checksum': '8198de09fc5940cf7495e2657039ee72',
'checksum_type': 'md5',
'buildroot_id': 1
},
{
'extra': {
'image': {
'arch': 'x86_64'
},
'docker': {
'repositories': [
'brew-pulp-docker:8888/myproject/hello-world:0.0.1-9',
],
'parent_id': 'sha256:bf203442',
'id': '123456',
}
},
'checksum': '58a52e6f3ed52818603c2744b4e2b0a2',
'filename': 'test.x86_64.tar.gz',
'buildroot_id': 1,
'components': [
{
'name': 'tzdata',
'sigmd5': 'd9dc4e4f205428bc08a52e602747c1e9',
'arch': 'noarch',
'epoch': None,
'version': '2016d',
'signature': '199e2f91fd431d51',
'release': '1.el7',
'type': 'rpm'
},
{
'name': 'setup',
'sigmd5': 'b1e5ca72c71f94112cd9fb785b95d4da',
'arch': 'noarch',
'epoch': None,
'version': '2.8.71',
'signature': '199e2f91fd431d51',
'release': '6.el7',
'type': 'rpm'
},
],
'type': 'docker-image',
'checksum_type': 'md5',
'arch': 'x86_64',
'filesize': 71268781
}
]
}
}
workflow.plugin_workspace = {
OrchestrateBuildPlugin.key: {
WORKSPACE_KEY_UPLOAD_DIR: 'test-dir',
WORKSPACE_KEY_BUILD_INFO: {
'x86_64': BuildInfo('help.md')
}
}
}
return tasker, workflow
