def test_get_methods(self, fallback, method):
tasker, workflow = self.prepare()
workflow.plugin_workspace[ReactorConfigPlugin.key] = {}
if fallback is False:
workflow.plugin_workspace[ReactorConfigPlugin.key][WORKSPACE_CONF_KEY] = \
ReactorConfig(yaml.safe_load(REACTOR_CONFIG_MAP))
else:
if fallback:
fall_source = ReactorConfig(yaml.safe_load(REACTOR_CONFIG_MAP))
else:
fall_source = ReactorConfig(yaml.safe_load("version: 1"))
method_name = 'get_' + method
real_method = getattr(atomic_reactor.plugins.pre_reactor_config,
method_name)
if fallback is True:
output = real_method(workflow, fall_source.conf[method])
else:
if fallback is False:
output = real_method(workflow)
else:
with pytest.raises(KeyError):
real_method(workflow)
return
expected = yaml.safe_load(REACTOR_CONFIG_MAP)[method]
if method == 'registries':
registries_cm = {}
for registry in expected:
reguri = RegistryURI(registry.get('url'))
regdict = {}
regdict['version'] = reguri.version
if registry.get('auth'):
regdict['secret'] = registry['auth']['cfg_path']
regdict['insecure'] = registry.get('insecure', False)
regdict['expected_media_types'] = registry.get(
'expected_media_types', [])
registries_cm[reguri.docker_uri] = regdict
if fallback:
output = real_method(workflow, registries_cm)
assert output == registries_cm
return
if method == 'source_registry':
expect = {
'uri': RegistryURI(expected['url']),
'insecure': expected.get('insecure', False)
}
if fallback:
output = real_method(workflow, expect)
assert output['insecure'] == expect['insecure']
assert output['uri'].uri == expect['uri'].uri
return
assert output == expected
def remove_pulp_plugins(self):
def has_plugin(self, phase, target_plugin):
return self.find_plugin(phase, target_plugin) >= 0
phases = ('postbuild_plugins', 'exit_plugins')
pulp_registry = self.get_value('pulp')
koji_hub = self.get_value('koji', {}).get('hub_url')
has_pulp_pull = False
for phase in phases:
if not (pulp_registry and koji_hub):
self.remove_plugin(phase, PLUGIN_PULP_PULL_KEY,
'no pulp or koji available')
if not pulp_registry:
self.remove_plugin('postbuild_plugins', PLUGIN_PULP_PUSH_KEY,
'no pulp available')
self.remove_plugin('postbuild_plugins', PLUGIN_PULP_SYNC_KEY,
'no pulp available')
self.remove_plugin('postbuild_plugins', PLUGIN_PULP_TAG_KEY,
'no pulp available')
self.remove_plugin('exit_plugins', PLUGIN_DELETE_FROM_REG_KEY,
'no pulp available')
self.remove_plugin('exit_plugins', PLUGIN_PULP_PUBLISH_KEY,
'no pulp available')
else:
docker_registry = None
all_registries = self.get_value('registries', {})
versions = self.get_value('content_versions', ['v1', 'v2'])
for registry in all_registries:
reguri = RegistryURI(registry.get('url'))
if reguri.version == 'v2':
# First specified v2 registry is the one we'll tell pulp
# to sync from. Keep the http prefix -- pulp wants it.
docker_registry = registry
break
if 'v1' not in versions:
self.remove_plugin('postbuild_plugins', PLUGIN_PULP_PUSH_KEY,
'v1 content not enabled')
if docker_registry:
source_registry_str = self.get_value('source_registry',
{}).get('url')
perform_delete = (
source_registry_str is None
or RegistryURI(source_registry_str).uri != reguri.uri)
if not perform_delete:
self.remove_plugin('exit_plugins',
PLUGIN_DELETE_FROM_REG_KEY,
'no delete needed')
else:
self.remove_plugin('postbuild_plugins', PLUGIN_PULP_SYNC_KEY,
'no V2 pulp available')
self.remove_plugin('exit_plugins', PLUGIN_DELETE_FROM_REG_KEY,
'no V2 pulp available')
def test_get_methods(self, parse_from, method, tmpdir, caplog,
monkeypatch):
if parse_from == 'raw':
conf = Configuration(raw_config=yaml.safe_load(REACTOR_CONFIG_MAP))
elif parse_from == 'env':
monkeypatch.setenv(REACTOR_CONFIG_ENV_NAME,
dedent(REACTOR_CONFIG_MAP))
conf = Configuration(env_name=REACTOR_CONFIG_ENV_NAME)
elif parse_from == 'file':
filename = str(tmpdir.join('config.yaml'))
with open(filename, 'w') as fp:
fp.write(dedent(REACTOR_CONFIG_MAP))
conf = Configuration(config_path=filename)
real_attr = getattr(conf, method)
output = real_attr
reactor_config_map = yaml.safe_load(REACTOR_CONFIG_MAP)
if method == 'registry':
expected = reactor_config_map['registry']
else:
expected = reactor_config_map[method]
if method == 'registry':
# since there will only be exactly one registry
registry = expected
reguri = RegistryURI(registry.get('url'))
regdict = {'uri': reguri.docker_uri, 'version': reguri.version}
regdict['secret'] = reactor_config_map['registries_cfg_path']
regdict['insecure'] = registry.get('insecure', False)
regdict['expected_media_types'] = registry.get(
'expected_media_types', [])
assert output == regdict
return
if method == 'source_registry':
expect = {
'uri': RegistryURI(expected['url']),
'insecure': expected.get('insecure', False)
}
assert output['insecure'] == expect['insecure']
assert output['uri'].uri == expect['uri'].uri
return
assert output == expected
os.environ.pop(REACTOR_CONFIG_ENV_NAME, None)
if parse_from == 'raw':
log_msg = "reading config from raw_config kwarg"
elif parse_from == 'env':
log_msg = f"reading config from {REACTOR_CONFIG_ENV_NAME} env variable"
elif parse_from == 'file':
log_msg = f"reading config from {filename}"
assert log_msg in caplog.text
def __init__(self,
tasker,
workflow,
parent_registry=None,
parent_registry_insecure=False,
check_platforms=False):
"""
constructor
:param tasker: DockerTasker instance
:param workflow: DockerBuildWorkflow instance
:param parent_registry: registry to enforce pulling from
:param parent_registry_insecure: allow connecting to the registry over plain http
:param check_platforms: validate parent images provide all platforms expected for the build
"""
# call parent constructor
super(PullBaseImagePlugin, self).__init__(tasker, workflow)
self.check_platforms = check_platforms
source_registry = get_source_registry(
self.workflow, {
'uri':
RegistryURI(parent_registry) if parent_registry else None,
'insecure': parent_registry_insecure
})
if source_registry.get('uri'):
self.parent_registry = source_registry['uri'].docker_uri
self.parent_registry_insecure = source_registry['insecure']
else:
self.parent_registry = None
self.parent_registry_insecure = False
def get_source_registry(workflow, fallback=NO_FALLBACK):
try:
source_registry = get_config(workflow).conf['source_registry']
except KeyError:
if fallback != NO_FALLBACK:
return fallback
raise
return {
'uri': RegistryURI(source_registry['url']),
'insecure': source_registry.get('insecure', False)
}
def registry(self):
all_registries = self._get_value(ReactorConfigKeys.REGISTRIES_KEY)
registry = all_registries[0]
reguri = RegistryURI(registry.get('url'))
regdict = {'uri': reguri.docker_uri, 'version': reguri.version}
if registry.get('auth'):
regdict['secret'] = registry['auth']['cfg_path']
regdict['insecure'] = registry.get('insecure', False)
regdict['expected_media_types'] = registry.get('expected_media_types',
[])
return regdict
def registry(self):
registry = self._get_value(ReactorConfigKeys.REGISTRY_KEY)
reguri = RegistryURI(registry.get('url'))
if reguri.version == 'v2':
info = {
'version': reguri.version,
'uri': reguri.docker_uri,
'insecure': registry.get('insecure', False),
'expected_media_types': registry.get('expected_media_types',
[]),
}
if (cfg_path := self.registries_cfg_path) is not None:
info['secret'] = cfg_path
return info
def docker_registry(self):
all_registries = self._get_value(ReactorConfigKeys.REGISTRIES_KEY)
for registry in all_registries:
reguri = RegistryURI(registry.get('url'))
if reguri.version == 'v2':
regdict = {
'url': reguri.uri,
'insecure': registry.get('insecure', False)
}
if registry.get('auth'):
regdict['secret'] = registry['auth']['cfg_path']
return regdict
raise RuntimeError("Expected V2 registry but none in REACTOR_CONFIG")
def get_source_registry(workflow, fallback=NO_FALLBACK):
try:
source_registry = get_config(workflow).conf['source_registry']
except KeyError:
if fallback != NO_FALLBACK:
return fallback
raise
regdict = {
'uri': RegistryURI(source_registry['url']),
'insecure': source_registry.get('insecure', False),
'dockercfg_path': None
}
if source_registry.get('auth'):
regdict['dockercfg_path'] = source_registry['auth']['cfg_path']
return regdict
def __init__(self,
tasker,
workflow,
parent_registry=None,
parent_registry_insecure=False,
check_platforms=False,
inspect_only=False,
parent_images_digests=None):
"""
constructor
:param tasker: ContainerTasker instance
:param workflow: DockerBuildWorkflow instance
:param parent_registry: registry to enforce pulling from
:param parent_registry_insecure: allow connecting to the registry over plain http
:param check_platforms: validate parent images provide all platforms expected for the build
:param inspect_only: bool, if set to True, base images will not be pulled
:param parent_images_digests: dict, parent images manifest digests
"""
# call parent constructor
super(PullBaseImagePlugin, self).__init__(tasker, workflow)
self.check_platforms = check_platforms
self.inspect_only = inspect_only
source_registry = get_source_registry(
self.workflow, {
'uri':
RegistryURI(parent_registry) if parent_registry else None,
'insecure': parent_registry_insecure
})
if source_registry.get('uri'):
self.parent_registry = source_registry['uri'].docker_uri
self.parent_registry_insecure = source_registry['insecure']
self.parent_registry_dockercfg_path = source_registry.get(
'dockercfg_path', None)
else:
self.parent_registry = None
self.parent_registry_insecure = False
self.parent_registry_dockercfg_path = None
if parent_images_digests:
metadata = self.workflow.builder.parent_images_digests
metadata.update(parent_images_digests)
self.manifest_list_cache = {}
def get_registries(workflow, fallback=NO_FALLBACK):
try:
all_registries = get_config(workflow).conf['registries']
except KeyError:
if fallback != NO_FALLBACK:
return fallback
raise
registries_cm = {}
for registry in all_registries:
reguri = RegistryURI(registry.get('url'))
regdict = {}
regdict['version'] = reguri.version
if registry.get('auth'):
regdict['secret'] = registry['auth']['cfg_path']
regdict['insecure'] = registry.get('insecure', False)
registries_cm[reguri.docker_uri] = regdict
return registries_cm
def get_docker_registry(workflow, fallback=NO_FALLBACK):
try:
all_registries = get_config(workflow).conf['registries']
except KeyError:
if fallback != NO_FALLBACK:
return fallback
raise
for registry in all_registries:
reguri = RegistryURI(registry.get('url'))
if reguri.version == 'v2':
regdict = {
'url': reguri.uri,
'insecure': registry.get('insecure', False)
}
if registry.get('auth'):
regdict['secret'] = registry['auth']['cfg_path']
return regdict
raise RuntimeError("Expected V2 registry but none in REACTOR_CONFIG")
def set_data_from_reactor_config(self, reactor_config_data):
"""
Sets data from reactor config
"""
super(BuildRequestV2,
self).set_data_from_reactor_config(reactor_config_data)
if not reactor_config_data:
if self.user_params.flatpak and not self.user_params.base_image:
raise OsbsValidationException(
"Flatpak base_image must be be set in container.yaml or reactor config"
)
else:
self._set_flatpak(reactor_config_data)
if not self.source_registry:
raise RuntimeError(
'mandatory "source_registry" is not defined in reactor_config')
source_registry = RegistryURI(self.source_registry['url']).docker_uri
self._update_trigger_imagestreamtag(source_registry)
self._update_imagestream_name(source_registry)
def adjust_config_kwargs(self):
koji_fallback = {
'hub_url': self.config_kwargs.get('koji_hub'),
'root_url': self.config_kwargs.get('koji_root')
}
koji_map = get_koji(self.workflow, koji_fallback)
self.config_kwargs['koji_hub'] = koji_map['hub_url']
self.config_kwargs['koji_root'] = koji_map['root_url']
odcs_fallback = {
'api_url': self.config_kwargs.get('odcs_url'),
'insecure': self.config_kwargs.get('odcs_insecure')
}
odcs_map = get_odcs(self.workflow, odcs_fallback)
self.config_kwargs['odcs_url'] = odcs_map['api_url']
self.config_kwargs['odcs_insecure'] = odcs_map.get('insecure', False)
smtp_fallback = {
'host': self.config_kwargs.get('smtp_host'),
'from_address': self.config_kwargs.get('smtp_from'),
'domain': self.config_kwargs.get('smtp_email_domain'),
'send_to_submitter': self.config_kwargs.get('smtp_to_submitter'),
'send_to_pkg_owner': self.config_kwargs.get('smtp_to_pkgowner')
}
additional_addresses = self.config_kwargs.get('smtp_error_addresses')
error_addresses = self.config_kwargs.get('smtp_additional_addresses')
smtp_fallback['additional_addresses'] =\
additional_addresses.split(',') if additional_addresses else ()
smtp_fallback['error_addresses'] = error_addresses.split(
',') if error_addresses else ()
smtp_map = get_smtp(self.workflow, smtp_fallback)
self.config_kwargs['smtp_additional_addresses'] =\
','.join(smtp_map.get('additional_addresses', ()))
self.config_kwargs['smtp_email_domain'] = smtp_map.get('domain')
self.config_kwargs['smtp_error_addresses'] = ','.join(
smtp_map.get('error_addresses', ()))
self.config_kwargs['smtp_from'] = smtp_map['from_address']
self.config_kwargs['smtp_host'] = smtp_map['host']
self.config_kwargs['smtp_to_pkgowner'] = smtp_map.get(
'send_to_pkg_owner', False)
self.config_kwargs['smtp_to_submitter'] = smtp_map.get(
'send_to_submitter', False)
source_reg = self.config_kwargs.get('source_registry_uri')
souce_registry = get_source_registry(
self.workflow,
{'uri': RegistryURI(source_reg) if source_reg else None})['uri']
self.config_kwargs[
'source_registry_uri'] = souce_registry.uri if souce_registry else None
artifacts = self.config_kwargs.get('artifacts_allowed_domains')
self.config_kwargs['artifacts_allowed_domains'] =\
','.join(get_artifacts_allowed_domains(
self.workflow, artifacts.split(',') if artifacts else ()))
equal_labels_fallback = []
equal_labels_str = self.config_kwargs.get('equal_labels')
if equal_labels_str:
label_groups = [x.strip() for x in equal_labels_str.split(',')]
for label_group in label_groups:
equal_labels_fallback.append(
[label.strip() for label in label_group.split(':')])
equal_labels = get_image_equal_labels(self.workflow,
equal_labels_fallback)
if equal_labels:
equal_labels_sets = []
for equal_set in equal_labels:
equal_labels_sets.append(':'.join(equal_set))
equal_labels_string = ','.join(equal_labels_sets)
self.config_kwargs['equal_labels'] = equal_labels_string
self.config_kwargs['prefer_schema1_digest'] =\
get_prefer_schema1_digest(self.workflow,
self.config_kwargs.get('prefer_schema1_digest'))
registry_api_ver = self.config_kwargs.get('registry_api_versions')
self.config_kwargs['registry_api_versions'] =\
','.join(get_content_versions(self.workflow,
registry_api_ver.split(',') if registry_api_ver else ()))
self.config_kwargs['yum_proxy'] =\
get_yum_proxy(self.workflow, self.config_kwargs.get('yum_proxy'))
self.config_kwargs['sources_command'] =\
get_sources_command(self.workflow, self.config_kwargs.get('sources_command'))
def _as_source_registry(self, registry):
return {
'uri': RegistryURI(registry['url']),
'insecure': registry.get('insecure', False),
'dockercfg_path': registry.get('auth', {}).get('cfg_path', None)
}
class TestConfiguration(object):
@pytest.mark.parametrize(('config', 'exc'), [
("""\
registries:
- url: https://container-registry.example.com/v2
auth:
cfg_path: /var/run/secrets/atomic-reactor/v2-registry-dockercfg
""", None),
("""\
registries:
- url: https://container-registry.example.com/v2
auth:
cfg_path: /var/run/secrets/atomic-reactor/v2-registry-dockercfg
- url: https://another-container-registry.example.com/
auth:
cfg_path: /var/run/secrets/atomic-reactor/another-registry-dockercfg
""", None),
("""\
registries:
- url: https://old-container-registry.example.com/v1
auth:
cfg_path: /var/run/secrets/atomic-reactor/v1-registry-dockercfg
""", OsbsValidationException),
("""\
registries:
- url: https://wrong-container-registry.example.com/v3
auth:
cfg_path: /var/run/secrets/atomic-reactor/wrong-registry-dockercfg
""", RuntimeError),
])
def test_get_docker_registry(self, config, exc):
required_config = """\
version: 1
koji:
hub_url: /
root_url: ''
auth: {}
openshift:
url: openshift_url
source_registry:
url: source_registry.com
"""
config += "\n" + required_config
config_json = read_yaml(config, 'schemas/config.json')
expected = {
'url': 'https://container-registry.example.com',
'insecure': False,
'secret': '/var/run/secrets/atomic-reactor/v2-registry-dockercfg'
}
conf = Configuration(raw_config=config_json)
if exc is None:
docker_registry = conf.docker_registry
assert docker_registry == expected
else:
with pytest.raises(exc):
getattr(conf, 'docker_registry')
@pytest.mark.parametrize(('config', 'expected'), [
("""\
pull_registries: []
""", []),
("""\
pull_registries:
- url: registry.io
""", [
{
"uri": RegistryURI("registry.io"),
"insecure": False,
"dockercfg_path": None
},
]),
("""\
pull_registries:
- url: https://registry.io
""", [
{
"uri": RegistryURI("https://registry.io"),
"insecure": False,
"dockercfg_path": None
},
]),
("""\
pull_registries:
- url: registry.io
auth:
cfg_path: /var/run/secrets/atomic-reactor/v2-registry-dockercfg
""", [
{
"uri":
RegistryURI("registry.io"),
"insecure":
False,
"dockercfg_path":
"/var/run/secrets/atomic-reactor/v2-registry-dockercfg"
},
]),
("""\
pull_registries:
- url: registry.io
insecure: true
auth:
cfg_path: /var/run/secrets/atomic-reactor/v2-registry-dockercfg
""", [
{
"uri":
RegistryURI("registry.io"),
"insecure":
True,
"dockercfg_path":
"/var/run/secrets/atomic-reactor/v2-registry-dockercfg"
},
]),
("""\
pull_registries:
- url: registry.io
insecure: true
auth:
cfg_path: /var/run/secrets/atomic-reactor/v2-registry-dockercfg
- url: registry.org
""", [
{
"uri":
RegistryURI("registry.io"),
"insecure":
True,
"dockercfg_path":
"/var/run/secrets/atomic-reactor/v2-registry-dockercfg"
},
{
"uri": RegistryURI("registry.org"),
"insecure": False,
"dockercfg_path": None
},
]),
])
def test_get_pull_registries(self, config, expected):
config += "\n" + REQUIRED_CONFIG
config_json = read_yaml(config, 'schemas/config.json')
conf = Configuration(raw_config=config_json)
pull_registries = conf.pull_registries
# RegistryURI does not implement equality, check URI as string
for reg in pull_registries + expected:
reg['uri'] = reg['uri'].uri
assert pull_registries == expected
@pytest.mark.parametrize(
('config', 'expected_slots_dir', 'expected_enabled_hosts'), [
("""\
remote_hosts:
slots_dir: path/foo
pools:
x86_64:
remote-host1.x86_64:
enabled: true
auth: foo
username: bar
slots: 1
socket_path: /user/foo/podman.sock
remote-host2.x86_64:
enabled: false
auth: foo
username: bar
slots: 2
socket_path: /user/foo/podman.sock
ppc64le:
remote-host3.ppc64le:
enabled: true
auth: foo
username: bar
slots: 3
socket_path: /user/foo/podman.sock
""", 'path/foo', {
'x86_64': ['remote-host1.x86_64'],
'ppc64le': ['remote-host3.ppc64le']
}),
])
def test_get_remote_hosts(self, config, expected_slots_dir,
expected_enabled_hosts):
config += "\n" + REQUIRED_CONFIG
config_json = read_yaml(config, 'schemas/config.json')
conf = Configuration(raw_config=config_json)
remote_hosts = conf.remote_hosts
assert expected_slots_dir == remote_hosts['slots_dir']
pools = remote_hosts['pools']
assert len(pools), 'Remote hosts do not have 2 architectures'
assert len(
pools['x86_64']) == 2, '2 entries expected for x86_64 architecture'
assert sorted(pools['x86_64']) == sorted(
['remote-host1.x86_64', 'remote-host2.x86_64'])
assert len(
pools['ppc64le']) == 1, '1 entry expected for ppc64le architecture'
host1_x86_64 = pools['x86_64']['remote-host1.x86_64']
assert host1_x86_64['auth'] == 'foo', 'Unexpected SSH key path'
assert host1_x86_64[
'socket_path'] == '/user/foo/podman.sock', 'Unexpected socket path'
host2_x86_64 = pools['x86_64']['remote-host2.x86_64']
assert host2_x86_64['username'] == 'bar', 'Unexpected user name'
host3_ppc64le = pools['ppc64le']['remote-host3.ppc64le']
assert host3_ppc64le['slots'] == 3, 'Unexpected number of slots'
for arch in ['x86_64', 'ppc64le']:
enabled_hosts = [
host for host, items in pools[arch].items() if items['enabled']
]
assert enabled_hosts == expected_enabled_hosts[arch]
@pytest.mark.parametrize('config, error', [
("""\
remote_hosts: []
""", "is not of type {!r}".format("object")),
("""\
remote_hosts:
slots_dir: path/foo
""", "{!r} is a required property".format("pools")),
("""\
remote_hosts:
pools:
x86_64:
remote-host1.x86_64:
enabled: true
auth: foo
username: bar
slots: 1
socket_path: /user/foo/podman.sock
""", "{!r} is a required property".format("slots_dir")),
("""\
remote_hosts:
pools:
amd-64:
remote-host1:
enabled: true
auth: foo
username: bar
slots: 1
socket_path: /user/foo/podman.sock
""", "{!r} does not match any of the regexes".format("amd-64")),
("""\
remote_hosts:
slots_dir: path/foo
pools:
s390x:
remote-host1:
auth: foo
username: bar
slots: 1
socket_path: /user/foo/podman.sock
""", "{!r} is a required property".format("enabled")),
("""\
remote_hosts:
slots_dir: path/foo
pools:
s390x:
remote-host1.s390x:
enabled: true
username: bar
slots: 1
socket_path: /user/foo/podman.sock
""", "{!r} is a required property".format("auth")),
("""\
remote_hosts:
slots_dir: path/foo
pools:
s390x:
remote-host1.s390x:
enabled: true
auth: foo
slots: 1
socket_path: /user/foo/podman.sock
""", "{!r} is a required property".format("username")),
("""\
remote_hosts:
slots_dir: path/foo
pools:
s390x:
remote-host1.s390x:
enabled: true
auth: foo
username: bar
slots: 1
""", "{!r} is a required property".format("socket_path")),
("""\
remote_hosts:
slots_dir: path/foo
pools:
s390x:
remote-host1.s390x:
enabled: true
auth: foo
username: bar
socket_path: /user/foo/podman.sock
""", "{!r} is a required property".format("slots")),
("""\
remote_hosts:
slots_dir: path/foo
pools:
aarch64:
remote-host1.@aarch64@@:
enabled: true
auth: foo
username: bar
socket_path: /user/foo/podman.sock
""", "{!r} does not match any of the regexes".format(
"remote-host1.@aarch64@@")),
])
def test_get_remote_hosts_schema_validation(self, config, error):
config += "\n" + REQUIRED_CONFIG
with pytest.raises(OsbsValidationException) as exc_info:
read_yaml(config, 'schemas/config.json')
assert error in str(exc_info.value)
@pytest.mark.parametrize('config, error', [
("""\
pull_registries: {}
""", "is not of type {!r}".format("array")),
("""\
pull_registries:
- insecure: false
""", "{!r} is a required property".format("url")),
("""\
pull_registries:
- url: registry.io
auth: {}
""", "{!r} is a required property".format("cfg_path")),
])
def test_get_pull_registries_schema_validation(self, config, error):
config += "\n" + REQUIRED_CONFIG
with pytest.raises(OsbsValidationException) as exc_info:
read_yaml(config, 'schemas/config.json')
assert error in str(exc_info.value)
def test_filename(self, tmpdir):
filename = os.path.join(str(tmpdir), 'config.yaml')
with open(filename, 'w') as fp:
fp.write(dedent(REQUIRED_CONFIG))
Configuration(config_path=filename)
def test_no_schema_resource(self, tmpdir, caplog):
class FakeProvider(object):
def get_resource_stream(self, pkg, rsc):
raise IOError
# pkg_resources.resource_stream() cannot be mocked directly
# Instead mock the module-level function it calls.
(flexmock(pkg_resources).should_receive('get_provider').and_return(
FakeProvider()))
filename = os.path.join(str(tmpdir), 'config.yaml')
with open(filename, 'w'):
pass
with caplog.at_level(logging.ERROR), pytest.raises(Exception):
Configuration(config_path=filename)
captured_errs = [x.message for x in caplog.records]
assert "unable to extract JSON schema, cannot validate" in captured_errs
@pytest.mark.parametrize(
'schema',
[
# Invalid JSON
'{',
# Invalid schema
'{"properties": {"any": null}}',
])
def test_invalid_schema_resource(self, tmpdir, caplog, schema):
class FakeProvider(object):
def get_resource_stream(self, pkg, rsc):
return io.BufferedReader(io.BytesIO(schema))
# pkg_resources.resource_stream() cannot be mocked directly
# Instead mock the module-level function it calls.
(flexmock(pkg_resources).should_receive('get_provider').and_return(
FakeProvider()))
filename = os.path.join(str(tmpdir), 'config.yaml')
with open(filename, 'w'):
pass
with caplog.at_level(logging.ERROR), pytest.raises(Exception):
Configuration(config_path=filename)
captured_errs = [x.message for x in caplog.records]
assert any("cannot validate" in x for x in captured_errs)
@pytest.mark.parametrize(('config', 'errors'), [
("""\
clusters:
foo:
bar: 1
plat/form:
- name: foo
max_concurrent_builds: 1
""", [
"validation error: .clusters.foo: "
"validating 'type' has failed "
"(None is not of type %r)" % u'array',
"validation error: .clusters.bar: "
"validating 'type' has failed "
"(1 is not of type %r)" % u'array',
re.compile("validation error: .clusters: "
"validating 'additionalProperties' has failed"),
]),
("""\
clusters:
foo:
- name: 1
max_concurrent_builds: 1
- name: blah
max_concurrent_builds: one
- name: "2" # quoting prevents error
max_concurrent_builds: 2
- name: negative
max_concurrent_builds: -1
""", [
"validation error: .clusters.foo[0].name: "
"validating 'type' has failed "
"(1 is not of type %r)" % u'string',
"validation error: .clusters.foo[1].max_concurrent_builds: "
"validating 'type' has failed "
"('one' is not of type %r)" % u'integer',
re.compile(
r"validation error: \.clusters\.foo\[3\]\.max_concurrent_builds: "
r"validating 'minimum' has failed "
r"\(-1(\.0)? is less than the minimum of 0\)"),
]),
("""\
clusters:
foo:
- name: blah
max_concurrent_builds: 1
enabled: never
""", [
"validation error: .clusters.foo[0].enabled: "
"validating 'type' has failed "
"('never' is not of type %r)" % u'boolean',
]),
("""\
clusters:
foo:
# missing name
- nam: bar
max_concurrent_builds: 1
# missing max_concurrent_builds
- name: baz
max_concurrrent_builds: 2
- name: bar
max_concurrent_builds: 4
extra: false
""", [
"validation error: .clusters.foo[0]: validating 'required' has failed "
"(%r is a required property)" % u'name',
"validation error: .clusters.foo[1]: validating 'required' has failed "
"(%r is a required property)" % u'max_concurrent_builds',
"validation error: .clusters.foo[2]: validating 'additionalProperties' has failed "
"(Additional properties are not allowed ('extra' was unexpected))",
])
])
def test_bad_cluster_config(self, tmpdir, caplog, config, errors):
config += "\n" + REQUIRED_CONFIG
filename = os.path.join(str(tmpdir), 'config.yaml')
with open(filename, 'w') as fp:
fp.write(dedent(config))
with caplog.at_level(
logging.DEBUG,
logger='osbs'), pytest.raises(OsbsValidationException):
Configuration(config_path=filename)
captured_errs = [x.message for x in caplog.records]
for error in errors:
try:
# Match regexp
assert any(filter(error.match, captured_errs))
except AttributeError:
# String comparison
assert error in captured_errs
def test_bad_version(self, tmpdir):
filename = os.path.join(str(tmpdir), 'config.yaml')
with open(filename, 'w') as fp:
fp.write(
dedent("""\
version: 2
koji:
hub_url: /
root_url: ''
auth: {}
openshift:
url: openshift_url
source_registry:
url: source_registry.com
registries:
- url: registry_url
"""))
with pytest.raises(ValueError):
Configuration(config_path=filename)
@pytest.mark.parametrize(
('config', 'clusters', 'defined_platforms'),
[
# Default config
("", [], []),
# Unknown key
("""\
special: foo
""", [], []),
("""\
clusters:
all_disabled:
- name: foo
max_concurrent_builds: 2
enabled: false
platform:
- name: one
max_concurrent_builds: 4
- name: two
max_concurrent_builds: 8
enabled: true
- name: three
max_concurrent_builds: 16
enabled: false
""", [
('one', 4),
('two', 8),
], ['all_disabled', 'platform']),
])
def test_good_cluster_config(self, tmpdir, config, clusters,
defined_platforms):
config += "\n" + REQUIRED_CONFIG
filename = os.path.join(str(tmpdir), 'config.yaml')
with open(filename, 'w') as fp:
fp.write(dedent(config))
conf = Configuration(config_path=filename)
enabled = conf.get_enabled_clusters_for_platform('platform')
assert {(x.name, x.max_concurrent_builds)
for x in enabled} == set(clusters)
for platform in defined_platforms:
assert conf.cluster_defined_for_platform(platform)
@pytest.mark.parametrize('default', (
'release',
'beta',
'unsigned',
))
def test_odcs_config(self, tmpdir, default):
config = """\
odcs:
signing_intents:
- name: release
keys: [R123, R234]
- name: beta
keys: [R123, B456, B457]
- name: unsigned
keys: []
default_signing_intent: {default}
api_url: http://odcs.example.com
auth:
ssl_certs_dir: /var/run/secrets/atomic-reactor/odcssecret
""".format(default=default)
config += "\n" + REQUIRED_CONFIG
filename = str(tmpdir.join('config.yaml'))
with open(filename, 'w') as fp:
fp.write(dedent(config))
conf = Configuration(config_path=filename)
odcs_config = conf.odcs_config
assert odcs_config.default_signing_intent == default
unsigned_intent = {
'name': 'unsigned',
'keys': [],
'restrictiveness': 0
}
beta_intent = {
'name': 'beta',
'keys': ['R123', 'B456', 'B457'],
'restrictiveness': 1
}
release_intent = {
'name': 'release',
'keys': ['R123', 'R234'],
'restrictiveness': 2
}
assert odcs_config.signing_intents == [
unsigned_intent, beta_intent, release_intent
]
assert odcs_config.get_signing_intent_by_name(
'release') == release_intent
assert odcs_config.get_signing_intent_by_name('beta') == beta_intent
assert odcs_config.get_signing_intent_by_name(
'unsigned') == unsigned_intent
with pytest.raises(ValueError):
odcs_config.get_signing_intent_by_name('missing')
assert odcs_config.get_signing_intent_by_keys(['R123', 'R234'
])['name'] == 'release'
assert odcs_config.get_signing_intent_by_keys(
'R123 R234')['name'] == 'release'
assert odcs_config.get_signing_intent_by_keys(['R123'
])['name'] == 'release'
assert odcs_config.get_signing_intent_by_keys(
'R123')['name'] == 'release'
assert odcs_config.get_signing_intent_by_keys(['R123', 'B456'
])['name'] == 'beta'
assert odcs_config.get_signing_intent_by_keys(['B456', 'R123'
])['name'] == 'beta'
assert odcs_config.get_signing_intent_by_keys(
'B456 R123')['name'] == 'beta'
assert odcs_config.get_signing_intent_by_keys(
'R123 B456 ')['name'] == 'beta'
assert odcs_config.get_signing_intent_by_keys(['B456'
])['name'] == 'beta'
assert odcs_config.get_signing_intent_by_keys('B456')['name'] == 'beta'
assert odcs_config.get_signing_intent_by_keys([])['name'] == 'unsigned'
assert odcs_config.get_signing_intent_by_keys('')['name'] == 'unsigned'
with pytest.raises(ValueError):
assert odcs_config.get_signing_intent_by_keys(['missing'])
with pytest.raises(ValueError):
assert odcs_config.get_signing_intent_by_keys(
['R123', 'R234', 'B457'])
def test_odcs_config_invalid_default_signing_intent(self, tmpdir):
config = """\
odcs:
signing_intents:
- name: release
keys: [R123]
- name: beta
keys: [R123, B456]
- name: unsigned
keys: []
default_signing_intent: spam
api_url: http://odcs.example.com
auth:
ssl_certs_dir: /var/run/secrets/atomic-reactor/odcssecret
"""
config += "\n" + REQUIRED_CONFIG
filename = str(tmpdir.join('config.yaml'))
with open(filename, 'w') as fp:
fp.write(dedent(config))
conf = Configuration(config_path=filename)
with pytest.raises(ValueError) as exc_info:
getattr(conf, 'odcs_config')
message = str(exc_info.value)
assert message == dedent("""\
unknown signing intent name "spam", valid names: unsigned, beta, release
""".rstrip())
def test_odcs_config_deprecated_signing_intent(self, tmpdir, caplog):
config = """\
odcs:
signing_intents:
- name: release
keys: [R123]
deprecated_keys: [R122]
default_signing_intent: release
api_url: http://odcs.example.com
auth:
ssl_certs_dir: /var/run/secrets/atomic-reactor/odcssecret
"""
config += "\n" + REQUIRED_CONFIG
filename = str(tmpdir.join('config.yaml'))
with open(filename, 'w') as fp:
fp.write(dedent(config))
conf = Configuration(config_path=filename)
odcs_config = conf.odcs_config
signing_intent = odcs_config.get_signing_intent_by_keys(['R123'])
assert signing_intent['name'] == 'release'
assert 'contain deprecated entries' not in caplog.text
signing_intent = odcs_config.get_signing_intent_by_keys(
['R123', 'R122'])
assert signing_intent['name'] == 'release'
assert 'contain deprecated entries' in caplog.text
@pytest.mark.parametrize('parse_from', ['env', 'file', 'raw'])
@pytest.mark.parametrize('method', [
'odcs', 'smtp', 'artifacts_allowed_domains',
'yum_repo_allowed_domains', 'image_labels',
'image_label_info_url_format', 'image_equal_labels',
'fail_on_digest_mismatch', 'openshift', 'group_manifests',
'platform_descriptors', 'prefer_schema1_digest', 'content_versions',
'registry', 'yum_proxy', 'source_registry', 'sources_command',
'required_secrets', 'worker_token_secrets', 'clusters', 'hide_files',
'skip_koji_check_for_base_image', 'deep_manifest_list_inspection'
])
def test_get_methods(self, parse_from, method, tmpdir, caplog):
if parse_from == 'raw':
conf = Configuration(raw_config=yaml.safe_load(REACTOR_CONFIG_MAP))
elif parse_from == 'env':
os.environ[REACTOR_CONFIG_ENV_NAME] = dedent(REACTOR_CONFIG_MAP)
conf = Configuration(env_name=REACTOR_CONFIG_ENV_NAME)
elif parse_from == 'file':
filename = str(tmpdir.join('config.yaml'))
with open(filename, 'w') as fp:
fp.write(dedent(REACTOR_CONFIG_MAP))
conf = Configuration(config_path=filename)
real_attr = getattr(conf, method)
output = real_attr
if method == 'registry':
# map `registry` to `registries` in config so that neither use nor devs are confused
expected = yaml.safe_load(REACTOR_CONFIG_MAP)['registries']
else:
expected = yaml.safe_load(REACTOR_CONFIG_MAP)[method]
if method == 'registry':
# since there will only be exactly one registry
registry = expected[0]
reguri = RegistryURI(registry.get('url'))
regdict = {'uri': reguri.docker_uri, 'version': reguri.version}
if registry.get('auth'):
regdict['secret'] = registry['auth']['cfg_path']
regdict['insecure'] = registry.get('insecure', False)
regdict['expected_media_types'] = registry.get(
'expected_media_types', [])
assert output == regdict
return
if method == 'source_registry':
expect = {
'uri': RegistryURI(expected['url']),
'insecure': expected.get('insecure', False)
}
assert output['insecure'] == expect['insecure']
assert output['uri'].uri == expect['uri'].uri
return
assert output == expected
os.environ.pop(REACTOR_CONFIG_ENV_NAME, None)
if parse_from == 'raw':
log_msg = "reading config from raw_config kwarg"
elif parse_from == 'env':
log_msg = f"reading config from {REACTOR_CONFIG_ENV_NAME} env variable"
elif parse_from == 'file':
log_msg = f"reading config from {filename}"
assert log_msg in caplog.text
@pytest.mark.parametrize(('config', 'expect'), [
("""\
platform_descriptors:
- platform: x86_64
architecture: amd64
""", {
'x86_64': 'amd64',
'ppc64le': 'ppc64le'
}),
])
def test_get_platform_to_goarch_mapping(self, config, expect):
config += "\n" + REQUIRED_CONFIG
config_json = read_yaml(config, 'schemas/config.json')
conf = Configuration(raw_config=config_json)
platform_to_goarch = conf.platform_to_goarch_mapping
goarch_to_platform = conf.goarch_to_platform_mapping
for plat, goarch in expect.items():
assert platform_to_goarch[plat] == goarch
assert goarch_to_platform[goarch] == plat
@pytest.mark.parametrize(('config', 'expect'), [
("""\
build_image_override:
ppc64le: registry.example.com/buildroot-ppc64le:latest
arm: registry.example.com/buildroot-arm:latest
""", {
'ppc64le': 'registry.example.com/buildroot-ppc64le:latest',
'arm': 'registry.example.com/buildroot-arm:latest'
}),
])
def test_get_build_image_override(self, config, expect):
config += "\n" + REQUIRED_CONFIG
config_json = read_yaml(config, 'schemas/config.json')
conf = Configuration(raw_config=config_json)
build_image_override = conf.build_image_override
assert build_image_override == expect
@pytest.mark.parametrize(('config', 'expect'), [
("""\
flatpak:
base_image: fedora:latest
""", "fedora:latest"),
("""\
""", None),
("""\
flatpak: {}
""", None),
])
def test_get_flatpak_base_image(self, config, expect):
config += "\n" + REQUIRED_CONFIG
config_json = read_yaml(config, 'schemas/config.json')
conf = Configuration(raw_config=config_json)
if expect:
base_image = conf.flatpak_base_image
assert base_image == expect
else:
with pytest.raises(KeyError):
getattr(conf, 'flatpak_base_image')
@pytest.mark.parametrize(('config', 'expect'), [
("""\
flatpak:
metadata: labels
""", "labels"),
("""\
""", None),
("""\
flatpak: {}
""", None),
])
def test_get_flatpak_metadata(self, config, expect):
config += "\n" + REQUIRED_CONFIG
config_json = read_yaml(config, 'schemas/config.json')
conf = Configuration(raw_config=config_json)
if expect:
base_image = conf.flatpak_metadata
assert base_image == expect
else:
with pytest.raises(KeyError):
getattr(conf, 'flatpak_metadata')
@pytest.mark.parametrize(('config', 'raise_error'), [
("""\
koji:
hub_url: https://koji.example.com/hub
root_url: https://koji.example.com/root
auth:
proxyuser: proxyuser
krb_principal: krb_principal
krb_keytab_path: /tmp/krb_keytab
""", False),
("""\
koji:
hub_url: https://koji.example.com/hub
root_url: https://koji.example.com/root
auth:
proxyuser: proxyuser
krb_principal: krb_principal
krb_keytab_path: /tmp/krb_keytab
use_fast_upload: false
""", False),
("""\
koji:
hub_url: https://koji.example.com/hub
root_url: https://koji.example.com/root
auth:
proxyuser: proxyuser
ssl_certs_dir: /var/certs
""", False),
("""\
koji:
hub_url: https://koji.example.com/hub
root_url: https://koji.example.com/root
auth:
proxyuser: proxyuser
""", False),
("""\
koji:
hub_url: https://koji.example.com/hub
root_url: https://koji.example.com/root
auth:
""", True),
("""\
koji:
hub_url: https://koji.example.com/hub
root_url: https://koji.example.com/root
auth:
proxyuser: proxyuser
krb_principal: krb_principal
krb_keytab_path: /tmp/krb_keytab
ssl_certs_dir: /var/certs
""", True),
("""\
koji:
hub_url: https://koji.example.com/hub
root_url: https://koji.example.com/root
auth:
proxyuser: proxyuser
krb_keytab_path: /tmp/krb_keytab
""", True),
("""\
koji:
hub_url: https://koji.example.com/hub
root_url: https://koji.example.com/root
auth:
proxyuser: proxyuser
krb_principal: krb_principal
""", True),
("""\
koji:
hub_url: https://koji.example.com/hub
root_url: https://koji.example.com/root
auth:
proxyuser: proxyuser
krb_principal: krb_principal
ssl_certs_dir: /var/certs
""", True),
("""\
koji:
hub_url: https://koji.example.com/hub
root_url: https://koji.example.com/root
auth:
proxyuser: proxyuser
krb_keytab_path: /tmp/krb_keytab
ssl_certs_dir: /var/certs
""", True),
])
def test_get_koji_session(self, config, raise_error):
required_config = """\
version: 1
source_registry:
url: source_registry.com
registries:
- url: registry_url
openshift:
url: openshift_url
"""
config += "\n" + required_config
if raise_error:
with pytest.raises(Exception):
read_yaml(config, 'schemas/config.json')
return
config_json = read_yaml(config, 'schemas/config.json')
auth_info = {
"proxyuser": config_json['koji']['auth'].get('proxyuser'),
"ssl_certs_dir": config_json['koji']['auth'].get('ssl_certs_dir'),
"krb_principal": config_json['koji']['auth'].get('krb_principal'),
"krb_keytab": config_json['koji']['auth'].get('krb_keytab_path')
}
use_fast_upload = config_json['koji'].get('use_fast_upload', True)
conf = Configuration(raw_config=config_json)
(flexmock(atomic_reactor.utils.koji).should_receive(
'create_koji_session').with_args(
config_json['koji']['hub_url'], auth_info,
use_fast_upload).once().and_return(True))
get_koji_session(conf)
@pytest.mark.parametrize('root_url',
('https://koji.example.com/root',
'https://koji.example.com/root/', None))
def test_get_koji_path_info(self, root_url):
config = {
'version': 1,
'koji': {
'hub_url': 'https://koji.example.com/hub',
'auth': {
'ssl_certs_dir': '/var/certs'
}
},
'openshift': {
'url': 'openshift_url'
},
'source_registry': {
'url': 'source_registry'
},
'registries': [{
'url': 'registry_url'
}]
}
expected_root_url = 'https://koji.example.com/root'
if root_url:
config['koji']['root_url'] = root_url
config_yaml = yaml.safe_dump(config)
expect_error = not root_url
if expect_error:
with pytest.raises(Exception):
read_yaml(config_yaml, 'schemas/config.json')
return
parsed_config = read_yaml(config_yaml, 'schemas/config.json')
conf = Configuration(raw_config=parsed_config)
(flexmock(koji.PathInfo).should_receive('__init__').with_args(
topdir=expected_root_url).once())
getattr(conf, 'koji_path_info')
@pytest.mark.parametrize(('config', 'raise_error'), [
("""\
odcs:
api_url: https://odcs.example.com/api/1
auth:
ssl_certs_dir: /var/run/secrets/atomic-reactor/odcssecret
signing_intents:
- name: release
keys: [R123]
default_signing_intent: default
timeout: 3600
""", False),
("""\
odcs:
api_url: https://odcs.example.com/api/1
auth:
ssl_certs_dir: nonexistent
signing_intents:
- name: release
keys: [R123]
default_signing_intent: default
""", False),
("""\
odcs:
api_url: https://odcs.example.com/api/1
auth:
openidc_dir: /var/run/open_idc
signing_intents:
- name: release
keys: [R123]
default_signing_intent: default
""", False),
("""\
odcs:
api_url: https://odcs.example.com/api/1
auth:
openidc_dir: /var/run/open_idc
ssl_certs_dir: /var/run/secrets/atomic-reactor/odcssecret
signing_intents:
- name: release
keys: [R123]
default_signing_intent: default
""", True),
("""\
odcs:
api_url: https://odcs.example.com/api/1
auth:
openidc_dir: /var/run/open_idc
signing_intents:
- name: release
keys: [R123]
""", True),
("""\
odcs:
api_url: https://odcs.example.com/api/1
auth:
openidc_dir: /var/run/open_idc
default_signing_intent: default
""", True),
("""\
odcs:
auth:
openidc_dir: /var/run/open_idc
signing_intents:
- name: release
keys: [R123]
default_signing_intent: default
""", True),
])
def test_get_odcs_session(self, tmpdir, config, raise_error):
config += "\n" + REQUIRED_CONFIG
if raise_error:
with pytest.raises(Exception):
read_yaml(config, 'schemas/config.json')
return
config_json = read_yaml(config, 'schemas/config.json')
auth_info = {
'insecure': config_json['odcs'].get('insecure', False),
'timeout': config_json['odcs'].get('timeout', None),
}
if 'openidc_dir' in config_json['odcs']['auth']:
config_json['odcs']['auth']['openidc_dir'] = str(tmpdir)
filename = str(tmpdir.join('token'))
with open(filename, 'w') as fp:
fp.write("my_token")
auth_info['token'] = "my_token"
ssl_dir_raise = False
if 'ssl_certs_dir' in config_json['odcs']['auth']:
if config_json['odcs']['auth']['ssl_certs_dir'] != "nonexistent":
config_json['odcs']['auth']['ssl_certs_dir'] = str(tmpdir)
filename = str(tmpdir.join('cert'))
with open(filename, 'w') as fp:
fp.write("my_cert")
auth_info['cert'] = filename
else:
ssl_dir_raise = True
conf = Configuration(raw_config=config_json)
if not ssl_dir_raise:
(flexmock(atomic_reactor.utils.odcs.ODCSClient).should_receive(
'__init__').with_args(config_json['odcs']['api_url'],
**auth_info).once().and_return(None))
get_odcs_session(conf)
else:
with pytest.raises(KeyError):
get_odcs_session(conf)
@pytest.mark.parametrize(('config', 'raise_error'), [
("""\
smtp:
host: smtp.example.com
from_address: [email protected]
""", False),
("""\
smtp:
from_address: [email protected]
""", True),
("""\
smtp:
host: smtp.example.com
""", True),
("""\
smtp:
""", True),
])
def test_get_smtp_session(self, config, raise_error):
config += "\n" + REQUIRED_CONFIG
if raise_error:
with pytest.raises(Exception):
read_yaml(config, 'schemas/config.json')
return
config_json = read_yaml(config, 'schemas/config.json')
conf = Configuration(raw_config=config_json)
(flexmock(smtplib.SMTP).should_receive('__init__').with_args(
config_json['smtp']['host']).once().and_return(None))
get_smtp_session(conf)
@pytest.mark.parametrize(('config', 'error'), [
("""\
cachito:
api_url: https://cachito.example.com
auth:
ssl_certs_dir: /var/run/secrets/atomic-reactor/cachitosecret
timeout: 1000
""", False),
("""\
cachito:
api_url: https://cachito.example.com
insecure: true
auth:
ssl_certs_dir: /var/run/secrets/atomic-reactor/cachitosecret
""", False),
("""\
cachito:
api_url: https://cachito.example.com
auth:
""", OsbsValidationException),
("""\
cachito:
api_url: https://cachito.example.com
""", OsbsValidationException),
("""\
cachito:
auth:
ssl_certs_dir: /var/run/secrets/atomic-reactor/cachitosecret
""", OsbsValidationException),
("""\
cachito:
api_url: https://cachito.example.com
auth:
ssl_certs_dir: /var/run/secrets/atomic-reactor/cachitosecret
spam: ham
""", OsbsValidationException),
("""\
cachito:
api_url: https://cachito.example.com
auth:
ssl_certs_dir: nonexistent
""", False),
])
def test_get_cachito_session(self, tmpdir, config, error):
config += "\n" + REQUIRED_CONFIG
if error:
with pytest.raises(error):
read_yaml(config, 'schemas/config.json')
return
config_json = read_yaml(config, 'schemas/config.json')
auth_info = {
'insecure': config_json['cachito'].get('insecure', False),
'timeout': config_json['cachito'].get('timeout'),
}
ssl_dir_raise = False
if 'ssl_certs_dir' in config_json['cachito']['auth']:
if config_json['cachito']['auth']['ssl_certs_dir'] != "nonexistent":
config_json['cachito']['auth']['ssl_certs_dir'] = str(tmpdir)
filename = str(tmpdir.join('cert'))
with open(filename, 'w') as fp:
fp.write("my_cert")
auth_info['cert'] = filename
else:
ssl_dir_raise = True
conf = Configuration(raw_config=config_json)
if not ssl_dir_raise:
(flexmock(atomic_reactor.utils.cachito.CachitoAPI).should_receive(
'__init__').with_args(config_json['cachito']['api_url'],
**auth_info).once().and_return(None))
get_cachito_session(conf)
else:
with pytest.raises(RuntimeError,
match="Cachito ssl_certs_dir doesn't exist"):
get_cachito_session(conf)
@pytest.mark.parametrize(('config', 'raise_error'), [
("""\
openshift:
url: https://openshift.example.com
auth:
ssl_certs_dir: /var/run/secrets/atomic-reactor/odcssecret
""", False),
("""\
openshift:
url: https://openshift.example.com
""", False),
("""\
openshift:
url: https://openshift.example.com
auth:
krb_principal: principal
krb_keytab_path: /var/keytab
""", False),
("""\
openshift:
url: https://openshift.example.com
auth:
krb_principal: principal
krb_keytab_path: /var/keytab
krb_cache_path: /var/krb/cache
""", False),
("""\
openshift:
url: https://openshift.example.com
auth:
enable: True
""", False),
("""\
openshift:
url: https://openshift.example.com
auth:
krb_keytab_path: /var/keytab
""", True),
("""\
openshift:
url: https://openshift.example.com
auth:
krb_principal: principal
""", True),
("""\
openshift:
auth:
ssl_certs_dir: /var/run/secrets/atomic-reactor/odcssecret
""", True),
("""\
openshift:
auth:
krb_principal: principal
krb_keytab_path: /var/keytab
""", True),
("""\
openshift:
url: https://openshift.example.com
auth:
""", True),
("""\
openshift:
auth:
ssl_certs_dir: /var/run/secrets/atomic-reactor/odcssecret
""", True),
])
def test_get_openshift_session(self, config, raise_error):
required_config = """\
version: 1
koji:
hub_url: /
root_url: ''
auth: {}
source_registry:
url: source_registry.com
registries:
- url: registry_url
"""
config += "\n" + required_config
if raise_error:
with pytest.raises(Exception):
read_yaml(config, 'schemas/config.json')
return
config_json = read_yaml(config, 'schemas/config.json')
auth_info = {
'openshift_url': config_json['openshift']['url'],
'verify_ssl': not config_json['openshift'].get('insecure', False),
'use_auth': False,
'conf_file': None,
'namespace': 'namespace',
}
if config_json['openshift'].get('auth'):
if config_json['openshift']['auth'].get('krb_keytab_path'):
auth_info['kerberos_keytab'] =\
config_json['openshift']['auth'].get('krb_keytab_path')
if config_json['openshift']['auth'].get('krb_principal'):
auth_info['kerberos_principal'] =\
config_json['openshift']['auth'].get('krb_principal')
if config_json['openshift']['auth'].get('krb_cache_path'):
auth_info['kerberos_ccache'] =\
config_json['openshift']['auth'].get('krb_cache_path')
if config_json['openshift']['auth'].get('ssl_certs_dir'):
auth_info['client_cert'] =\
os.path.join(config_json['openshift']['auth'].get('ssl_certs_dir'), 'cert')
auth_info['client_key'] =\
os.path.join(config_json['openshift']['auth'].get('ssl_certs_dir'), 'key')
auth_info['use_auth'] = config_json['openshift']['auth'].get(
'enable', False)
(flexmock(osbs.conf.Configuration).should_call('__init__').with_args(
**auth_info).once())
(flexmock(osbs.api.OSBS).should_call('__init__').once())
conf = Configuration(raw_config=config_json)
get_openshift_session(conf, 'namespace')
@pytest.mark.parametrize(
'config, valid',
[
("""\
operator_manifests:
allowed_registries: null
""", True), # minimal valid example, allows all registries
("""\
operator_manifests:
allowed_registries:
- foo
- bar
repo_replacements:
- registry: foo
package_mappings_url: https://somewhere.net/mapping.yaml
registry_post_replace:
- old: foo
new: bar
""", True), # all known properties
("""\
operator_manifests: null
""", False), # has to be a dict
("""\
operator_manifests: {}
""", False), # allowed_registries is required
("""\
operator_manifests:
allowed_registries: []
""", False), # if not null, allowed_registries must not be empty
("""\
operator_manifests:
allowed_registries: null
something_else: null
""", False), # additional properties not allowed
("""\
operator_manifests:
allowed_registries: null
registry_post_replace:
- old: foo
""", False), # missing replacement registry
("""\
operator_manifests:
allowed_registries: null
registry_post_replace:
- new: foo
""", False), # missing original registry
("""\
operator_manifests:
allowed_registries: null
repo_replacements:
- registry: foo
""", False), # missing package mappings url
("""\
operator_manifests:
allowed_registries: null
repo_replacements:
- package_mappings_url: https://somewhere.net/mapping.yaml
""", False), # missing registry
("""\
operator_manifests:
allowed_registries: null,
repo_replacements:
- registry: foo
package_mappings_url: mapping.yaml
""", False), # package mappings url is not a url
])
def test_get_operator_manifests(self, tmpdir, config, valid):
config += "\n" + REQUIRED_CONFIG
if valid:
read_yaml(config, 'schemas/config.json')
else:
with pytest.raises(OsbsValidationException):
read_yaml(config, 'schemas/config.json')
return
filename = os.path.join(str(tmpdir), 'config.yaml')
with open(filename, 'w') as fp:
fp.write(dedent(config))
conf = Configuration(config_path=filename)
operator_config = conf.operator_manifests
assert isinstance(operator_config, dict)
assert "allowed_registries" in operator_config
@pytest.mark.parametrize(
'config, valid',
[
("""\
build_env_vars: []
""", True),
("""\
build_env_vars:
- name: HTTP_PROXY
value: example.proxy.net
- name: NO_PROXY
value: localhost
""", True),
("""\
build_env_vars:
- name: FOO
value: 1
""", False), # values must be strings
("""\
build_env_vars:
- name: FOO
""", False), # values must be defined
])
def test_validate_build_env_vars(self, config, valid):
# Only test schema validation, atomic-reactor has no additional support
# for build_env_vars (osbs-client does, however)
config += "\n" + REQUIRED_CONFIG
if valid:
read_yaml(config, 'schemas/config.json')
else:
with pytest.raises(OsbsValidationException):
read_yaml(config, 'schemas/config.json')
@pytest.mark.parametrize(('images_exist', 'organization'), [
(True, None),
(True, 'organization'),
(False, None),
(False, 'organization'),
])
def test_update_dockerfile_images_from_config(self, tmp_path, images_exist,
organization):
config = REQUIRED_CONFIG
if organization:
config += "\nregistries_organization: " + organization
config_yaml = tmp_path / 'config.yaml'
config_yaml.write_text(dedent(config), "utf-8")
if images_exist:
parent_images = ['parent:latest', 'base:latest']
if organization:
expect_images = [
ImageName.parse(
'source_registry.com/organization/base:latest'),
ImageName.parse(
'source_registry.com/organization/parent:latest')
]
else:
expect_images = [
ImageName.parse('source_registry.com/base:latest'),
ImageName.parse('source_registry.com/parent:latest')
]
else:
parent_images = []
dockerfile_images = DockerfileImages(parent_images)
conf = Configuration(config_path=str(config_yaml))
conf.update_dockerfile_images_from_config(dockerfile_images)
if images_exist:
assert len(dockerfile_images) == 2
assert dockerfile_images.keys() == expect_images
else:
assert not dockerfile_images
def _as_registry(self, registry):
return {
'uri': RegistryURI(registry['url']),
'insecure': registry.get('insecure', False),
'dockercfg_path': self.registries_cfg_path,
}
def workflow(self,
build_process_failed=False,
registries=None,
registry_types=None,
platforms=None,
platform_descriptors=None,
group=True,
fail=False,
limit_media_types=None):
tag_conf = TagConf()
tag_conf.add_unique_image(self.TEST_UNIQUE_IMAGE)
if platform_descriptors is None:
platform_descriptors = [
{
'platform': 'x86_64',
'architecture': 'amd64'
},
{
'platform': 'ppc64le',
'architecture': 'ppc64le'
},
{
'platform': 's390x',
'architecture': 's390x'
},
]
if platforms is None:
platforms = [
descriptor['platform'] for descriptor in platform_descriptors
]
no_amd64 = 'x86_64' not in platforms
keep_types = False
if registries or registry_types:
keep_types = True
if registries is None and registry_types is None:
registry_types = [
MEDIA_TYPE_DOCKER_V2_SCHEMA1, MEDIA_TYPE_DOCKER_V2_SCHEMA2,
MEDIA_TYPE_DOCKER_V2_MANIFEST_LIST, MEDIA_TYPE_OCI_V1,
MEDIA_TYPE_OCI_V1_INDEX
]
if registries is None:
registries = [{
'url': 'https://container-registry.example.com/v2',
'version': 'v2',
'insecure': True,
'expected_media_types': registry_types
}]
conf = {
'version': 1,
'registries': registries,
}
if limit_media_types is not None:
conf['source_container'] = {
'limit_media_types': limit_media_types,
}
if platform_descriptors:
conf['platform_descriptors'] = platform_descriptors
for registry in registries:
def get_manifest(request):
media_types = request.headers.get('Accept', '').split(',')
content_type = media_types[0]
return 200, {'Content-Type': content_type}, '{}'
url_regex = "r'" + registry['url'] + ".*/manifests/.*'"
url = re.compile(url_regex)
responses.add_callback(responses.GET, url, callback=get_manifest)
expected_types = registry.get('expected_media_types',
registry_types or [])
if fail == "bad_results":
response_types = []
elif not keep_types and no_amd64:
response_types = [MEDIA_TYPE_DOCKER_V2_MANIFEST_LIST]
else:
response_types = expected_types
reguri = RegistryURI(registry['url']).docker_uri
if re.match('http(s)?://', reguri):
urlbase = reguri
else:
urlbase = 'https://{0}'.format(reguri)
actual_v2_url = urlbase + "/v2/foo/manifests/unique-tag"
if fail == "bad_results":
response = requests.Response()
(flexmock(response,
raise_for_status=lambda: None,
status_code=requests.codes.ok,
json={},
headers={'Content-Type': 'application/json'}))
v1_response = response
v1_oci_response = response
v1_oci_index_response = response
v2_response = response
v2_list_response = response
else:
v1_response = self.config_response_none
v1_oci_response = self.config_response_none
v1_oci_index_response = self.config_response_none
v2_response = self.config_response_none
v2_list_response = self.config_response_none
if MEDIA_TYPE_DOCKER_V2_SCHEMA1 in response_types:
v1_response = self.config_response_config_v1
if MEDIA_TYPE_DOCKER_V2_SCHEMA2 in response_types:
v2_response = self.config_response_config_v2
if MEDIA_TYPE_DOCKER_V2_MANIFEST_LIST in response_types:
v2_list_response = self.config_response_config_v2_list
if MEDIA_TYPE_OCI_V1 in response_types:
v1_oci_response = self.config_response_config_oci_v1
if MEDIA_TYPE_OCI_V1_INDEX in response_types:
v1_oci_index_response = self.config_response_config_oci_v1_index
v2_header_v1 = {'Accept': MEDIA_TYPE_DOCKER_V2_SCHEMA1}
v2_header_v2 = {'Accept': MEDIA_TYPE_DOCKER_V2_SCHEMA2}
manifest_header = {'Accept': MEDIA_TYPE_DOCKER_V2_MANIFEST_LIST}
(flexmock(requests.Session).should_receive('get').with_args(
actual_v2_url,
headers=v2_header_v1,
auth=HTTPRegistryAuth,
verify=False).and_return(v1_response))
(flexmock(requests.Session).should_receive('get').with_args(
actual_v2_url,
headers=v2_header_v2,
auth=HTTPRegistryAuth,
verify=False).and_return(v2_response))
(flexmock(requests.Session).should_receive('get').with_args(
actual_v2_url,
headers={
'Accept': MEDIA_TYPE_OCI_V1
},
auth=HTTPRegistryAuth,
verify=False).and_return(v1_oci_response))
(flexmock(requests.Session).should_receive('get').with_args(
actual_v2_url,
headers={
'Accept': MEDIA_TYPE_OCI_V1_INDEX
},
auth=HTTPRegistryAuth,
verify=False).and_return(v1_oci_index_response))
(flexmock(requests.Session).should_receive('get').with_args(
actual_v2_url,
headers=manifest_header,
auth=HTTPRegistryAuth,
verify=False).and_return(v2_list_response))
digests = {'media_type': MEDIA_TYPE_DOCKER_V2_MANIFEST_LIST}
if not group:
digests = {'media_type': MEDIA_TYPE_DOCKER_V2_SCHEMA2}
plugins_results = {
PLUGIN_CHECK_AND_SET_PLATFORMS_KEY: platforms,
PLUGIN_GROUP_MANIFESTS_KEY: digests,
}
mock_get_retry_session()
builder = flexmock()
setattr(builder, 'image_id', 'sha256:(old)')
flexmock(tag_conf=tag_conf)
wf_data = ImageBuildWorkflowData()
wf_data.tag_conf = tag_conf
wf_data.plugins_results = plugins_results
return flexmock(data=wf_data,
builder=builder,
conf=Configuration(raw_config=conf),
build_process_failed=build_process_failed)
def value(self, val): # pylint: disable=W0221
BuildParam.value.fset(self, RegistryURI(val) if val else None)
def value(self, val): # pylint: disable=W0221
registry_uris = [RegistryURI(uri) for uri in val]
BuildParam.value.fset(self, registry_uris) # pylint: disable=no-member
def workflow(self, build_process_failed=False, registries=None, registry_types=None,
platforms=None, platform_descriptors=None, group=True, no_amd64=False,
fail=False):
tag_conf = TagConf()
tag_conf.add_unique_image(self.TEST_UNIQUE_IMAGE)
push_conf = PushConf()
if platform_descriptors is None:
platform_descriptors = [
{'platform': 'x86_64', 'architecture': 'amd64'},
{'platform': 'ppc64le', 'architecture': 'ppc64le'},
{'platform': 's390x', 'architecture': 's390x'},
]
if platforms is None:
platforms = [descriptor['platform'] for descriptor in platform_descriptors]
if registries is None and registry_types is None:
registry_types = [MEDIA_TYPE_DOCKER_V1, MEDIA_TYPE_DOCKER_V2_SCHEMA1,
MEDIA_TYPE_DOCKER_V2_SCHEMA2, MEDIA_TYPE_DOCKER_V2_MANIFEST_LIST]
if registries is None:
registries = [{
'url': 'https://container-registry.example.com/v2',
'version': 'v2',
'insecure': True,
'expected_media_types': registry_types
}]
conf = {
ReactorConfigKeys.VERSION_KEY: 1,
'registries': registries,
}
if platform_descriptors:
conf['platform_descriptors'] = platform_descriptors
plugin_workspace = {
ReactorConfigPlugin.key: {
WORKSPACE_CONF_KEY: ReactorConfig(conf)
}
}
flexmock(HTTPRegistryAuth).should_receive('__new__').and_return(None)
mock_auth = None
for registry in registries:
def get_manifest(request):
media_types = request.headers.get('Accept', '').split(',')
content_type = media_types[0]
return (200, {'Content-Type': content_type}, '{}')
url_regex = "r'" + registry['url'] + ".*/manifests/.*'"
url = re.compile(url_regex)
responses.add_callback(responses.GET, url, callback=get_manifest)
expected_types = registry.get('expected_media_types', [])
if fail == "bad_results":
response_types = [MEDIA_TYPE_DOCKER_V1]
elif no_amd64:
response_types = [MEDIA_TYPE_DOCKER_V2_MANIFEST_LIST]
else:
response_types = expected_types
reguri = RegistryURI(registry['url']).docker_uri
if re.match('http(s)?://', reguri):
urlbase = reguri
else:
urlbase = 'https://{0}'.format(reguri)
actual_v2_url = urlbase + "/v2/foo/manifests/unique-tag"
actual_v1_url = urlbase + "/v1/repositories/foo/tags/unique-tag"
v1_response = self.config_response_none
v2_response = self.config_response_none
v2_list_response = self.config_response_none
if MEDIA_TYPE_DOCKER_V2_SCHEMA1 in response_types:
v1_response = self.config_response_config_v1
if MEDIA_TYPE_DOCKER_V2_SCHEMA2 in response_types:
v2_response = self.config_response_config_v2
if MEDIA_TYPE_DOCKER_V2_MANIFEST_LIST in response_types:
v2_list_response = self.config_response_config_v2_list
v2_header_v1 = {'Accept': MEDIA_TYPE_DOCKER_V2_SCHEMA1}
v2_header_v2 = {'Accept': MEDIA_TYPE_DOCKER_V2_SCHEMA2}
manifest_header = {'Accept': MEDIA_TYPE_DOCKER_V2_MANIFEST_LIST}
(flexmock(requests.Session)
.should_receive('get')
.with_args(actual_v2_url, headers=v2_header_v1,
auth=mock_auth, verify=False)
.and_return(v1_response))
(flexmock(requests.Session)
.should_receive('get')
.with_args(actual_v2_url, headers=v2_header_v2,
auth=mock_auth, verify=False)
.and_return(v2_response))
(flexmock(requests.Session)
.should_receive('get')
.with_args(actual_v2_url, headers={'Accept': MEDIA_TYPE_OCI_V1},
auth=mock_auth, verify=False)
.and_return(self.config_response_none))
(flexmock(requests.Session)
.should_receive('get')
.with_args(actual_v2_url, headers={'Accept': MEDIA_TYPE_OCI_V1_INDEX},
auth=mock_auth, verify=False)
.and_return(self.config_response_none))
(flexmock(requests.Session)
.should_receive('get')
.with_args(actual_v2_url, headers=manifest_header,
auth=mock_auth, verify=False)
.and_return(v2_list_response))
if MEDIA_TYPE_DOCKER_V1 in response_types:
(flexmock(requests.Session)
.should_receive('get')
.with_args(actual_v1_url, headers={'Accept': MEDIA_TYPE_DOCKER_V1},
auth=mock_auth, verify=False)
.and_return(self.config_response_v1))
digests = {'digest': None} if group else {}
prebuild_results = {PLUGIN_CHECK_AND_SET_PLATFORMS_KEY: platforms}
postbuild_results = {PLUGIN_GROUP_MANIFESTS_KEY: digests}
mock_get_retry_session()
builder = flexmock()
setattr(builder, 'image_id', 'sha256:(old)')
return flexmock(tag_conf=tag_conf,
push_conf=push_conf,
builder=builder,
build_process_failed=build_process_failed,
plugin_workspace=plugin_workspace,
prebuild_results=prebuild_results,
postbuild_results=postbuild_results)
