def __init__(self,
build_json_store,
outer_template=None,
customize_conf=None):
"""
:param build_json_store: str, path to directory with JSON build files
:param outer_template: str, path to outer template JSON
:param customize_conf: str, path to customize configuration JSON
"""
self.build_json_store = build_json_store
self._outer_template_path = outer_template or DEFAULT_OUTER_TEMPLATE
self._customize_conf_path = customize_conf or DEFAULT_CUSTOMIZE_CONF
self.build_json = None # rendered template
self._template = None # template loaded from filesystem
self._resource_limits = None
self._openshift_required_version = parse_version('3.6.0')
self._repo_info = None
# For the koji "scratch" build type
self.scratch = None
self.isolated = None
self.is_auto = None
self.skip_build = None
self.base_image = None
self.scratch_build_node_selector = None
self.explicit_build_node_selector = None
self.auto_build_node_selector = None
self.isolated_build_node_selector = None
self.is_auto = None
# forward reference
self.platform_node_selector = None
self.user_params = BuildUserParams(build_json_store, customize_conf)
self.osbs_api = None
self.source_registry = None
self.organization = None
self.triggered_after_koji_task = None
def __init__(self, build_json_store, outer_template=None, customize_conf=None):
"""
:param build_json_store: str, path to directory with JSON build files
:param outer_template: str, path to outer template JSON
:param customize_conf: str, path to customize configuration JSON
"""
super(BuildRequestV2, self).__init__(
build_json_store,
outer_template=outer_template or DEFAULT_OUTER_TEMPLATE
)
self._customize_conf_path = customize_conf or DEFAULT_CUSTOMIZE_CONF
self.build_json = None # rendered template
self._repo_info = None
self.isolated = None
self.is_auto = None
self.skip_build = None
self.base_image = None
self.scratch_build_node_selector = None
self.explicit_build_node_selector = None
self.auto_build_node_selector = None
self.isolated_build_node_selector = None
self.is_auto = None
# forward reference
self.platform_node_selector = None
self.user_params = BuildUserParams(self.build_json_store, customize_conf)
self.triggered_after_koji_task = None
def test_user_params_bad_none_flatpak(self, missing_arg):
kwargs = self.get_minimal_kwargs()
kwargs['flatpak'] = False
kwargs.pop(missing_arg)
with pytest.raises(OsbsValidationException):
BuildUserParams.make_params(**kwargs)
def test_render_prod_custom_site_plugin_enable(self, tmpdir):
# Test to make sure that when we attempt to enable a plugin, it is
# actually enabled in the JSON for the build_request after running
# build_request.render()
self.mock_repo_info()
sample_params = get_sample_prod_params()
user_params = BuildUserParams(INPUTS_PATH)
user_params.set_params(**sample_params)
plugins_conf = PluginsConfiguration(user_params)
plugin_type = "exit_plugins"
plugin_name = "testing_exit_plugin"
plugin_args = {"foo": "bar"}
plugins_conf.pt.customize_conf['enable_plugins'].append({
"plugin_type":
plugin_type,
"plugin_name":
plugin_name,
"plugin_args":
plugin_args
})
build_json = plugins_conf.render()
plugins = get_plugins_from_build_json(build_json)
assert {
"name": plugin_name,
"args": plugin_args
} in plugins[plugin_type]
def test_v2_compose_ids_and_signing_intent(self, signing_intent,
compose_ids, yum_repourls, exc):
kwargs = self.get_minimal_kwargs()
if signing_intent:
kwargs['signing_intent'] = signing_intent
if compose_ids:
kwargs['compose_ids'] = compose_ids
if yum_repourls:
kwargs['yum_repourls'] = yum_repourls
kwargs.update({
'git_uri': 'https://github.com/user/reponame.git',
'git_branch': 'master',
})
if exc:
with pytest.raises(exc):
BuildUserParams.make_params(**kwargs)
else:
spec = BuildUserParams.make_params(**kwargs)
if yum_repourls:
assert spec.yum_repourls == yum_repourls
if signing_intent:
assert spec.signing_intent == signing_intent
if compose_ids:
assert spec.compose_ids == compose_ids
def test_v2_image_tag(self, rand, timestr, platform):
kwargs = self.get_minimal_kwargs()
kwargs.update({
'component': 'foo',
'koji_target': 'tothepoint',
})
if platform:
kwargs['platform'] = platform
(flexmock(sys.modules['osbs.build.user_params']).should_receive(
'utcnow').once().and_return(
datetime.datetime.strptime(timestr, '%Y%m%d%H%M%S')))
(flexmock(random).should_receive('randrange').once().with_args(
10**(len(rand) - 1), 10**len(rand)).and_return(int(rand)))
spec = BuildUserParams()
spec.set_params(**kwargs)
img_tag = '{user}/{component}:{koji_target}-{random_number}-{time_string}'
if platform:
img_tag += '-{platform}'
img_tag = img_tag.format(random_number=rand,
time_string=timestr,
**kwargs)
assert spec.image_tag.value == img_tag
def test_user_params_bad_compose_ids(self):
kwargs = self.get_minimal_kwargs()
kwargs['compose_ids'] = True
spec = BuildUserParams()
with pytest.raises(OsbsValidationException):
spec.set_params(**kwargs)
def get_sample_user_params(build_type=BUILD_TYPE_ORCHESTRATOR,
conf_args=None,
extra_args=None):
sample_params = get_sample_prod_params(build_type, conf_args, extra_args)
user_params = BuildUserParams(INPUTS_PATH)
user_params.set_params(**sample_params)
return user_params
def test_from_json_continue(self):
spec = BuildUserParams()
expected_json = {
"arrangement_version": REACTOR_CONFIG_ARRANGEMENT_VERSION,
"base_image": "buildroot:old",
"build_image": "buildroot:latest",
"build_json_dir": "build_dir",
"build_type": "worker",
"component": TEST_COMPONENT,
"compose_ids": [1, 2],
"customize_conf": "prod_customize.json",
"filesystem_koji_task_id": TEST_FILESYSTEM_KOJI_TASK_ID,
"git_branch": TEST_GIT_BRANCH,
"git_ref": TEST_GIT_REF,
"git_uri": TEST_GIT_URI,
"image_tag": "latest",
"imagestream_name": "name_label",
"koji_parent_build": "fedora-26-9",
"koji_target": "tothepoint",
"name": "path-master-cd1e4",
"orchestrator_deadline": 4,
"platform": "x86_64",
"platforms": ["x86_64"],
"reactor_config_map": "reactor-config-map",
"reactor_config_override": "reactor-config-override",
"release": "29",
"trigger_imagestreamtag": "buildroot:old",
"this is not a valid key": "this is not a valid field",
"user": TEST_USER,
"worker_deadline": 3,
"triggered_after_koji_task": 12345,
}
spec.from_json(json.dumps(expected_json))
def test_render_tag_from_config(self, tmpdir, from_container_yaml,
extra_args, has_platform_tag, extra_tags,
primary_tags, floating_tags):
kwargs = get_sample_prod_params(BUILD_TYPE_WORKER)
kwargs.pop('platforms', None)
kwargs.pop('platform', None)
expected_primary = set(primary_tags)
expected_floating = set(floating_tags)
exclude_for_override = set(['latest', '{version}'])
if from_container_yaml:
expected_floating -= exclude_for_override
extra_args['tags_from_yaml'] = from_container_yaml
extra_args['additional_tags'] = extra_tags
kwargs.update(extra_args)
user_params = BuildUserParams(INPUTS_PATH)
user_params.set_params(**kwargs)
build_json = PluginsConfiguration(user_params).render()
plugins = get_plugins_from_build_json(build_json)
assert get_plugin(plugins, 'postbuild_plugins', 'tag_from_config')
tag_suffixes = plugin_value_get(plugins, 'postbuild_plugins',
'tag_from_config', 'args',
'tag_suffixes')
assert len(tag_suffixes['unique']) == 1
if has_platform_tag:
unique_tag_suffix = tag_suffixes['unique'][0]
assert unique_tag_suffix.endswith('-x86_64') == has_platform_tag
assert len(tag_suffixes['primary']) == len(expected_primary)
assert set(tag_suffixes['primary']) == expected_primary
assert len(tag_suffixes['floating']) == len(expected_floating)
assert set(tag_suffixes['floating']) == expected_floating
def test_user_params_bad_build_from(self):
# does not have an "image:" prefix:
conf_args = {'build_from': 'registry.example.com/buildroot'}
kwargs = self.get_minimal_kwargs(conf_args=conf_args)
with pytest.raises(OsbsValidationException) as e:
BuildUserParams.make_params(**kwargs)
assert 'build_from must be "source_type:source_value"' in str(e.value)
def get_sample_user_params(extra_args=None,
build_type=BUILD_TYPE_ORCHESTRATOR):
sample_params = get_sample_prod_params(build_type)
if extra_args:
sample_params.update(extra_args)
user_params = BuildUserParams(INPUTS_PATH)
user_params.set_params(**sample_params)
return user_params
def test_user_params_bad_build_from(self):
kwargs = self.get_minimal_kwargs()
# does not have an "image:" prefix:
kwargs['build_from'] = 'registry.example.com/buildroot'
spec = BuildUserParams()
with pytest.raises(OsbsValidationException) as e:
spec.set_params(**kwargs)
assert 'build_from must be "source_type:source_value"' in str(e.value)
def test_missing_build_opts(self):
conf_args = {
'build_from': None,
'build_image': None,
'build_imagestream': None,
}
kwargs = self.get_minimal_kwargs(conf_args=conf_args)
with pytest.raises(OsbsValidationException):
BuildUserParams.make_params(**kwargs)
def test_v2_spec_name2(self):
kwargs = self.get_minimal_kwargs()
kwargs.update({
'git_uri': TEST_GIT_URI,
'git_branch': TEST_GIT_BRANCH,
})
spec = BuildUserParams()
spec.set_params(**kwargs)
assert spec.name.value.startswith('path-master')
def __init__(self,
build_json_store,
outer_template=None,
customize_conf=None):
"""
:param build_json_store: str, path to directory with JSON build files
:param outer_template: str, path to outer template JSON
:param customize_conf: str, path to customize configuration JSON
"""
super(BuildRequestV2, self).__init__(build_json_store=build_json_store,
outer_template=outer_template,
customize_conf=customize_conf)
self.spec = None
self.user_params = BuildUserParams(build_json_store, customize_conf)
self.osbs_api = None
def __init__(self,
osbs_api,
outer_template=None,
customize_conf=None,
user_params=None,
build_json_store=None,
repo_info=None):
"""
:param build_json_store: str, path to directory with JSON build files
:param outer_template: str, path to outer template JSON
:param customize_conf: str, path to customize configuration JSON
:param repo_info: RepoInfo, git repo data for the build
"""
if user_params:
assert isinstance(user_params, BuildUserParams)
else:
user_params = BuildUserParams()
super(BuildRequestV2, self).__init__(
osbs_api,
outer_template=outer_template or DEFAULT_OUTER_TEMPLATE,
user_params=user_params,
build_json_store=build_json_store,
)
self._customize_conf_path = customize_conf or DEFAULT_CUSTOMIZE_CONF
self.build_json = None # rendered template
self.repo_info = repo_info
def test_v2_spec_name2(self):
git_args = {'git_branch': TEST_GIT_BRANCH}
kwargs = self.get_minimal_kwargs(git_args=git_args)
spec = BuildUserParams.make_params(**kwargs)
assert spec.name.startswith('path-master')
def test_validate_missing_required(self):
kwargs = self.get_minimal_kwargs()
kwargs['user'] = None
spec = BuildUserParams.make_params(**kwargs)
with pytest.raises(OsbsValidationException):
spec.validate()
def get_user_params(self, component=None, req_labels=None, **kwargs):
req_labels = req_labels or {}
user_component = component or req_labels[Labels.LABEL_TYPE_COMPONENT]
return BuildUserParams.make_params(
build_conf=self.os_conf,
component=user_component,
name_label=req_labels[Labels.LABEL_TYPE_NAME],
**kwargs)
def test_render_tag_from_config(self, tmpdir, from_container_yaml,
extra_args, has_platform_tag, extra_tags,
primary_tags):
kwargs = get_sample_prod_params(BUILD_TYPE_WORKER)
kwargs.pop('platforms', None)
kwargs.pop('platform', None)
expected_primary = set(primary_tags)
exclude_for_override = set(['latest', '{version}'])
if extra_tags and not from_container_yaml:
with open(os.path.join(str(tmpdir), ADDITIONAL_TAGS_FILE),
'w') as f:
f.write('\n'.join(extra_tags))
kwargs.update(extra_args)
if from_container_yaml:
if extra_tags:
expected_primary -= exclude_for_override
(flexmock(utils).should_receive('get_repo_info').with_args(
TEST_GIT_URI, TEST_GIT_REF,
git_branch=TEST_GIT_BRANCH).and_return(
RepoInfo(additional_tags=AdditionalTagsConfig(
tags=extra_tags))))
else:
(flexmock(utils).should_receive('get_repo_info').with_args(
TEST_GIT_URI, TEST_GIT_REF,
git_branch=TEST_GIT_BRANCH).and_return(
RepoInfo(additional_tags=AdditionalTagsConfig(
dir_path=str(tmpdir)))))
user_params = BuildUserParams(INPUTS_PATH)
user_params.set_params(**kwargs)
build_json = PluginsConfiguration(user_params).render()
plugins = get_plugins_from_build_json(build_json)
assert get_plugin(plugins, 'postbuild_plugins', 'tag_from_config')
tag_suffixes = plugin_value_get(plugins, 'postbuild_plugins',
'tag_from_config', 'args',
'tag_suffixes')
assert len(tag_suffixes['unique']) == 1
if has_platform_tag:
unique_tag_suffix = tag_suffixes['unique'][0]
assert unique_tag_suffix.endswith('-x86_64') == has_platform_tag
assert len(tag_suffixes['primary']) == len(expected_primary)
assert set(tag_suffixes['primary']) == expected_primary
def test_make_params_keeps_defaults(self):
kwargs = self.get_minimal_kwargs()
params = BuildUserParams.make_params(**kwargs)
assert params.arrangement_version == REACTOR_CONFIG_ARRANGEMENT_VERSION
assert params.buildroot_is_imagestream is False
assert params.customize_conf == DEFAULT_CUSTOMIZE_CONF
# assert params.git_ref == 'master' # set from repo_info
assert params.include_koji_repo is False
def get_plugins_from_buildrequest(self, build_request, template):
conf_kwargs = {
'build_from': 'image:test',
'reactor_config_map': 'reactor-config-map',
}
kwargs = {
'git_uri': TEST_GIT_URI,
'git_ref': TEST_GIT_REF,
'git_branch': TEST_GIT_BRANCH,
'user': '******',
'build_type': template.split('_')[0],
'build_conf': Configuration(conf_file=None, **conf_kwargs),
'base_image': 'test',
'name_label': 'test',
}
user_params = BuildUserParams()
user_params.set_params(**kwargs)
build_request.set_params(user_params)
return PluginsConfiguration(build_request.user_params).pt.template
def test_prod_custom_base_image(self, tmpdir):
kwargs = get_sample_prod_params()
kwargs['base_image'] = 'koji/image-build'
kwargs['yum_repourls'] = ["http://example.com/my.repo"]
self.mock_repo_info()
user_params = BuildUserParams(INPUTS_PATH)
user_params.set_params(**kwargs)
build_json = PluginsConfiguration(user_params).render()
plugins = get_plugins_from_build_json(build_json)
get_plugin(plugins, 'prebuild_plugins', 'pull_base_image')
add_filesystem_args = plugin_value_get(plugins, 'prebuild_plugins',
'add_filesystem', 'args')
assert add_filesystem_args['repos'] == kwargs['yum_repourls']
assert add_filesystem_args['from_task_id'] == kwargs[
'filesystem_koji_task_id']
assert add_filesystem_args['koji_target'] == kwargs['koji_target']
def test_render_all_code_paths(self, caplog):
# Alter the plugins configuration so that all code paths are exercised
sample_params = get_sample_prod_params()
sample_params['scratch'] = True
sample_params['parent_images_digests'] = True
user_params = BuildUserParams(INPUTS_PATH)
user_params.set_params(**sample_params)
plugins_conf = PluginsConfiguration(user_params)
plugins_conf.pt.customize_conf['disable_plugins'].append({
"plugin_type":
"postbuild_plugins",
"plugin_name":
"tag_from_config"
})
plugins_conf.pt.customize_conf['disable_plugins'].append({
"plugin_type":
"prebuild_plugins",
"plugin_name":
"add_labels_in_dockerfile"
})
plugins_conf.pt.customize_conf['disable_plugins'].append(
{
"bad_plugin_type": "postbuild_plugins",
"bad_plugin_name": "tag_from_config"
}, )
plugins_conf.pt.customize_conf['enable_plugins'].append(
{
"bad_plugin_type": "postbuild_plugins",
"bad_plugin_name": "tag_from_config"
}, )
build_json = plugins_conf.render()
plugins = get_plugins_from_build_json(build_json)
log_messages = [l.getMessage() for l in caplog.records]
assert 'no tag suffix placeholder' in log_messages
assert 'Invalid custom configuration found for disable_plugins' in log_messages
assert 'Invalid custom configuration found for enable_plugins' in log_messages
assert plugin_value_get(plugins, 'prebuild_plugins', 'pull_base_image',
'args', 'parent_images_digests')
def test_user_params_bad_json(self):
required_json = json.dumps(
{
'git_ref': 'master',
'kind': 'build_user_params',
},
sort_keys=True)
spec = BuildUserParams()
spec.from_json(None)
assert spec.to_json() == required_json
spec.from_json("")
assert spec.to_json() == required_json
assert '{}'.format(spec)
def test_render_prod_custom_site_plugin_override(self):
# Test to make sure that when we attempt to override a plugin's args,
# they are actually overridden in the JSON for the build_request
# after running build_request.render()
self.mock_repo_info()
sample_params = get_sample_prod_params()
base_user_params = BuildUserParams.make_params(**sample_params)
base_plugins_conf = PluginsConfiguration(base_user_params)
base_build_json = base_plugins_conf.render()
base_plugins = get_plugins_from_build_json(base_build_json)
plugin_type = "exit_plugins"
plugin_name = "koji_import"
plugin_args = {"foo": "bar"}
for plugin_dict in base_plugins[plugin_type]:
if plugin_dict['name'] == plugin_name:
plugin_index = base_plugins[plugin_type].index(plugin_dict)
user_params = BuildUserParams.make_params(**sample_params)
plugins_conf = PluginsConfiguration(user_params)
plugins_conf.pt.customize_conf['enable_plugins'].append({
"plugin_type":
plugin_type,
"plugin_name":
plugin_name,
"plugin_args":
plugin_args
})
build_json = plugins_conf.render()
plugins = get_plugins_from_build_json(build_json)
assert {
"name": plugin_name,
"args": plugin_args
} in plugins[plugin_type]
assert base_plugins[plugin_type][plugin_index]['name'] == \
plugin_name
assert plugins[plugin_type][plugin_index]['name'] == plugin_name
def test_worker_custom_base_image(self, tmpdir):
self.mock_repo_info()
kwargs = get_sample_prod_params(BUILD_TYPE_WORKER)
kwargs['base_image'] = 'koji/image-build'
kwargs['yum_repourls'] = ["http://example.com/my.repo"]
kwargs.pop('platforms', None)
kwargs['platform'] = 'ppc64le'
user_params = BuildUserParams(INPUTS_PATH)
user_params.set_params(**kwargs)
build_json = PluginsConfiguration(user_params).render()
plugins = get_plugins_from_build_json(build_json)
get_plugin(plugins, 'prebuild_plugins', 'pull_base_image')
add_filesystem_args = plugin_value_get(plugins, 'prebuild_plugins',
'add_filesystem', 'args')
assert add_filesystem_args['repos'] == kwargs['yum_repourls']
assert add_filesystem_args['from_task_id'] == kwargs[
'filesystem_koji_task_id']
assert add_filesystem_args['architecture'] == kwargs['platform']
assert add_filesystem_args['koji_target'] == kwargs['koji_target']
def test_user_params_bad_json(self):
required_json = json.dumps(
{
'arrangement_version': 6,
'customize_conf': 'worker_customize.json',
'git_ref': 'master'
},
sort_keys=True)
spec = BuildUserParams()
spec.from_json(None)
assert spec.to_json() == required_json
spec.from_json("")
assert spec.to_json() == required_json
assert '{}'.format(spec)
def test_render_prod_custom_site_plugin_disable(self):
# Test to make sure that when we attempt to disable a plugin, it is
# actually disabled in the JSON for the build_request after running
# build_request.render()
sample_params = get_sample_prod_params()
user_params = BuildUserParams(INPUTS_PATH)
user_params.set_params(**sample_params)
plugins_conf = PluginsConfiguration(user_params)
plugin_type = "postbuild_plugins"
plugin_name = "tag_from_config"
plugins_conf.pt.customize_conf['disable_plugins'].append({
"plugin_type":
plugin_type,
"plugin_name":
plugin_name
})
build_json = plugins_conf.render()
plugins = get_plugins_from_build_json(build_json)
for plugin in plugins[plugin_type]:
if plugin['name'] == plugin_name:
assert False
