def test_v2_compose_ids_and_signing_intent(self, signing_intent,
compose_ids, yum_repourls, exc):
kwargs = self.get_minimal_kwargs()
if signing_intent:
kwargs['signing_intent'] = signing_intent
if compose_ids:
kwargs['compose_ids'] = compose_ids
if yum_repourls:
kwargs['yum_repourls'] = yum_repourls
kwargs.update({
'git_uri': 'https://github.com/user/reponame.git',
'git_branch': 'master',
})
spec = BuildUserParams()
if exc:
with pytest.raises(exc):
spec.set_params(**kwargs)
else:
spec.set_params(**kwargs)
if yum_repourls:
assert spec.yum_repourls.value == yum_repourls
if signing_intent:
assert spec.signing_intent.value == signing_intent
if compose_ids:
assert spec.compose_ids.value == compose_ids
def test_user_params_bad_compose_ids(self):
kwargs = self.get_minimal_kwargs()
kwargs['compose_ids'] = True
spec = BuildUserParams()
with pytest.raises(OsbsValidationException):
spec.set_params(**kwargs)
def test_v2_image_tag(self, rand, timestr, platform):
kwargs = self.get_minimal_kwargs()
kwargs.update({
'component': 'foo',
'koji_target': 'tothepoint',
})
if platform:
kwargs['platform'] = platform
(flexmock(sys.modules['osbs.build.user_params']).should_receive(
'utcnow').once().and_return(
datetime.datetime.strptime(timestr, '%Y%m%d%H%M%S')))
(flexmock(random).should_receive('randrange').once().with_args(
10**(len(rand) - 1), 10**len(rand)).and_return(int(rand)))
spec = BuildUserParams()
spec.set_params(**kwargs)
img_tag = '{user}/{component}:{koji_target}-{random_number}-{time_string}'
if platform:
img_tag += '-{platform}'
img_tag = img_tag.format(random_number=rand,
time_string=timestr,
**kwargs)
assert spec.image_tag.value == img_tag
def get_sample_user_params(build_type=BUILD_TYPE_ORCHESTRATOR,
conf_args=None,
extra_args=None):
sample_params = get_sample_prod_params(build_type, conf_args, extra_args)
user_params = BuildUserParams(INPUTS_PATH)
user_params.set_params(**sample_params)
return user_params
def test_render_prod_custom_site_plugin_enable(self, tmpdir):
# Test to make sure that when we attempt to enable a plugin, it is
# actually enabled in the JSON for the build_request after running
# build_request.render()
self.mock_repo_info()
sample_params = get_sample_prod_params()
user_params = BuildUserParams(INPUTS_PATH)
user_params.set_params(**sample_params)
plugins_conf = PluginsConfiguration(user_params)
plugin_type = "exit_plugins"
plugin_name = "testing_exit_plugin"
plugin_args = {"foo": "bar"}
plugins_conf.pt.customize_conf['enable_plugins'].append({
"plugin_type":
plugin_type,
"plugin_name":
plugin_name,
"plugin_args":
plugin_args
})
build_json = plugins_conf.render()
plugins = get_plugins_from_build_json(build_json)
assert {
"name": plugin_name,
"args": plugin_args
} in plugins[plugin_type]
def test_render_tag_from_config(self, tmpdir, from_container_yaml,
extra_args, has_platform_tag, extra_tags,
primary_tags, floating_tags):
kwargs = get_sample_prod_params(BUILD_TYPE_WORKER)
kwargs.pop('platforms', None)
kwargs.pop('platform', None)
expected_primary = set(primary_tags)
expected_floating = set(floating_tags)
exclude_for_override = set(['latest', '{version}'])
if from_container_yaml:
expected_floating -= exclude_for_override
extra_args['tags_from_yaml'] = from_container_yaml
extra_args['additional_tags'] = extra_tags
kwargs.update(extra_args)
user_params = BuildUserParams(INPUTS_PATH)
user_params.set_params(**kwargs)
build_json = PluginsConfiguration(user_params).render()
plugins = get_plugins_from_build_json(build_json)
assert get_plugin(plugins, 'postbuild_plugins', 'tag_from_config')
tag_suffixes = plugin_value_get(plugins, 'postbuild_plugins',
'tag_from_config', 'args',
'tag_suffixes')
assert len(tag_suffixes['unique']) == 1
if has_platform_tag:
unique_tag_suffix = tag_suffixes['unique'][0]
assert unique_tag_suffix.endswith('-x86_64') == has_platform_tag
assert len(tag_suffixes['primary']) == len(expected_primary)
assert set(tag_suffixes['primary']) == expected_primary
assert len(tag_suffixes['floating']) == len(expected_floating)
assert set(tag_suffixes['floating']) == expected_floating
def get_sample_user_params(extra_args=None,
build_type=BUILD_TYPE_ORCHESTRATOR):
sample_params = get_sample_prod_params(build_type)
if extra_args:
sample_params.update(extra_args)
user_params = BuildUserParams(INPUTS_PATH)
user_params.set_params(**sample_params)
return user_params
def test_user_params_bad_none_flatpak(self, missing_arg):
kwargs = self.get_minimal_kwargs()
kwargs['flatpak'] = False
kwargs.pop(missing_arg)
spec = BuildUserParams()
with pytest.raises(OsbsValidationException):
spec.set_params(**kwargs)
def test_user_params_bad_build_from(self):
kwargs = self.get_minimal_kwargs()
# does not have an "image:" prefix:
kwargs['build_from'] = 'registry.example.com/buildroot'
spec = BuildUserParams()
with pytest.raises(OsbsValidationException) as e:
spec.set_params(**kwargs)
assert 'build_from must be "source_type:source_value"' in str(e.value)
def test_v2_spec_name2(self):
kwargs = self.get_minimal_kwargs()
kwargs.update({
'git_uri': TEST_GIT_URI,
'git_branch': TEST_GIT_BRANCH,
})
spec = BuildUserParams()
spec.set_params(**kwargs)
assert spec.name.value.startswith('path-master')
def test_validate_missing_required(self):
kwargs = {
'base_image': 'base_image',
'git_uri': TEST_GIT_URI,
'name_label': 'name_label',
'build_from': 'image:buildroot:latest',
}
spec = BuildUserParams()
spec.set_params(**kwargs)
with pytest.raises(OsbsValidationException):
spec.validate()
def test_render_tag_from_config(self, tmpdir, from_container_yaml,
extra_args, has_platform_tag, extra_tags,
primary_tags):
kwargs = get_sample_prod_params(BUILD_TYPE_WORKER)
kwargs.pop('platforms', None)
kwargs.pop('platform', None)
expected_primary = set(primary_tags)
exclude_for_override = set(['latest', '{version}'])
if extra_tags and not from_container_yaml:
with open(os.path.join(str(tmpdir), ADDITIONAL_TAGS_FILE),
'w') as f:
f.write('\n'.join(extra_tags))
kwargs.update(extra_args)
if from_container_yaml:
if extra_tags:
expected_primary -= exclude_for_override
(flexmock(utils).should_receive('get_repo_info').with_args(
TEST_GIT_URI, TEST_GIT_REF,
git_branch=TEST_GIT_BRANCH).and_return(
RepoInfo(additional_tags=AdditionalTagsConfig(
tags=extra_tags))))
else:
(flexmock(utils).should_receive('get_repo_info').with_args(
TEST_GIT_URI, TEST_GIT_REF,
git_branch=TEST_GIT_BRANCH).and_return(
RepoInfo(additional_tags=AdditionalTagsConfig(
dir_path=str(tmpdir)))))
user_params = BuildUserParams(INPUTS_PATH)
user_params.set_params(**kwargs)
build_json = PluginsConfiguration(user_params).render()
plugins = get_plugins_from_build_json(build_json)
assert get_plugin(plugins, 'postbuild_plugins', 'tag_from_config')
tag_suffixes = plugin_value_get(plugins, 'postbuild_plugins',
'tag_from_config', 'args',
'tag_suffixes')
assert len(tag_suffixes['unique']) == 1
if has_platform_tag:
unique_tag_suffix = tag_suffixes['unique'][0]
assert unique_tag_suffix.endswith('-x86_64') == has_platform_tag
assert len(tag_suffixes['primary']) == len(expected_primary)
assert set(tag_suffixes['primary']) == expected_primary
def get_plugins_from_buildrequest(self, build_request, template):
conf_kwargs = {
'build_from': 'image:test',
'reactor_config_map': 'reactor-config-map',
}
kwargs = {
'git_uri': TEST_GIT_URI,
'git_ref': TEST_GIT_REF,
'git_branch': TEST_GIT_BRANCH,
'user': '******',
'build_type': template.split('_')[0],
'build_conf': Configuration(conf_file=None, **conf_kwargs),
'base_image': 'test',
'name_label': 'test',
}
user_params = BuildUserParams()
user_params.set_params(**kwargs)
build_request.set_params(user_params)
return PluginsConfiguration(build_request.user_params).pt.template
def test_prod_custom_base_image(self, tmpdir):
kwargs = get_sample_prod_params()
kwargs['base_image'] = 'koji/image-build'
kwargs['yum_repourls'] = ["http://example.com/my.repo"]
self.mock_repo_info()
user_params = BuildUserParams(INPUTS_PATH)
user_params.set_params(**kwargs)
build_json = PluginsConfiguration(user_params).render()
plugins = get_plugins_from_build_json(build_json)
get_plugin(plugins, 'prebuild_plugins', 'pull_base_image')
add_filesystem_args = plugin_value_get(plugins, 'prebuild_plugins',
'add_filesystem', 'args')
assert add_filesystem_args['repos'] == kwargs['yum_repourls']
assert add_filesystem_args['from_task_id'] == kwargs[
'filesystem_koji_task_id']
assert add_filesystem_args['koji_target'] == kwargs['koji_target']
def test_render_prod_custom_site_plugin_override(self):
# Test to make sure that when we attempt to override a plugin's args,
# they are actually overridden in the JSON for the build_request
# after running build_request.render()
self.mock_repo_info()
sample_params = get_sample_prod_params()
base_user_params = BuildUserParams(INPUTS_PATH)
base_user_params.set_params(**sample_params)
base_plugins_conf = PluginsConfiguration(base_user_params)
base_build_json = base_plugins_conf.render()
base_plugins = get_plugins_from_build_json(base_build_json)
plugin_type = "exit_plugins"
plugin_name = "koji_import"
plugin_args = {"foo": "bar"}
for plugin_dict in base_plugins[plugin_type]:
if plugin_dict['name'] == plugin_name:
plugin_index = base_plugins[plugin_type].index(plugin_dict)
user_params = BuildUserParams(INPUTS_PATH)
user_params.set_params(**sample_params)
plugins_conf = PluginsConfiguration(user_params)
plugins_conf.pt.customize_conf['enable_plugins'].append({
"plugin_type":
plugin_type,
"plugin_name":
plugin_name,
"plugin_args":
plugin_args
})
build_json = plugins_conf.render()
plugins = get_plugins_from_build_json(build_json)
assert {
"name": plugin_name,
"args": plugin_args
} in plugins[plugin_type]
assert base_plugins[plugin_type][plugin_index]['name'] == \
plugin_name
assert plugins[plugin_type][plugin_index]['name'] == plugin_name
def test_render_all_code_paths(self, caplog):
# Alter the plugins configuration so that all code paths are exercised
sample_params = get_sample_prod_params()
sample_params['scratch'] = True
sample_params['parent_images_digests'] = True
user_params = BuildUserParams(INPUTS_PATH)
user_params.set_params(**sample_params)
plugins_conf = PluginsConfiguration(user_params)
plugins_conf.pt.customize_conf['disable_plugins'].append({
"plugin_type":
"postbuild_plugins",
"plugin_name":
"tag_from_config"
})
plugins_conf.pt.customize_conf['disable_plugins'].append({
"plugin_type":
"prebuild_plugins",
"plugin_name":
"add_labels_in_dockerfile"
})
plugins_conf.pt.customize_conf['disable_plugins'].append(
{
"bad_plugin_type": "postbuild_plugins",
"bad_plugin_name": "tag_from_config"
}, )
plugins_conf.pt.customize_conf['enable_plugins'].append(
{
"bad_plugin_type": "postbuild_plugins",
"bad_plugin_name": "tag_from_config"
}, )
build_json = plugins_conf.render()
plugins = get_plugins_from_build_json(build_json)
log_messages = [l.getMessage() for l in caplog.records]
assert 'no tag suffix placeholder' in log_messages
assert 'Invalid custom configuration found for disable_plugins' in log_messages
assert 'Invalid custom configuration found for enable_plugins' in log_messages
assert plugin_value_get(plugins, 'prebuild_plugins', 'pull_base_image',
'args', 'parent_images_digests')
def test_worker_custom_base_image(self, tmpdir):
self.mock_repo_info()
kwargs = get_sample_prod_params(BUILD_TYPE_WORKER)
kwargs['base_image'] = 'koji/image-build'
kwargs['yum_repourls'] = ["http://example.com/my.repo"]
kwargs.pop('platforms', None)
kwargs['platform'] = 'ppc64le'
user_params = BuildUserParams(INPUTS_PATH)
user_params.set_params(**kwargs)
build_json = PluginsConfiguration(user_params).render()
plugins = get_plugins_from_build_json(build_json)
get_plugin(plugins, 'prebuild_plugins', 'pull_base_image')
add_filesystem_args = plugin_value_get(plugins, 'prebuild_plugins',
'add_filesystem', 'args')
assert add_filesystem_args['repos'] == kwargs['yum_repourls']
assert add_filesystem_args['from_task_id'] == kwargs[
'filesystem_koji_task_id']
assert add_filesystem_args['architecture'] == kwargs['platform']
assert add_filesystem_args['koji_target'] == kwargs['koji_target']
def test_render_prod_custom_site_plugin_disable(self):
# Test to make sure that when we attempt to disable a plugin, it is
# actually disabled in the JSON for the build_request after running
# build_request.render()
sample_params = get_sample_prod_params()
user_params = BuildUserParams(INPUTS_PATH)
user_params.set_params(**sample_params)
plugins_conf = PluginsConfiguration(user_params)
plugin_type = "postbuild_plugins"
plugin_name = "tag_from_config"
plugins_conf.pt.customize_conf['disable_plugins'].append({
"plugin_type":
plugin_type,
"plugin_name":
plugin_name
})
build_json = plugins_conf.render()
plugins = get_plugins_from_build_json(build_json)
for plugin in plugins[plugin_type]:
if plugin['name'] == plugin_name:
assert False
class BuildRequestV2(BaseBuildRequest):
"""
Wraps logic for creating build inputs
"""
def __init__(self, build_json_store, outer_template=None, customize_conf=None):
"""
:param build_json_store: str, path to directory with JSON build files
:param outer_template: str, path to outer template JSON
:param customize_conf: str, path to customize configuration JSON
"""
super(BuildRequestV2, self).__init__(
build_json_store,
outer_template=outer_template or DEFAULT_OUTER_TEMPLATE
)
self._customize_conf_path = customize_conf or DEFAULT_CUSTOMIZE_CONF
self.build_json = None # rendered template
self._repo_info = None
self.isolated = None
self.is_auto = None
self.skip_build = None
self.base_image = None
self.scratch_build_node_selector = None
self.explicit_build_node_selector = None
self.auto_build_node_selector = None
self.isolated_build_node_selector = None
self.is_auto = None
# forward reference
self.platform_node_selector = None
self.user_params = BuildUserParams(self.build_json_store, customize_conf)
self.triggered_after_koji_task = None
# Override
def set_params(self, **kwargs):
"""
set parameters in the user parameters
these parameters are accepted:
:param git_uri: str, uri of the git repository for the source
:param git_ref: str, commit ID of the branch to be pulled
:param git_branch: str, branch name of the branch to be pulled
:param base_image: str, name of the parent image
:param name_label: str, label of the parent image
:param user: str, name of the user requesting the build
:param component: str, name of the component
:param release: str,
:param build_image: str,
:param build_imagestream: str,
:param build_from: str,
:param build_type: str, orchestrator or worker
:param platforms: list of str, platforms to build on
:param platform: str, platform
:param koji_target: str, koji tag with packages used to build the image
:param koji_task_id: str, koji ID
:param koji_parent_build: str,
:param koji_upload_dir: str, koji directory where the completed image will be uploaded
:param flatpak: if we should build a Flatpak OCI Image
:param flatpak_base_image: str, name of the Flatpack OCI Image
:param reactor_config_map: str, name of the config map containing the reactor environment
:param reactor_config_override: dict, data structure for reactor config to be injected as
an environment variable into a worker build;
when used, reactor_config_map is ignored.
:param yum_repourls: list of str, uris of the yum repos to pull from
:param signing_intent: bool, True to sign the resulting image
:param compose_ids: list of int, ODCS composes to use instead of generating new ones
:param filesystem_koji_task_id: int, Koji Task that created the base filesystem
:param platform_node_selector: dict, a nodeselector for a user_paramsific platform
:param scratch_build_node_selector: dict, a nodeselector for scratch builds
:param explicit_build_node_selector: dict, a nodeselector for explicit builds
:param auto_build_node_selector: dict, a nodeselector for auto builds
:param isolated_build_node_selector: dict, a nodeselector for isolated builds
:param operator_manifests_extract_platform: str, indicates which platform should upload
operator manifests to koji
:param parent_images_digests: dict, mapping image digests to names and platforms
:param worker_deadline: int, worker completion deadline in hours
:param orchestrator_deadline: int, orchestrator deadline in hours
:param skip_build: bool, if we should skip build and just set buildconfig for autorebuilds
:param triggered_after_koji_task: int, koji task ID from which was autorebuild triggered
"""
super(BuildRequestV2, self).set_params(**kwargs)
# When true, it indicates build was automatically started by
# OpenShift via a trigger, for instance ImageChangeTrigger
self.is_auto = kwargs.pop('is_auto', False)
self.triggered_after_koji_task = kwargs.get('triggered_after_koji_task')
self.skip_build = kwargs.pop('skip_build', False)
# An isolated build is meant to patch a certain release and not
# update transient tags in container registry
self.isolated = kwargs.get('isolated')
self.validate_build_variation()
self.base_image = kwargs.get('base_image')
self.platform_node_selector = kwargs.get('platform_node_selector', {})
self.scratch_build_node_selector = kwargs.get('scratch_build_node_selector', {})
self.explicit_build_node_selector = kwargs.get('explicit_build_node_selector', {})
self.auto_build_node_selector = kwargs.get('auto_build_node_selector', {})
self.isolated_build_node_selector = kwargs.get('isolated_build_node_selector', {})
logger.debug("now setting params '%s' for user_params", kwargs)
self.user_params.set_params(**kwargs)
# Override
@property
def inner_template(self):
raise RuntimeError('inner_template not supported in BuildRequestV2')
# Override
@property
def customize_conf(self):
raise RuntimeError('customize_conf not supported in BuildRequestV2')
# Override
@property
def dj(self):
raise RuntimeError('DockJson not supported in BuildRequestV2')
# Override
@property
def trigger_imagestreamtag(self):
return self.user_params.trigger_imagestreamtag.value
def _set_flatpak(self, reactor_config_data):
flatpak_key = 'flatpak'
flatpak_base_image_key = 'base_image'
if self.user_params.flatpak.value:
flatpack_base_image = (
reactor_config_data
.get(flatpak_key, {})
.get(flatpak_base_image_key, None)
)
if flatpack_base_image:
self.base_image = flatpack_base_image
self.user_params.base_image.value = flatpack_base_image
else:
raise OsbsValidationException("flatpak_base_image must be provided")
def set_required_secrets(self, reactor_config_data):
"""
Sets required secrets into build config
"""
req_secrets_key = 'required_secrets'
token_secrets_key = 'worker_token_secrets'
required_secrets = reactor_config_data.get(req_secrets_key, [])
token_secrets = reactor_config_data.get(token_secrets_key, [])
if self.user_params.build_type.value == BUILD_TYPE_ORCHESTRATOR:
required_secrets += token_secrets
self._set_required_secrets(required_secrets)
def set_data_from_reactor_config(self, reactor_config_data):
"""
Sets data from reactor config
"""
super(BuildRequestV2, self).set_data_from_reactor_config(reactor_config_data)
if not reactor_config_data:
if self.user_params.flatpak.value:
raise OsbsValidationException("flatpak_base_image must be provided")
else:
return
self._set_flatpak(reactor_config_data)
def render(self, validate=True):
super(BuildRequestV2, self).render(validate=validate)
self.template['spec']['source']['git']['uri'] = self.user_params.git_uri.value
self.template['spec']['source']['git']['ref'] = self.user_params.git_ref.value
if self.has_ist_trigger():
imagechange = self.template['spec']['triggers'][0]['imageChange']
imagechange['from']['name'] = self.user_params.trigger_imagestreamtag.value
# Set git-repo-name and git-full-name labels
repo_name = git_repo_humanish_part_from_uri(self.user_params.git_uri.value)
# Use the repo name to differentiate different repos, but include the full url as an
# optional filter.
self.set_label('git-repo-name', repo_name)
self.set_label('git-branch', self.user_params.git_branch.value)
self.set_label('git-full-repo', self.user_params.git_uri.value)
koji_task_id = self.user_params.koji_task_id.value
if koji_task_id is not None:
# keep also original task for all manual builds with task
# that way when delegated task for autorebuild will be used
# we will still keep track of it
if self.triggered_after_koji_task is None:
self.set_label('original-koji-task-id', str(koji_task_id))
# Set template.spec.strategy.customStrategy.env[] USER_PARAMS
# Adjust triggers for custom base image
if (self.template['spec'].get('triggers', []) and
(self.is_custom_base_image() or self.is_from_scratch_image())):
if self.is_custom_base_image():
logger.info("removing triggers from request because custom base image")
elif self.is_from_scratch_image():
logger.info('removing from request because FROM scratch image')
del self.template['spec']['triggers']
self.adjust_for_repo_info()
self.adjust_for_isolated(self.user_params.release)
self.render_node_selectors(self.user_params.build_type.value)
self._set_deadline()
# Log build json
# Return build json
self.build_json = self.template
logger.debug(self.build_json)
return self.build_json
def validate_build_variation(self):
variations = (self.scratch, self.is_auto, self.isolated)
if variations.count(True) > 1:
raise OsbsValidationException(
'Build variations are mutually exclusive. '
'Must set either scratch, is_auto, isolated, or none. ')
def set_repo_info(self, repo_info):
self._repo_info = repo_info
@property
def build_id(self):
return self.build_json['metadata']['name']
def has_ist_trigger(self):
"""Return True if this BuildConfig has ImageStreamTag trigger."""
triggers = self.template['spec'].get('triggers', [])
if not triggers:
return False
for trigger in triggers:
if trigger['type'] == 'ImageChange' and \
trigger['imageChange']['from']['kind'] == 'ImageStreamTag':
return True
return False
def render_node_selectors(self, build_type):
# for worker builds set nodeselectors
if build_type == BUILD_TYPE_WORKER:
# auto or explicit build selector
if self.is_auto:
self.template['spec']['nodeSelector'] = self.auto_build_node_selector
# scratch build nodeselector
elif self.scratch:
self.template['spec']['nodeSelector'] = self.scratch_build_node_selector
# isolated build nodeselector
elif self.isolated:
self.template['spec']['nodeSelector'] = self.isolated_build_node_selector
# explicit build nodeselector
else:
self.template['spec']['nodeSelector'] = self.explicit_build_node_selector
# platform nodeselector
if self.platform_node_selector:
self.template['spec']['nodeSelector'].update(self.platform_node_selector)
def _set_deadline(self):
if self.user_params.build_type.value == BUILD_TYPE_WORKER:
deadline_hours = self.user_params.worker_deadline.value
else:
deadline_hours = self.user_params.orchestrator_deadline.value
self.set_deadline(deadline_hours)
def adjust_for_repo_info(self):
if not self._repo_info:
logger.warning('repo info not set')
return
if not self._repo_info.configuration.is_autorebuild_enabled():
logger.info('autorebuild is disabled in repo configuration, removing triggers')
self.template['spec'].pop('triggers', None)
else:
labels = Labels(self._repo_info.dockerfile_parser.labels)
add_timestamp = self._repo_info.configuration.autorebuild.\
get('add_timestamp_to_release', False)
if add_timestamp:
logger.info('add_timestamp_to_release is enabled for autorebuilds,'
'skipping release check in dockerfile')
return
try:
labels.get_name_and_value(Labels.LABEL_TYPE_RELEASE)
except KeyError:
# As expected, release label not set in Dockerfile
pass
else:
raise RuntimeError('when autorebuild is enabled in repo configuration, '
'"release" label must not be set in Dockerfile')
def adjust_for_isolated(self, release):
if not self.isolated:
return
self.template['spec'].pop('triggers', None)
if not release.value:
raise OsbsValidationException('The release parameter is required for isolated builds.')
if not ISOLATED_RELEASE_FORMAT.match(release.value):
raise OsbsValidationException(
'For isolated builds, the release value must be in the format: {}'
.format(ISOLATED_RELEASE_FORMAT.pattern))
self.set_label('isolated', 'true')
self.set_label('isolated-release', release.value)
def is_custom_base_image(self):
"""
Returns whether or not this is a build from a custom base image
"""
return bool(re.match('^koji/image-build(:.*)?$',
self.base_image or ''))
def is_from_scratch_image(self):
"""
Returns whether or not this is a build `FROM scratch`
"""
return self.base_image == 'scratch'
class BuildRequestV2(BuildRequest):
"""
Wraps logic for creating build inputs
"""
def __init__(self,
build_json_store,
outer_template=None,
customize_conf=None):
"""
:param build_json_store: str, path to directory with JSON build files
:param outer_template: str, path to outer template JSON
:param customize_conf: str, path to customize configuration JSON
"""
super(BuildRequestV2, self).__init__(build_json_store=build_json_store,
outer_template=outer_template,
customize_conf=customize_conf)
self.spec = None
self.user_params = BuildUserParams(build_json_store, customize_conf)
self.osbs_api = None
# Override
def set_params(self, **kwargs):
"""
set parameters in the user parameters
these parameters are accepted:
:param git_uri: str, uri of the git repository for the source
:param git_ref: str, commit ID of the branch to be pulled
:param git_branch: str, branch name of the branch to be pulled
:param base_image: str, name of the parent image
:param name_label: str, label of the parent image
:param user: str, name of the user requesting the build
:param component: str, name of the component
:param release: str,
:param build_image: str,
:param build_imagestream: str,
:param build_from: str,
:param build_type: str, orchestrator or worker
:param platforms: list of str, platforms to build on
:param platform: str, platform
:param koji_target: str, koji tag with packages used to build the image
:param koji_task_id: str, koji ID
:param koji_parent_build: str,
:param koji_upload_dir: str, koji directory where the completed image will be uploaded
:param flatpak: if we should build a Flatpak OCI Image
:param flatpak_base_image: str, name of the Flatpack OCI Image
:param reactor_config_map: str, name of the config map containing the reactor environment
:param reactor_config_override: dict, data structure for reactor config to be injected as
an environment variable into a worker build;
when used, reactor_config_map is ignored.
:param yum_repourls: list of str, uris of the yum repos to pull from
:param signing_intent: bool, True to sign the resulting image
:param compose_ids: list of int, ODCS composes to use instead of generating new ones
:param filesystem_koji_task_id: int, Koji Task that created the base filesystem
:param platform_node_selector: dict, a nodeselector for a user_paramsific platform
:param scratch_build_node_selector: dict, a nodeselector for scratch builds
:param explicit_build_node_selector: dict, a nodeselector for explicit builds
:param auto_build_node_selector: dict, a nodeselector for auto builds
:param isolated_build_node_selector: dict, a nodeselector for isolated builds
"""
# Here we cater to the koji "scratch" build type, this will disable
# all plugins that might cause importing of data to koji
self.scratch = kwargs.get('scratch')
# When true, it indicates build was automatically started by
# OpenShift via a trigger, for instance ImageChangeTrigger
self.is_auto = kwargs.pop('is_auto', False)
# An isolated build is meant to patch a certain release and not
# update transient tags in container registry
self.isolated = kwargs.get('isolated')
self.osbs_api = kwargs.pop('osbs_api', None)
self.validate_build_variation()
self.base_image = kwargs.get('base_image')
self.platform_node_selector = kwargs.get('platform_node_selector', {})
self.scratch_build_node_selector = kwargs.get(
'scratch_build_node_selector', {})
self.explicit_build_node_selector = kwargs.get(
'explicit_build_node_selector', {})
self.auto_build_node_selector = kwargs.get('auto_build_node_selector',
{})
self.isolated_build_node_selector = kwargs.get(
'isolated_build_node_selector', {})
logger.debug("now setting params '%s' for user_params", kwargs)
self.user_params.set_params(**kwargs)
# Override
@property
def inner_template(self):
raise RuntimeError('inner_template not supported in BuildRequestV2')
# Override
@property
def customize_conf(self):
raise RuntimeError('customize_conf not supported in BuildRequestV2')
# Override
@property
def dj(self):
raise RuntimeError('DockJson not supported in BuildRequestV2')
# Override
@property
def trigger_imagestreamtag(self):
return self.user_params.trigger_imagestreamtag.value
def adjust_for_scratch(self):
"""
Scratch builds must not affect subsequent builds,
and should not be imported into Koji.
"""
if self.scratch:
self.template['spec'].pop('triggers', None)
self.set_label('scratch', 'true')
def set_reactor_config(self):
reactor_config_override = self.user_params.reactor_config_override.value
reactor_config_map = self.user_params.reactor_config_map.value
if not reactor_config_map and not reactor_config_override:
return
custom = self.template['spec']['strategy']['customStrategy']
if reactor_config_override:
reactor_config = {
'name': 'REACTOR_CONFIG',
'value': yaml.safe_dump(reactor_config_override)
}
elif reactor_config_map:
reactor_config = {
'name': 'REACTOR_CONFIG',
'valueFrom': {
'configMapKeyRef': {
'name': reactor_config_map,
'key': 'config.yaml'
}
}
}
custom['env'].append(reactor_config)
def set_required_secrets(self):
"""
Sets required secrets
"""
reactor_config_override = self.user_params.reactor_config_override.value
reactor_config_map = self.user_params.reactor_config_map.value
if not reactor_config_map and not reactor_config_override:
return
req_secrets_key = 'required_secrets'
token_secrets_key = 'worker_token_secrets'
required_secrets = []
token_secrets = []
if reactor_config_override:
data = reactor_config_override
required_secrets = data.get(req_secrets_key, [])
token_secrets = data.get(token_secrets_key, [])
elif reactor_config_map:
config_map = self.osbs_api.get_config_map(reactor_config_map)
required_secrets = config_map.get_data_by_key('config.yaml').get(
req_secrets_key, [])
token_secrets = config_map.get_data_by_key('config.yaml').get(
token_secrets_key, [])
if self.user_params.build_type.value == BUILD_TYPE_ORCHESTRATOR:
required_secrets += token_secrets
if not required_secrets:
return
secrets = self.template['spec']['strategy'][
'customStrategy'].setdefault('secrets', [])
existing = set(secret_mount['secretSource']['name']
for secret_mount in secrets)
required_secrets = set(required_secrets)
already_set = required_secrets.intersection(existing)
if already_set:
logger.debug("secrets %s are already set", already_set)
for secret in required_secrets - existing:
secret_path = os.path.join(SECRETS_PATH, secret)
logger.info("Configuring %s secret at %s", secret, secret_path)
secrets.append({
'secretSource': {
'name': secret,
},
'mountPath': secret_path,
})
def render(self, validate=True):
# the api is required for BuildRequestV2
# can't check that its an OSBS object because of the circular import
if not self.osbs_api:
raise OsbsValidationException
# Validate BuildUserParams
if validate:
self.user_params.validate()
self.render_name(self.user_params.name.value,
self.user_params.image_tag.value,
self.user_params.platform.value)
self.render_resource_limits()
self.template['spec']['source']['git'][
'uri'] = self.user_params.git_uri.value
self.template['spec']['source']['git'][
'ref'] = self.user_params.git_ref.value
self.template['spec']['output']['to'][
'name'] = self.user_params.image_tag.value
if self.has_ist_trigger():
imagechange = self.template['spec']['triggers'][0]['imageChange']
imagechange['from'][
'name'] = self.user_params.trigger_imagestreamtag.value
custom_strategy = self.template['spec']['strategy']['customStrategy']
if self.user_params.build_imagestream.value:
custom_strategy['from']['kind'] = 'ImageStreamTag'
custom_strategy['from'][
'name'] = self.user_params.build_imagestream.value
else:
custom_strategy['from'][
'name'] = self.user_params.build_image.value
# Set git-repo-name and git-full-name labels
repo_name = git_repo_humanish_part_from_uri(
self.user_params.git_uri.value)
# Use the repo name to differentiate different repos, but include the full url as an
# optional filter.
self.set_label('git-repo-name', repo_name)
self.set_label('git-branch', self.user_params.git_branch.value)
self.set_label('git-full-repo', self.user_params.git_uri.value)
koji_task_id = self.user_params.koji_task_id.value
if koji_task_id is not None:
self.set_label('koji-task-id', str(koji_task_id))
# Set template.spec.strategy.customStrategy.env[] USER_PARAMS
# Set required_secrets based on reactor_config
# Set worker_token_secrets based on reactor_config, if any
self.set_reactor_config()
self.set_required_secrets()
# Adjust triggers for custom base image
if self.template['spec'].get('triggers',
[]) and self.is_custom_base_image():
del self.template['spec']['triggers']
self.adjust_for_repo_info()
self.adjust_for_scratch()
self.adjust_for_isolated(self.user_params.release)
self.render_node_selectors(self.user_params.build_type.value)
# Set our environment variables
custom_strategy['env'].append({
'name': 'USER_PARAMS',
'value': self.user_params.to_json(),
})
# delete the ATOMIC_REACTOR_PLUGINS placeholder
for (index, env) in enumerate(custom_strategy['env']):
if env['name'] == 'ATOMIC_REACTOR_PLUGINS':
del custom_strategy['env'][index]
break
# Log build json
# Return build json
self.build_json = self.template
logger.debug(self.build_json)
return self.build_json
def test_v2_all_values_and_json(self):
# all values that BuildUserParams stores
param_kwargs = {
# 'arrangement_version': self.arrangement_version, # calculated value
'base_image': 'buildroot:old',
# 'build_from': 'buildroot:old', # only one of build_*
# 'build_json_dir': self.build_json_dir, # init paramater
'build_image': 'buildroot:latest',
# 'build_imagestream': 'buildroot:name_label',
'build_type': BUILD_TYPE_WORKER,
'component': TEST_COMPONENT,
'compose_ids': [1, 2],
'filesystem_koji_task_id': TEST_FILESYSTEM_KOJI_TASK_ID,
'flatpak': False,
# 'flatpak_base_image': self.flatpak_base_image, # not used with false flatpack
'git_branch': TEST_GIT_BRANCH,
'git_ref': TEST_GIT_REF,
'git_uri': TEST_GIT_URI,
'image_tag': 'user/None:none-0-0',
'imagestream_name': TEST_IMAGESTREAM,
'isolated': False,
'koji_parent_build': 'fedora-26-9',
'koji_target': 'tothepoint',
"orchestrator_deadline": 4,
'parent_images_digests': {
'registry.fedorahosted.org/fedora:29': {
'x86_64':
'registry.fedorahosted.org/fedora@sha256:8b96f2f9f88179a065738b2b37'
'35e386efb2534438c2a2f45b74358c0f344c81'
}
},
# 'name': self.name, # calculated value
'platform': 'x86_64',
'platforms': [
'x86_64',
],
'reactor_config_map': 'reactor-config-map',
'reactor_config_override': 'reactor-config-override',
'release': '29',
'scratch': False,
'signing_intent': False,
'task_id': TEST_KOJI_TASK_ID,
'trigger_imagestreamtag': 'base_image:latest',
'user': TEST_USER,
# 'yum_repourls': , # not used with compose_ids
"worker_deadline": 3,
}
# additional values that BuildUserParams requires but stores under different names
param_kwargs.update({
'name_label': 'name_label',
})
rand = '12345'
timestr = '20170731111111'
(flexmock(sys.modules['osbs.build.user_params']).should_receive(
'utcnow').once().and_return(
datetime.datetime.strptime(timestr, '%Y%m%d%H%M%S')))
(flexmock(random).should_receive('randrange').once().with_args(
10**(len(rand) - 1), 10**len(rand)).and_return(int(rand)))
build_json_dir = 'inputs'
spec = BuildUserParams(build_json_dir)
spec.set_params(**param_kwargs)
expected_json = {
"arrangement_version":
REACTOR_CONFIG_ARRANGEMENT_VERSION,
"base_image":
"buildroot:old",
"build_image":
"buildroot:latest",
"build_json_dir":
build_json_dir,
"build_type":
"worker",
"component":
TEST_COMPONENT,
"compose_ids": [1, 2],
"customize_conf":
"worker_customize.json",
"filesystem_koji_task_id":
TEST_FILESYSTEM_KOJI_TASK_ID,
"git_branch":
TEST_GIT_BRANCH,
"git_ref":
TEST_GIT_REF,
"git_uri":
TEST_GIT_URI,
"image_tag":
"{}/{}:tothepoint-{}-{}-x86_64".format(TEST_USER, TEST_COMPONENT,
rand, timestr),
"imagestream_name":
"name_label",
"kind":
"build_user_params",
"koji_parent_build":
"fedora-26-9",
"koji_target":
"tothepoint",
"name":
"path-master-cd1e4",
"orchestrator_deadline":
4,
'parent_images_digests': {
'registry.fedorahosted.org/fedora:29': {
'x86_64':
'registry.fedorahosted.org/fedora@sha256:8b96f2f9f88179a065738b2b37'
'35e386efb2534438c2a2f45b74358c0f344c81'
}
},
"platform":
"x86_64",
"platforms": ["x86_64"],
"reactor_config_map":
"reactor-config-map",
"reactor_config_override":
"reactor-config-override",
"release":
"29",
"trigger_imagestreamtag":
"buildroot:old",
"user":
TEST_USER,
"worker_deadline":
3,
}
assert spec.to_json() == json.dumps(expected_json, sort_keys=True)
spec2 = BuildUserParams()
spec2.from_json(spec.to_json())
assert spec2.to_json() == json.dumps(expected_json, sort_keys=True)
def test_render_orchestrate_build(self, tmpdir, platforms, build_from,
build_image, build_imagestream,
worker_build_image, additional_kwargs,
koji_parent_build, valid):
phase = 'buildstep_plugins'
plugin = 'orchestrate_build'
kwargs = {
'git_uri': TEST_GIT_URI,
'git_ref': TEST_GIT_REF,
'git_branch': TEST_GIT_BRANCH,
'user': "******",
'component': TEST_COMPONENT,
'base_image': 'fedora:latest',
'name_label': 'fedora/resultingimage',
'platforms': platforms,
'build_type': BUILD_TYPE_ORCHESTRATOR,
'reactor_config_map': 'reactor-config-map',
'reactor_config_override': 'reactor-config-override',
}
if build_image:
kwargs['build_image'] = build_image
if build_imagestream:
kwargs['build_imagestream'] = build_imagestream
if build_from:
kwargs['build_from'] = build_from
if koji_parent_build:
kwargs['koji_parent_build'] = koji_parent_build
kwargs.update(additional_kwargs)
self.mock_repo_info()
user_params = BuildUserParams(INPUTS_PATH)
if valid:
user_params.set_params(**kwargs)
build_json = PluginsConfiguration(user_params).render()
else:
with pytest.raises(OsbsValidationException):
user_params.set_params(**kwargs)
build_json = PluginsConfiguration(user_params).render()
return
plugins = get_plugins_from_build_json(build_json)
if platforms is None:
platforms = {}
assert plugin_value_get(plugins, phase, plugin, 'args',
'platforms') == platforms or {}
build_kwargs = plugin_value_get(plugins, phase, plugin, 'args',
'build_kwargs')
assert build_kwargs[
'arrangement_version'] == REACTOR_CONFIG_ARRANGEMENT_VERSION
assert build_kwargs.get('koji_parent_build') == koji_parent_build
assert build_kwargs.get('reactor_config_map') == 'reactor-config-map'
assert build_kwargs.get(
'reactor_config_override') == 'reactor-config-override'
worker_config_kwargs = plugin_value_get(plugins, phase, plugin, 'args',
'config_kwargs')
worker_config = Configuration(conf_file=None, **worker_config_kwargs)
if worker_build_image is KeyError:
assert 'build_image' not in worker_config_kwargs
assert not worker_config.get_build_image()
else:
assert worker_config_kwargs['build_image'] == worker_build_image
assert worker_config.get_build_image() == worker_build_image
if kwargs.get('flatpak', False):
assert kwargs.get('flatpak') is True
class BuildRequestV2(object):
"""
Wraps logic for creating build inputs
"""
def __init__(self,
build_json_store,
outer_template=None,
customize_conf=None):
"""
:param build_json_store: str, path to directory with JSON build files
:param outer_template: str, path to outer template JSON
:param customize_conf: str, path to customize configuration JSON
"""
self.build_json_store = build_json_store
self._outer_template_path = outer_template or DEFAULT_OUTER_TEMPLATE
self._customize_conf_path = customize_conf or DEFAULT_CUSTOMIZE_CONF
self.build_json = None # rendered template
self._template = None # template loaded from filesystem
self._resource_limits = None
self._openshift_required_version = parse_version('3.6.0')
self._repo_info = None
# For the koji "scratch" build type
self.scratch = None
self.isolated = None
self.is_auto = None
self.skip_build = None
self.base_image = None
self.scratch_build_node_selector = None
self.explicit_build_node_selector = None
self.auto_build_node_selector = None
self.isolated_build_node_selector = None
self.is_auto = None
# forward reference
self.platform_node_selector = None
self.user_params = BuildUserParams(build_json_store, customize_conf)
self.osbs_api = None
self.source_registry = None
self.organization = None
self.triggered_after_koji_task = None
# Override
def set_params(self, **kwargs):
"""
set parameters in the user parameters
these parameters are accepted:
:param git_uri: str, uri of the git repository for the source
:param git_ref: str, commit ID of the branch to be pulled
:param git_branch: str, branch name of the branch to be pulled
:param base_image: str, name of the parent image
:param name_label: str, label of the parent image
:param user: str, name of the user requesting the build
:param component: str, name of the component
:param release: str,
:param build_image: str,
:param build_imagestream: str,
:param build_from: str,
:param build_type: str, orchestrator or worker
:param platforms: list of str, platforms to build on
:param platform: str, platform
:param koji_target: str, koji tag with packages used to build the image
:param koji_task_id: str, koji ID
:param koji_parent_build: str,
:param koji_upload_dir: str, koji directory where the completed image will be uploaded
:param flatpak: if we should build a Flatpak OCI Image
:param flatpak_base_image: str, name of the Flatpack OCI Image
:param reactor_config_map: str, name of the config map containing the reactor environment
:param reactor_config_override: dict, data structure for reactor config to be injected as
an environment variable into a worker build;
when used, reactor_config_map is ignored.
:param yum_repourls: list of str, uris of the yum repos to pull from
:param signing_intent: bool, True to sign the resulting image
:param compose_ids: list of int, ODCS composes to use instead of generating new ones
:param filesystem_koji_task_id: int, Koji Task that created the base filesystem
:param platform_node_selector: dict, a nodeselector for a user_paramsific platform
:param scratch_build_node_selector: dict, a nodeselector for scratch builds
:param explicit_build_node_selector: dict, a nodeselector for explicit builds
:param auto_build_node_selector: dict, a nodeselector for auto builds
:param isolated_build_node_selector: dict, a nodeselector for isolated builds
:param operator_manifests_extract_platform: str, indicates which platform should upload
operator manifests to koji
:param parent_images_digests: dict, mapping image digests to names and platforms
:param worker_deadline: int, worker completion deadline in hours
:param orchestrator_deadline: int, orchestrator deadline in hours
:param skip_build: bool, if we should skip build and just set buildconfig for autorebuilds
:param triggered_after_koji_task: int, koji task ID from which was autorebuild triggered
"""
# Here we cater to the koji "scratch" build type, this will disable
# all plugins that might cause importing of data to koji
self.scratch = kwargs.get('scratch')
# When true, it indicates build was automatically started by
# OpenShift via a trigger, for instance ImageChangeTrigger
self.is_auto = kwargs.pop('is_auto', False)
self.triggered_after_koji_task = kwargs.get(
'triggered_after_koji_task')
self.skip_build = kwargs.pop('skip_build', False)
# An isolated build is meant to patch a certain release and not
# update transient tags in container registry
self.isolated = kwargs.get('isolated')
self.osbs_api = kwargs.pop('osbs_api', None)
self.validate_build_variation()
self.base_image = kwargs.get('base_image')
self.platform_node_selector = kwargs.get('platform_node_selector', {})
self.scratch_build_node_selector = kwargs.get(
'scratch_build_node_selector', {})
self.explicit_build_node_selector = kwargs.get(
'explicit_build_node_selector', {})
self.auto_build_node_selector = kwargs.get('auto_build_node_selector',
{})
self.isolated_build_node_selector = kwargs.get(
'isolated_build_node_selector', {})
logger.debug("now setting params '%s' for user_params", kwargs)
self.user_params.set_params(**kwargs)
# Override
@property
def inner_template(self):
raise RuntimeError('inner_template not supported in BuildRequestV2')
# Override
@property
def customize_conf(self):
raise RuntimeError('customize_conf not supported in BuildRequestV2')
# Override
@property
def dj(self):
raise RuntimeError('DockJson not supported in BuildRequestV2')
# Override
@property
def trigger_imagestreamtag(self):
return self.user_params.trigger_imagestreamtag.value
def adjust_for_scratch(self):
"""
Scratch builds must not affect subsequent builds,
and should not be imported into Koji.
"""
if self.scratch:
self.template['spec'].pop('triggers', None)
self.set_label('scratch', 'true')
def set_reactor_config(self):
reactor_config_override = self.user_params.reactor_config_override.value
reactor_config_map = self.user_params.reactor_config_map.value
if not reactor_config_map and not reactor_config_override:
return
custom = self.template['spec']['strategy']['customStrategy']
if reactor_config_override:
reactor_config = {
'name': 'REACTOR_CONFIG',
'value': yaml.safe_dump(reactor_config_override)
}
elif reactor_config_map:
reactor_config = {
'name': 'REACTOR_CONFIG',
'valueFrom': {
'configMapKeyRef': {
'name': reactor_config_map,
'key': 'config.yaml'
}
}
}
custom['env'].append(reactor_config)
def set_data_from_reactor_config(self):
"""
Sets data from reactor config
"""
reactor_config_override = self.user_params.reactor_config_override.value
reactor_config_map = self.user_params.reactor_config_map.value
data = None
if reactor_config_override:
data = reactor_config_override
elif reactor_config_map:
config_map = self.osbs_api.get_config_map(reactor_config_map)
data = config_map.get_data_by_key('config.yaml')
if not data:
if self.user_params.flatpak.value:
raise OsbsValidationException(
"flatpak_base_image must be provided")
else:
return
source_registry_key = 'source_registry'
registry_organization_key = 'registries_organization'
req_secrets_key = 'required_secrets'
token_secrets_key = 'worker_token_secrets'
flatpak_key = 'flatpak'
flatpak_base_image_key = 'base_image'
if source_registry_key in data:
self.source_registry = data[source_registry_key]
if registry_organization_key in data:
self.organization = data[registry_organization_key]
if self.user_params.flatpak.value:
flatpack_base_image = data.get(flatpak_key,
{}).get(flatpak_base_image_key,
None)
if flatpack_base_image:
self.base_image = flatpack_base_image
self.user_params.base_image.value = flatpack_base_image
else:
raise OsbsValidationException(
"flatpak_base_image must be provided")
required_secrets = data.get(req_secrets_key, [])
token_secrets = data.get(token_secrets_key, [])
self._set_required_secrets(required_secrets, token_secrets)
def _set_required_secrets(self, required_secrets, token_secrets):
"""
Sets required secrets
"""
if self.user_params.build_type.value == BUILD_TYPE_ORCHESTRATOR:
required_secrets += token_secrets
if not required_secrets:
return
secrets = self.template['spec']['strategy'][
'customStrategy'].setdefault('secrets', [])
existing = set(secret_mount['secretSource']['name']
for secret_mount in secrets)
required_secrets = set(required_secrets)
already_set = required_secrets.intersection(existing)
if already_set:
logger.debug("secrets %s are already set", already_set)
for secret in required_secrets - existing:
secret_path = os.path.join(SECRETS_PATH, secret)
logger.info("Configuring %s secret at %s", secret, secret_path)
secrets.append({
'secretSource': {
'name': secret,
},
'mountPath': secret_path,
})
def render(self, validate=True):
# the api is required for BuildRequestV2
# can't check that its an OSBS object because of the circular import
if not self.osbs_api:
raise OsbsValidationException
# Validate BuildUserParams
if validate:
self.user_params.validate()
self.render_name(self.user_params.name.value,
self.user_params.image_tag.value,
self.user_params.platform.value)
self.render_resource_limits()
self.template['spec']['source']['git'][
'uri'] = self.user_params.git_uri.value
self.template['spec']['source']['git'][
'ref'] = self.user_params.git_ref.value
self.template['spec']['output']['to'][
'name'] = self.user_params.image_tag.value
if self.has_ist_trigger():
imagechange = self.template['spec']['triggers'][0]['imageChange']
imagechange['from'][
'name'] = self.user_params.trigger_imagestreamtag.value
custom_strategy = self.template['spec']['strategy']['customStrategy']
if self.user_params.build_imagestream.value:
custom_strategy['from']['kind'] = 'ImageStreamTag'
custom_strategy['from'][
'name'] = self.user_params.build_imagestream.value
else:
custom_strategy['from'][
'name'] = self.user_params.build_image.value
# Set git-repo-name and git-full-name labels
repo_name = git_repo_humanish_part_from_uri(
self.user_params.git_uri.value)
# Use the repo name to differentiate different repos, but include the full url as an
# optional filter.
self.set_label('git-repo-name', repo_name)
self.set_label('git-branch', self.user_params.git_branch.value)
self.set_label('git-full-repo', self.user_params.git_uri.value)
koji_task_id = self.user_params.koji_task_id.value
if koji_task_id is not None:
self.set_label('koji-task-id', str(koji_task_id))
# keep also original task for all manual builds with task
# that way when delegated task for autorebuild will be used
# we will still keep track of it
if self.triggered_after_koji_task is None:
self.set_label('original-koji-task-id', str(koji_task_id))
# Set template.spec.strategy.customStrategy.env[] USER_PARAMS
# Set required_secrets based on reactor_config
# Set worker_token_secrets based on reactor_config, if any
self.set_reactor_config()
self.set_data_from_reactor_config()
# Adjust triggers for custom base image
if (self.template['spec'].get('triggers', []) and
(self.is_custom_base_image() or self.is_from_scratch_image())):
if self.is_custom_base_image():
logger.info(
"removing triggers from request because custom base image")
elif self.is_from_scratch_image():
logger.info('removing from request because FROM scratch image')
del self.template['spec']['triggers']
self.adjust_for_repo_info()
self.adjust_for_scratch()
self.adjust_for_isolated(self.user_params.release)
self.render_node_selectors(self.user_params.build_type.value)
self.set_deadline(self.user_params.build_type.value)
# Set our environment variables
custom_strategy['env'].append({
'name': 'USER_PARAMS',
'value': self.user_params.to_json(),
})
# delete the ATOMIC_REACTOR_PLUGINS placeholder
for (index, env) in enumerate(custom_strategy['env']):
if env['name'] == 'ATOMIC_REACTOR_PLUGINS':
del custom_strategy['env'][index]
break
# Log build json
# Return build json
self.build_json = self.template
logger.debug(self.build_json)
return self.build_json
def validate_build_variation(self):
variations = (self.scratch, self.is_auto, self.isolated)
if variations.count(True) > 1:
raise OsbsValidationException(
'Build variations are mutually exclusive. '
'Must set either scratch, is_auto, isolated, or none. ')
def set_resource_limits(self, cpu=None, memory=None, storage=None):
if self._resource_limits is None:
self._resource_limits = {}
if cpu is not None:
self._resource_limits['cpu'] = cpu
if memory is not None:
self._resource_limits['memory'] = memory
if storage is not None:
self._resource_limits['storage'] = storage
def set_openshift_required_version(self, openshift_required_version):
if openshift_required_version is not None:
self._openshift_required_version = openshift_required_version
def set_repo_info(self, repo_info):
self._repo_info = repo_info
@property
def build_id(self):
return self.build_json['metadata']['name']
@property
def template(self):
if self._template is None:
path = os.path.join(self.build_json_store,
self._outer_template_path)
logger.debug("loading template from path %s", path)
try:
with open(path, "r") as fp:
self._template = json.load(fp)
except (IOError, OSError) as ex:
raise OsbsException("Can't open template '%s': %s" %
(path, repr(ex)))
return self._template
def has_ist_trigger(self):
"""Return True if this BuildConfig has ImageStreamTag trigger."""
triggers = self.template['spec'].get('triggers', [])
if not triggers:
return False
for trigger in triggers:
if trigger['type'] == 'ImageChange' and \
trigger['imageChange']['from']['kind'] == 'ImageStreamTag':
return True
return False
def set_label(self, name, value):
if not value:
value = ''
self.template['metadata'].setdefault('labels', {})
value = sanitize_strings_for_openshift(value)
self.template['metadata']['labels'][name] = value
def render_name(self, name, image_tag, platform):
"""Sets the Build/BuildConfig object name"""
if self.scratch or self.isolated:
name = image_tag
# Platform name may contain characters not allowed by OpenShift.
if platform:
platform_suffix = '-{}'.format(platform)
if name.endswith(platform_suffix):
name = name[:-len(platform_suffix)]
_, salt, timestamp = name.rsplit('-', 2)
if self.scratch:
name = 'scratch-{}-{}'.format(salt, timestamp)
elif self.isolated:
name = 'isolated-{}-{}'.format(salt, timestamp)
# !IMPORTANT! can't be too long: https://github.com/openshift/origin/issues/733
self.template['metadata']['name'] = name
def render_node_selectors(self, build_type):
# for worker builds set nodeselectors
if build_type == BUILD_TYPE_WORKER:
# auto or explicit build selector
if self.is_auto:
self.template['spec'][
'nodeSelector'] = self.auto_build_node_selector
# scratch build nodeselector
elif self.scratch:
self.template['spec'][
'nodeSelector'] = self.scratch_build_node_selector
# isolated build nodeselector
elif self.isolated:
self.template['spec'][
'nodeSelector'] = self.isolated_build_node_selector
# explicit build nodeselector
else:
self.template['spec'][
'nodeSelector'] = self.explicit_build_node_selector
# platform nodeselector
if self.platform_node_selector:
self.template['spec']['nodeSelector'].update(
self.platform_node_selector)
def render_resource_limits(self):
if self._resource_limits is not None:
resources = self.template['spec'].get('resources', {})
limits = resources.get('limits', {})
limits.update(self._resource_limits)
resources['limits'] = limits
self.template['spec']['resources'] = resources
def set_deadline(self, build_type):
if build_type == BUILD_TYPE_WORKER:
deadline_hours = self.user_params.worker_deadline.value
else:
deadline_hours = self.user_params.orchestrator_deadline.value
if deadline_hours > 0:
deadline_seconds = deadline_hours * 3600
self.template['spec'][
'completionDeadlineSeconds'] = deadline_seconds
logger.info("setting completion_deadline to %s hours (%s seconds)",
deadline_hours, deadline_seconds)
def adjust_for_repo_info(self):
if not self._repo_info:
logger.warning('repo info not set')
return
if not self._repo_info.configuration.is_autorebuild_enabled():
logger.info(
'autorebuild is disabled in repo configuration, removing triggers'
)
self.template['spec'].pop('triggers', None)
else:
labels = Labels(self._repo_info.dockerfile_parser.labels)
add_timestamp = self._repo_info.configuration.autorebuild.\
get('add_timestamp_to_release', False)
if add_timestamp:
logger.info(
'add_timestamp_to_release is enabled for autorebuilds,'
'skipping release check in dockerfile')
return
try:
labels.get_name_and_value(Labels.LABEL_TYPE_RELEASE)
except KeyError:
# As expected, release label not set in Dockerfile
pass
else:
raise RuntimeError(
'when autorebuild is enabled in repo configuration, '
'"release" label must not be set in Dockerfile')
def adjust_for_isolated(self, release):
if not self.isolated:
return
self.template['spec'].pop('triggers', None)
if not release.value:
raise OsbsValidationException(
'The release parameter is required for isolated builds.')
if not ISOLATED_RELEASE_FORMAT.match(release.value):
raise OsbsValidationException(
'For isolated builds, the release value must be in the format: {}'
.format(ISOLATED_RELEASE_FORMAT.pattern))
self.set_label('isolated', 'true')
self.set_label('isolated-release', release.value)
def is_custom_base_image(self):
"""
Returns whether or not this is a build from a custom base image
"""
return bool(re.match('^koji/image-build(:.*)?$', self.base_image
or ''))
def is_from_scratch_image(self):
"""
Returns whether or not this is a build `FROM scratch`
"""
return self.base_image == 'scratch'