def __init__(self, build_json_store, outer_template=None, customize_conf=None): """ :param build_json_store: str, path to directory with JSON build files :param outer_template: str, path to outer template JSON :param customize_conf: str, path to customize configuration JSON """ self.build_json_store = build_json_store self._outer_template_path = outer_template or DEFAULT_OUTER_TEMPLATE self._customize_conf_path = customize_conf or DEFAULT_CUSTOMIZE_CONF self.build_json = None # rendered template self._template = None # template loaded from filesystem self._resource_limits = None self._openshift_required_version = parse_version('3.6.0') self._repo_info = None # For the koji "scratch" build type self.scratch = None self.isolated = None self.is_auto = None self.skip_build = None self.base_image = None self.scratch_build_node_selector = None self.explicit_build_node_selector = None self.auto_build_node_selector = None self.isolated_build_node_selector = None self.is_auto = None # forward reference self.platform_node_selector = None self.user_params = BuildUserParams(build_json_store, customize_conf) self.osbs_api = None self.source_registry = None self.organization = None self.triggered_after_koji_task = None
def __init__(self, build_json_store, outer_template=None, customize_conf=None): """ :param build_json_store: str, path to directory with JSON build files :param outer_template: str, path to outer template JSON :param customize_conf: str, path to customize configuration JSON """ super(BuildRequestV2, self).__init__( build_json_store, outer_template=outer_template or DEFAULT_OUTER_TEMPLATE ) self._customize_conf_path = customize_conf or DEFAULT_CUSTOMIZE_CONF self.build_json = None # rendered template self._repo_info = None self.isolated = None self.is_auto = None self.skip_build = None self.base_image = None self.scratch_build_node_selector = None self.explicit_build_node_selector = None self.auto_build_node_selector = None self.isolated_build_node_selector = None self.is_auto = None # forward reference self.platform_node_selector = None self.user_params = BuildUserParams(self.build_json_store, customize_conf) self.triggered_after_koji_task = None
def test_user_params_bad_none_flatpak(self, missing_arg): kwargs = self.get_minimal_kwargs() kwargs['flatpak'] = False kwargs.pop(missing_arg) with pytest.raises(OsbsValidationException): BuildUserParams.make_params(**kwargs)
def test_render_prod_custom_site_plugin_enable(self, tmpdir): # Test to make sure that when we attempt to enable a plugin, it is # actually enabled in the JSON for the build_request after running # build_request.render() self.mock_repo_info() sample_params = get_sample_prod_params() user_params = BuildUserParams(INPUTS_PATH) user_params.set_params(**sample_params) plugins_conf = PluginsConfiguration(user_params) plugin_type = "exit_plugins" plugin_name = "testing_exit_plugin" plugin_args = {"foo": "bar"} plugins_conf.pt.customize_conf['enable_plugins'].append({ "plugin_type": plugin_type, "plugin_name": plugin_name, "plugin_args": plugin_args }) build_json = plugins_conf.render() plugins = get_plugins_from_build_json(build_json) assert { "name": plugin_name, "args": plugin_args } in plugins[plugin_type]
def test_v2_compose_ids_and_signing_intent(self, signing_intent, compose_ids, yum_repourls, exc): kwargs = self.get_minimal_kwargs() if signing_intent: kwargs['signing_intent'] = signing_intent if compose_ids: kwargs['compose_ids'] = compose_ids if yum_repourls: kwargs['yum_repourls'] = yum_repourls kwargs.update({ 'git_uri': 'https://github.com/user/reponame.git', 'git_branch': 'master', }) if exc: with pytest.raises(exc): BuildUserParams.make_params(**kwargs) else: spec = BuildUserParams.make_params(**kwargs) if yum_repourls: assert spec.yum_repourls == yum_repourls if signing_intent: assert spec.signing_intent == signing_intent if compose_ids: assert spec.compose_ids == compose_ids
def test_v2_image_tag(self, rand, timestr, platform): kwargs = self.get_minimal_kwargs() kwargs.update({ 'component': 'foo', 'koji_target': 'tothepoint', }) if platform: kwargs['platform'] = platform (flexmock(sys.modules['osbs.build.user_params']).should_receive( 'utcnow').once().and_return( datetime.datetime.strptime(timestr, '%Y%m%d%H%M%S'))) (flexmock(random).should_receive('randrange').once().with_args( 10**(len(rand) - 1), 10**len(rand)).and_return(int(rand))) spec = BuildUserParams() spec.set_params(**kwargs) img_tag = '{user}/{component}:{koji_target}-{random_number}-{time_string}' if platform: img_tag += '-{platform}' img_tag = img_tag.format(random_number=rand, time_string=timestr, **kwargs) assert spec.image_tag.value == img_tag
def test_user_params_bad_compose_ids(self): kwargs = self.get_minimal_kwargs() kwargs['compose_ids'] = True spec = BuildUserParams() with pytest.raises(OsbsValidationException): spec.set_params(**kwargs)
def get_sample_user_params(build_type=BUILD_TYPE_ORCHESTRATOR, conf_args=None, extra_args=None): sample_params = get_sample_prod_params(build_type, conf_args, extra_args) user_params = BuildUserParams(INPUTS_PATH) user_params.set_params(**sample_params) return user_params
def test_from_json_continue(self): spec = BuildUserParams() expected_json = { "arrangement_version": REACTOR_CONFIG_ARRANGEMENT_VERSION, "base_image": "buildroot:old", "build_image": "buildroot:latest", "build_json_dir": "build_dir", "build_type": "worker", "component": TEST_COMPONENT, "compose_ids": [1, 2], "customize_conf": "prod_customize.json", "filesystem_koji_task_id": TEST_FILESYSTEM_KOJI_TASK_ID, "git_branch": TEST_GIT_BRANCH, "git_ref": TEST_GIT_REF, "git_uri": TEST_GIT_URI, "image_tag": "latest", "imagestream_name": "name_label", "koji_parent_build": "fedora-26-9", "koji_target": "tothepoint", "name": "path-master-cd1e4", "orchestrator_deadline": 4, "platform": "x86_64", "platforms": ["x86_64"], "reactor_config_map": "reactor-config-map", "reactor_config_override": "reactor-config-override", "release": "29", "trigger_imagestreamtag": "buildroot:old", "this is not a valid key": "this is not a valid field", "user": TEST_USER, "worker_deadline": 3, "triggered_after_koji_task": 12345, } spec.from_json(json.dumps(expected_json))
def test_render_tag_from_config(self, tmpdir, from_container_yaml, extra_args, has_platform_tag, extra_tags, primary_tags, floating_tags): kwargs = get_sample_prod_params(BUILD_TYPE_WORKER) kwargs.pop('platforms', None) kwargs.pop('platform', None) expected_primary = set(primary_tags) expected_floating = set(floating_tags) exclude_for_override = set(['latest', '{version}']) if from_container_yaml: expected_floating -= exclude_for_override extra_args['tags_from_yaml'] = from_container_yaml extra_args['additional_tags'] = extra_tags kwargs.update(extra_args) user_params = BuildUserParams(INPUTS_PATH) user_params.set_params(**kwargs) build_json = PluginsConfiguration(user_params).render() plugins = get_plugins_from_build_json(build_json) assert get_plugin(plugins, 'postbuild_plugins', 'tag_from_config') tag_suffixes = plugin_value_get(plugins, 'postbuild_plugins', 'tag_from_config', 'args', 'tag_suffixes') assert len(tag_suffixes['unique']) == 1 if has_platform_tag: unique_tag_suffix = tag_suffixes['unique'][0] assert unique_tag_suffix.endswith('-x86_64') == has_platform_tag assert len(tag_suffixes['primary']) == len(expected_primary) assert set(tag_suffixes['primary']) == expected_primary assert len(tag_suffixes['floating']) == len(expected_floating) assert set(tag_suffixes['floating']) == expected_floating
def test_user_params_bad_build_from(self): # does not have an "image:" prefix: conf_args = {'build_from': 'registry.example.com/buildroot'} kwargs = self.get_minimal_kwargs(conf_args=conf_args) with pytest.raises(OsbsValidationException) as e: BuildUserParams.make_params(**kwargs) assert 'build_from must be "source_type:source_value"' in str(e.value)
def get_sample_user_params(extra_args=None, build_type=BUILD_TYPE_ORCHESTRATOR): sample_params = get_sample_prod_params(build_type) if extra_args: sample_params.update(extra_args) user_params = BuildUserParams(INPUTS_PATH) user_params.set_params(**sample_params) return user_params
def test_user_params_bad_build_from(self): kwargs = self.get_minimal_kwargs() # does not have an "image:" prefix: kwargs['build_from'] = 'registry.example.com/buildroot' spec = BuildUserParams() with pytest.raises(OsbsValidationException) as e: spec.set_params(**kwargs) assert 'build_from must be "source_type:source_value"' in str(e.value)
def test_missing_build_opts(self): conf_args = { 'build_from': None, 'build_image': None, 'build_imagestream': None, } kwargs = self.get_minimal_kwargs(conf_args=conf_args) with pytest.raises(OsbsValidationException): BuildUserParams.make_params(**kwargs)
def test_v2_spec_name2(self): kwargs = self.get_minimal_kwargs() kwargs.update({ 'git_uri': TEST_GIT_URI, 'git_branch': TEST_GIT_BRANCH, }) spec = BuildUserParams() spec.set_params(**kwargs) assert spec.name.value.startswith('path-master')
def __init__(self, build_json_store, outer_template=None, customize_conf=None): """ :param build_json_store: str, path to directory with JSON build files :param outer_template: str, path to outer template JSON :param customize_conf: str, path to customize configuration JSON """ super(BuildRequestV2, self).__init__(build_json_store=build_json_store, outer_template=outer_template, customize_conf=customize_conf) self.spec = None self.user_params = BuildUserParams(build_json_store, customize_conf) self.osbs_api = None
def __init__(self, osbs_api, outer_template=None, customize_conf=None, user_params=None, build_json_store=None, repo_info=None): """ :param build_json_store: str, path to directory with JSON build files :param outer_template: str, path to outer template JSON :param customize_conf: str, path to customize configuration JSON :param repo_info: RepoInfo, git repo data for the build """ if user_params: assert isinstance(user_params, BuildUserParams) else: user_params = BuildUserParams() super(BuildRequestV2, self).__init__( osbs_api, outer_template=outer_template or DEFAULT_OUTER_TEMPLATE, user_params=user_params, build_json_store=build_json_store, ) self._customize_conf_path = customize_conf or DEFAULT_CUSTOMIZE_CONF self.build_json = None # rendered template self.repo_info = repo_info
def test_v2_spec_name2(self): git_args = {'git_branch': TEST_GIT_BRANCH} kwargs = self.get_minimal_kwargs(git_args=git_args) spec = BuildUserParams.make_params(**kwargs) assert spec.name.startswith('path-master')
def test_validate_missing_required(self): kwargs = self.get_minimal_kwargs() kwargs['user'] = None spec = BuildUserParams.make_params(**kwargs) with pytest.raises(OsbsValidationException): spec.validate()
def get_user_params(self, component=None, req_labels=None, **kwargs): req_labels = req_labels or {} user_component = component or req_labels[Labels.LABEL_TYPE_COMPONENT] return BuildUserParams.make_params( build_conf=self.os_conf, component=user_component, name_label=req_labels[Labels.LABEL_TYPE_NAME], **kwargs)
def test_render_tag_from_config(self, tmpdir, from_container_yaml, extra_args, has_platform_tag, extra_tags, primary_tags): kwargs = get_sample_prod_params(BUILD_TYPE_WORKER) kwargs.pop('platforms', None) kwargs.pop('platform', None) expected_primary = set(primary_tags) exclude_for_override = set(['latest', '{version}']) if extra_tags and not from_container_yaml: with open(os.path.join(str(tmpdir), ADDITIONAL_TAGS_FILE), 'w') as f: f.write('\n'.join(extra_tags)) kwargs.update(extra_args) if from_container_yaml: if extra_tags: expected_primary -= exclude_for_override (flexmock(utils).should_receive('get_repo_info').with_args( TEST_GIT_URI, TEST_GIT_REF, git_branch=TEST_GIT_BRANCH).and_return( RepoInfo(additional_tags=AdditionalTagsConfig( tags=extra_tags)))) else: (flexmock(utils).should_receive('get_repo_info').with_args( TEST_GIT_URI, TEST_GIT_REF, git_branch=TEST_GIT_BRANCH).and_return( RepoInfo(additional_tags=AdditionalTagsConfig( dir_path=str(tmpdir))))) user_params = BuildUserParams(INPUTS_PATH) user_params.set_params(**kwargs) build_json = PluginsConfiguration(user_params).render() plugins = get_plugins_from_build_json(build_json) assert get_plugin(plugins, 'postbuild_plugins', 'tag_from_config') tag_suffixes = plugin_value_get(plugins, 'postbuild_plugins', 'tag_from_config', 'args', 'tag_suffixes') assert len(tag_suffixes['unique']) == 1 if has_platform_tag: unique_tag_suffix = tag_suffixes['unique'][0] assert unique_tag_suffix.endswith('-x86_64') == has_platform_tag assert len(tag_suffixes['primary']) == len(expected_primary) assert set(tag_suffixes['primary']) == expected_primary
def test_make_params_keeps_defaults(self): kwargs = self.get_minimal_kwargs() params = BuildUserParams.make_params(**kwargs) assert params.arrangement_version == REACTOR_CONFIG_ARRANGEMENT_VERSION assert params.buildroot_is_imagestream is False assert params.customize_conf == DEFAULT_CUSTOMIZE_CONF # assert params.git_ref == 'master' # set from repo_info assert params.include_koji_repo is False
def get_plugins_from_buildrequest(self, build_request, template): conf_kwargs = { 'build_from': 'image:test', 'reactor_config_map': 'reactor-config-map', } kwargs = { 'git_uri': TEST_GIT_URI, 'git_ref': TEST_GIT_REF, 'git_branch': TEST_GIT_BRANCH, 'user': '******', 'build_type': template.split('_')[0], 'build_conf': Configuration(conf_file=None, **conf_kwargs), 'base_image': 'test', 'name_label': 'test', } user_params = BuildUserParams() user_params.set_params(**kwargs) build_request.set_params(user_params) return PluginsConfiguration(build_request.user_params).pt.template
def test_prod_custom_base_image(self, tmpdir): kwargs = get_sample_prod_params() kwargs['base_image'] = 'koji/image-build' kwargs['yum_repourls'] = ["http://example.com/my.repo"] self.mock_repo_info() user_params = BuildUserParams(INPUTS_PATH) user_params.set_params(**kwargs) build_json = PluginsConfiguration(user_params).render() plugins = get_plugins_from_build_json(build_json) get_plugin(plugins, 'prebuild_plugins', 'pull_base_image') add_filesystem_args = plugin_value_get(plugins, 'prebuild_plugins', 'add_filesystem', 'args') assert add_filesystem_args['repos'] == kwargs['yum_repourls'] assert add_filesystem_args['from_task_id'] == kwargs[ 'filesystem_koji_task_id'] assert add_filesystem_args['koji_target'] == kwargs['koji_target']
def test_render_all_code_paths(self, caplog): # Alter the plugins configuration so that all code paths are exercised sample_params = get_sample_prod_params() sample_params['scratch'] = True sample_params['parent_images_digests'] = True user_params = BuildUserParams(INPUTS_PATH) user_params.set_params(**sample_params) plugins_conf = PluginsConfiguration(user_params) plugins_conf.pt.customize_conf['disable_plugins'].append({ "plugin_type": "postbuild_plugins", "plugin_name": "tag_from_config" }) plugins_conf.pt.customize_conf['disable_plugins'].append({ "plugin_type": "prebuild_plugins", "plugin_name": "add_labels_in_dockerfile" }) plugins_conf.pt.customize_conf['disable_plugins'].append( { "bad_plugin_type": "postbuild_plugins", "bad_plugin_name": "tag_from_config" }, ) plugins_conf.pt.customize_conf['enable_plugins'].append( { "bad_plugin_type": "postbuild_plugins", "bad_plugin_name": "tag_from_config" }, ) build_json = plugins_conf.render() plugins = get_plugins_from_build_json(build_json) log_messages = [l.getMessage() for l in caplog.records] assert 'no tag suffix placeholder' in log_messages assert 'Invalid custom configuration found for disable_plugins' in log_messages assert 'Invalid custom configuration found for enable_plugins' in log_messages assert plugin_value_get(plugins, 'prebuild_plugins', 'pull_base_image', 'args', 'parent_images_digests')
def test_user_params_bad_json(self): required_json = json.dumps( { 'git_ref': 'master', 'kind': 'build_user_params', }, sort_keys=True) spec = BuildUserParams() spec.from_json(None) assert spec.to_json() == required_json spec.from_json("") assert spec.to_json() == required_json assert '{}'.format(spec)
def test_render_prod_custom_site_plugin_override(self): # Test to make sure that when we attempt to override a plugin's args, # they are actually overridden in the JSON for the build_request # after running build_request.render() self.mock_repo_info() sample_params = get_sample_prod_params() base_user_params = BuildUserParams.make_params(**sample_params) base_plugins_conf = PluginsConfiguration(base_user_params) base_build_json = base_plugins_conf.render() base_plugins = get_plugins_from_build_json(base_build_json) plugin_type = "exit_plugins" plugin_name = "koji_import" plugin_args = {"foo": "bar"} for plugin_dict in base_plugins[plugin_type]: if plugin_dict['name'] == plugin_name: plugin_index = base_plugins[plugin_type].index(plugin_dict) user_params = BuildUserParams.make_params(**sample_params) plugins_conf = PluginsConfiguration(user_params) plugins_conf.pt.customize_conf['enable_plugins'].append({ "plugin_type": plugin_type, "plugin_name": plugin_name, "plugin_args": plugin_args }) build_json = plugins_conf.render() plugins = get_plugins_from_build_json(build_json) assert { "name": plugin_name, "args": plugin_args } in plugins[plugin_type] assert base_plugins[plugin_type][plugin_index]['name'] == \ plugin_name assert plugins[plugin_type][plugin_index]['name'] == plugin_name
def test_worker_custom_base_image(self, tmpdir): self.mock_repo_info() kwargs = get_sample_prod_params(BUILD_TYPE_WORKER) kwargs['base_image'] = 'koji/image-build' kwargs['yum_repourls'] = ["http://example.com/my.repo"] kwargs.pop('platforms', None) kwargs['platform'] = 'ppc64le' user_params = BuildUserParams(INPUTS_PATH) user_params.set_params(**kwargs) build_json = PluginsConfiguration(user_params).render() plugins = get_plugins_from_build_json(build_json) get_plugin(plugins, 'prebuild_plugins', 'pull_base_image') add_filesystem_args = plugin_value_get(plugins, 'prebuild_plugins', 'add_filesystem', 'args') assert add_filesystem_args['repos'] == kwargs['yum_repourls'] assert add_filesystem_args['from_task_id'] == kwargs[ 'filesystem_koji_task_id'] assert add_filesystem_args['architecture'] == kwargs['platform'] assert add_filesystem_args['koji_target'] == kwargs['koji_target']
def test_user_params_bad_json(self): required_json = json.dumps( { 'arrangement_version': 6, 'customize_conf': 'worker_customize.json', 'git_ref': 'master' }, sort_keys=True) spec = BuildUserParams() spec.from_json(None) assert spec.to_json() == required_json spec.from_json("") assert spec.to_json() == required_json assert '{}'.format(spec)
def test_render_prod_custom_site_plugin_disable(self): # Test to make sure that when we attempt to disable a plugin, it is # actually disabled in the JSON for the build_request after running # build_request.render() sample_params = get_sample_prod_params() user_params = BuildUserParams(INPUTS_PATH) user_params.set_params(**sample_params) plugins_conf = PluginsConfiguration(user_params) plugin_type = "postbuild_plugins" plugin_name = "tag_from_config" plugins_conf.pt.customize_conf['disable_plugins'].append({ "plugin_type": plugin_type, "plugin_name": plugin_name }) build_json = plugins_conf.render() plugins = get_plugins_from_build_json(build_json) for plugin in plugins[plugin_type]: if plugin['name'] == plugin_name: assert False