def test_add_flatpak_labels(workflow, source_dir, labels, expected): if labels is not None: data = {'flatpak': {'labels': labels}} else: data = {} container_yaml = yaml.dump(data) mock_workflow(workflow, source_dir, container_yaml) runner = (MockEnv(workflow).for_plugin( AddFlatpakLabelsPlugin.key).create_runner()) runner.run() def check_last_line_in_df(build_dir): lines = build_dir.dockerfile_path.read_text().splitlines() if expected: assert lines[-1] == expected else: assert lines[-1] == "CMD sleep 1000" workflow.build_dir.for_each_platform(check_last_line_in_df)
def test_invalid_repourl(workflow, build_dir): """Plugin should raise RuntimeError with repo details when invalid URL is used """ wrong_repo_url = "http://example.com/nope/repo" workflow = prepare(workflow, build_dir, yum_repourls={'x86_64': [wrong_repo_url]}) runner = (MockEnv(workflow).for_plugin(InjectYumReposPlugin.key, args={ 'inject_proxy': None }).create_runner()) (flexmock(YumRepo).should_receive('fetch').and_raise( Exception, 'Oh noes, repo is not working!')) with pytest.raises(PluginFailedException) as exc: runner.run() msg = "Failed to fetch yum repo {repo}".format(repo=wrong_repo_url) assert msg in str(exc.value)
def mock_env( tmpdir, docker_tasker, platform='x86_64', base_layers=0, icm_url=CACHITO_ICM_URL, r_c_m_override=None, ): inspection_data = { INSPECT_ROOTFS: { INSPECT_ROOTFS_LAYERS: list(range(base_layers)) } } if r_c_m_override is None: r_c_m = { 'version': 1, 'cachito': { 'api_url': CACHITO_URL, 'auth': { 'ssl_certs_dir': str(tmpdir), }, }, } else: r_c_m = r_c_m_override env = (MockEnv().for_plugin( 'prebuild', AddImageContentManifestPlugin.key, { 'remote_source_icm_url': icm_url }).set_reactor_config(r_c_m).make_orchestrator()) tmpdir.join('cert').write('') env.workflow.builder.set_inspection_data(inspection_data) env.workflow.user_params['platform'] = platform # Ensure to succeed in reading the content_sets.yml env.workflow.source.get_build_file_path = lambda: (str(tmpdir), str(tmpdir) ) return env.create_runner(docker_tasker)
def create_runner(workflow, ssl_certs=False, principal=None, keytab=None, poll_interval=0.01, proxy_user=None, use_args=True, koji_target='koji-target'): add_koji_map_in_workflow(workflow, hub_url='', ssl_certs_dir='/' if ssl_certs else None, krb_keytab=keytab, krb_principal=principal, proxyuser=proxy_user) args = {'target': koji_target} if poll_interval is not None: args['poll_interval'] = poll_interval if not use_args: args = {'target': koji_target} return (MockEnv(workflow).for_plugin(KojiTagBuildPlugin.key, args=args).create_runner())
def test_get_built_images_multiple_manifest_types(workflow): MockEnv(workflow).set_check_platforms_result(["x86_64"]) workflow.data.tag_conf.add_unique_image(UNIQUE_IMAGE) flexmock(ManifestUtil).should_receive("__init__") # and do nothing, this test doesn't use it ( flexmock(RegistryClient) .should_receive("get_manifest_digests") .with_args(ImageName.parse(f"{UNIQUE_IMAGE}-x86_64"), versions=("v2", "oci")) .and_return(ManifestDigest({"v2": make_digest("foo"), "oci": make_digest("bar")})) ) plugin = GroupManifestsPlugin(workflow) session = RegistrySession(REGISTRY_V2) expect_error = ( f"Expected to find a single manifest digest for {UNIQUE_IMAGE}-x86_64, " "but found multiple: {'v2': .*, 'oci': .*}" ) with pytest.raises(RuntimeError, match=expect_error): plugin.get_built_images(session)
def mock_env(workflow, r_c_m=None, domains_override=None): if not r_c_m: r_c_m = { 'version': 1, 'koji': { 'hub_url': KOJI_HUB, 'root_url': KOJI_ROOT, 'auth': {} }, 'pnc': { 'base_api_url': PNC_ROOT, 'get_artifact_path': 'artifacts/{}', }, } if domains_override: r_c_m.setdefault('artifacts_allowed_domains', domains_override) env = (MockEnv(workflow).for_plugin( FetchMavenArtifactsPlugin.key).set_reactor_config(r_c_m)) workflow.build_dir.init_build_dirs(["aarch64", "x86_64"], workflow.source) return env
def test_failed_build(workflow, source_dir, caplog, user_params): """ Test if proper error state is returned when build inside build container failed """ (flexmock(subprocess).should_receive('check_output').and_raise( subprocess.CalledProcessError(1, 'cmd', output='stub stdout'))) some_dir = workflow.build_dir.path / 'some_dir' some_dir.mkdir() fetch_sources_result = { 'image_sources_dir': str(some_dir), 'remote_sources_dir': str(some_dir), 'maven_sources_dir': str(some_dir), } runner = (MockEnv(workflow).for_plugin( SourceContainerPlugin.key).set_plugin_result( FetchSourcesPlugin.key, fetch_sources_result).create_runner()) with pytest.raises(PluginFailedException, match="BSI utility failed"): runner.run() assert 'BSI failed with output:' in caplog.text assert 'stub stdout' in caplog.text
def test_skip_plugin(workflow, caplog): reactor_config = { 'registry': { 'url': LOCALHOST_REGISTRY, 'insecure': True, 'auth': {}, }, } runner = (MockEnv(workflow).for_plugin( TagAndPushPlugin.key).set_reactor_config( reactor_config).create_runner()) results = runner.run()[TagAndPushPlugin.key] assert 'not a flatpak or source build, skipping plugin' in caplog.text assert 'pushed_images' in results assert 'repositories' in results assert not results['pushed_images'] repositories = results['repositories'] assert 'primary' in repositories assert 'unique' in repositories assert 'floating' in repositories assert not repositories['primary'] assert not repositories['unique'] assert not repositories['floating']
def test_rpmqa_plugin_success(caplog, workflow, build_dir, base_from_scratch, ignore_autogenerated): (flexmock(retries).should_receive("run_cmd").replace_with( mock_oc_image_extract)) (flexmock(_RandomNameSequence).should_receive("__next__").times( 4).and_return('abcdef12')) (flexmock(subprocess).should_receive("check_output").times(4).and_return( "\n".join(PACKAGE_LIST_WITH_AUTOGENERATED))) platforms = ['x86_64', 's390x', 'ppc64le', 'aarch64'] workflow.build_dir.init_build_dirs(platforms, workflow.source) workflow.data.tag_conf.add_unique_image(f'registry.com/{TEST_IMAGE}') (MockEnv(workflow).for_plugin(RPMqaPlugin.key).set_plugin_args({ "ignore_autogenerated_gpg_keys": ignore_autogenerated["ignore"] }).set_dockerfile_images( ['scratch'] if base_from_scratch else []).create_runner().run()) for platform in platforms: assert workflow.data.image_components[platform] == parse_rpm_output( ignore_autogenerated["package_list"])
def test_include_koji(workflow, build_dir, caplog, parent_images, base_from_scratch, target, expect_success, root, koji_ssl_certs, expected_string, proxy): prepare(workflow, build_dir, include_koji_repo=True, koji_ssl_certs=koji_ssl_certs, yum_proxy=proxy, root_url=root) dockerfile_images = [] if parent_images: dockerfile_images.append('parent_image:latest') if base_from_scratch: dockerfile_images.append('scratch') workflow.data.dockerfile_images = DockerfileImages(dockerfile_images) runner = (MockEnv(workflow).for_plugin(InjectYumReposPlugin.key, args={ 'target': target }).create_runner()) if target == KOJI_TARGET_NO_INFO and parent_images: with pytest.raises(PluginFailedException) as exc: runner.run() assert f"Provided target '{target}' doesn't exist!" in str(exc.value) assert f"provided target '{target}' doesn't exist" in caplog.text else: runner.run() if not parent_images: log_msg = "Skipping plugin, from scratch stage(s) can't add repos" assert log_msg in caplog.text return if not expect_success: return if proxy: assert f"Setting yum proxy to {proxy}" in caplog.text if koji_ssl_certs: for file_path, expected in [(workflow.koji_session.cert_path, 'cert'), (workflow.koji_session.serverca_path, 'serverca')]: assert os.path.isfile(file_path) with open(file_path, 'r') as fd: assert fd.read() == expected repos_path = workflow.build_dir.any_platform.path / RELATIVE_REPOS_PATH repofile = 'target-?????.repo' files = os.listdir(repos_path) assert len(files) == 1 assert fnmatch(next(iter(files)), repofile) with open(repos_path / files[0], 'r') as f: content = f.read() assert content.startswith("[atomic-reactor-koji-plugin-target]\n") assert "gpgcheck=0\n" in content assert "enabled=1\n" in content assert "name=atomic-reactor-koji-plugin-target\n" in content assert "baseurl=%s/repos/tag/2/$basearch\n" % root in content if proxy: assert "proxy=%s" % proxy in content if expected_string: assert expected_string in content
def test_flatpak_create_oci(workflow, config_name, flatpak_metadata, breakage): # Check that we actually have flatpak available have_flatpak = False try: output = subprocess.check_output(['flatpak', '--version'], universal_newlines=True) m = re.search(r'(\d+)\.(\d+)\.(\d+)', output) if m and (int(m.group(1)), int(m.group(2)), int( m.group(3))) >= (0, 9, 7): have_flatpak = True except (subprocess.CalledProcessError, OSError): pytest.skip(msg='flatpak not available') if not have_flatpak: return # Check if we have skopeo try: subprocess.check_output(['skopeo', '--version']) except (subprocess.CalledProcessError, OSError): pytest.skip(msg='skopeo not available') config = CONFIGS[config_name] runner = (MockEnv(workflow).for_plugin( FlatpakCreateOciPlugin.key).set_reactor_config({ 'flatpak': { 'metadata': flatpak_metadata } }).create_runner()) platforms = ['x86_64', 'aarch64', 's390x', 'ppc64le'] workflow.user_params['flatpak'] = True write_docker_file(config, workflow.source.path) workflow.build_dir.init_build_dirs(platforms, workflow.source) mock_extract_filesystem_call = functools.partial(mock_extract_filesystem, config) (flexmock(ImageUtil).should_receive( 'extract_filesystem_layer').replace_with(mock_extract_filesystem_call)) for image_platform in platforms: image_path = workflow.build_dir.platform_dir( image_platform).exported_squashed_image os.mknod(image_path) if breakage == 'no_runtime': # Copy the parts of the config we are going to change config = dict(config) config['modules'] = dict(config['modules']) config['modules']['eog'] = dict(config['modules']['eog']) module_config = config['modules']['eog'] mmd = Modulemd.ModuleStream.read_string(module_config['metadata'], strict=True) mmd.clear_dependencies() mmd.add_dependencies(Modulemd.Dependencies()) mmd_index = Modulemd.ModuleIndex.new() mmd_index.add_module_stream(mmd) module_config['metadata'] = mmd_index.dump_to_string() expected_exception = 'Failed to identify runtime module' elif breakage == 'run_cmd_failed': for platform in platforms: platform_dir = workflow.build_dir.platform_dir(platform) skopeo_cmd = [ 'skopeo', 'copy', f'oci:{str(platform_dir.path)}' f'/flatpak-oci-image:app/org.gnome.eog/x86_64/stable', '--format=v2s2', f'docker-archive:{platform_dir.exported_squashed_image}' ] flexmock(retries).should_receive('run_cmd').with_args( skopeo_cmd).and_raise( subprocess.CalledProcessError( 1, ["skopeo", "..."], output=b'something went wrong')) expected_exception = 'skopeo copy failed with output:' else: assert breakage is None expected_exception = None setup_flatpak_composes(workflow) source = setup_flatpak_source_info(config) (flexmock(FlatpakUtil).should_receive( 'get_flatpak_source_info').and_return(source)) if expected_exception: with pytest.raises(PluginFailedException, match=expected_exception): runner.run() else: builder = FlatpakBuilder(source, workflow.build_dir.any_platform.path, 'var/tmp/flatpak-build', parse_manifest=parse_rpm_output, flatpak_metadata=FLATPAK_METADATA_ANNOTATIONS) with NamedTemporaryFile(dir=workflow.build_dir.any_platform.path) as f: f.write( config['filesystem_contents']['/var/tmp/flatpak-build.rpm_qf']) f.flush() expected_components = builder.get_components(f.name) results = runner.run() x86_64_results = results[FlatpakCreateOciPlugin.key][platforms[0]] dir_metadata = x86_64_results assert workflow.data.image_components[ platforms[0]] == expected_components assert dir_metadata['type'] == IMAGE_TYPE_OCI for image_platform in platforms: image_path = workflow.build_dir.platform_dir( image_platform).exported_squashed_image assert image_path.exists() # Check that the correct labels and annotations were written labels, annotations = load_labels_and_annotations(dir_metadata) if config_name == 'app': assert labels['name'] == 'eog' assert labels['com.redhat.component'] == 'eog' assert labels['version'] == 'f28' assert labels['release'] == '20170629213428' elif config_name == 'runtime': # runtime assert labels['name'] == 'flatpak-runtime' assert labels[ 'com.redhat.component'] == 'flatpak-runtime-container' assert labels['version'] == 'f28' assert labels['release'] == '20170701152209' else: assert labels['name'] == 'flatpak-sdk' assert labels['com.redhat.component'] == 'flatpak-sdk-container' assert labels['version'] == 'f28' assert labels['release'] == '20170701152209' if flatpak_metadata == 'annotations': assert annotations.get( 'org.flatpak.ref') == config['expected_ref_name'] assert 'org.flatpak.ref' not in labels elif flatpak_metadata == 'labels': assert 'org.flatpak.ref' not in annotations assert labels.get('org.flatpak.ref') == config['expected_ref_name'] elif flatpak_metadata == 'both': assert annotations.get( 'org.flatpak.ref') == config['expected_ref_name'] assert labels.get('org.flatpak.ref') == config['expected_ref_name'] # Check that the expected files ended up in the flatpak # Flatpak versions before 1.6 require annotations to be present, and Flatpak # versions 1.6 and later require labels to be present. Skip the remaining # checks unless we have both annotations and labels. if flatpak_metadata != 'both': return inspector = DefaultInspector(str(workflow.build_dir.any_platform.path), dir_metadata) files = inspector.list_files() assert sorted(files) == config['expected_contents'] metadata_lines = inspector.cat_file('/metadata').split('\n') assert any( re.match(r'runtime=org.fedoraproject.Platform/.*/f28$', line) for line in metadata_lines) assert any( re.match(r'sdk=org.fedoraproject.Sdk/.*/f28$', line) for line in metadata_lines) if config_name == 'app': # Check that the desktop file was rewritten output = inspector.cat_file( '/export/share/applications/org.gnome.eog.desktop') lines = output.split('\n') assert 'Icon=org.gnome.eog' in lines assert 'name=org.gnome.eog' in metadata_lines assert 'tags=Viewer' in metadata_lines assert 'command=eog2' in metadata_lines elif config_name == 'runtime': # runtime # Check that permissions have been normalized assert inspector.get_file_perms('/files/etc/shadow') == '-00644' assert inspector.get_file_perms('/files/bin/mount') == '-00755' assert inspector.get_file_perms('/files/share/foo') == 'd00755' assert 'name=org.fedoraproject.Platform' in metadata_lines else: # SDK assert 'name=org.fedoraproject.Sdk' in metadata_lines
def mock_env( workflow, *, df_content: str, base_inspect: ImageInspectionData, # (dict[str, str] | str | None), but some tests intentionally pass the wrong type labels_plugin_arg: Optional[Any] = None, labels_reactor_conf: Optional[Dict[str, str]] = None, # list[list[str]], but same as above eq_conf: Optional[Any] = None, dont_overwrite: Optional[List[str]] = None, dont_overwrite_if_in_dockerfile: Optional[List[str]] = None, aliases: Optional[Dict[str, str]] = None, auto_labels: Optional[List[str]] = None, ) -> MockEnv: env = MockEnv(workflow).for_plugin(AddLabelsPlugin.key) args = { 'labels': labels_plugin_arg, 'dont_overwrite': dont_overwrite, 'auto_labels': auto_labels, 'aliases': aliases, 'equal_labels': eq_conf, } if dont_overwrite_if_in_dockerfile is not None: args[ 'dont_overwrite_if_in_dockerfile'] = dont_overwrite_if_in_dockerfile env.set_plugin_args(args) df_path = Path(workflow.source.path) / "Dockerfile" df_path.write_text(df_content) def get_build(pipeline_run_name): start_time_json = {'status': {'startTime': TIME}} return start_time_json flexmock(OSBS, get_build=get_build) config_kwargs = { 'namespace': workflow.namespace, 'verify_ssl': True, 'openshift_url': 'http://example.com/', 'use_auth': True, 'conf_file': None, } (flexmock(osbs.conf.Configuration).should_call("__init__").with_args( **config_kwargs)) openshift_map = { 'url': 'http://example.com/', 'insecure': False, 'auth': { 'enable': True }, } rcm = {'version': 1, 'openshift': openshift_map} if labels_reactor_conf is not None: rcm["image_labels"] = deepcopy(labels_reactor_conf) if eq_conf is not None: rcm["image_equal_labels"] = eq_conf workflow.conf.conf = rcm workflow.build_dir.init_build_dirs(["aarch64", "x86_64"], workflow.source) env.set_dockerfile_images( workflow.build_dir.any_platform.dockerfile.parent_images) flexmock(workflow.imageutil).should_receive( 'base_image_inspect').and_return(base_inspect) flexmock(workflow.source).should_receive('get_vcs_info').and_return( VcsInfo(vcs_type="git", vcs_url=DOCKERFILE_GIT, vcs_ref=DOCKERFILE_SHA1)) return env
def mock_env( workflow, *, df_content: str, help_md: Optional[HelpMdFile] = None, plugin_args: Optional[Dict[str, str]] = None, mock_md2man: Callable[..., MockedPopen] = mock_md2man_success("man file content"), ) -> PluginsRunner: def get_build(pipeline_run_name): start_time_json = {'status': {'startTime': TIME}} return start_time_json flexmock(OSBS, get_build=get_build) config_kwargs = { 'namespace': workflow.namespace, 'verify_ssl': True, 'openshift_url': 'http://example.com/', 'use_auth': True, 'conf_file': None, } (flexmock(osbs.conf.Configuration) .should_call("__init__") .with_args(**config_kwargs)) openshift_map = { 'url': 'http://example.com/', 'insecure': False, 'auth': {'enable': True}, } rcm = {'version': 1, 'openshift': openshift_map} workflow.conf.conf = rcm env = MockEnv(workflow).for_plugin(AddHelpPlugin.key, plugin_args) source_dir = Path(workflow.source.path) (source_dir / "Dockerfile").write_text(df_content) if help_md: (source_dir / help_md.name).write_text(help_md.content) workflow.build_dir.init_build_dirs(["aarch64", "x86_64"], workflow.source) if help_md: def check_popen(*args, **kwargs): cmd = args[0] in_path, out_path = parse_md2man_args(cmd) assert in_path.name == help_md.name assert out_path.name == AddHelpPlugin.man_filename return mock_md2man(*args, **kwargs) (flexmock(subprocess) .should_receive("Popen") .once() .replace_with(check_popen)) else: (flexmock(subprocess) .should_receive("Popen") .never()) return env.create_runner()
def test_metadata_plugin_source(image_id, verify_media_results, expected_media_results, workflow): sources_for_nvr = 'image_build' sources_for_koji_build_id = '12345' fetch_sources_result = { 'sources_for_koji_build_id': sources_for_koji_build_id, 'sources_for_nvr': sources_for_nvr, 'image_sources_dir': 'source_dir', } env = (MockEnv(workflow) .for_plugin(StoreMetadataPlugin.key) .set_plugin_args({"url": "http://example.com/"}) .set_plugin_result(PLUGIN_FETCH_SOURCES_KEY, fetch_sources_result) .set_plugin_result(VerifyMediaTypesPlugin.key, verify_media_results)) prepare(workflow) if image_id: workflow.data.koji_source_manifest = {'config': {'digest': image_id}} workflow.fs_watcher._data = dict(fs_data=None) initial_timestamp = datetime.now() timestamp = (initial_timestamp + timedelta(seconds=3)).isoformat() workflow.data.plugins_timestamps = { PLUGIN_FETCH_SOURCES_KEY: timestamp, } workflow.data.plugins_durations = { PLUGIN_FETCH_SOURCES_KEY: 3.03, } workflow.data.plugins_errors = {} output = env.create_runner().run() assert StoreMetadataPlugin.key in output annotations = output[StoreMetadataPlugin.key]["annotations"] assert "filesystem" in annotations assert "fs_data" in annotations['filesystem'] assert "image-id" in annotations assert is_string_type(annotations['image-id']) assert annotations['image-id'] == (image_id if image_id else '') assert "digests" in annotations assert is_string_type(annotations['digests']) digests = json.loads(annotations['digests']) expected = [{ "registry": LOCALHOST_REGISTRY, "repository": TEST_IMAGE, "tag": 'latest', "digest": DIGEST_NOT_USED, "version": "v1" }, { "registry": LOCALHOST_REGISTRY, "repository": TEST_IMAGE, "tag": 'latest', "digest": DIGEST1, "version": "v2" }, { "registry": LOCALHOST_REGISTRY, "repository": "namespace/image", "tag": 'asd123', "digest": DIGEST_NOT_USED, "version": "v1" }, { "registry": LOCALHOST_REGISTRY, "repository": "namespace/image", "tag": 'asd123', "digest": DIGEST2, "version": "v2" }] assert all(digest in expected for digest in digests) assert all(digest in digests for digest in expected) assert "plugins-metadata" in annotations assert "errors" in annotations["plugins-metadata"] assert "durations" in annotations["plugins-metadata"] assert "timestamps" in annotations["plugins-metadata"] plugins_metadata = json.loads(annotations["plugins-metadata"]) assert PLUGIN_FETCH_SOURCES_KEY in plugins_metadata["durations"] if expected_media_results: media_types = expected_media_results assert sorted(json.loads(annotations['media-types'])) == sorted(list(set(media_types))) else: assert 'media-types' not in annotations
def test_skip_plugin(workflow, build_dir, caplog): workflow = mock_workflow(workflow, build_dir, "", user_params={}) (MockEnv(workflow).for_plugin( FlatpakUpdateDockerfilePlugin.key).create_runner().run()) assert 'not flatpak build, skipping plugin' in caplog.text
def test_tag_and_push_plugin(workflow, monkeypatch, caplog, image_name, should_raise, missing_v2, use_secret, file_name, dockerconfig_contents): workflow.user_params['flatpak'] = True platforms = ['x86_64', 'ppc64le', 's390x', 'aarch64'] workflow.data.tag_conf.add_unique_image(ImageName.parse(image_name)) workflow.build_dir.init_build_dirs(platforms, workflow.source) secret_path = None if use_secret: temp_dir = mkdtemp() with open(os.path.join(temp_dir, file_name), "w+") as dockerconfig: dockerconfig.write(json.dumps(dockerconfig_contents)) dockerconfig.flush() secret_path = temp_dir # Add a mock OCI image to 'flatpak_create_oci' results; this forces the tag_and_push # plugin to push with skopeo flatpak_create_oci_result: Dict[str, Any] = {} # Since we are always mocking the push for now, we can get away with a stub image for current_platform in platforms: metadata = deepcopy(IMAGE_METADATA_OCI) metadata['ref_name'] = f'app/org.gnome.eog/{current_platform}/master' flatpak_create_oci_result[current_platform] = metadata manifest_latest_url = "https://{}/v2/{}/manifests/latest".format( LOCALHOST_REGISTRY, TEST_IMAGE) manifest_url = "https://{}/v2/{}/manifests/{}".format( LOCALHOST_REGISTRY, TEST_IMAGE, DIGEST_V2) # We return our v2 manifest in the mocked v1 response as a placeholder - only the # digest matters anyways manifest_response_v1 = requests.Response() (flexmock(manifest_response_v1, status_code=200, headers={ 'Content-Type': 'application/vnd.docker.distribution.manifest.v1+json', 'Docker-Content-Digest': DIGEST_V1 })) manifest_response_v2 = requests.Response() (flexmock(manifest_response_v2, status_code=200, headers={ 'Content-Type': 'application/vnd.docker.distribution.manifest.v2+json', 'Docker-Content-Digest': DIGEST_V2 })) manifest_response_v2_list = requests.Response() (flexmock(manifest_response_v2_list, raise_for_status=lambda: None, headers={ 'Content-Type': 'application/vnd.docker.distribution.manifest.list.v2+json', })) if should_raise: (flexmock(retries).should_receive('run_cmd').and_raise( subprocess.CalledProcessError(1, 'echo', output=b'something went wrong'))) else: (flexmock(retries).should_receive('run_cmd').and_return(0)) manifest_unknown_response = requests.Response() (flexmock(manifest_unknown_response, status_code=404, json={"errors": [{ "code": "MANIFEST_UNKNOWN" }]})) def custom_get(method, url, headers, **kwargs): if url.startswith(manifest_latest_url): # For a manifest stored as v2 or v1, the docker registry defaults to # returning a v1 manifest if a v2 manifest is not explicitly requested if headers[ 'Accept'] == 'application/vnd.docker.distribution.manifest.v2+json': if missing_v2: return manifest_unknown_response else: return manifest_response_v2 elif headers[ 'Accept'] == 'application/vnd.docker.distribution.manifest.list.v2+json': return manifest_response_v2_list else: return manifest_response_v1 if url == manifest_url: if missing_v2: return manifest_unknown_response else: return manifest_response_v2 mock_get_retry_session() (flexmock( requests.Session).should_receive('request').replace_with(custom_get)) (flexmock(time).should_receive('sleep').and_return(None)) reactor_config = { 'registries': [{ 'url': LOCALHOST_REGISTRY, 'insecure': True, 'auth': { 'cfg_path': secret_path }, }] } runner = (MockEnv(workflow).for_plugin( TagAndPushPlugin.key).set_reactor_config( reactor_config).set_plugin_result( CheckAndSetPlatformsPlugin.key, platforms).set_plugin_result( FlatpakCreateOciPlugin.key, flatpak_create_oci_result).create_runner()) add_koji_map_in_workflow(workflow, hub_url='', root_url='') if should_raise: with pytest.raises(PluginFailedException): runner.run() else: runner.run() assert workflow.conf.registry repos_annotations = get_repositories_annotations( workflow.data.tag_conf) assert workflow.data.annotations['repositories'] == repos_annotations if MOCK: # we only test this when mocking docker because we don't expect # running actual docker against v2 registry if missing_v2: assert "Retrying push because V2 schema 2" in caplog.text
def test_group_manifests(workflow, source_dir, schema_version, test_name, group, foreign_layers, per_platform_images, expected_exception, user_params): test_images = ['namespace/httpd:2.4', 'namespace/httpd:latest'] goarch = { 'ppc64le': 'powerpc', 'x86_64': 'amd64', } registry_conf = {REGISTRY_V2: {'version': 'v2', 'insecure': True}} temp_dir = mkdtemp(dir=str(source_dir)) with open(os.path.join(temp_dir, ".dockercfg"), "w+") as dockerconfig: dockerconfig_contents = { REGISTRY_V2: { "username": "******", "password": DOCKER0_REGISTRY } } dockerconfig.write(json.dumps(dockerconfig_contents)) dockerconfig.flush() registry_conf[REGISTRY_V2]['secret'] = temp_dir registry_images_conf = { platform: {REGISTRY_V2: images} for platform, images in per_platform_images.items() } mocked_registries, platform_digests = mock_registries(registry_conf, registry_images_conf, schema_version=schema_version, foreign_layers=foreign_layers) some_per_platform_image = next( image for images in per_platform_images.values() for image in images ) # NOTE: this test assumes that all the images in per_platform_images follow the format of # {noarch_image}-{platform}. If they don't, this test will fail with cryptic errors noarch_image, *_ = some_per_platform_image.rsplit("-", 1) mock_environment(workflow, unique_image=noarch_image, primary_images=test_images) platform_descriptors_list = [] for platform, arch in goarch.items(): new_plat = { 'platform': platform, 'architecture': arch, } platform_descriptors_list.append(new_plat) runner = ( MockEnv(workflow) .for_plugin(GroupManifestsPlugin.key) .set_check_platforms_result(list(per_platform_images.keys())) .set_reactor_config( { 'version': 1, 'group_manifests': group, 'registry': { 'url': f'https://{REGISTRY_V2}/{registry_conf[REGISTRY_V2]["version"]}', 'auth': True, }, 'registries_cfg_path': str(temp_dir), 'platform_descriptors': platform_descriptors_list, } ) .create_runner() ) if expected_exception is None: results = runner.run() manifest_type, list_type = { 'v2': ( 'application/vnd.docker.distribution.manifest.v2+json', 'application/vnd.docker.distribution.manifest.list.v2+json', ), 'oci': ( 'application/vnd.oci.image.manifest.v1+json', 'application/vnd.oci.image.index.v1+json', ), }[schema_version] def verify_manifest_in_repository(registry, repo, manifest, platform, tag=None): config = 'config-' + platform assert registry.get_blob(repo, make_digest(config)) == config layer = 'layer-' + platform assert registry.get_blob(repo, make_digest(layer)) == layer assert registry.get_manifest(repo, make_digest(manifest)) == manifest if tag is not None: assert registry.get_manifest(repo, tag) == manifest if group: source_builds = {} source_manifests = {} for platform in per_platform_images: build = platform_digests[platform]['digests'][0] source_builds[platform] = build source_registry = mocked_registries[build['registry']] source_manifests[platform] = source_registry.get_manifest(build['repository'], build['digest']) for registry, conf in registry_conf.items(): target_registry = mocked_registries[registry] for image in test_images: name, tag = image.split(':') if tag not in target_registry.get_repo(name)['tags']: continue raw_manifest_list = to_text(target_registry.get_manifest(name, tag)) manifest_list = json.loads(raw_manifest_list, object_pairs_hook=OrderedDict) # Check if the manifest list is sorted assert json.dumps(manifest_list, indent=4, sort_keys=True, separators=(',', ': ')) == raw_manifest_list arch_list = [m['platform']['architecture'] for m in manifest_list['manifests']] assert arch_list == sorted(arch_list) assert manifest_list['mediaType'] == list_type assert manifest_list['schemaVersion'] == 2 manifests = manifest_list['manifests'] assert all(d['mediaType'] == manifest_type for d in manifests) assert all(d['platform']['os'] == 'linux' for d in manifests) for platform in platform_digests: descs = [d for d in manifests if d['platform']['architecture'] == goarch[platform]] assert len(descs) == 1 assert descs[0]['digest'] == source_builds[platform]['digest'] verify_manifest_in_repository(target_registry, name, source_manifests[platform], platform) else: platforms = list(platform_digests) assert len(platforms) == 1 platform = platforms[0] source_build = platform_digests[platform]['digests'][0] source_registry = mocked_registries[source_build['registry']] source_manifest = source_registry.get_manifest(source_build['repository'], source_build['digest']) for registry, conf in registry_conf.items(): if conf['version'] == 'v1': continue target_registry = mocked_registries[registry] for image in get_primary_images(workflow): repo = image.to_str(registry=False, tag=False) if image.tag not in target_registry.get_repo(repo)['tags']: continue verify_manifest_in_repository(target_registry, repo, source_manifest, platform, image.tag) for image in get_floating_images(workflow): repo = image.to_str(registry=False, tag=False) assert image.tag not in target_registry.get_repo(repo)['tags'] # Check that plugin returns ManifestDigest object plugin_results = results[GroupManifestsPlugin.key] result_digest = plugin_results["manifest_digest"] assert isinstance(result_digest, ManifestDigest) result_digest = plugin_results["manifest_digest"] assert isinstance(result_digest, ManifestDigest) assert plugin_results["media_type"] assert plugin_results["manifest"] else: with pytest.raises(PluginFailedException) as ex: runner.run() assert expected_exception in str(ex.value)
def test_running_build(workflow, caplog, sources_dir, sources_dir_exists, sources_dir_empty, remote_dir, remote_dir_exists, remote_dir_empty, maven_dir, maven_dir_exists, maven_dir_empty, export_failed): """ Test if proper result is returned and if plugin works """ build_sources_dir = workflow.build_dir.source_container_sources_dir sources_dir_path = build_sources_dir / sources_dir if sources_dir_exists: sources_dir_path.mkdir() if not sources_dir_empty: os.mknod(sources_dir_path / 'stub.srpm') remote_dir_path = build_sources_dir / remote_dir if remote_dir_exists: remote_dir_path.mkdir() if not remote_dir_empty: os.mknod(remote_dir_path / 'remote-sources-first.tar.gz') os.mknod(remote_dir_path / 'remote-sources-second.tar.gz') maven_dir_path = build_sources_dir / maven_dir if maven_dir_exists: maven_dir_path.mkdir() if not maven_dir_empty: os.mkdir(maven_dir_path / 'maven-sources-1') os.mknod(maven_dir_path / 'maven-sources-1' / 'maven-sources-1.tar.gz') workflow.build_dir.init_build_dirs(["noarch"], workflow.source) fetch_sources_result = { 'image_sources_dir': str(sources_dir_path), 'remote_sources_dir': str(remote_dir_path), 'maven_sources_dir': str(maven_dir_path), } runner = (MockEnv(workflow).for_plugin( SourceContainerPlugin.key).set_plugin_result( FetchSourcesPlugin.key, fetch_sources_result).create_runner()) temp_image_output_dir = workflow.build_dir.source_container_output_dir exported_image_file = workflow.build_dir.any_platform.exported_squashed_image temp_image_export_dir = exported_image_file.parent tempfile_chain = (flexmock(tempfile).should_receive("mkdtemp").and_return( str(temp_image_output_dir))) tempfile_chain.and_return(str(temp_image_export_dir)) temp_image_export_dir.mkdir(parents=True, exist_ok=True) temp_image_output_dir.joinpath('blobs', 'sha256').mkdir(parents=True, exist_ok=True) # temp dir created by bsi flexmock(os).should_receive('getcwd').and_return( str(workflow.build_dir.path)) temp_bsi_dir = workflow.build_dir.path / 'SrcImg' temp_bsi_dir.mkdir() def check_run_skopeo(args): """Mocked call to skopeo""" assert args[0] == 'skopeo' assert args[1] == 'copy' assert args[2] == 'oci:%s' % temp_image_output_dir assert args[3] == f'docker-archive:{exported_image_file}' if export_failed: raise subprocess.CalledProcessError(returncode=1, cmd=args, output="Failed") return '' def check_check_output(args, **kwargs): """Mocked check_output call for bsi""" args_expect = ['bsi', '-d'] drivers = set() if sources_dir and sources_dir_exists: drivers.add('sourcedriver_rpm_dir') if remote_dir and remote_dir_exists: drivers.add('sourcedriver_extra_src_dir') if maven_dir and maven_dir_exists: drivers.add('sourcedriver_extra_src_dir') args_expect.append(','.join(drivers)) if sources_dir and sources_dir_exists: args_expect.append('-s') args_expect.append(str(sources_dir_path)) if remote_dir and remote_dir_exists: for count in range(len(os.listdir(remote_dir_path))): args_expect.append('-e') args_expect.append( str(remote_dir_path / f"remote_source_{count}")) if maven_dir and maven_dir_exists: for maven_subdir in os.listdir(maven_dir_path): args_expect.append('-e') args_expect.append(str(maven_dir_path / maven_subdir)) args_expect.append('-o') args_expect.append(str(temp_image_output_dir)) assert args == args_expect return 'stub stdout' any_sources = any( [sources_dir_exists, remote_dir_exists, maven_dir_exists]) (flexmock(retries).should_receive("run_cmd").times( 1 if any_sources else 0).replace_with(check_run_skopeo)) (flexmock(subprocess).should_receive("check_output").times( 1 if any_sources else 0).replace_with(check_check_output)) blob_sha = "f568c411849e21aa3917973f1c5b120f6b52fe69b1944dfb977bc11bed6fbb6d" index_json = { "schemaVersion": 2, "manifests": [{ "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:%s" % blob_sha, "size": 645, "annotations": { "org.opencontainers.image.ref.name": "latest-source" }, "platform": { "architecture": "amd64", "os": "linux" } }] } blob_json = {"schemaVersion": 2, "layers": []} expected_exported_image_metadata = {} temp_image_output_dir.joinpath("index.json").write_text( json.dumps(index_json), "utf-8") temp_image_output_dir.joinpath("blobs", "sha256", blob_sha).write_text( json.dumps(blob_json), "utf-8") if not export_failed: export_tar = workflow.build_dir.any_platform.exported_squashed_image with open(export_tar, "wb") as f: with tarfile.TarFile(mode="w", fileobj=f) as tf: for f in os.listdir(temp_image_output_dir): tf.add(str(temp_image_output_dir / f), f) expected_exported_image_metadata = get_exported_image_metadata( str(export_tar), IMAGE_TYPE_DOCKER_ARCHIVE) if not any([sources_dir_exists, remote_dir_exists, maven_dir_exists]): with pytest.raises(PluginFailedException) as exc_info: runner.run() err_msg = exc_info.value.args[0] assert re.search("No SRPMs directory", err_msg) assert re.search("No Remote source directory", err_msg) assert re.search("No Maven source directory", err_msg) err_msg = f"No SRPMs directory '{sources_dir_path}' available" err_msg += f"\nNo Remote source directory '{remote_dir_path}' available" err_msg += f"\nNo Maven source directory '{maven_dir_path}' available" # Since Python 3.7 logger adds additional whitespaces by default -> checking without them assert re.sub(r'\s+', " ", err_msg) in re.sub(r'\s+', " ", caplog.text) elif export_failed: with pytest.raises(PluginFailedException): runner.run() else: runner.run() result = workflow.data.plugins_results[SourceContainerPlugin.key] assert result.keys() == {'image_metadata', 'logs'} assert result['logs'] == ['stub stdout'] assert result['image_metadata'] == expected_exported_image_metadata assert 'stub stdout' in caplog.text empty_srpm_msg = f"SRPMs directory '{sources_dir_path}' is empty" empty_remote_msg = f"Remote source directory '{remote_dir_path}' is empty" empty_maven_msg = f"Maven source directory '{maven_dir_path}' is empty" if sources_dir_exists and sources_dir_empty: assert empty_srpm_msg in caplog.text else: assert empty_srpm_msg not in caplog.text if remote_dir_exists and remote_dir_empty: assert empty_remote_msg in caplog.text else: assert empty_remote_msg not in caplog.text if maven_dir_exists and maven_dir_empty: assert empty_maven_msg in caplog.text else: assert empty_maven_msg not in caplog.text remove_srpm_msg = f"Will remove directory with downloaded srpms: {sources_dir_path}" remove_remote_msg = f"Will remove directory with downloaded remote sources: " \ f"{remote_dir_path}" remove_maven_msg = f"Will remove directory with downloaded maven sources: " \ f"{maven_dir_path}" if sources_dir_exists: assert remove_srpm_msg in caplog.text else: assert remove_srpm_msg not in caplog.text if remote_dir_exists: assert remove_remote_msg in caplog.text else: assert remove_remote_msg not in caplog.text if maven_dir_exists: assert remove_maven_msg in caplog.text else: assert remove_maven_msg not in caplog.text remove_unpacked_msg = f"Will remove unpacked image directory: {temp_image_output_dir}" assert remove_unpacked_msg in caplog.text remove_tmpbsi_msg = f"Will remove BSI temporary directory: {temp_bsi_dir}" assert remove_tmpbsi_msg in caplog.text
def test_skip_plugin(workflow, caplog): workflow.user_params['scratch'] = True (MockEnv(workflow).for_plugin( CompareComponentsPlugin.key).set_dockerfile_images( ['fedora:36']).set_scratch(True).create_runner().run()) assert 'scratch build, skipping plugin' in caplog.text
def run_plugin(workflow): result = MockEnv(workflow).for_plugin( ChangeFromPlugin.key).create_runner().run() return result[ChangeFromPlugin.key]
def test_render_mail(self, workflow, source_dir, caplog, manual_cancel, to_koji_submitter, koji_integration, success, has_repositories, has_store_metadata_results, annotations, empty_repositories, expect_error): git_source_url = 'git_source_url' git_source_ref = '123423431234123' VcsInfo = namedtuple('VcsInfo', ['vcs_type', 'vcs_url', 'vcs_ref']) session = MockedClientSession('', has_kerberos=True) pathinfo = MockedPathInfo('https://koji') (flexmock(pathinfo) .should_receive('work') .and_raise(RuntimeError, "xyz")) flexmock(koji, ClientSession=lambda hub, opts: session, PathInfo=pathinfo) kwargs = { 'url': 'https://something.com', 'smtp_host': 'smtp.bar.com', 'from_address': '*****@*****.**', 'to_koji_submitter': to_koji_submitter, 'to_koji_pkgowner': False } workflow.data.plugins_results[KojiImportPlugin.key] = MOCK_KOJI_BUILD_ID workflow.user_params['koji_task_id'] = MOCK_KOJI_TASK_ID mock_dockerfile(workflow) flexmock(workflow.source, get_vcs_info=VcsInfo(vcs_type='git', vcs_url=git_source_url, vcs_ref=git_source_ref)) MockEnv(workflow).mock_build_outcome(failed=True, cancelled=manual_cancel) if has_store_metadata_results: if annotations: if empty_repositories: mock_store_metadata_results(workflow, {'repositories': {}}) else: mock_store_metadata_results(workflow) if not has_repositories: result = workflow.data.plugins_results[StoreMetadataPlugin.key] del result['annotations']['repositories'] else: mock_store_metadata_results(workflow, {}) smtp_map = { 'from_address': '*****@*****.**', 'host': 'smtp.bar.com', 'send_to_submitter': to_koji_submitter, 'send_to_pkg_owner': False, } rcm = {'version': 1, 'smtp': smtp_map, 'openshift': {'url': 'https://something.com'}} workflow.conf = Configuration(raw_config=rcm) add_koji_map_in_workflow(workflow, hub_url='/' if koji_integration else None, root_url='https://koji/', ssl_certs_dir='/certs') p = SendMailPlugin(workflow, **kwargs) # Submitter is updated in _get_receivers_list try: p._get_receivers_list() except RuntimeError as ex: # Only valid exception is a RuntimeError when there are no # recipients available assert str(ex) == 'No recipients found' if expect_error: with pytest.raises(ValueError): p._render_mail(success, manual_cancel) return subject, body, logs = p._render_mail(success, manual_cancel) if manual_cancel: status = 'Canceled' assert not logs elif success: status = 'Succeeded' assert not logs else: status = 'Failed' if not has_repositories or empty_repositories: exp_subject = '%s building image %s' % (status, MOCK_NAME_LABEL) exp_body = [ 'Image Name: ' + MOCK_NAME_LABEL, 'Repositories: ', ] else: exp_subject = '%s building image foo/bar' % status exp_body = [ 'Image Name: foo/bar', 'Repositories: ', ' foo/bar:baz', ' foo/bar:spam', ] result = workflow.data.plugins_results[StoreMetadataPlugin.key] if 'repositories' not in result['annotations']: assert "repositories is not included in annotations" in caplog.text common_body = [ 'Status: ' + status, 'Submitted by: ', 'Task id: ' + str(MOCK_KOJI_TASK_ID), 'Source url: ' + git_source_url, 'Source ref: ' + git_source_ref, ] exp_body.extend(common_body) if koji_integration and to_koji_submitter: exp_body[-4] += MOCK_KOJI_SUBMITTER_EMAIL else: exp_body[-4] += SendMailPlugin.DEFAULT_SUBMITTER if koji_integration: exp_body.insert(-2, "Logs: <not available>") assert subject == exp_subject assert body == '\n'.join(exp_body)
def test_flatpak_update_dockerfile(workflow, build_dir, config_name, breakage): config = CONFIGS[config_name] container_yaml = config['container_yaml'] workflow = mock_workflow(workflow, build_dir, container_yaml) if breakage == 'branch_mismatch': config = deepcopy(config) base_module = config['modules'][config['base_module']] base_module['metadata'] = base_module['metadata'].replace( 'branch: f28', 'branch: MISMATCH') expected_exception = "Mismatch for 'branch'" elif breakage == 'no_compose': config = deepcopy(config) config['odcs_composes'] = [] expected_exception = "Can't find main module" else: assert breakage is None expected_exception = None mock_koji_session(config) # composes run by resolve_composes plugin setup_flatpak_composes(workflow, config) secrets_path = build_dir / "secret" secrets_path.mkdir() secrets_path.joinpath("token").write_text("green_eggs_and_ham", "utf-8") rcm = { 'version': 1, 'odcs': { 'api_url': ODCS_URL, 'auth': { 'openidc_dir': secrets_path }, 'signing_intents': [ { 'name': 'unsigned', 'keys': [], }, { 'name': 'release', 'keys': ['R123', 'R234'], }, { 'name': 'beta', 'keys': ['R123', 'B456', 'B457'], }, ], 'default_signing_intent': 'unsigned' }, 'koji': { 'auth': {}, 'hub_url': 'https://koji.example.com/hub' } } runner = (MockEnv(workflow).for_plugin( FlatpakUpdateDockerfilePlugin.key).set_reactor_config( rcm).create_runner()) if expected_exception: with pytest.raises(PluginFailedException) as ex: runner.run() assert expected_exception in str(ex.value) else: runner.run() assert os.path.exists(workflow.build_dir.any_platform.dockerfile_path) df = workflow.build_dir.any_platform.dockerfile.content m = re.search(r'module enable\s*(.*?)\s*$', df, re.MULTILINE) assert m enabled_modules = sorted(m.group(1).split()) if config_name == 'app': assert enabled_modules == ['eog:f28', 'flatpak-runtime:f28'] else: assert enabled_modules == ['flatpak-runtime:f28'] includepkgs_path = os.path.join(workflow.build_dir.any_platform.path, 'atomic-reactor-includepkgs') assert os.path.exists(includepkgs_path) with open(includepkgs_path) as f: includepkgs = f.read() assert 'librsvg2' in includepkgs if config_name == 'app': assert 'eog-0:3.28.3-1.module_2123+73a9ef6f.x86_64' in includepkgs assert os.path.exists( os.path.join(workflow.build_dir.any_platform.path, 'cleanup.sh')) resolve_comp_result = workflow.data.plugins_results.get( PLUGIN_RESOLVE_COMPOSES_KEY) flatpak_util = FlatpakUtil(workflow_config=workflow.conf, source_config=workflow.source.config, composes=resolve_comp_result['composes']) compose_info = flatpak_util.get_flatpak_compose_info() assert compose_info.source_spec == config['source_spec'] if config_name == 'app': assert compose_info.main_module.name == 'eog' assert compose_info.main_module.stream == 'f28' assert compose_info.main_module.version == '20170629213428' assert (compose_info.main_module.mmd.get_summary("C") == 'Eye of GNOME Application Module') assert compose_info.main_module.rpms == [ 'eog-0:3.28.3-1.module_2123+73a9ef6f.src.rpm', 'eog-0:3.28.3-1.module_2123+73a9ef6f.x86_64.rpm', 'eog-0:3.28.3-1.module_2123+73a9ef6f.ppc64le.rpm', ] source_info = flatpak_util.get_flatpak_source_info() assert source_info.base_module.name == config['base_module']
def test_flatpak_create_dockerfile(workflow, source_dir, config_name, override_base_image, breakage): config = CONFIGS[config_name] modules = None if breakage == 'no_modules': modules = [] expected_exception = "a module is required for Flatpaks" elif breakage == 'multiple_modules': modules = ['eog:f28:20170629213428', 'flatpak-common:f28:123456'] expected_exception = None # Just a warning else: assert breakage is None expected_exception = None data = yaml.safe_load(config['container_yaml']) if override_base_image is not None: data['flatpak']['base_image'] = override_base_image if modules is not None: data['compose']['modules'] = modules container_yaml = yaml.dump(data) platforms = ["x86_64", "s390x"] mock_workflow(workflow, source_dir, container_yaml, platforms) base_image = "registry.fedoraproject.org/fedora:latest" reactor_config = { 'version': 1, 'flatpak': { 'base_image': base_image }, 'source_registry': { 'url': 'source_registry' }, } runner = (MockEnv(workflow).for_plugin( FlatpakCreateDockerfilePlugin.key).set_reactor_config( reactor_config).create_runner()) if expected_exception: with pytest.raises(PluginFailedException) as ex: runner.run() assert expected_exception in str(ex.value) else: runner.run() flatpak_util = FlatpakUtil(workflow_config=None, source_config=workflow.source.config) source_spec = flatpak_util.get_flatpak_source_spec() assert source_spec == config['source_spec'] expect_base_image = override_base_image if override_base_image else base_image for platform in platforms: build_dir = BuildDir(workflow.build_dir.path / platform, platform) df = build_dir.dockerfile_path.read_text("utf-8") assert "FROM " + expect_base_image in df assert 'name="{}"'.format(config['name']) in df assert 'com.redhat.component="{}"'.format( config['component']) in df assert "RUN rm -f /etc/yum.repos.d/*" in df assert "ADD atomic-reactor-repos/* /etc/yum.repos.d/" in df
def test_metadata_plugin(workflow, source_dir, help_results, expected_help_results, base_from_scratch, verify_media_results, expected_media_results): if base_from_scratch: df_content = dedent("""\ FROM fedora RUN yum install -y python-django CMD blabla FROM scratch RUN yum install -y python """) else: df_content = dedent("""\ FROM fedora RUN yum install -y python-django CMD blabla """) prepare(workflow) mock_dockerfile(workflow, df_content) dockerfile = workflow.build_dir.any_platform.dockerfile_with_parent_env( workflow.imageutil.base_image_inspect() ) df_images = DockerfileImages(dockerfile.parent_images) for parent in dockerfile.parent_images: if parent != 'scratch': df_images[parent] = "sha256:spamneggs" env = (MockEnv(workflow) .for_plugin(StoreMetadataPlugin.key) .set_plugin_args({"url": "http://example.com/"}) .set_dockerfile_images(df_images) .set_plugin_result(RPMqaPlugin.key, "rpm1\nrpm2") .set_plugin_result(VerifyMediaTypesPlugin.key, verify_media_results) .set_plugin_result(AddHelpPlugin.key, help_results)) if help_results is not None: workflow.data.annotations['help_file'] = help_results['help_file'] workflow.fs_watcher._data = dict(fs_data=None) initial_timestamp = datetime.now() timestamp = (initial_timestamp + timedelta(seconds=3)).isoformat() workflow.data.plugins_timestamps = { RPMqaPlugin.key: timestamp, } workflow.data.plugins_durations = { RPMqaPlugin.key: 3.03, } workflow.data.plugins_errors = {} output = env.create_runner().run() assert StoreMetadataPlugin.key in output annotations = output[StoreMetadataPlugin.key]["annotations"] assert "dockerfile" in annotations assert is_string_type(annotations['dockerfile']) assert "commit_id" in annotations assert is_string_type(annotations['commit_id']) assert annotations['commit_id'] == 'commit' assert "base-image-id" in annotations assert is_string_type(annotations['base-image-id']) assert "base-image-name" in annotations assert is_string_type(annotations['base-image-name']) assert "parent_images" in annotations assert is_string_type(annotations['parent_images']) if base_from_scratch: assert annotations["base-image-name"] == "" assert annotations["base-image-id"] == "" assert '"scratch": "scratch"' in annotations['parent_images'] else: assert annotations["base-image-name"] ==\ workflow.data.dockerfile_images.original_base_image assert annotations["base-image-id"] != "" assert (workflow.data.dockerfile_images.base_image.to_str() in annotations['parent_images']) assert "image-id" in annotations assert is_string_type(annotations['image-id']) assert "filesystem" in annotations assert "fs_data" in annotations['filesystem'] assert "digests" in annotations assert is_string_type(annotations['digests']) digests = json.loads(annotations['digests']) expected = [{ "registry": LOCALHOST_REGISTRY, "repository": TEST_IMAGE, "tag": 'latest', "digest": DIGEST_NOT_USED, "version": "v1" }, { "registry": LOCALHOST_REGISTRY, "repository": TEST_IMAGE, "tag": 'latest', "digest": DIGEST1, "version": "v2" }, { "registry": LOCALHOST_REGISTRY, "repository": "namespace/image", "tag": 'asd123', "digest": DIGEST_NOT_USED, "version": "v1" }, { "registry": LOCALHOST_REGISTRY, "repository": "namespace/image", "tag": 'asd123', "digest": DIGEST2, "version": "v2" }] assert all(digest in expected for digest in digests) assert all(digest in digests for digest in expected) assert "plugins-metadata" in annotations assert "errors" in annotations["plugins-metadata"] assert "durations" in annotations["plugins-metadata"] assert "timestamps" in annotations["plugins-metadata"] plugins_metadata = json.loads(annotations["plugins-metadata"]) assert "all_rpm_packages" in plugins_metadata["durations"] if expected_help_results is False: assert 'help_file' not in annotations else: assert json.loads(annotations['help_file']) == expected_help_results if expected_media_results: media_types = expected_media_results assert sorted(json.loads(annotations['media-types'])) == sorted(list(set(media_types))) else: assert 'media-types' not in annotations
def mock_env(tmpdir, docker_tasker, has_appregistry_label=False, appregistry_label=False, has_bundle_label=True, bundle_label=True, has_archive=True, scratch=False, orchestrator=False, selected_platform=True, empty_archive=False, remove_fails=False, change_csv_content=False, multiple_csv=False): repo_dir = tmpdir.join('test-operator').mkdir() mock_dockerfile(repo_dir, has_appregistry_label=has_appregistry_label, appregistry_label=appregistry_label, has_bundle_label=has_bundle_label, bundle_label=bundle_label) manifests_dir = mock_manifests_dir(repo_dir) env = (MockEnv().for_plugin( 'postbuild', ExportOperatorManifestsPlugin.key).set_scratch(scratch)) if orchestrator: env.make_orchestrator() class MockSource(object): @property def manifests_dir(self): return manifests_dir # Set a new source object, only the manifests_dir property is required for tests. setattr(env.workflow, 'source', MockSource()) env.workflow.builder.set_df_path(str(repo_dir)) mock_stream = generate_archive(tmpdir, empty_archive, change_csv_content, multiple_csv) if selected_platform: env.set_plugin_args({ 'operator_manifests_extract_platform': machine(), 'platform': machine() }) (flexmock(docker_tasker.tasker.d.wrapped).should_receive( 'create_container').with_args(env.workflow.image, command=["/bin/bash"]).and_return( {'Id': CONTAINER_ID})) if remove_fails: (flexmock(docker_tasker.tasker.d.wrapped).should_receive( 'remove_container').with_args(CONTAINER_ID).and_raise( Exception('error'))) else: (flexmock(docker_tasker.tasker.d.wrapped).should_receive( 'remove_container').with_args(CONTAINER_ID)) if has_archive: (flexmock(docker_tasker.tasker.d.wrapped).should_receive( 'get_archive').with_args(CONTAINER_ID, '/manifests').and_return(mock_stream, {})) elif has_archive is not None: response = Response() response.status_code = 404 (flexmock(docker_tasker.tasker.d.wrapped).should_receive( 'get_archive').with_args(CONTAINER_ID, '/manifests').and_raise( NotFound('Not found', response=response))) else: response = Response() response.status_code = 500 (flexmock(docker_tasker.tasker.d.wrapped).should_receive( 'get_archive').with_args(CONTAINER_ID, '/manifests').and_raise( Exception('error'))) return env.create_runner(docker_tasker)
def test_hide_files_multi_stage(self, workflow): df_content = dedent("""\ FROM sha256:123456 as builder RUN blah USER custom_user RUN bluh FROM sha256:654321 as unused RUN bleh FROM sha256:123456 RUN yum install -y python-flask USER custom_user2 CMD /bin/bash """) hide_files = {'tmpdir': '/tmp', 'files': ['/etc/yum.repos.d/repo_ignore_1.repo', '/etc/yum.repos.d/repo_ignore_2.repo']} parent_images = [ 'sha256:123456', 'sha256:654321', ] self.prepare(workflow, df_content, hide_files=hide_files, parent_images=parent_images, inherited_user="******") (MockEnv(workflow) .for_plugin(HideFilesPlugin.key) .create_runner() .run()) expected_df_content = dedent("""\ FROM sha256:123456 as builder USER root RUN mv -f /etc/yum.repos.d/repo_ignore_1.repo /tmp || : RUN mv -f /etc/yum.repos.d/repo_ignore_2.repo /tmp || : USER inherited_user RUN blah USER custom_user RUN bluh USER root RUN mv -fZ /tmp/repo_ignore_1.repo /etc/yum.repos.d/repo_ignore_1.repo || : RUN mv -fZ /tmp/repo_ignore_2.repo /etc/yum.repos.d/repo_ignore_2.repo || : USER custom_user FROM sha256:654321 as unused USER root RUN mv -f /etc/yum.repos.d/repo_ignore_1.repo /tmp || : RUN mv -f /etc/yum.repos.d/repo_ignore_2.repo /tmp || : USER inherited_user RUN bleh USER root RUN mv -fZ /tmp/repo_ignore_1.repo /etc/yum.repos.d/repo_ignore_1.repo || : RUN mv -fZ /tmp/repo_ignore_2.repo /etc/yum.repos.d/repo_ignore_2.repo || : USER inherited_user FROM sha256:123456 USER root RUN mv -f /etc/yum.repos.d/repo_ignore_1.repo /tmp || : RUN mv -f /etc/yum.repos.d/repo_ignore_2.repo /tmp || : USER inherited_user RUN yum install -y python-flask USER custom_user2 CMD /bin/bash USER root RUN mv -fZ /tmp/repo_ignore_1.repo /etc/yum.repos.d/repo_ignore_1.repo || : RUN mv -fZ /tmp/repo_ignore_2.repo /etc/yum.repos.d/repo_ignore_2.repo || : USER custom_user2 """) check_df_content(expected_df_content, workflow)
def test_rpmqa_plugin_exception(workflow): platforms = ['x86_64', 's390x', 'ppc64le', 'aarch64'] workflow.build_dir.init_build_dirs(platforms, workflow.source) runner = MockEnv(workflow).for_plugin(RPMqaPlugin.key).create_runner() with pytest.raises(PluginFailedException): runner.run()
def test_tag_and_push_plugin_oci(workflow, monkeypatch, is_source_build, v2s2, unsupported_image_type, use_secret, fail_push, caplog): sources_koji_id = '123456' sources_koji_target = 'source_target' sources_koji_repo = 'namespace/container_build_image' sources_koji_pull_spec = 'registry_url/{}@sha256:987654321'.format( sources_koji_repo) sources_random_number = 1234567890 sources_timestamp = datetime(year=2019, month=12, day=12) current_platform = platform.processor() or 'x86_64' sources_tagname = '{}-{}-{}-{}'.format( sources_koji_target, sources_random_number, sources_timestamp.strftime('%Y%m%d%H%M%S'), current_platform) secret_path = None if use_secret: temp_dir = mkdtemp() with open(os.path.join(temp_dir, ".dockercfg"), "w+") as dockerconfig: dockerconfig_contents = { LOCALHOST_REGISTRY: { "username": "******", "email": "*****@*****.**", "password": "******" } } dockerconfig.write(json.dumps(dockerconfig_contents)) dockerconfig.flush() secret_path = temp_dir reactor_config = { 'registries': [ { 'url': LOCALHOST_REGISTRY, 'insecure': True, 'auth': { 'cfg_path': secret_path }, }, ], } env = (MockEnv(workflow).for_plugin(TagAndPushPlugin.key).set_plugin_args({ 'koji_target': sources_koji_target }).set_reactor_config(reactor_config)) add_koji_map_in_workflow(workflow, hub_url='', root_url='') wf_data = workflow.data if is_source_build: platforms = ['x86_64'] workflow.build_dir.init_build_dirs(platforms, workflow.source) env.set_plugin_result( FetchSourcesPlugin.key, {'sources_for_koji_build_id': sources_koji_id}, ) env.set_plugin_result( SourceContainerPlugin.key, {'image_metadata': deepcopy(IMAGE_METADATA_DOCKER_ARCHIVE)}, ) else: platforms = ['x86_64', 'ppc64le', 's390x', 'aarch64'] wf_data.tag_conf.add_unique_image(f'{LOCALHOST_REGISTRY}/{TEST_IMAGE}') workflow.user_params['flatpak'] = True workflow.build_dir.init_build_dirs(platforms, workflow.source) env.set_plugin_result(CheckAndSetPlatformsPlugin.key, platforms) class MockedClientSession(object): def __init__(self, hub, opts=None): pass def getBuild(self, build_info): if is_source_build: assert build_info == sources_koji_id return { 'extra': { 'image': { 'index': { 'pull': [sources_koji_pull_spec] } } } } else: return None def krb_login(self, *args, **kwargs): return True session = MockedClientSession('') flexmock(koji, ClientSession=session) flexmock(random).should_receive('randrange').and_return( sources_random_number) flexmock(osbs.utils).should_receive('utcnow').and_return(sources_timestamp) if is_source_build: media_type = 'application/vnd.docker.distribution.manifest.v2+json' else: media_type = 'application/vnd.oci.image.manifest.v1+json' ref_name = "app/org.gnome.eog/x86_64/master" if not is_source_build: # Add a mock OCI image to 'flatpak_create_oci' results; this forces the tag_and_push # plugin to push with skopeo flatpak_create_oci_result: Dict[str, Any] = {} # No need to create image archives, just need to mock its metadata for current_platform in platforms: if unsupported_image_type: image_type = 'unsupported_type' else: image_type = IMAGE_TYPE_OCI metadata = deepcopy(IMAGE_METADATA_OCI) metadata['ref_name'] = ref_name.replace('x86_64', current_platform) metadata['type'] = image_type flatpak_create_oci_result[current_platform] = metadata env.set_plugin_result(FlatpakCreateOciPlugin.key, flatpak_create_oci_result) # Mock the call to skopeo def check_run_skopeo(args): if fail_push: raise subprocess.CalledProcessError(returncode=1, cmd=args, output="Failed") assert args[0] == 'skopeo' if use_secret: assert '--authfile=' + os.path.join(secret_path, '.dockercfg') in args assert '--dest-tls-verify=false' in args if is_source_build: assert args[ -2] == 'docker-archive://' + IMAGE_METADATA_DOCKER_ARCHIVE[ 'path'] output_image = 'docker://{}/{}:{}'.format(LOCALHOST_REGISTRY, sources_koji_repo, sources_tagname) assert args[-1] == output_image else: current_platform = args[-1].split('-')[-1] assert args[-2] == ('oci:' + IMAGE_METADATA_OCI['path'] + ':' + ref_name.replace('x86_64', current_platform)) assert args[-1].startswith('docker://' + LOCALHOST_REGISTRY + f'/{TEST_IMAGE_NAME}') assert '--format=v2s2' in args return '' (flexmock(retries).should_receive("run_cmd").replace_with(check_run_skopeo) ) # Mock out the response from the registry once the OCI image is uploaded manifest_latest_url = "https://{}/v2/{}/manifests/latest".format( LOCALHOST_REGISTRY, TEST_IMAGE) manifest_url = "https://{}/v2/{}/manifests/{}".format( LOCALHOST_REGISTRY, TEST_IMAGE, DIGEST_OCI) manifest_source_tag_url = "https://{}/v2/{}/manifests/{}".format( LOCALHOST_REGISTRY, sources_koji_repo, sources_tagname) manifest_source_digest_url = "https://{}/v2/{}/manifests/{}".format( LOCALHOST_REGISTRY, sources_koji_repo, DIGEST_OCI) manifest_response = requests.Response() (flexmock(manifest_response, raise_for_status=lambda: None, json={}, headers={ 'Content-Type': media_type, 'Docker-Content-Digest': DIGEST_OCI })) manifest_unacceptable_response = requests.Response() (flexmock(manifest_unacceptable_response, status_code=404, json={"errors": [{ "code": "MANIFEST_UNKNOWN" }]})) def custom_get(method, url, headers, **kwargs): if url.startswith( manifest_latest_url) or url == manifest_source_tag_url: if headers['Accept'] == media_type: if is_source_build and not v2s2: return manifest_unacceptable_response else: return manifest_response else: return manifest_unacceptable_response if url == manifest_url or url == manifest_source_digest_url: return manifest_response mock_get_retry_session() (flexmock( requests.Session).should_receive('request').replace_with(custom_get)) if fail_push or unsupported_image_type or (is_source_build and not v2s2): with pytest.raises(PluginFailedException): env.create_runner().run() if not fail_push and is_source_build and not v2s2: assert "Unable to fetch v2 schema 2 digest for" in caplog.text if unsupported_image_type and not fail_push: assert ( 'Attempt to push unsupported image type unsupported_type with skopeo' in caplog.text) else: env.create_runner().run() assert workflow.conf.registry repos_annotations = get_repositories_annotations(wf_data.tag_conf) assert wf_data.annotations['repositories'] == repos_annotations
def mock_env(workflow): return MockEnv(workflow).for_plugin(CheckBaseImagePlugin.key)