def test_dangling_volumes_removed(docker_tasker, caplog): mock_docker() workflow = DockerBuildWorkflow(TEST_IMAGE, source=SOURCE) workflow.source = StubSource() workflow.builder = StubInsideBuilder().for_workflow(workflow) workflow.builder.set_base_from_scratch(False) runner = PostBuildPluginsRunner(docker_tasker, workflow, [{ "name": PostBuildRPMqaPlugin.key, "args": { 'image_id': TEST_IMAGE } }]) runner.run() logs = {} for record in caplog.records: logs.setdefault(record.levelno, []).append(record.message) assert "container_id = 'f8ee920b2db5e802da2583a13a4edbf0523ca5fff6b6d6454c1fd6db5f38014d'" \ in logs[logging.DEBUG] expected_volumes = [u'test', u'conflict_exception', u'real_exception'] assert "volumes = {}".format(expected_volumes) in logs[logging.DEBUG] for volume in expected_volumes: assert "removing volume '{}'".format(volume) in logs[logging.INFO] assert 'ignoring a conflict when removing volume conflict_exception' in logs[ logging.DEBUG]
def test_distgit_fetch_artefacts_plugin(tmpdir, workflow): # noqa command = 'fedpkg sources' expected_command = ['fedpkg', 'sources'] workflow.source = StubSource() workflow.source.path = str(tmpdir) initial_dir = os.getcwd() assert initial_dir != str(tmpdir) def assert_tmpdir(*args, **kwargs): assert os.getcwd() == str(tmpdir) (flexmock( pre_pyrpkg_fetch_artefacts.subprocess).should_receive('check_call'). with_args(expected_command).replace_with(assert_tmpdir).once()) workflow.conf.conf['sources_command'] = command runner = PreBuildPluginsRunner(workflow, [{ 'name': DistgitFetchArtefactsPlugin.key, }]) runner.run() assert os.getcwd() == initial_dir
def test_rpmqa_plugin(caplog, docker_tasker, base_from_scratch, remove_container_error, ignore_autogenerated): should_raise_error = {} if remove_container_error: should_raise_error['remove_container'] = None mock_docker(should_raise_error=should_raise_error) workflow = DockerBuildWorkflow(TEST_IMAGE, source=SOURCE) workflow.source = StubSource() workflow.builder = StubInsideBuilder().for_workflow(workflow) workflow.builder.set_base_from_scratch(base_from_scratch) flexmock(docker.APIClient, logs=mock_logs) runner = PostBuildPluginsRunner( docker_tasker, workflow, [{ "name": PostBuildRPMqaPlugin.key, "args": { 'image_id': TEST_IMAGE, "ignore_autogenerated_gpg_keys": ignore_autogenerated["ignore"] } }]) results = runner.run() if base_from_scratch: log_msg = "from scratch can't run rpmqa" assert log_msg in caplog.text assert results[PostBuildRPMqaPlugin.key] is None assert workflow.image_components is None else: assert results[ PostBuildRPMqaPlugin.key] == ignore_autogenerated["package_list"] assert workflow.image_components == parse_rpm_output( ignore_autogenerated["package_list"])
def __init__(self): self.workflow = DockerBuildWorkflow(source=MOCK_SOURCE) self.workflow.source = StubSource() self.workflow.builder = StubInsideBuilder().for_workflow(self.workflow) self.workflow.builder.tasker = flexmock() self._phase = None self._plugin_key = None
def test_distgit_fetch_artefacts_skip(tmpdir, workflow, caplog): # noqa workflow.source = StubSource() workflow.source.path = str(tmpdir) (MockEnv(workflow).for_plugin( DistgitFetchArtefactsPlugin.key).create_runner().run()) log_msg = 'no sources command configuration, skipping plugin' assert log_msg in caplog.text
def prepare(df_path): if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow("test-image", source=SOURCE) workflow.source = StubSource() workflow.builder = ( StubInsideBuilder().for_workflow(workflow).set_df_path(df_path)) return tasker, workflow
def test_distgit_fetch_artefacts_skip(tmpdir, workflow, caplog): # noqa workflow.source = StubSource() workflow.source.path = str(tmpdir) runner = PreBuildPluginsRunner(workflow, [{ 'name': DistgitFetchArtefactsPlugin.key, }]) runner.run() log_msg = 'no sources command configuration, skipping plugin' assert log_msg in caplog.text
def prepare(): if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow(SOURCE, "test-image") workflow.source = StubSource() workflow.builder = StubInsideBuilder().for_workflow(workflow) session = MockedClientSession(hub='', opts=None) workflow.koji_session = session flexmock(koji, ClientSession=session, PathInfo=MockedPathInfo) return tasker, workflow
def mock_workflow(tmpdir): workflow = DockerBuildWorkflow(TEST_IMAGE, source={ "provider": "git", "uri": "asd" }) workflow.source = StubSource() builder = StubInsideBuilder().for_workflow(workflow) builder.set_df_path(str(tmpdir)) builder.tasker = flexmock() workflow.builder = flexmock(builder) return workflow
def prepare(df_path, inherited_user=''): if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow(source=SOURCE) workflow.source = StubSource() workflow.builder = ( StubInsideBuilder().for_workflow(workflow).set_dockerfile_images( df_parser(df_path).parent_images).set_df_path( df_path).set_inspection_data({ INSPECT_CONFIG: { 'User': inherited_user, }, })) return tasker, workflow
def mock_workflow(tmpdir, for_orchestrator=False): workflow = DockerBuildWorkflow( TEST_IMAGE, source={"provider": "git", "uri": "asd"} ) workflow.source = StubSource() builder = StubInsideBuilder().for_workflow(workflow) builder.set_df_path(str(tmpdir)) builder.tasker = flexmock() workflow.builder = flexmock(builder) if for_orchestrator: workflow.buildstep_plugins_conf = [{'name': PLUGIN_BUILD_ORCHESTRATE_KEY}] return workflow
def prepare(): if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow({ "provider": "git", "uri": DOCKERFILE_GIT }, "test-image") workflow.source = StubSource() workflow.builder = StubInsideBuilder().for_workflow(workflow) (flexmock(requests.Response, content=repocontent).should_receive( 'raise_for_status').and_return(None)) (flexmock(requests.Session, get=lambda *_: requests.Response())) mock_get_retry_session() return tasker, workflow
def test_rpmqa_plugin_exception(docker_tasker): # noqa mock_docker() workflow = DockerBuildWorkflow(TEST_IMAGE, source=SOURCE) workflow.source = StubSource() workflow.builder = StubInsideBuilder().for_workflow(workflow) flexmock(docker.APIClient, logs=mock_logs_raise) runner = PostBuildPluginsRunner(docker_tasker, workflow, [{ "name": PostBuildRPMqaPlugin.key, "args": { 'image_id': TEST_IMAGE } }]) with pytest.raises(PluginFailedException): runner.run()
def prepare(scratch=False): if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow(source={"provider": "git", "uri": DOCKERFILE_GIT}) workflow.source = StubSource() workflow.builder = StubInsideBuilder().for_workflow(workflow) workflow.builder.set_dockerfile_images([]) workflow.user_params['scratch'] = scratch (flexmock(requests.Response, content=repocontent) .should_receive('raise_for_status') .and_return(None)) (flexmock(requests.Session, get=lambda *_: requests.Response())) mock_get_retry_session() return tasker, workflow
def prepare(df_path, inherited_user=''): if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow(SOURCE, "test-image") workflow.source = StubSource() workflow.builder = (StubInsideBuilder().for_workflow(workflow).set_df_path( df_path).set_inspection_data({ INSPECT_CONFIG: { 'User': inherited_user, }, })) (flexmock(requests.Response, content=repocontent).should_receive( 'raise_for_status').and_return(None)) (flexmock(requests.Session, get=lambda *_: requests.Response())) return tasker, workflow
def test_empty_logs_failure(docker_tasker): # noqa mock_docker() workflow = DockerBuildWorkflow(source=SOURCE) workflow.source = StubSource() workflow.builder = get_builder(workflow) flexmock(docker.APIClient, logs=mock_logs_empty) runner = PostBuildPluginsRunner(docker_tasker, workflow, [{ "name": PostBuildRPMqaPlugin.key, "args": { 'image_id': TEST_IMAGE } }]) with pytest.raises(PluginFailedException) as exc_info: runner.run() assert 'Unable to gather list of installed packages in container' in str( exc_info.value)
def mock_workflow(): """ Provide just enough structure that workflow can be used to run the plugin. Defaults below are solely to enable that purpose; tests where those values matter should provide their own. """ workflow = DockerBuildWorkflow(SOURCE, "mock:default_built") workflow.source = StubSource() builder = StubInsideBuilder().for_workflow(workflow) builder.set_df_path('/mock-path') base_image_name = ImageName.parse("mock:tag") builder.parent_images[ImageName.parse("mock:base")] = base_image_name builder.base_image = base_image_name builder.tasker = flexmock() workflow.builder = flexmock(builder) return workflow
def test_empty_logs_retry(docker_tasker): # noqa mock_docker() workflow = DockerBuildWorkflow(SOURCE, TEST_IMAGE) workflow.source = StubSource() workflow.builder = StubInsideBuilder().for_workflow(workflow) workflow.builder.set_base_from_scratch(False) flexmock(docker.APIClient, logs=mock_logs_retry) runner = PostBuildPluginsRunner(docker_tasker, workflow, [{ "name": PostBuildRPMqaPlugin.key, "args": { 'image_id': TEST_IMAGE } }]) results = runner.run() assert results[PostBuildRPMqaPlugin.key] == PACKAGE_LIST assert workflow.image_components == parse_rpm_output(PACKAGE_LIST)
def prepare(testfile="test-image", check_repo_retval=0): if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow(SOURCE, testfile) workflow.source = StubSource() workflow.builder = StubInsideBuilder() workflow.tag_conf = StubTagConf().set_images([ ImageName(repo="image-name1"), ImageName(namespace="prefix", repo="image-name2"), ImageName(repo="image-name3", tag="asd"), ]) # Mock dockpulp and docker dockpulp.Pulp = flexmock(dockpulp.Pulp) dockpulp.Pulp.registry = 'registry.example.com' (flexmock(dockpulp.imgutils).should_receive('get_metadata').with_args( object).and_return([{ 'id': 'foo' }])) (flexmock(dockpulp.imgutils).should_receive('get_manifest').with_args( object).and_return([{ 'id': 'foo' }])) (flexmock(dockpulp.imgutils).should_receive('get_versions').with_args( object).and_return({'id': '1.6.0'})) (flexmock(dockpulp.imgutils).should_receive('check_repo').and_return( check_repo_retval)) (flexmock(dockpulp.Pulp).should_receive('set_certs').with_args( object, object)) (flexmock(dockpulp.Pulp).should_receive('login').with_args(object, object)) (flexmock(dockpulp.Pulp).should_receive('getRepos').with_args( list, fields=list, distributors=True).and_return([{ "id": "redhat-image-name1" }, { "id": "redhat-prefix-image-name2" }])) (flexmock(dockpulp.Pulp).should_receive('createRepo')) mock_docker() return tasker, workflow
def prepare(scratch=False): if MOCK: mock_docker() build_json = {'metadata': {'labels': {'scratch': scratch}}} flexmock(util).should_receive('get_build_json').and_return(build_json) tasker = DockerTasker() workflow = DockerBuildWorkflow("test-image", source={ "provider": "git", "uri": DOCKERFILE_GIT }) workflow.source = StubSource() workflow.builder = StubInsideBuilder().for_workflow(workflow) (flexmock(requests.Response, content=repocontent).should_receive( 'raise_for_status').and_return(None)) (flexmock(requests.Session, get=lambda *_: requests.Response())) mock_get_retry_session() return tasker, workflow
def test_distgit_fetch_artefacts_failure(tmpdir, workflow): # noqa command = 'fedpkg sources' expected_command = ['fedpkg', 'sources'] workflow.source = StubSource() workflow.source.path = str(tmpdir) initial_dir = os.getcwd() assert initial_dir != str(tmpdir) (flexmock(pyrpkg_fetch_artefacts.subprocess).should_receive('check_call'). with_args(expected_command).and_raise(RuntimeError).once()) workflow.conf.conf['sources_command'] = command runner = (MockEnv(workflow).for_plugin( DistgitFetchArtefactsPlugin.key).create_runner()) with pytest.raises(PluginFailedException): runner.run() assert os.getcwd() == initial_dir
def mock_workflow(tmpdir, orchestrator, user_config=None, site_config=None): workflow = DockerBuildWorkflow(TEST_IMAGE, source={ 'provider': 'git', 'uri': 'asd' }) workflow.source = StubSource() workflow.source.path = str(tmpdir) workflow.source.config = make_user_config(user_config) workflow.builder = (StubInsideBuilder().for_workflow(workflow).set_df_path( str(tmpdir))) if orchestrator: workflow.buildstep_plugins_conf = [{ 'name': PLUGIN_BUILD_ORCHESTRATE_KEY }] workflow.plugin_workspace[ReactorConfigPlugin.key] = { WORKSPACE_CONF_KEY: make_reactor_config(site_config or {}) } return workflow
def test_rpmqa_plugin_skip(docker_tasker): # noqa """ Test skipping the plugin if workflow.image_components is already set """ mock_docker() workflow = DockerBuildWorkflow(TEST_IMAGE, source=SOURCE) workflow.source = StubSource() workflow.builder = StubInsideBuilder().for_workflow(workflow) image_components = {'type': 'rpm', 'name': 'something'} setattr(workflow, 'image_components', image_components) flexmock(docker.APIClient, logs=mock_logs_raise) runner = PostBuildPluginsRunner(docker_tasker, workflow, [{ "name": PostBuildRPMqaPlugin.key, "args": { 'image_id': TEST_IMAGE } }]) results = runner.run() assert results[PostBuildRPMqaPlugin.key] is None assert workflow.image_components == image_components
def test_dangling_volumes_removed(docker_tasker, request): fake_logger = FakeLogger() existing_logger = atomic_reactor.core.logger def restore_logger(): atomic_reactor.core.logger = existing_logger request.addfinalizer(restore_logger) atomic_reactor.core.logger = fake_logger mock_docker() workflow = DockerBuildWorkflow(SOURCE, TEST_IMAGE) workflow.source = StubSource() workflow.builder = StubInsideBuilder().for_workflow(workflow) workflow.builder.set_base_from_scratch(False) runner = PostBuildPluginsRunner(docker_tasker, workflow, [{ "name": PostBuildRPMqaPlugin.key, "args": { 'image_id': TEST_IMAGE } }]) runner.run() assert ("container_id = '%s'", u'f8ee920b2db5e802da2583a13a4edbf0523ca5fff6b6d6454c1fd6db5f38014d') \ in fake_logger.debugs expected_volumes = [u'test', u'conflict_exception', u'real_exception'] assert ("volumes = %s", expected_volumes) in fake_logger.debugs assert ("removing volume '%s'", u'test') in fake_logger.infos assert ("removing volume '%s'", u'conflict_exception') in fake_logger.infos assert ("removing volume '%s'", u'real_exception') in fake_logger.infos assert ('ignoring a conflict when removing volume %s', 'conflict_exception') in \ fake_logger.debugs
def prepare(workflow, df_path): workflow.source = StubSource() workflow.builder = StubInsideBuilder().for_workflow(workflow) workflow.builder.set_df_path(df_path)
def workflow(workflow): workflow.data.dockerfile_images = DockerfileImages(['source_registry.com/fedora:26']) workflow.source = StubSource() return workflow
def test_rpmqa_plugin_base_from_scratch(caplog, tmpdir, docker_tasker, packages_exists, rpm_failed, get_archive_raises): mock_docker() workflow = DockerBuildWorkflow(source=SOURCE) workflow.source = StubSource() workflow.builder = get_builder(workflow, True) mock_reactor_config(workflow, list_rpms_from_scratch=True) if get_archive_raises is None: mock_stream = generate_archive(tmpdir, empty=not packages_exists) (flexmock(docker_tasker.tasker.d.wrapped).should_receive( 'get_archive').and_return(mock_stream, {})) else: response = flexmock(content="abc", status_code=123) (flexmock(docker_tasker.tasker.d.wrapped).should_receive( 'get_archive').and_raise(get_archive_raises, 'get archive failed', response)) runner = PostBuildPluginsRunner( docker_tasker, workflow, [{ "name": PostBuildRPMqaPlugin.key, "args": { 'image_id': TEST_IMAGE, "ignore_autogenerated_gpg_keys": True } }]) if get_archive_raises is None and packages_exists and rpm_failed: (flexmock(subprocess).should_receive("check_output").once().and_raise( Exception, 'rpm query failed')) elif get_archive_raises is None and packages_exists and not rpm_failed: (flexmock(subprocess).should_receive("check_output").once().and_return( "\n".join(PACKAGE_LIST_WITH_AUTOGENERATED))) if get_archive_raises == APIError: results = runner.run() log_msg = 'Could not extract rpmdb in' assert log_msg in caplog.text assert results[PostBuildRPMqaPlugin.key] is None assert workflow.image_components is None elif get_archive_raises == Exception: with pytest.raises(Exception, match='get archive failed'): runner.run() log_msg = 'Get archive failed while extracting rpmdb in' assert log_msg in caplog.text assert workflow.image_components is None else: if not packages_exists: results = runner.run() log_msg = '{} does not exist in rpmdb'.format(RPMDB_PACKAGES_NAME) assert log_msg in caplog.text assert results[PostBuildRPMqaPlugin.key] is None assert workflow.image_components is None else: log_msg_getting = 'getting rpms from rpmdb:' if rpm_failed: with pytest.raises(Exception, match='rpm query failed'): runner.run() assert log_msg_getting in caplog.text assert workflow.image_components is None log_msg = 'Failed to get rpms from rpmdb:' assert log_msg in caplog.text else: results = runner.run() assert log_msg_getting in caplog.text assert results[PostBuildRPMqaPlugin.key] == PACKAGE_LIST assert workflow.image_components == parse_rpm_output( PACKAGE_LIST) log_msg = "from scratch, list_rpms_from_scratch is True, trying get rpmdb" assert log_msg in caplog.text
def mock_environment(tmpdir, session=None, name=None, component=None, version=None, release=None, source=None, build_process_failed=False, docker_registry=True, pulp_registries=0, blocksize=None, task_states=None, additional_tags=None, has_config=None, prefer_schema1_digest=True): if session is None: session = MockedClientSession('', task_states=None) if source is None: source = GitSource('git', 'git://hostname/path') if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow(SOURCE, "test-image") base_image_id = '123456parent-id' workflow.source = StubSource() workflow.builder = StubInsideBuilder().for_workflow(workflow) workflow.builder.image_id = '123456imageid' workflow.builder.set_inspection_data({'Id': base_image_id}) setattr(workflow, 'tag_conf', TagConf()) with open(os.path.join(str(tmpdir), 'Dockerfile'), 'wt') as df: df.write( 'FROM base\n' 'LABEL BZComponent={component} com.redhat.component={component}\n' 'LABEL Version={version} version={version}\n' 'LABEL Release={release} release={release}\n'.format( component=component, version=version, release=release)) workflow.builder.set_df_path(df.name) if name and version: workflow.tag_conf.add_unique_image( 'user/test-image:{v}-timestamp'.format(v=version)) if name and version and release: workflow.tag_conf.add_primary_images([ "{0}:{1}-{2}".format(name, version, release), "{0}:{1}".format(name, version), "{0}:latest".format(name) ]) if additional_tags: workflow.tag_conf.add_primary_images( ["{0}:{1}".format(name, tag) for tag in additional_tags]) flexmock(subprocess, Popen=fake_Popen) flexmock(koji, ClientSession=lambda hub, opts: session) flexmock(GitSource) setattr(workflow, 'source', source) setattr(workflow.source, 'lg', X()) setattr(workflow.source.lg, 'commit_id', '123456') setattr(workflow, 'push_conf', PushConf()) if docker_registry: docker_reg = workflow.push_conf.add_docker_registry( 'docker.example.com') for image in workflow.tag_conf.images: tag = image.to_str(registry=False) if pulp_registries and prefer_schema1_digest: docker_reg.digests[tag] = ManifestDigest(v1=fake_digest(image), v2='sha256:not-used') else: docker_reg.digests[tag] = ManifestDigest(v1='sha256:not-used', v2=fake_digest(image)) if has_config: docker_reg.config = { 'config': { 'architecture': LOCAL_ARCH }, 'container_config': {} } for pulp_registry in range(pulp_registries): workflow.push_conf.add_pulp_registry('env', 'pulp.example.com') with open(os.path.join(str(tmpdir), 'image.tar.xz'), 'wt') as fp: fp.write('x' * 2**12) setattr(workflow, 'exported_image_sequence', [{ 'path': fp.name, 'type': IMAGE_TYPE_DOCKER_ARCHIVE }]) if build_process_failed: workflow.build_result = BuildResult( logs=["docker build log - \u2018 \u2017 \u2019 \n'"], fail_reason="not built") else: workflow.build_result = BuildResult( logs=["docker build log - \u2018 \u2017 \u2019 \n'"], image_id="id1234") workflow.prebuild_plugins_conf = {} workflow.image_components = parse_rpm_output([ "name1;1.0;1;" + LOCAL_ARCH + ";0;2000;" + FAKE_SIGMD5.decode() + ";23000;" "RSA/SHA256, Tue 30 Aug 2016 00:00:00, Key ID 01234567890abc;(none)", "name2;2.0;1;" + LOCAL_ARCH + ";0;3000;" + FAKE_SIGMD5.decode() + ";24000" "RSA/SHA256, Tue 30 Aug 2016 00:00:00, Key ID 01234567890abd;(none)", ]) return tasker, workflow