def test_dangling_volumes_removed(docker_tasker, caplog):
mock_docker()
workflow = DockerBuildWorkflow(TEST_IMAGE, source=SOURCE)
workflow.source = StubSource()
workflow.builder = StubInsideBuilder().for_workflow(workflow)
workflow.builder.set_base_from_scratch(False)
runner = PostBuildPluginsRunner(docker_tasker, workflow,
[{
"name": PostBuildRPMqaPlugin.key,
"args": {
'image_id': TEST_IMAGE
}
}])
runner.run()
logs = {}
for record in caplog.records:
logs.setdefault(record.levelno, []).append(record.message)
assert "container_id = 'f8ee920b2db5e802da2583a13a4edbf0523ca5fff6b6d6454c1fd6db5f38014d'" \
in logs[logging.DEBUG]
expected_volumes = [u'test', u'conflict_exception', u'real_exception']
assert "volumes = {}".format(expected_volumes) in logs[logging.DEBUG]
for volume in expected_volumes:
assert "removing volume '{}'".format(volume) in logs[logging.INFO]
assert 'ignoring a conflict when removing volume conflict_exception' in logs[
logging.DEBUG]
def test_distgit_fetch_artefacts_plugin(tmpdir, workflow): # noqa
command = 'fedpkg sources'
expected_command = ['fedpkg', 'sources']
workflow.source = StubSource()
workflow.source.path = str(tmpdir)
initial_dir = os.getcwd()
assert initial_dir != str(tmpdir)
def assert_tmpdir(*args, **kwargs):
assert os.getcwd() == str(tmpdir)
(flexmock(
pre_pyrpkg_fetch_artefacts.subprocess).should_receive('check_call').
with_args(expected_command).replace_with(assert_tmpdir).once())
workflow.conf.conf['sources_command'] = command
runner = PreBuildPluginsRunner(workflow,
[{
'name': DistgitFetchArtefactsPlugin.key,
}])
runner.run()
assert os.getcwd() == initial_dir
def test_rpmqa_plugin(caplog, docker_tasker, base_from_scratch,
remove_container_error, ignore_autogenerated):
should_raise_error = {}
if remove_container_error:
should_raise_error['remove_container'] = None
mock_docker(should_raise_error=should_raise_error)
workflow = DockerBuildWorkflow(TEST_IMAGE, source=SOURCE)
workflow.source = StubSource()
workflow.builder = StubInsideBuilder().for_workflow(workflow)
workflow.builder.set_base_from_scratch(base_from_scratch)
flexmock(docker.APIClient, logs=mock_logs)
runner = PostBuildPluginsRunner(
docker_tasker, workflow, [{
"name": PostBuildRPMqaPlugin.key,
"args": {
'image_id': TEST_IMAGE,
"ignore_autogenerated_gpg_keys": ignore_autogenerated["ignore"]
}
}])
results = runner.run()
if base_from_scratch:
log_msg = "from scratch can't run rpmqa"
assert log_msg in caplog.text
assert results[PostBuildRPMqaPlugin.key] is None
assert workflow.image_components is None
else:
assert results[
PostBuildRPMqaPlugin.key] == ignore_autogenerated["package_list"]
assert workflow.image_components == parse_rpm_output(
ignore_autogenerated["package_list"])
def __init__(self):
self.workflow = DockerBuildWorkflow(source=MOCK_SOURCE)
self.workflow.source = StubSource()
self.workflow.builder = StubInsideBuilder().for_workflow(self.workflow)
self.workflow.builder.tasker = flexmock()
self._phase = None
self._plugin_key = None
def test_distgit_fetch_artefacts_skip(tmpdir, workflow, caplog): # noqa
workflow.source = StubSource()
workflow.source.path = str(tmpdir)
(MockEnv(workflow).for_plugin(
DistgitFetchArtefactsPlugin.key).create_runner().run())
log_msg = 'no sources command configuration, skipping plugin'
assert log_msg in caplog.text
def prepare(df_path):
if MOCK:
mock_docker()
tasker = DockerTasker()
workflow = DockerBuildWorkflow("test-image", source=SOURCE)
workflow.source = StubSource()
workflow.builder = (
StubInsideBuilder().for_workflow(workflow).set_df_path(df_path))
return tasker, workflow
def test_distgit_fetch_artefacts_skip(tmpdir, workflow, caplog): # noqa
workflow.source = StubSource()
workflow.source.path = str(tmpdir)
runner = PreBuildPluginsRunner(workflow,
[{
'name': DistgitFetchArtefactsPlugin.key,
}])
runner.run()
log_msg = 'no sources command configuration, skipping plugin'
assert log_msg in caplog.text
def prepare():
if MOCK:
mock_docker()
tasker = DockerTasker()
workflow = DockerBuildWorkflow(SOURCE, "test-image")
workflow.source = StubSource()
workflow.builder = StubInsideBuilder().for_workflow(workflow)
session = MockedClientSession(hub='', opts=None)
workflow.koji_session = session
flexmock(koji, ClientSession=session, PathInfo=MockedPathInfo)
return tasker, workflow
def mock_workflow(tmpdir):
workflow = DockerBuildWorkflow(TEST_IMAGE,
source={
"provider": "git",
"uri": "asd"
})
workflow.source = StubSource()
builder = StubInsideBuilder().for_workflow(workflow)
builder.set_df_path(str(tmpdir))
builder.tasker = flexmock()
workflow.builder = flexmock(builder)
return workflow
def prepare(df_path, inherited_user=''):
if MOCK:
mock_docker()
tasker = DockerTasker()
workflow = DockerBuildWorkflow(source=SOURCE)
workflow.source = StubSource()
workflow.builder = (
StubInsideBuilder().for_workflow(workflow).set_dockerfile_images(
df_parser(df_path).parent_images).set_df_path(
df_path).set_inspection_data({
INSPECT_CONFIG: {
'User': inherited_user,
},
}))
return tasker, workflow
def mock_workflow(tmpdir, for_orchestrator=False):
workflow = DockerBuildWorkflow(
TEST_IMAGE,
source={"provider": "git", "uri": "asd"}
)
workflow.source = StubSource()
builder = StubInsideBuilder().for_workflow(workflow)
builder.set_df_path(str(tmpdir))
builder.tasker = flexmock()
workflow.builder = flexmock(builder)
if for_orchestrator:
workflow.buildstep_plugins_conf = [{'name': PLUGIN_BUILD_ORCHESTRATE_KEY}]
return workflow
def prepare():
if MOCK:
mock_docker()
tasker = DockerTasker()
workflow = DockerBuildWorkflow({
"provider": "git",
"uri": DOCKERFILE_GIT
}, "test-image")
workflow.source = StubSource()
workflow.builder = StubInsideBuilder().for_workflow(workflow)
(flexmock(requests.Response, content=repocontent).should_receive(
'raise_for_status').and_return(None))
(flexmock(requests.Session, get=lambda *_: requests.Response()))
mock_get_retry_session()
return tasker, workflow
def test_rpmqa_plugin_exception(docker_tasker): # noqa
mock_docker()
workflow = DockerBuildWorkflow(TEST_IMAGE, source=SOURCE)
workflow.source = StubSource()
workflow.builder = StubInsideBuilder().for_workflow(workflow)
flexmock(docker.APIClient, logs=mock_logs_raise)
runner = PostBuildPluginsRunner(docker_tasker, workflow,
[{
"name": PostBuildRPMqaPlugin.key,
"args": {
'image_id': TEST_IMAGE
}
}])
with pytest.raises(PluginFailedException):
runner.run()
def prepare(scratch=False):
if MOCK:
mock_docker()
tasker = DockerTasker()
workflow = DockerBuildWorkflow(source={"provider": "git", "uri": DOCKERFILE_GIT})
workflow.source = StubSource()
workflow.builder = StubInsideBuilder().for_workflow(workflow)
workflow.builder.set_dockerfile_images([])
workflow.user_params['scratch'] = scratch
(flexmock(requests.Response, content=repocontent)
.should_receive('raise_for_status')
.and_return(None))
(flexmock(requests.Session, get=lambda *_: requests.Response()))
mock_get_retry_session()
return tasker, workflow
def prepare(df_path, inherited_user=''):
if MOCK:
mock_docker()
tasker = DockerTasker()
workflow = DockerBuildWorkflow(SOURCE, "test-image")
workflow.source = StubSource()
workflow.builder = (StubInsideBuilder().for_workflow(workflow).set_df_path(
df_path).set_inspection_data({
INSPECT_CONFIG: {
'User': inherited_user,
},
}))
(flexmock(requests.Response, content=repocontent).should_receive(
'raise_for_status').and_return(None))
(flexmock(requests.Session, get=lambda *_: requests.Response()))
return tasker, workflow
def test_empty_logs_failure(docker_tasker): # noqa
mock_docker()
workflow = DockerBuildWorkflow(source=SOURCE)
workflow.source = StubSource()
workflow.builder = get_builder(workflow)
flexmock(docker.APIClient, logs=mock_logs_empty)
runner = PostBuildPluginsRunner(docker_tasker, workflow,
[{
"name": PostBuildRPMqaPlugin.key,
"args": {
'image_id': TEST_IMAGE
}
}])
with pytest.raises(PluginFailedException) as exc_info:
runner.run()
assert 'Unable to gather list of installed packages in container' in str(
exc_info.value)
def mock_workflow():
"""
Provide just enough structure that workflow can be used to run the plugin.
Defaults below are solely to enable that purpose; tests where those values
matter should provide their own.
"""
workflow = DockerBuildWorkflow(SOURCE, "mock:default_built")
workflow.source = StubSource()
builder = StubInsideBuilder().for_workflow(workflow)
builder.set_df_path('/mock-path')
base_image_name = ImageName.parse("mock:tag")
builder.parent_images[ImageName.parse("mock:base")] = base_image_name
builder.base_image = base_image_name
builder.tasker = flexmock()
workflow.builder = flexmock(builder)
return workflow
def test_empty_logs_retry(docker_tasker): # noqa
mock_docker()
workflow = DockerBuildWorkflow(SOURCE, TEST_IMAGE)
workflow.source = StubSource()
workflow.builder = StubInsideBuilder().for_workflow(workflow)
workflow.builder.set_base_from_scratch(False)
flexmock(docker.APIClient, logs=mock_logs_retry)
runner = PostBuildPluginsRunner(docker_tasker, workflow,
[{
"name": PostBuildRPMqaPlugin.key,
"args": {
'image_id': TEST_IMAGE
}
}])
results = runner.run()
assert results[PostBuildRPMqaPlugin.key] == PACKAGE_LIST
assert workflow.image_components == parse_rpm_output(PACKAGE_LIST)
def prepare(testfile="test-image", check_repo_retval=0):
if MOCK:
mock_docker()
tasker = DockerTasker()
workflow = DockerBuildWorkflow(SOURCE, testfile)
workflow.source = StubSource()
workflow.builder = StubInsideBuilder()
workflow.tag_conf = StubTagConf().set_images([
ImageName(repo="image-name1"),
ImageName(namespace="prefix", repo="image-name2"),
ImageName(repo="image-name3", tag="asd"),
])
# Mock dockpulp and docker
dockpulp.Pulp = flexmock(dockpulp.Pulp)
dockpulp.Pulp.registry = 'registry.example.com'
(flexmock(dockpulp.imgutils).should_receive('get_metadata').with_args(
object).and_return([{
'id': 'foo'
}]))
(flexmock(dockpulp.imgutils).should_receive('get_manifest').with_args(
object).and_return([{
'id': 'foo'
}]))
(flexmock(dockpulp.imgutils).should_receive('get_versions').with_args(
object).and_return({'id': '1.6.0'}))
(flexmock(dockpulp.imgutils).should_receive('check_repo').and_return(
check_repo_retval))
(flexmock(dockpulp.Pulp).should_receive('set_certs').with_args(
object, object))
(flexmock(dockpulp.Pulp).should_receive('login').with_args(object, object))
(flexmock(dockpulp.Pulp).should_receive('getRepos').with_args(
list, fields=list, distributors=True).and_return([{
"id":
"redhat-image-name1"
}, {
"id":
"redhat-prefix-image-name2"
}]))
(flexmock(dockpulp.Pulp).should_receive('createRepo'))
mock_docker()
return tasker, workflow
def prepare(scratch=False):
if MOCK:
mock_docker()
build_json = {'metadata': {'labels': {'scratch': scratch}}}
flexmock(util).should_receive('get_build_json').and_return(build_json)
tasker = DockerTasker()
workflow = DockerBuildWorkflow("test-image",
source={
"provider": "git",
"uri": DOCKERFILE_GIT
})
workflow.source = StubSource()
workflow.builder = StubInsideBuilder().for_workflow(workflow)
(flexmock(requests.Response, content=repocontent).should_receive(
'raise_for_status').and_return(None))
(flexmock(requests.Session, get=lambda *_: requests.Response()))
mock_get_retry_session()
return tasker, workflow
def test_distgit_fetch_artefacts_failure(tmpdir, workflow): # noqa
command = 'fedpkg sources'
expected_command = ['fedpkg', 'sources']
workflow.source = StubSource()
workflow.source.path = str(tmpdir)
initial_dir = os.getcwd()
assert initial_dir != str(tmpdir)
(flexmock(pyrpkg_fetch_artefacts.subprocess).should_receive('check_call').
with_args(expected_command).and_raise(RuntimeError).once())
workflow.conf.conf['sources_command'] = command
runner = (MockEnv(workflow).for_plugin(
DistgitFetchArtefactsPlugin.key).create_runner())
with pytest.raises(PluginFailedException):
runner.run()
assert os.getcwd() == initial_dir
def mock_workflow(tmpdir, orchestrator, user_config=None, site_config=None):
workflow = DockerBuildWorkflow(TEST_IMAGE,
source={
'provider': 'git',
'uri': 'asd'
})
workflow.source = StubSource()
workflow.source.path = str(tmpdir)
workflow.source.config = make_user_config(user_config)
workflow.builder = (StubInsideBuilder().for_workflow(workflow).set_df_path(
str(tmpdir)))
if orchestrator:
workflow.buildstep_plugins_conf = [{
'name': PLUGIN_BUILD_ORCHESTRATE_KEY
}]
workflow.plugin_workspace[ReactorConfigPlugin.key] = {
WORKSPACE_CONF_KEY: make_reactor_config(site_config or {})
}
return workflow
def test_rpmqa_plugin_skip(docker_tasker): # noqa
"""
Test skipping the plugin if workflow.image_components is already set
"""
mock_docker()
workflow = DockerBuildWorkflow(TEST_IMAGE, source=SOURCE)
workflow.source = StubSource()
workflow.builder = StubInsideBuilder().for_workflow(workflow)
image_components = {'type': 'rpm', 'name': 'something'}
setattr(workflow, 'image_components', image_components)
flexmock(docker.APIClient, logs=mock_logs_raise)
runner = PostBuildPluginsRunner(docker_tasker, workflow,
[{
"name": PostBuildRPMqaPlugin.key,
"args": {
'image_id': TEST_IMAGE
}
}])
results = runner.run()
assert results[PostBuildRPMqaPlugin.key] is None
assert workflow.image_components == image_components
def test_dangling_volumes_removed(docker_tasker, request):
fake_logger = FakeLogger()
existing_logger = atomic_reactor.core.logger
def restore_logger():
atomic_reactor.core.logger = existing_logger
request.addfinalizer(restore_logger)
atomic_reactor.core.logger = fake_logger
mock_docker()
workflow = DockerBuildWorkflow(SOURCE, TEST_IMAGE)
workflow.source = StubSource()
workflow.builder = StubInsideBuilder().for_workflow(workflow)
workflow.builder.set_base_from_scratch(False)
runner = PostBuildPluginsRunner(docker_tasker, workflow,
[{
"name": PostBuildRPMqaPlugin.key,
"args": {
'image_id': TEST_IMAGE
}
}])
runner.run()
assert ("container_id = '%s'",
u'f8ee920b2db5e802da2583a13a4edbf0523ca5fff6b6d6454c1fd6db5f38014d') \
in fake_logger.debugs
expected_volumes = [u'test', u'conflict_exception', u'real_exception']
assert ("volumes = %s", expected_volumes) in fake_logger.debugs
assert ("removing volume '%s'", u'test') in fake_logger.infos
assert ("removing volume '%s'", u'conflict_exception') in fake_logger.infos
assert ("removing volume '%s'", u'real_exception') in fake_logger.infos
assert ('ignoring a conflict when removing volume %s', 'conflict_exception') in \
fake_logger.debugs
def prepare(workflow, df_path):
workflow.source = StubSource()
workflow.builder = StubInsideBuilder().for_workflow(workflow)
workflow.builder.set_df_path(df_path)
def workflow(workflow):
workflow.data.dockerfile_images = DockerfileImages(['source_registry.com/fedora:26'])
workflow.source = StubSource()
return workflow
def test_rpmqa_plugin_base_from_scratch(caplog, tmpdir, docker_tasker,
packages_exists, rpm_failed,
get_archive_raises):
mock_docker()
workflow = DockerBuildWorkflow(source=SOURCE)
workflow.source = StubSource()
workflow.builder = get_builder(workflow, True)
mock_reactor_config(workflow, list_rpms_from_scratch=True)
if get_archive_raises is None:
mock_stream = generate_archive(tmpdir, empty=not packages_exists)
(flexmock(docker_tasker.tasker.d.wrapped).should_receive(
'get_archive').and_return(mock_stream, {}))
else:
response = flexmock(content="abc", status_code=123)
(flexmock(docker_tasker.tasker.d.wrapped).should_receive(
'get_archive').and_raise(get_archive_raises, 'get archive failed',
response))
runner = PostBuildPluginsRunner(
docker_tasker, workflow, [{
"name": PostBuildRPMqaPlugin.key,
"args": {
'image_id': TEST_IMAGE,
"ignore_autogenerated_gpg_keys": True
}
}])
if get_archive_raises is None and packages_exists and rpm_failed:
(flexmock(subprocess).should_receive("check_output").once().and_raise(
Exception, 'rpm query failed'))
elif get_archive_raises is None and packages_exists and not rpm_failed:
(flexmock(subprocess).should_receive("check_output").once().and_return(
"\n".join(PACKAGE_LIST_WITH_AUTOGENERATED)))
if get_archive_raises == APIError:
results = runner.run()
log_msg = 'Could not extract rpmdb in'
assert log_msg in caplog.text
assert results[PostBuildRPMqaPlugin.key] is None
assert workflow.image_components is None
elif get_archive_raises == Exception:
with pytest.raises(Exception, match='get archive failed'):
runner.run()
log_msg = 'Get archive failed while extracting rpmdb in'
assert log_msg in caplog.text
assert workflow.image_components is None
else:
if not packages_exists:
results = runner.run()
log_msg = '{} does not exist in rpmdb'.format(RPMDB_PACKAGES_NAME)
assert log_msg in caplog.text
assert results[PostBuildRPMqaPlugin.key] is None
assert workflow.image_components is None
else:
log_msg_getting = 'getting rpms from rpmdb:'
if rpm_failed:
with pytest.raises(Exception, match='rpm query failed'):
runner.run()
assert log_msg_getting in caplog.text
assert workflow.image_components is None
log_msg = 'Failed to get rpms from rpmdb:'
assert log_msg in caplog.text
else:
results = runner.run()
assert log_msg_getting in caplog.text
assert results[PostBuildRPMqaPlugin.key] == PACKAGE_LIST
assert workflow.image_components == parse_rpm_output(
PACKAGE_LIST)
log_msg = "from scratch, list_rpms_from_scratch is True, trying get rpmdb"
assert log_msg in caplog.text
def mock_environment(tmpdir,
session=None,
name=None,
component=None,
version=None,
release=None,
source=None,
build_process_failed=False,
docker_registry=True,
pulp_registries=0,
blocksize=None,
task_states=None,
additional_tags=None,
has_config=None,
prefer_schema1_digest=True):
if session is None:
session = MockedClientSession('', task_states=None)
if source is None:
source = GitSource('git', 'git://hostname/path')
if MOCK:
mock_docker()
tasker = DockerTasker()
workflow = DockerBuildWorkflow(SOURCE, "test-image")
base_image_id = '123456parent-id'
workflow.source = StubSource()
workflow.builder = StubInsideBuilder().for_workflow(workflow)
workflow.builder.image_id = '123456imageid'
workflow.builder.set_inspection_data({'Id': base_image_id})
setattr(workflow, 'tag_conf', TagConf())
with open(os.path.join(str(tmpdir), 'Dockerfile'), 'wt') as df:
df.write(
'FROM base\n'
'LABEL BZComponent={component} com.redhat.component={component}\n'
'LABEL Version={version} version={version}\n'
'LABEL Release={release} release={release}\n'.format(
component=component, version=version, release=release))
workflow.builder.set_df_path(df.name)
if name and version:
workflow.tag_conf.add_unique_image(
'user/test-image:{v}-timestamp'.format(v=version))
if name and version and release:
workflow.tag_conf.add_primary_images([
"{0}:{1}-{2}".format(name, version, release),
"{0}:{1}".format(name, version), "{0}:latest".format(name)
])
if additional_tags:
workflow.tag_conf.add_primary_images(
["{0}:{1}".format(name, tag) for tag in additional_tags])
flexmock(subprocess, Popen=fake_Popen)
flexmock(koji, ClientSession=lambda hub, opts: session)
flexmock(GitSource)
setattr(workflow, 'source', source)
setattr(workflow.source, 'lg', X())
setattr(workflow.source.lg, 'commit_id', '123456')
setattr(workflow, 'push_conf', PushConf())
if docker_registry:
docker_reg = workflow.push_conf.add_docker_registry(
'docker.example.com')
for image in workflow.tag_conf.images:
tag = image.to_str(registry=False)
if pulp_registries and prefer_schema1_digest:
docker_reg.digests[tag] = ManifestDigest(v1=fake_digest(image),
v2='sha256:not-used')
else:
docker_reg.digests[tag] = ManifestDigest(v1='sha256:not-used',
v2=fake_digest(image))
if has_config:
docker_reg.config = {
'config': {
'architecture': LOCAL_ARCH
},
'container_config': {}
}
for pulp_registry in range(pulp_registries):
workflow.push_conf.add_pulp_registry('env', 'pulp.example.com')
with open(os.path.join(str(tmpdir), 'image.tar.xz'), 'wt') as fp:
fp.write('x' * 2**12)
setattr(workflow, 'exported_image_sequence',
[{
'path': fp.name,
'type': IMAGE_TYPE_DOCKER_ARCHIVE
}])
if build_process_failed:
workflow.build_result = BuildResult(
logs=["docker build log - \u2018 \u2017 \u2019 \n'"],
fail_reason="not built")
else:
workflow.build_result = BuildResult(
logs=["docker build log - \u2018 \u2017 \u2019 \n'"],
image_id="id1234")
workflow.prebuild_plugins_conf = {}
workflow.image_components = parse_rpm_output([
"name1;1.0;1;" + LOCAL_ARCH + ";0;2000;" + FAKE_SIGMD5.decode() +
";23000;"
"RSA/SHA256, Tue 30 Aug 2016 00:00:00, Key ID 01234567890abc;(none)",
"name2;2.0;1;" + LOCAL_ARCH + ";0;3000;" + FAKE_SIGMD5.decode() +
";24000"
"RSA/SHA256, Tue 30 Aug 2016 00:00:00, Key ID 01234567890abd;(none)",
])
return tasker, workflow