def test_build_result(): with pytest.raises(AssertionError): BuildResult(fail_reason='it happens', image_id='spam') with pytest.raises(AssertionError): BuildResult(fail_reason='', image_id='spam') assert BuildResult(fail_reason='it happens').is_failed() assert not BuildResult(image_id='spam').is_failed() assert BuildResult(image_id='spam', logs=list('logs')).logs == list('logs') assert BuildResult(fail_reason='it happens').fail_reason == 'it happens' assert BuildResult(image_id='spam').image_id == 'spam' assert BuildResult(image_id='spam', annotations={ 'ham': 'mah' }).annotations == { 'ham': 'mah' } assert BuildResult(image_id='spam', labels={ 'ham': 'mah' }).labels == { 'ham': 'mah' } assert BuildResult(image_id='spam').is_image_available() assert not BuildResult(fail_reason='it happens').is_image_available() assert not BuildResult.make_remote_image_result().is_image_available() assert not BuildResult.make_remote_image_result().is_failed()
def run(self): release = self.get_release() platforms = self.get_platforms() koji_upload_dir = self.get_koji_upload_dir() task_id = self.get_fs_task_id() thread_pool = ThreadPool(len(platforms)) result = thread_pool.map_async( lambda cluster_info: self.do_worker_build(release, cluster_info, koji_upload_dir, task_id), [self.choose_cluster(platform) for platform in platforms] ) try: while not result.ready(): # The wait call is a blocking call which prevents signals # from being processed. Wait for short intervals instead # of a single long interval, so build cancellation can # be handled virtually immediately. result.wait(1) # Always clean up worker builds on any error to avoid # runaway worker builds (includes orchestrator build cancellation) except Exception: self.log.info('build cancelled, cancelling worker builds') if self.worker_builds: ThreadPool(len(self.worker_builds)).map( lambda bi: bi.cancel_build(), self.worker_builds) while not result.ready(): result.wait(1) raise annotations = {'worker-builds': { build_info.platform: build_info.get_annotations() for build_info in self.worker_builds if build_info.build }} self._apply_repositories(annotations) labels = self._make_labels() fail_reasons = { build_info.platform: build_info.get_fail_reason() for build_info in self.worker_builds if not build_info.build or not build_info.build.is_succeeded() } self.workflow.plugin_workspace[self.key] = { WORKSPACE_KEY_UPLOAD_DIR: koji_upload_dir, WORKSPACE_KEY_BUILD_INFO: {build_info.platform: build_info for build_info in self.worker_builds}, } if fail_reasons: return BuildResult(fail_reason=json.dumps(fail_reasons), annotations=annotations, labels=labels) return BuildResult.make_remote_image_result(annotations, labels=labels)
def run(self): if not self.platforms: raise RuntimeError("No enabled platform to build on") self.set_build_image() thread_pool = ThreadPool(len(self.platforms)) result = thread_pool.map_async(self.select_and_start_cluster, self.platforms) try: result.get() # Always clean up worker builds on any error to avoid # runaway worker builds (includes orchestrator build cancellation) except Exception: thread_pool.terminate() self.log.info('build cancelled, cancelling worker builds') if self.worker_builds: ThreadPool(len(self.worker_builds)).map( lambda bi: bi.cancel_build(), self.worker_builds) while not result.ready(): result.wait(1) raise else: thread_pool.close() thread_pool.join() annotations = { 'worker-builds': { build_info.platform: build_info.get_annotations() for build_info in self.worker_builds if build_info.build } } self._apply_repositories(annotations) labels = self._make_labels() fail_reasons = { build_info.platform: build_info.get_fail_reason() for build_info in self.worker_builds if not build_info.build or not build_info.build.is_succeeded() } workspace = self.workflow.plugin_workspace.setdefault(self.key, {}) workspace[WORKSPACE_KEY_UPLOAD_DIR] = self.koji_upload_dir workspace[WORKSPACE_KEY_BUILD_INFO] = { build_info.platform: build_info for build_info in self.worker_builds } if fail_reasons: return BuildResult(fail_reason=json.dumps(fail_reasons), annotations=annotations, labels=labels) return BuildResult.make_remote_image_result(annotations, labels=labels)
def test_tag_parse(tmpdir, docker_tasker, floating_tags, unique_tags, primary_tags, expected): df = df_parser(str(tmpdir)) df.content = DF_CONTENT_LABELS workflow = mock_workflow(tmpdir) setattr(workflow.builder, 'df_path', df.dockerfile_path) workflow.build_result = BuildResult.make_remote_image_result() base_inspect = { INSPECT_CONFIG: { 'Labels': { 'parentrelease': '7.4.1' }, 'Env': { 'parentrelease': '7.4.1' }, } } setattr(workflow.builder, 'base_image_inspect', base_inspect) mock_additional_tags_file(str(tmpdir), ['get_tags', 'file_tags']) if unique_tags is not None and primary_tags is not None and floating_tags is not None: input_tags = { 'unique': unique_tags, 'primary': primary_tags, 'floating': floating_tags, } else: input_tags = None runner = PostBuildPluginsRunner(docker_tasker, workflow, [{ 'name': TagFromConfigPlugin.key, 'args': { 'tag_suffixes': input_tags } }]) if expected is not None: results = runner.run() plugin_result = results[TagFromConfigPlugin.key] # Plugin should return the tags we expect assert plugin_result == expected # Workflow should have the expected tags configured for tag in expected: assert any(tag == str(image) for image in workflow.tag_conf.images) # Workflow should not have any other tags configured assert len(workflow.tag_conf.images) == len(expected) else: with pytest.raises(PluginFailedException): runner.run()
def test_tags_enclosed(tmpdir, docker_tasker, name, organization, expected): df = df_parser(str(tmpdir)) df.content = dedent("""\ FROM fedora LABEL "name"="{}" LABEL "version"="1.7" LABEL "release"="99" """.format(name)) workflow = mock_workflow(tmpdir) setattr(workflow.builder, 'df_path', df.dockerfile_path) workflow.build_result = BuildResult.make_remote_image_result() if organization: reactor_config = ReactorConfig({ 'version': 1, 'registries_organization': organization }) workflow.plugin_workspace[ReactorConfigPlugin.key] = { WORKSPACE_CONF_KEY: reactor_config } input_tags = { 'unique': ['foo', 'bar'], 'primary': ['{version}', '{version}-{release}'], } runner = PostBuildPluginsRunner(docker_tasker, workflow, [{ 'name': TagFromConfigPlugin.key, 'args': { 'tag_suffixes': input_tags } }]) results = runner.run() plugin_result = results[TagFromConfigPlugin.key] expected_tags = [ '{}:{}'.format(expected, tag) for tag in ['foo', 'bar', '1.7', '1.7-99'] ] # Plugin should return the tags we expect assert plugin_result == expected_tags # Workflow should have the expected tags configured for tag in expected_tags: assert any(tag == str(image) for image in workflow.tag_conf.images) # Workflow should not have any other tags configured assert len(workflow.tag_conf.images) == len(expected_tags)
def run(self): if not self.platforms: raise RuntimeError("No enabled platform to build on") self.set_build_image() thread_pool = ThreadPool(len(self.platforms)) result = thread_pool.map_async(self.select_and_start_cluster, self.platforms) try: result.get() # Always clean up worker builds on any error to avoid # runaway worker builds (includes orchestrator build cancellation) except Exception: thread_pool.terminate() self.log.info('build cancelled, cancelling worker builds') if self.worker_builds: ThreadPool(len(self.worker_builds)).map( lambda bi: bi.cancel_build(), self.worker_builds) while not result.ready(): result.wait(1) raise else: thread_pool.close() thread_pool.join() annotations = {'worker-builds': { build_info.platform: build_info.get_annotations() for build_info in self.worker_builds if build_info.build }} self._apply_repositories(annotations) labels = self._make_labels() fail_reasons = { build_info.platform: build_info.get_fail_reason() for build_info in self.worker_builds if not build_info.build or not build_info.build.is_succeeded() } workspace = self.workflow.plugin_workspace.setdefault(self.key, {}) workspace[WORKSPACE_KEY_UPLOAD_DIR] = self.koji_upload_dir workspace[WORKSPACE_KEY_BUILD_INFO] = {build_info.platform: build_info for build_info in self.worker_builds} if fail_reasons: return BuildResult(fail_reason=json.dumps(fail_reasons), annotations=annotations, labels=labels) return BuildResult.make_remote_image_result(annotations, labels=labels)
def test_build_result(): with pytest.raises(AssertionError): BuildResult(fail_reason='it happens', image_id='spam') with pytest.raises(AssertionError): BuildResult(fail_reason='', image_id='spam') assert BuildResult(fail_reason='it happens').is_failed() assert not BuildResult(image_id='spam').is_failed() assert BuildResult(image_id='spam', logs=list('logs')).logs == list('logs') assert BuildResult(fail_reason='it happens').fail_reason == 'it happens' assert BuildResult(image_id='spam').image_id == 'spam' assert BuildResult(image_id='spam', annotations={'ham': 'mah'}).annotations == {'ham': 'mah'} assert BuildResult(image_id='spam', labels={'ham': 'mah'}).labels == {'ham': 'mah'} assert BuildResult(image_id='spam').is_image_available() assert not BuildResult(fail_reason='it happens').is_image_available() assert not BuildResult.make_remote_image_result().is_image_available() assert not BuildResult.make_remote_image_result().is_failed()
def test_tag_parse(tmpdir, docker_tasker, floating_tags, unique_tags, primary_tags, expected): df = df_parser(str(tmpdir)) df.content = DF_CONTENT_LABELS workflow = mock_workflow(tmpdir) setattr(workflow.builder, 'df_path', df.dockerfile_path) workflow.build_result = BuildResult.make_remote_image_result() base_inspect = { INSPECT_CONFIG: { 'Labels': {'parentrelease': '7.4.1'}, 'Env': {'parentrelease': '7.4.1'}, } } setattr(workflow.builder, 'base_image_inspect', base_inspect) mock_additional_tags_file(str(tmpdir), ['get_tags', 'file_tags']) if unique_tags is not None and primary_tags is not None and floating_tags is not None: input_tags = { 'unique': unique_tags, 'primary': primary_tags, 'floating': floating_tags, } else: input_tags = None runner = PostBuildPluginsRunner( docker_tasker, workflow, [{'name': TagFromConfigPlugin.key, 'args': {'tag_suffixes': input_tags}}] ) if expected is not None: results = runner.run() plugin_result = results[TagFromConfigPlugin.key] # Plugin should return the tags we expect assert plugin_result == expected # Workflow should have the expected tags configured for tag in expected: assert any(tag == str(image) for image in workflow.tag_conf.images) # Workflow should not have any other tags configured assert len(workflow.tag_conf.images) == len(expected) else: with pytest.raises(PluginFailedException): runner.run()
def test_tags_enclosed(tmpdir, docker_tasker, name, organization, expected): df = df_parser(str(tmpdir)) df.content = dedent("""\ FROM fedora LABEL "name"="{}" LABEL "version"="1.7" LABEL "release"="99" """.format(name)) workflow = mock_workflow(tmpdir) setattr(workflow.builder, 'df_path', df.dockerfile_path) workflow.build_result = BuildResult.make_remote_image_result() if organization: reactor_config = ReactorConfig({ 'version': 1, 'registries_organization': organization }) workflow.plugin_workspace[ReactorConfigPlugin.key] = {WORKSPACE_CONF_KEY: reactor_config} input_tags = { 'unique': ['foo', 'bar'], 'primary': ['{version}', '{version}-{release}'], } runner = PostBuildPluginsRunner( docker_tasker, workflow, [{'name': TagFromConfigPlugin.key, 'args': {'tag_suffixes': input_tags}}] ) results = runner.run() plugin_result = results[TagFromConfigPlugin.key] expected_tags = ['{}:{}'.format(expected, tag) for tag in ['foo', 'bar', '1.7', '1.7-99']] # Plugin should return the tags we expect assert plugin_result == expected_tags # Workflow should have the expected tags configured for tag in expected_tags: assert any(tag == str(image) for image in workflow.tag_conf.images) # Workflow should not have any other tags configured assert len(workflow.tag_conf.images) == len(expected_tags)
def test_arrangementv4_repositories(tmpdir, group_manifests, restore, reactor_config_map): workflow = prepare(reactor_config_map=reactor_config_map) df_content = """ FROM fedora RUN yum install -y python-django CMD blabla""" df = df_parser(str(tmpdir)) df.content = df_content workflow.builder = X workflow.builder.df_path = df.dockerfile_path workflow.builder.df_dir = str(tmpdir) runner = ExitPluginsRunner(None, workflow, [{ 'name': StoreMetadataInOSv3Plugin.key, "args": { "url": "http://example.com/" } }]) worker_data = { 'repositories': { 'primary': ['worker:1'], 'unique': ['worker:unique'], }, } workflow.buildstep_result[OrchestrateBuildPlugin.key] = worker_data workflow.build_result = BuildResult.make_remote_image_result( annotations=worker_data) if group_manifests is not None: workflow.postbuild_results[ PLUGIN_GROUP_MANIFESTS_KEY] = group_manifests output = runner.run() assert StoreMetadataInOSv3Plugin.key in output annotations = output[StoreMetadataInOSv3Plugin.key]["annotations"] repositories = json.loads(annotations['repositories']) if restore: assert repositories != worker_data['repositories'] else: assert repositories == worker_data['repositories']
def test_arrangementv4_repositories(tmpdir, group_manifests, restore): workflow = prepare() df_content = """ FROM fedora RUN yum install -y python-django CMD blabla""" df = df_parser(str(tmpdir)) df.content = df_content workflow.builder = X workflow.builder.df_path = df.dockerfile_path workflow.builder.df_dir = str(tmpdir) runner = ExitPluginsRunner( None, workflow, [{ 'name': StoreMetadataInOSv3Plugin.key, "args": { "url": "http://example.com/" } }] ) worker_data = { 'repositories': { 'primary': ['worker:1'], 'unique': ['worker:unique'], }, } workflow.buildstep_result[OrchestrateBuildPlugin.key] = worker_data workflow.build_result = BuildResult.make_remote_image_result(annotations=worker_data) if group_manifests is not None: workflow.postbuild_results[PLUGIN_GROUP_MANIFESTS_KEY] = group_manifests output = runner.run() assert StoreMetadataInOSv3Plugin.key in output annotations = output[StoreMetadataInOSv3Plugin.key]["annotations"] repositories = json.loads(annotations['repositories']) if restore: assert repositories != worker_data['repositories'] else: assert repositories == worker_data['repositories']
def test_tag_parse(tmpdir, docker_tasker, unique_tags, primary_tags, expected): df = df_parser(str(tmpdir)) df.content = DF_CONTENT_LABELS workflow = mock_workflow(tmpdir) setattr(workflow.builder, 'df_path', df.dockerfile_path) workflow.build_result = BuildResult.make_remote_image_result() flexmock(workflow, base_image_inspect={ INSPECT_CONFIG: { 'Labels': { 'parentrelease': '7.4.1' }, 'Env': { 'parentrelease': '7.4.1' }, } }) mock_additional_tags_file(str(tmpdir), ['get_tags', 'file_tags']) if unique_tags is not None and primary_tags is not None: input_tags = {'unique': unique_tags, 'primary': primary_tags} else: input_tags = None runner = PostBuildPluginsRunner(docker_tasker, workflow, [{ 'name': TagFromConfigPlugin.key, 'args': { 'tag_suffixes': input_tags } }]) if expected is not None: results = runner.run() plugin_result = results[TagFromConfigPlugin.key] assert plugin_result == expected else: with pytest.raises(PluginFailedException): runner.run()
def test_tag_parse(tmpdir, docker_tasker, unique_tags, primary_tags, expected): df = df_parser(str(tmpdir)) df.content = DF_CONTENT_LABELS workflow = mock_workflow(tmpdir) setattr(workflow.builder, 'df_path', df.dockerfile_path) workflow.build_result = BuildResult.make_remote_image_result() flexmock(workflow, base_image_inspect={ INSPECT_CONFIG: { 'Labels': {'parentrelease': '7.4.1'}, 'Env': {'parentrelease': '7.4.1'}, } }) mock_additional_tags_file(str(tmpdir), ['get_tags', 'file_tags']) if unique_tags is not None and primary_tags is not None: input_tags = { 'unique': unique_tags, 'primary': primary_tags } else: input_tags = None runner = PostBuildPluginsRunner( docker_tasker, workflow, [{'name': TagFromConfigPlugin.key, 'args': {'tag_suffixes': input_tags}}] ) if expected is not None: results = runner.run() plugin_result = results[TagFromConfigPlugin.key] assert plugin_result == expected else: with pytest.raises(PluginFailedException): runner.run()
import inspect import signal import threading from time import sleep from atomic_reactor.inner import BuildResults, BuildResultsEncoder, BuildResultsJSONDecoder from atomic_reactor.inner import DockerBuildWorkflow BUILD_RESULTS_ATTRS = [ 'build_logs', 'built_img_inspect', 'built_img_info', 'base_img_info', 'base_plugins_output', 'built_img_plugins_output' ] DUMMY_BUILD_RESULT = BuildResult(image_id="image_id") DUMMY_FAILED_BUILD_RESULT = BuildResult(fail_reason='it happens') DUMMY_REMOTE_BUILD_RESULT = BuildResult.make_remote_image_result() def test_build_results_encoder(): results = BuildResults() expected_data = {} for attr in BUILD_RESULTS_ATTRS: setattr(results, attr, attr) expected_data[attr] = attr data = json.loads(json.dumps(results, cls=BuildResultsEncoder)) assert data == expected_data def test_build_results_decoder(): data = {}
from atomic_reactor.inner import BuildResults, BuildResultsEncoder, BuildResultsJSONDecoder from atomic_reactor.inner import DockerBuildWorkflow from atomic_reactor.inner import FSWatcher from atomic_reactor.constants import INSPECT_ROOTFS, INSPECT_ROOTFS_LAYERS BUILD_RESULTS_ATTRS = ['build_logs', 'built_img_inspect', 'built_img_info', 'base_img_info', 'base_plugins_output', 'built_img_plugins_output'] DUMMY_BUILD_RESULT = BuildResult(image_id="image_id") DUMMY_FAILED_BUILD_RESULT = BuildResult(fail_reason='it happens') DUMMY_REMOTE_BUILD_RESULT = BuildResult.make_remote_image_result() def test_build_results_encoder(): results = BuildResults() expected_data = {} for attr in BUILD_RESULTS_ATTRS: setattr(results, attr, attr) expected_data[attr] = attr data = json.loads(json.dumps(results, cls=BuildResultsEncoder)) assert data == expected_data def test_build_results_decoder(): data = {}