def mock_environment(tmpdir, session=None, build_process_failed=False, koji_build_id=None, use_import=False): if session is None: session = MockedClientSession('') if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow(SOURCE, 'test-image') setattr(workflow, 'builder', X()) setattr(workflow.builder, 'image_id', '123456imageid') setattr(workflow.builder, 'base_image', ImageName(repo='Fedora', tag='22')) setattr(workflow.builder, 'source', X()) setattr(workflow.builder.source, 'dockerfile_path', None) setattr(workflow.builder.source, 'path', None) flexmock(koji, ClientSession=lambda hub, opts: session) if build_process_failed: workflow.build_result = BuildResult(fail_reason="not built") else: workflow.build_result = BuildResult(image_id="id1234") if koji_build_id: if use_import: workflow.exit_results[KojiImportPlugin.key] = koji_build_id else: workflow.exit_results[KojiPromotePlugin.key] = koji_build_id (flexmock(time).should_receive('sleep').and_return(None)) return tasker, workflow
def run(self): """ build image inside current environment; it's expected this may run within (privileged) docker container Input: df_dir image Output: BuildResult built_image_info image_id """ builder = self.workflow.builder logs_gen = self.tasker.build_image_from_path(builder.df_dir, builder.image) self.log.debug('build is submitted, waiting for it to finish') command_result = wait_for_command(logs_gen) if command_result.is_failed(): return BuildResult(logs=command_result.logs, fail_reason=command_result.error_detail) else: image_id = builder.get_built_image_info()['Id'] return BuildResult(logs=command_result.logs, image_id=image_id)
def run(self): """ Build image inside current environment using buildah; It's expected this may run within (privileged) oci container. Returns: BuildResult """ builder = self.workflow.builder image = builder.image.to_str() kwargs = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) encoding_params = dict(encoding='utf-8', errors='replace') kwargs.update(encoding_params) ib_process = subprocess.Popen( ['buildah', 'bud', '-t', image, builder.df_dir], **kwargs) self.log.debug('buildah build has begun; waiting for it to finish') output = [] while True: poll = ib_process.poll() out = ib_process.stdout.readline() if out: self.log.info('%s', out.rstrip()) output.append(out) elif poll is not None: break if ib_process.returncode != 0: # in the case of an apparent failure, single out the last line to # include in the failure summary. err = output[ -1] if output else "<buildah had bad exit code but no output>" return BuildResult( logs=output, fail_reason="image build failed (rc={}): {}".format( ib_process.returncode, err), ) image_id = builder.get_built_image_info()['Id'] if ':' not in image_id: # Older versions of the daemon do not include the prefix image_id = 'sha256:{}'.format(image_id) # since we need no squash, export the image for local operations like squash would have self.log.info("fetching image %s from docker", image) output_path = os.path.join(self.workflow.source.workdir, EXPORTED_SQUASHED_IMAGE_NAME) with open(output_path, "w") as image_file: image_file.write(self.tasker.get_image(image).data) img_metadata = get_exported_image_metadata(output_path, IMAGE_TYPE_DOCKER_ARCHIVE) self.workflow.exported_image_sequence.append(img_metadata) return BuildResult(logs=output, image_id=image_id, skip_layer_squash=True)
def test_compress(self, tmpdir, caplog, source_build, method, load_exported_image, give_export, extension): if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow('test-image', source={ 'provider': 'git', 'uri': 'asd' }) workflow.builder = X() exp_img = os.path.join(str(tmpdir), 'img.tar') if source_build: workflow.build_result = BuildResult(oci_image_path="oci_path") else: workflow.build_result = BuildResult(image_id="12345") if load_exported_image and give_export: tarfile.open(exp_img, mode='w').close() workflow.exported_image_sequence.append({ 'path': exp_img, 'type': IMAGE_TYPE_DOCKER_ARCHIVE }) tasker = None # image provided, should not query docker runner = PostBuildPluginsRunner( tasker, workflow, [{ 'name': CompressPlugin.key, 'args': { 'method': method, 'load_exported_image': load_exported_image, }, }]) if not extension: with pytest.raises(Exception) as excinfo: runner.run() assert 'Unsupported compression format' in str(excinfo.value) return runner.run() if source_build and not (give_export and load_exported_image): assert 'skipping, no exported source image to compress' in caplog.text else: compressed_img = os.path.join( workflow.source.tmpdir, EXPORTED_COMPRESSED_IMAGE_NAME_TEMPLATE.format(extension)) assert os.path.exists(compressed_img) metadata = workflow.exported_image_sequence[-1] assert metadata['path'] == compressed_img assert metadata['type'] == IMAGE_TYPE_DOCKER_ARCHIVE assert 'uncompressed_size' in metadata assert isinstance(metadata['uncompressed_size'], integer_types) assert ", ratio: " in caplog.text
def run(self): release = self.get_release() platforms = self.get_platforms() koji_upload_dir = self.get_koji_upload_dir() task_id = self.get_fs_task_id() thread_pool = ThreadPool(len(platforms)) result = thread_pool.map_async( lambda cluster_info: self.do_worker_build(release, cluster_info, koji_upload_dir, task_id), [self.choose_cluster(platform) for platform in platforms] ) try: while not result.ready(): # The wait call is a blocking call which prevents signals # from being processed. Wait for short intervals instead # of a single long interval, so build cancellation can # be handled virtually immediately. result.wait(1) # Always clean up worker builds on any error to avoid # runaway worker builds (includes orchestrator build cancellation) except Exception: self.log.info('build cancelled, cancelling worker builds') if self.worker_builds: ThreadPool(len(self.worker_builds)).map( lambda bi: bi.cancel_build(), self.worker_builds) while not result.ready(): result.wait(1) raise annotations = {'worker-builds': { build_info.platform: build_info.get_annotations() for build_info in self.worker_builds if build_info.build }} self._apply_repositories(annotations) labels = self._make_labels() fail_reasons = { build_info.platform: build_info.get_fail_reason() for build_info in self.worker_builds if not build_info.build or not build_info.build.is_succeeded() } self.workflow.plugin_workspace[self.key] = { WORKSPACE_KEY_UPLOAD_DIR: koji_upload_dir, WORKSPACE_KEY_BUILD_INFO: {build_info.platform: build_info for build_info in self.worker_builds}, } if fail_reasons: return BuildResult(fail_reason=json.dumps(fail_reasons), annotations=annotations, labels=labels) return BuildResult.make_remote_image_result(annotations, labels=labels)
def run(self): if not self.platforms: raise RuntimeError("No enabled platform to build on") self.set_build_image() thread_pool = ThreadPool(len(self.platforms)) result = thread_pool.map_async(self.select_and_start_cluster, self.platforms) try: result.get() # Always clean up worker builds on any error to avoid # runaway worker builds (includes orchestrator build cancellation) except Exception: thread_pool.terminate() self.log.info('build cancelled, cancelling worker builds') if self.worker_builds: ThreadPool(len(self.worker_builds)).map( lambda bi: bi.cancel_build(), self.worker_builds) while not result.ready(): result.wait(1) raise else: thread_pool.close() thread_pool.join() annotations = { 'worker-builds': { build_info.platform: build_info.get_annotations() for build_info in self.worker_builds if build_info.build } } self._apply_repositories(annotations) labels = self._make_labels() fail_reasons = { build_info.platform: build_info.get_fail_reason() for build_info in self.worker_builds if not build_info.build or not build_info.build.is_succeeded() } workspace = self.workflow.plugin_workspace.setdefault(self.key, {}) workspace[WORKSPACE_KEY_UPLOAD_DIR] = self.koji_upload_dir workspace[WORKSPACE_KEY_BUILD_INFO] = { build_info.platform: build_info for build_info in self.worker_builds } if fail_reasons: return BuildResult(fail_reason=json.dumps(fail_reasons), annotations=annotations, labels=labels) return BuildResult.make_remote_image_result(annotations, labels=labels)
def mock_environment(tmpdir, primary_images=None, worker_annotations={}): if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow(SOURCE, "test-image") base_image_id = '123456parent-id' setattr(workflow, '_base_image_inspect', {'Id': base_image_id}) setattr(workflow, 'builder', X()) setattr(workflow.builder, 'image_id', '123456imageid') setattr(workflow.builder, 'base_image', ImageName(repo='Fedora', tag='22')) setattr(workflow.builder, 'source', X()) setattr(workflow.builder, 'built_image_info', {'ParentId': base_image_id}) setattr(workflow.builder.source, 'dockerfile_path', None) setattr(workflow.builder.source, 'path', None) setattr(workflow, 'tag_conf', TagConf()) if primary_images: workflow.tag_conf.add_primary_images(primary_images) workflow.tag_conf.add_unique_image(primary_images[0]) annotations = deepcopy(BUILD_ANNOTATIONS) if not worker_annotations: worker_annotations = {'ppc64le': PPC_ANNOTATIONS} for worker in worker_annotations: annotations['worker-builds'][worker] = deepcopy( worker_annotations[worker]) workflow.build_result = BuildResult(image_id='123456', annotations=annotations) return tasker, workflow
def test_get_primary_images(tag_conf, tag_annotation, expected): template_image = ImageName.parse('registry.example.com/fedora') workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image') for tag in tag_conf: image_name = ImageName.parse(str(template_image)) image_name.tag = tag workflow.tag_conf.add_primary_image(str(image_name)) annotations = {} for tag in tag_annotation: annotations.setdefault('repositories', {}).setdefault('primary', []) image_name = ImageName.parse(str(template_image)) image_name.tag = tag annotations['repositories']['primary'].append(str(image_name)) build_result = BuildResult(annotations=annotations, image_id='foo') workflow.build_result = build_result actual = get_primary_images(workflow) assert len(actual) == len(expected) for index, primary_image in enumerate(actual): assert primary_image.registry == template_image.registry assert primary_image.namespace == template_image.namespace assert primary_image.repo == template_image.repo assert primary_image.tag == expected[index]
def mock_environment(tmpdir, primary_images=None, annotations=None): if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow(source=SOURCE) base_image_id = '123456parent-id' setattr(workflow, '_base_image_inspect', {'Id': base_image_id}) setattr(workflow, 'builder', StubInsideBuilder()) setattr(workflow.builder, 'image_id', '123456imageid') setattr(workflow.builder, 'base_image', ImageName(repo='Fedora', tag='22')) setattr(workflow.builder, 'source', StubInsideBuilder()) setattr(workflow.builder, 'built_image_info', {'ParentId': base_image_id}) setattr(workflow.builder.source, 'dockerfile_path', None) setattr(workflow.builder.source, 'path', None) setattr(workflow, 'tag_conf', TagConf()) if primary_images: for image in primary_images: if '-' in ImageName.parse(image).tag: workflow.tag_conf.add_primary_image(image) workflow.tag_conf.add_unique_image(primary_images[0]) workflow.tag_conf.add_floating_image('namespace/httpd:floating') workflow.build_result = BuildResult(image_id='123456', annotations=annotations or {}) return tasker, workflow
def test_pulp_publish_delete(worker_builds_created, v1_image_ids, expected, caplog, reactor_config_map): tasker, workflow = prepare(success=False, v1_image_ids=v1_image_ids) if not worker_builds_created: workflow.build_result = BuildResult(fail_reason="not built") if reactor_config_map: pulp_map = {'name': 'pulp_registry_name', 'auth': {}} workflow.plugin_workspace[ReactorConfigPlugin.key] = {} workflow.plugin_workspace[ReactorConfigPlugin.key][WORKSPACE_CONF_KEY] =\ ReactorConfig({'version': 1, 'pulp': pulp_map}) plugin = PulpPublishPlugin(tasker, workflow, 'pulp_registry_name') msg = "removing ppc64le_v1_image_id from" (flexmock(dockpulp.Pulp).should_receive('crane').never()) if expected: (flexmock(dockpulp.Pulp).should_receive('remove').with_args( unicode, unicode)) else: (flexmock(dockpulp.Pulp).should_receive('remove').never()) crane_images = plugin.run() assert crane_images == [] if expected and worker_builds_created: assert msg in caplog.text() else: assert msg not in caplog.text()
def mock_environment(tmpdir, workflow, primary_images=None, floating_images=None, manifest_results=None, annotations=None): if MOCK: mock_docker() tasker = DockerTasker() base_image_id = '123456parent-id' setattr(workflow, '_base_image_inspect', {'Id': base_image_id}) setattr(workflow, 'builder', StubInsideBuilder()) setattr(workflow.builder, 'image_id', '123456imageid') setattr(workflow.builder, 'base_image', ImageName(repo='Fedora', tag='22')) setattr(workflow.builder, 'source', StubInsideBuilder()) setattr(workflow.builder, 'built_image_info', {'ParentId': base_image_id}) setattr(workflow.builder.source, 'dockerfile_path', None) setattr(workflow.builder.source, 'path', None) if primary_images: for image in primary_images: if '-' in ImageName.parse(image).tag: workflow.tag_conf.add_primary_image(image) workflow.tag_conf.add_unique_image(primary_images[0]) if floating_images: workflow.tag_conf.add_floating_images(floating_images) workflow.build_result = BuildResult(image_id='123456', annotations=annotations or {}) workflow.postbuild_results = {} if manifest_results: workflow.postbuild_results[PLUGIN_GROUP_MANIFESTS_KEY] = manifest_results return tasker, workflow
def prepare(insecure_registry=None, namespace=None, primary_images_tag_conf=DEFAULT_TAGS_AMOUNT, primary_images_annotations=DEFAULT_TAGS_AMOUNT): """ Boiler-plate test set-up """ if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow(SOURCE, "test-image") setattr(workflow, 'builder', X()) setattr(workflow.builder, 'image_id', 'asd123') setattr(workflow.builder, 'source', X()) setattr(workflow.builder.source, 'dockerfile_path', None) setattr(workflow.builder.source, 'path', None) version_release_primary_image = 'registry.example.com/fedora:version-release' annotations = None if primary_images_annotations: primary_images = [ 'registry.example.com/fedora:annotation_{}'.format(x) for x in range(primary_images_annotations) ] primary_images.append(version_release_primary_image) annotations = {'repositories': {'primary': primary_images}} annotations build_result = BuildResult(annotations=annotations, image_id='foo') setattr(workflow, 'build_result', build_result) if primary_images_tag_conf: primary_images = [ 'registry.example.com/fedora:tag_conf_{}'.format(x) for x in range(primary_images_tag_conf) ] primary_images.append(version_release_primary_image) workflow.tag_conf.add_primary_images(primary_images) fake_conf = osbs.conf.Configuration(conf_file=None, openshift_url='/') expectation = flexmock(osbs.conf).should_receive('Configuration').and_return(fake_conf) if namespace: expectation.with_args(conf_file=None, namespace=namespace, verify_ssl=False, openshift_url="", use_auth=False, build_json_dir="") runner = PostBuildPluginsRunner(tasker, workflow, [{ 'name': ImportImagePlugin.key, 'args': { 'imagestream': TEST_IMAGESTREAM, 'docker_image_repo': TEST_REPO, 'url': '', 'build_json_dir': "", 'verify_ssl': False, 'use_auth': False, 'insecure_registry': insecure_registry, }}]) return runner
def prepare(insecure_registry=None, retry_delay=0, import_attempts=3, namespace=None): """ Boiler-plate test set-up """ if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow(SOURCE, "test-image") setattr(workflow, 'builder', X()) setattr(workflow.builder, 'image_id', 'asd123') setattr(workflow.builder, 'source', X()) setattr(workflow.builder.source, 'dockerfile_path', None) setattr(workflow.builder.source, 'path', None) a = { 'repositories': { 'primary': ['one', 'two', 'three', 'four', 'five', 'six'] } } build_result = BuildResult(annotations=a, image_id='foo') setattr(workflow, 'build_result', build_result) fake_conf = osbs.conf.Configuration(conf_file=None, openshift_uri='/') expectation = flexmock( osbs.conf).should_receive('Configuration').and_return(fake_conf) if namespace: expectation.with_args(conf_file=None, namespace=namespace, verify_ssl=False, openshift_url="", openshift_uri="", use_auth=False, build_json_dir="") runner = PostBuildPluginsRunner( tasker, workflow, [{ 'name': ImportImagePlugin.key, 'args': { 'imagestream': TEST_IMAGESTREAM, 'docker_image_repo': TEST_REPO, 'url': '', 'build_json_dir': "", 'verify_ssl': False, 'use_auth': False, 'insecure_registry': insecure_registry, 'retry_delay': retry_delay, 'import_attempts': import_attempts, } }]) return runner
def test_tag_parse(tmpdir, docker_tasker, floating_tags, unique_tags, primary_tags, expected): df = df_parser(str(tmpdir)) df.content = DF_CONTENT_LABELS workflow = mock_workflow(tmpdir) setattr(workflow.builder, 'df_path', df.dockerfile_path) workflow.build_result = BuildResult.make_remote_image_result() base_inspect = { INSPECT_CONFIG: { 'Labels': { 'parentrelease': '7.4.1' }, 'Env': { 'parentrelease': '7.4.1' }, } } setattr(workflow.builder, 'base_image_inspect', base_inspect) mock_additional_tags_file(str(tmpdir), ['get_tags', 'file_tags']) if unique_tags is not None and primary_tags is not None and floating_tags is not None: input_tags = { 'unique': unique_tags, 'primary': primary_tags, 'floating': floating_tags, } else: input_tags = None runner = PostBuildPluginsRunner(docker_tasker, workflow, [{ 'name': TagFromConfigPlugin.key, 'args': { 'tag_suffixes': input_tags } }]) if expected is not None: results = runner.run() plugin_result = results[TagFromConfigPlugin.key] # Plugin should return the tags we expect assert plugin_result == expected # Workflow should have the expected tags configured for tag in expected: assert any(tag == str(image) for image in workflow.tag_conf.images) # Workflow should not have any other tags configured assert len(workflow.tag_conf.images) == len(expected) else: with pytest.raises(PluginFailedException): runner.run()
def run(self): """Build image inside current environment. Returns: BuildResult """ fetch_sources_result = self.workflow.prebuild_results.get( PLUGIN_FETCH_SOURCES_KEY, {}) source_data_dir = fetch_sources_result.get('image_sources_dir') if not source_data_dir or not os.path.isdir(source_data_dir): err_msg = "No SRPMs directory '{}' available".format( source_data_dir) self.log.error(err_msg) return BuildResult(logs=err_msg, fail_reason=err_msg) if not os.listdir(source_data_dir): self.log.warning("SRPMs directory '%s' is empty", source_data_dir) image_output_dir = tempfile.mkdtemp() cmd = [ 'bsi', '-d', 'sourcedriver_rpm_dir', '-s', '{}'.format(source_data_dir), '-o', '{}'.format(image_output_dir) ] try: output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: self.log.error("BSI failed with output:\n%s", e.output) return BuildResult( logs=e.output, fail_reason='BSI utility failed build source image') self.log.debug("Build log:\n%s\n", output) self.export_image(image_output_dir) return BuildResult(logs=output, oci_image_path=image_output_dir, skip_layer_squash=True)
def test_tags_enclosed(tmpdir, docker_tasker, name, organization, expected): df = df_parser(str(tmpdir)) df.content = dedent("""\ FROM fedora LABEL "name"="{}" LABEL "version"="1.7" LABEL "release"="99" """.format(name)) workflow = mock_workflow(tmpdir) setattr(workflow.builder, 'df_path', df.dockerfile_path) workflow.build_result = BuildResult.make_remote_image_result() if organization: reactor_config = ReactorConfig({ 'version': 1, 'registries_organization': organization }) workflow.plugin_workspace[ReactorConfigPlugin.key] = { WORKSPACE_CONF_KEY: reactor_config } input_tags = { 'unique': ['foo', 'bar'], 'primary': ['{version}', '{version}-{release}'], } runner = PostBuildPluginsRunner(docker_tasker, workflow, [{ 'name': TagFromConfigPlugin.key, 'args': { 'tag_suffixes': input_tags } }]) results = runner.run() plugin_result = results[TagFromConfigPlugin.key] expected_tags = [ '{}:{}'.format(expected, tag) for tag in ['foo', 'bar', '1.7', '1.7-99'] ] # Plugin should return the tags we expect assert plugin_result == expected_tags # Workflow should have the expected tags configured for tag in expected_tags: assert any(tag == str(image) for image in workflow.tag_conf.images) # Workflow should not have any other tags configured assert len(workflow.tag_conf.images) == len(expected_tags)
def run(self): if not self.platforms: raise RuntimeError("No enabled platform to build on") self.set_build_image() thread_pool = ThreadPool(len(self.platforms)) result = thread_pool.map_async(self.select_and_start_cluster, self.platforms) try: result.get() # Always clean up worker builds on any error to avoid # runaway worker builds (includes orchestrator build cancellation) except Exception: thread_pool.terminate() self.log.info('build cancelled, cancelling worker builds') if self.worker_builds: ThreadPool(len(self.worker_builds)).map( lambda bi: bi.cancel_build(), self.worker_builds) while not result.ready(): result.wait(1) raise else: thread_pool.close() thread_pool.join() annotations = {'worker-builds': { build_info.platform: build_info.get_annotations() for build_info in self.worker_builds if build_info.build }} self._apply_repositories(annotations) labels = self._make_labels() fail_reasons = { build_info.platform: build_info.get_fail_reason() for build_info in self.worker_builds if not build_info.build or not build_info.build.is_succeeded() } workspace = self.workflow.plugin_workspace.setdefault(self.key, {}) workspace[WORKSPACE_KEY_UPLOAD_DIR] = self.koji_upload_dir workspace[WORKSPACE_KEY_BUILD_INFO] = {build_info.platform: build_info for build_info in self.worker_builds} if fail_reasons: return BuildResult(fail_reason=json.dumps(fail_reasons), annotations=annotations, labels=labels) return BuildResult.make_remote_image_result(annotations, labels=labels)
def run(self): """ build image inside current environment; it's expected this may run within (privileged) docker container Input: df_dir image Output: BuildResult built_image_info image_id """ builder = self.workflow.builder allow_repo_dir_in_dockerignore(builder.df_dir) logs_gen = self.tasker.build_image_from_path( builder.df_dir, builder.image, buildargs=builder.buildargs) self.log.debug('build is submitted, waiting for it to finish') try: command_result = wait_for_command(logs_gen) except docker.errors.APIError as ex: return BuildResult(logs=[], fail_reason=ex.explanation) if command_result.is_failed(): return BuildResult(logs=command_result.logs, fail_reason=command_result.error) else: image_id = builder.get_built_image_info()['Id'] if ':' not in image_id: # Older versions of the daemon do not include the prefix image_id = 'sha256:{}'.format(image_id) return BuildResult(logs=command_result.logs, image_id=image_id)
def test_build_result(): with pytest.raises(AssertionError): BuildResult(fail_reason='it happens', image_id='spam') with pytest.raises(AssertionError): BuildResult(fail_reason='', image_id='spam') assert BuildResult(fail_reason='it happens').is_failed() assert not BuildResult(image_id='spam').is_failed() assert BuildResult(image_id='spam', logs=list('logs')).logs == list('logs') assert BuildResult(fail_reason='it happens').fail_reason == 'it happens' assert BuildResult(image_id='spam').image_id == 'spam' assert BuildResult(image_id='spam', annotations={'ham': 'mah'}).annotations == {'ham': 'mah'} assert BuildResult(image_id='spam', labels={'ham': 'mah'}).labels == {'ham': 'mah'} assert BuildResult(image_id='spam').is_image_available() assert not BuildResult(fail_reason='it happens').is_image_available() assert not BuildResult.make_remote_image_result().is_image_available() assert not BuildResult.make_remote_image_result().is_failed()
def test_tag_parse(tmpdir, docker_tasker, floating_tags, unique_tags, primary_tags, expected): df = df_parser(str(tmpdir)) df.content = DF_CONTENT_LABELS workflow = mock_workflow(tmpdir) setattr(workflow.builder, 'df_path', df.dockerfile_path) workflow.build_result = BuildResult.make_remote_image_result() base_inspect = { INSPECT_CONFIG: { 'Labels': {'parentrelease': '7.4.1'}, 'Env': {'parentrelease': '7.4.1'}, } } setattr(workflow.builder, 'base_image_inspect', base_inspect) mock_additional_tags_file(str(tmpdir), ['get_tags', 'file_tags']) if unique_tags is not None and primary_tags is not None and floating_tags is not None: input_tags = { 'unique': unique_tags, 'primary': primary_tags, 'floating': floating_tags, } else: input_tags = None runner = PostBuildPluginsRunner( docker_tasker, workflow, [{'name': TagFromConfigPlugin.key, 'args': {'tag_suffixes': input_tags}}] ) if expected is not None: results = runner.run() plugin_result = results[TagFromConfigPlugin.key] # Plugin should return the tags we expect assert plugin_result == expected # Workflow should have the expected tags configured for tag in expected: assert any(tag == str(image) for image in workflow.tag_conf.images) # Workflow should not have any other tags configured assert len(workflow.tag_conf.images) == len(expected) else: with pytest.raises(PluginFailedException): runner.run()
def test_bad_inspect_data(tmpdir, docker_tasker, inspect, error): workflow = mock_workflow(tmpdir) if inspect is not None: workflow.built_image_inspect = {INSPECT_CONFIG: inspect} workflow.build_result = BuildResult(image_id=IMPORTED_IMAGE_ID) mock_additional_tags_file(str(tmpdir), ['spam', 'bacon']) runner = PostBuildPluginsRunner(docker_tasker, workflow, [{ 'name': TagFromConfigPlugin.key }]) with pytest.raises(PluginFailedException) as exc: runner.run() assert error in str(exc)
def test_tags_enclosed(tmpdir, docker_tasker, name, organization, expected): df = df_parser(str(tmpdir)) df.content = dedent("""\ FROM fedora LABEL "name"="{}" LABEL "version"="1.7" LABEL "release"="99" """.format(name)) workflow = mock_workflow(tmpdir) setattr(workflow.builder, 'df_path', df.dockerfile_path) workflow.build_result = BuildResult.make_remote_image_result() if organization: reactor_config = ReactorConfig({ 'version': 1, 'registries_organization': organization }) workflow.plugin_workspace[ReactorConfigPlugin.key] = {WORKSPACE_CONF_KEY: reactor_config} input_tags = { 'unique': ['foo', 'bar'], 'primary': ['{version}', '{version}-{release}'], } runner = PostBuildPluginsRunner( docker_tasker, workflow, [{'name': TagFromConfigPlugin.key, 'args': {'tag_suffixes': input_tags}}] ) results = runner.run() plugin_result = results[TagFromConfigPlugin.key] expected_tags = ['{}:{}'.format(expected, tag) for tag in ['foo', 'bar', '1.7', '1.7-99']] # Plugin should return the tags we expect assert plugin_result == expected_tags # Workflow should have the expected tags configured for tag in expected_tags: assert any(tag == str(image) for image in workflow.tag_conf.images) # Workflow should not have any other tags configured assert len(workflow.tag_conf.images) == len(expected_tags)
def test_tag_from_config_plugin_generated(tmpdir, docker_tasker, tags, name, expected): workflow = mock_workflow(tmpdir) workflow.built_image_inspect = {INSPECT_CONFIG: {'Labels': {'Name': name}}} workflow.build_result = BuildResult(image_id=IMPORTED_IMAGE_ID) # Simulate missing additional-tags file. if tags is not None: mock_additional_tags_file(str(tmpdir), tags) runner = PostBuildPluginsRunner(docker_tasker, workflow, [{ 'name': TagFromConfigPlugin.key }]) results = runner.run() plugin_result = results[TagFromConfigPlugin.key] assert plugin_result == expected
def test_remove_worker_metadata_no_worker_build(tmpdir, caplog, user_params): """Don't traceback with missing worker builds, without worker builds plugin should just skip""" workflow = mock_workflow(tmpdir) annotations = None workflow.build_result = BuildResult(annotations=annotations, image_id="id1234") runner = ExitPluginsRunner( None, workflow, [{ 'name': PLUGIN_REMOVE_WORKER_METADATA_KEY, "args": {} }] ) runner.run() assert "No build annotations found, skipping plugin" in caplog.text assert "Traceback" not in caplog.text
def test_arrangementv4_repositories(tmpdir, group_manifests, restore, reactor_config_map): workflow = prepare(reactor_config_map=reactor_config_map) df_content = """ FROM fedora RUN yum install -y python-django CMD blabla""" df = df_parser(str(tmpdir)) df.content = df_content workflow.builder = X workflow.builder.df_path = df.dockerfile_path workflow.builder.df_dir = str(tmpdir) runner = ExitPluginsRunner(None, workflow, [{ 'name': StoreMetadataInOSv3Plugin.key, "args": { "url": "http://example.com/" } }]) worker_data = { 'repositories': { 'primary': ['worker:1'], 'unique': ['worker:unique'], }, } workflow.buildstep_result[OrchestrateBuildPlugin.key] = worker_data workflow.build_result = BuildResult.make_remote_image_result( annotations=worker_data) if group_manifests is not None: workflow.postbuild_results[ PLUGIN_GROUP_MANIFESTS_KEY] = group_manifests output = runner.run() assert StoreMetadataInOSv3Plugin.key in output annotations = output[StoreMetadataInOSv3Plugin.key]["annotations"] repositories = json.loads(annotations['repositories']) if restore: assert repositories != worker_data['repositories'] else: assert repositories == worker_data['repositories']
def test_arrangementv4_repositories(tmpdir, group_manifests, restore): workflow = prepare() df_content = """ FROM fedora RUN yum install -y python-django CMD blabla""" df = df_parser(str(tmpdir)) df.content = df_content workflow.builder = X workflow.builder.df_path = df.dockerfile_path workflow.builder.df_dir = str(tmpdir) runner = ExitPluginsRunner( None, workflow, [{ 'name': StoreMetadataInOSv3Plugin.key, "args": { "url": "http://example.com/" } }] ) worker_data = { 'repositories': { 'primary': ['worker:1'], 'unique': ['worker:unique'], }, } workflow.buildstep_result[OrchestrateBuildPlugin.key] = worker_data workflow.build_result = BuildResult.make_remote_image_result(annotations=worker_data) if group_manifests is not None: workflow.postbuild_results[PLUGIN_GROUP_MANIFESTS_KEY] = group_manifests output = runner.run() assert StoreMetadataInOSv3Plugin.key in output annotations = output[StoreMetadataInOSv3Plugin.key]["annotations"] repositories = json.loads(annotations['repositories']) if restore: assert repositories != worker_data['repositories'] else: assert repositories == worker_data['repositories']
def test_tag_parse(tmpdir, docker_tasker, unique_tags, primary_tags, expected): df = df_parser(str(tmpdir)) df.content = DF_CONTENT_LABELS workflow = mock_workflow(tmpdir) setattr(workflow.builder, 'df_path', df.dockerfile_path) workflow.build_result = BuildResult.make_remote_image_result() flexmock(workflow, base_image_inspect={ INSPECT_CONFIG: { 'Labels': { 'parentrelease': '7.4.1' }, 'Env': { 'parentrelease': '7.4.1' }, } }) mock_additional_tags_file(str(tmpdir), ['get_tags', 'file_tags']) if unique_tags is not None and primary_tags is not None: input_tags = {'unique': unique_tags, 'primary': primary_tags} else: input_tags = None runner = PostBuildPluginsRunner(docker_tasker, workflow, [{ 'name': TagFromConfigPlugin.key, 'args': { 'tag_suffixes': input_tags } }]) if expected is not None: results = runner.run() plugin_result = results[TagFromConfigPlugin.key] assert plugin_result == expected else: with pytest.raises(PluginFailedException): runner.run()
def test_pulp_publish_delete(worker_builds_created, v1_image_ids, expected, caplog): tasker, workflow = prepare(success=False, v1_image_ids=v1_image_ids) if not worker_builds_created: workflow.build_result = BuildResult(fail_reason="not built") plugin = PulpPublishPlugin(tasker, workflow, 'pulp_registry_name') msg = "removing ppc64le_v1_image_id from" (flexmock(dockpulp.Pulp).should_receive('crane').never()) if expected: (flexmock(dockpulp.Pulp).should_receive('remove').with_args( unicode, unicode)) else: (flexmock(dockpulp.Pulp).should_receive('remove').never()) crane_images = plugin.run() assert crane_images == [] if expected and worker_builds_created: assert msg in caplog.text() else: assert msg not in caplog.text()
def test_tag_parse(tmpdir, docker_tasker, unique_tags, primary_tags, expected): df = df_parser(str(tmpdir)) df.content = DF_CONTENT_LABELS workflow = mock_workflow(tmpdir) setattr(workflow.builder, 'df_path', df.dockerfile_path) workflow.build_result = BuildResult.make_remote_image_result() flexmock(workflow, base_image_inspect={ INSPECT_CONFIG: { 'Labels': {'parentrelease': '7.4.1'}, 'Env': {'parentrelease': '7.4.1'}, } }) mock_additional_tags_file(str(tmpdir), ['get_tags', 'file_tags']) if unique_tags is not None and primary_tags is not None: input_tags = { 'unique': unique_tags, 'primary': primary_tags } else: input_tags = None runner = PostBuildPluginsRunner( docker_tasker, workflow, [{'name': TagFromConfigPlugin.key, 'args': {'tag_suffixes': input_tags}}] ) if expected is not None: results = runner.run() plugin_result = results[TagFromConfigPlugin.key] assert plugin_result == expected else: with pytest.raises(PluginFailedException): runner.run()
class DockerBuildWorkflow(object): """ This class defines a workflow for building images: 1. pull image from registry 2. tag it properly if needed 3. obtain source 4. build image 5. tag it 6. push it to registries """ def __init__(self, source, image, prebuild_plugins=None, prepublish_plugins=None, postbuild_plugins=None, exit_plugins=None, plugin_files=None, openshift_build_selflink=None, client_version=None, buildstep_plugins=None, **kwargs): """ :param source: dict, where/how to get source code to put in image :param image: str, tag for built image ([registry/]image_name[:tag]) :param prebuild_plugins: dict, arguments for pre-build plugins :param prepublish_plugins: dict, arguments for test-build plugins :param postbuild_plugins: dict, arguments for post-build plugins :param plugin_files: list of str, load plugins also from these files :param openshift_build_selflink: str, link to openshift build (if we're actually running on openshift) without the actual hostname/IP address :param client_version: str, osbs-client version used to render build json :param buildstep_plugins: dict, arguments for build-step plugins """ self.source = get_source_instance_for(source, tmpdir=tempfile.mkdtemp()) self.image = image self.prebuild_plugins_conf = prebuild_plugins self.buildstep_plugins_conf = buildstep_plugins self.prepublish_plugins_conf = prepublish_plugins self.postbuild_plugins_conf = postbuild_plugins self.exit_plugins_conf = exit_plugins self.prebuild_results = {} self.buildstep_result = {} self.postbuild_results = {} self.prepub_results = {} self.exit_results = {} self.build_result = BuildResult(fail_reason="not built") self.plugin_workspace = {} self.plugins_timestamps = {} self.plugins_durations = {} self.plugins_errors = {} self.autorebuild_canceled = False self.build_canceled = False self.plugin_failed = False self.plugin_files = plugin_files self.kwargs = kwargs self.builder = None self.built_image_inspect = None self.layer_sizes = [] self._base_image_inspect = None self.pulled_base_images = set() # When an image is exported into tarball, it can then be processed by various plugins. # Each plugin that transforms the image should save it as a new file and append it to # the end of exported_image_sequence. Other plugins should then operate with last # member of this structure. Example: # [{'path': '/tmp/foo.tar', 'size': 12345678, 'md5sum': '<md5>', 'sha256sum': '<sha256>'}] # You can use util.get_exported_image_metadata to create a dict to append to this list. self.exported_image_sequence = [] self.tag_conf = TagConf() self.push_conf = PushConf() # mapping of downloaded files; DON'T PUT ANYTHING BIG HERE! # "path/to/file" -> "content" self.files = {} self.openshift_build_selflink = openshift_build_selflink # List of RPMs that go into the final result, as per rpm_util.parse_rpm_output self.image_components = None if client_version: logger.debug("build json was built by osbs-client %s", client_version) if kwargs: logger.warning("unprocessed keyword arguments: %s", kwargs) @property def build_process_failed(self): """ Has any aspect of the build process failed? """ return self.build_result.is_failed() or self.plugin_failed # inspect base image lazily just before it's needed - pre plugins may change the base image @property def base_image_inspect(self): if self._base_image_inspect is None: try: self._base_image_inspect = self.builder.tasker.inspect_image( self.builder.base_image) except docker.errors.NotFound: # If the base image cannot be found throw KeyError - as this property should behave # like a dict raise KeyError("Unprocessed base image Dockerfile cannot be inspected") return self._base_image_inspect def throw_canceled_build_exception(self, *args, **kwargs): self.build_canceled = True raise BuildCanceledException("Build was canceled") def build_docker_image(self): """ build docker image :return: BuildResult """ self.builder = InsideBuilder(self.source, self.image) try: signal.signal(signal.SIGTERM, self.throw_canceled_build_exception) # time to run pre-build plugins, so they can access cloned repo logger.info("running pre-build plugins") prebuild_runner = PreBuildPluginsRunner(self.builder.tasker, self, self.prebuild_plugins_conf, plugin_files=self.plugin_files) try: prebuild_runner.run() except PluginFailedException as ex: logger.error("one or more prebuild plugins failed: %s", ex) raise except AutoRebuildCanceledException as ex: logger.info(str(ex)) self.autorebuild_canceled = True raise logger.info("running buildstep plugins") buildstep_runner = BuildStepPluginsRunner(self.builder.tasker, self, self.buildstep_plugins_conf, plugin_files=self.plugin_files) try: self.build_result = buildstep_runner.run() if self.build_result.is_failed(): raise PluginFailedException(self.build_result.fail_reason) except PluginFailedException as ex: self.builder.is_built = False logger.error('buildstep plugin failed: %s', ex) raise self.builder.is_built = True if self.build_result.is_image_available(): self.builder.image_id = self.build_result.image_id # run prepublish plugins prepublish_runner = PrePublishPluginsRunner(self.builder.tasker, self, self.prepublish_plugins_conf, plugin_files=self.plugin_files) try: prepublish_runner.run() except PluginFailedException as ex: logger.error("one or more prepublish plugins failed: %s", ex) raise if self.build_result.is_image_available(): self.built_image_inspect = self.builder.inspect_built_image() history = self.builder.tasker.d.history(self.builder.image_id) diff_ids = self.built_image_inspect[INSPECT_ROOTFS][INSPECT_ROOTFS_LAYERS] # diff_ids is ordered oldest first # history is ordered newest first # We want layer_sizes to be ordered oldest first self.layer_sizes = [{"diff_id": diff_id, "size": layer['Size']} for (diff_id, layer) in zip(diff_ids, reversed(history))] postbuild_runner = PostBuildPluginsRunner(self.builder.tasker, self, self.postbuild_plugins_conf, plugin_files=self.plugin_files) try: postbuild_runner.run() except PluginFailedException as ex: logger.error("one or more postbuild plugins failed: %s", ex) raise return self.build_result except Exception as ex: logger.debug("caught exception (%r) so running exit plugins", ex) raise finally: # We need to make sure all exit plugins are executed signal.signal(signal.SIGTERM, lambda *args: None) exit_runner = ExitPluginsRunner(self.builder.tasker, self, self.exit_plugins_conf, plugin_files=self.plugin_files) try: exit_runner.run(keep_going=True) except PluginFailedException as ex: logger.error("one or more exit plugins failed: %s", ex) raise finally: self.source.remove_tmpdir() signal.signal(signal.SIGTERM, signal.SIG_DFL)
def __init__(self, source, image, prebuild_plugins=None, prepublish_plugins=None, postbuild_plugins=None, exit_plugins=None, plugin_files=None, openshift_build_selflink=None, client_version=None, buildstep_plugins=None, **kwargs): """ :param source: dict, where/how to get source code to put in image :param image: str, tag for built image ([registry/]image_name[:tag]) :param prebuild_plugins: dict, arguments for pre-build plugins :param prepublish_plugins: dict, arguments for test-build plugins :param postbuild_plugins: dict, arguments for post-build plugins :param plugin_files: list of str, load plugins also from these files :param openshift_build_selflink: str, link to openshift build (if we're actually running on openshift) without the actual hostname/IP address :param client_version: str, osbs-client version used to render build json :param buildstep_plugins: dict, arguments for build-step plugins """ self.source = get_source_instance_for(source, tmpdir=tempfile.mkdtemp()) self.image = image self.prebuild_plugins_conf = prebuild_plugins self.buildstep_plugins_conf = buildstep_plugins self.prepublish_plugins_conf = prepublish_plugins self.postbuild_plugins_conf = postbuild_plugins self.exit_plugins_conf = exit_plugins self.prebuild_results = {} self.buildstep_result = {} self.postbuild_results = {} self.prepub_results = {} self.exit_results = {} self.build_result = BuildResult(fail_reason="not built") self.plugin_workspace = {} self.plugins_timestamps = {} self.plugins_durations = {} self.plugins_errors = {} self.autorebuild_canceled = False self.build_canceled = False self.plugin_failed = False self.plugin_files = plugin_files self.fs_watcher = FSWatcher() self.kwargs = kwargs self.builder = None self.built_image_inspect = None self.layer_sizes = [] self._base_image_inspect = None self.default_image_build_method = CONTAINER_DEFAULT_BUILD_METHOD self.pulled_base_images = set() # When an image is exported into tarball, it can then be processed by various plugins. # Each plugin that transforms the image should save it as a new file and append it to # the end of exported_image_sequence. Other plugins should then operate with last # member of this structure. Example: # [{'path': '/tmp/foo.tar', 'size': 12345678, 'md5sum': '<md5>', 'sha256sum': '<sha256>'}] # You can use util.get_exported_image_metadata to create a dict to append to this list. self.exported_image_sequence = [] self.tag_conf = TagConf() self.push_conf = PushConf() # mapping of downloaded files; DON'T PUT ANYTHING BIG HERE! # "path/to/file" -> "content" self.files = {} self.openshift_build_selflink = openshift_build_selflink # List of RPMs that go into the final result, as per rpm_util.parse_rpm_output self.image_components = None if client_version: logger.debug("build json was built by osbs-client %s", client_version) if kwargs: logger.warning("unprocessed keyword arguments: %s", kwargs)
class DockerBuildWorkflow(object): """ This class defines a workflow for building images: 1. pull image from registry 2. tag it properly if needed 3. obtain source 4. build image 5. tag it 6. push it to registries """ def __init__(self, source, image, prebuild_plugins=None, prepublish_plugins=None, postbuild_plugins=None, exit_plugins=None, plugin_files=None, openshift_build_selflink=None, client_version=None, buildstep_plugins=None, **kwargs): """ :param source: dict, where/how to get source code to put in image :param image: str, tag for built image ([registry/]image_name[:tag]) :param prebuild_plugins: dict, arguments for pre-build plugins :param prepublish_plugins: dict, arguments for test-build plugins :param postbuild_plugins: dict, arguments for post-build plugins :param plugin_files: list of str, load plugins also from these files :param openshift_build_selflink: str, link to openshift build (if we're actually running on openshift) without the actual hostname/IP address :param client_version: str, osbs-client version used to render build json :param buildstep_plugins: dict, arguments for build-step plugins """ self.source = get_source_instance_for(source, tmpdir=tempfile.mkdtemp()) self.image = image self.prebuild_plugins_conf = prebuild_plugins self.buildstep_plugins_conf = buildstep_plugins self.prepublish_plugins_conf = prepublish_plugins self.postbuild_plugins_conf = postbuild_plugins self.exit_plugins_conf = exit_plugins self.prebuild_results = {} self.buildstep_result = {} self.postbuild_results = {} self.prepub_results = {} self.exit_results = {} self.build_result = BuildResult(fail_reason="not built") self.plugin_workspace = {} self.plugins_timestamps = {} self.plugins_durations = {} self.plugins_errors = {} self.autorebuild_canceled = False self.build_canceled = False self.plugin_failed = False self.plugin_files = plugin_files self.fs_watcher = FSWatcher() self.kwargs = kwargs self.builder = None self.built_image_inspect = None self.layer_sizes = [] self._base_image_inspect = None self.default_image_build_method = CONTAINER_DEFAULT_BUILD_METHOD self.pulled_base_images = set() # When an image is exported into tarball, it can then be processed by various plugins. # Each plugin that transforms the image should save it as a new file and append it to # the end of exported_image_sequence. Other plugins should then operate with last # member of this structure. Example: # [{'path': '/tmp/foo.tar', 'size': 12345678, 'md5sum': '<md5>', 'sha256sum': '<sha256>'}] # You can use util.get_exported_image_metadata to create a dict to append to this list. self.exported_image_sequence = [] self.tag_conf = TagConf() self.push_conf = PushConf() # mapping of downloaded files; DON'T PUT ANYTHING BIG HERE! # "path/to/file" -> "content" self.files = {} self.openshift_build_selflink = openshift_build_selflink # List of RPMs that go into the final result, as per rpm_util.parse_rpm_output self.image_components = None if client_version: logger.debug("build json was built by osbs-client %s", client_version) if kwargs: logger.warning("unprocessed keyword arguments: %s", kwargs) @property def build_process_failed(self): """ Has any aspect of the build process failed? """ return self.build_result.is_failed() or self.plugin_failed # inspect base image lazily just before it's needed - pre plugins may change the base image @property def base_image_inspect(self): if self._base_image_inspect is None: try: self._base_image_inspect = self.builder.tasker.inspect_image( self.builder.base_image) except docker.errors.NotFound: # If the base image cannot be found throw KeyError - as this property should behave # like a dict raise KeyError("Unprocessed base image Dockerfile cannot be inspected") return self._base_image_inspect def throw_canceled_build_exception(self, *args, **kwargs): self.build_canceled = True raise BuildCanceledException("Build was canceled") def build_docker_image(self): """ build docker image :return: BuildResult """ self.builder = InsideBuilder(self.source, self.image) try: self.fs_watcher.start() signal.signal(signal.SIGTERM, self.throw_canceled_build_exception) # time to run pre-build plugins, so they can access cloned repo logger.info("running pre-build plugins") prebuild_runner = PreBuildPluginsRunner(self.builder.tasker, self, self.prebuild_plugins_conf, plugin_files=self.plugin_files) try: prebuild_runner.run() except PluginFailedException as ex: logger.error("one or more prebuild plugins failed: %s", ex) raise except AutoRebuildCanceledException as ex: logger.info(str(ex)) self.autorebuild_canceled = True raise logger.info("running buildstep plugins") buildstep_runner = BuildStepPluginsRunner(self.builder.tasker, self, self.buildstep_plugins_conf, plugin_files=self.plugin_files) try: self.build_result = buildstep_runner.run() if self.build_result.is_failed(): raise PluginFailedException(self.build_result.fail_reason) except PluginFailedException as ex: self.builder.is_built = False logger.error('buildstep plugin failed: %s', ex) raise self.builder.is_built = True if self.build_result.is_image_available(): self.builder.image_id = self.build_result.image_id # run prepublish plugins prepublish_runner = PrePublishPluginsRunner(self.builder.tasker, self, self.prepublish_plugins_conf, plugin_files=self.plugin_files) try: prepublish_runner.run() except PluginFailedException as ex: logger.error("one or more prepublish plugins failed: %s", ex) raise if self.build_result.is_image_available(): self.built_image_inspect = self.builder.inspect_built_image() history = self.builder.tasker.d.history(self.builder.image_id) diff_ids = self.built_image_inspect[INSPECT_ROOTFS][INSPECT_ROOTFS_LAYERS] # diff_ids is ordered oldest first # history is ordered newest first # We want layer_sizes to be ordered oldest first self.layer_sizes = [{"diff_id": diff_id, "size": layer['Size']} for (diff_id, layer) in zip(diff_ids, reversed(history))] postbuild_runner = PostBuildPluginsRunner(self.builder.tasker, self, self.postbuild_plugins_conf, plugin_files=self.plugin_files) try: postbuild_runner.run() except PluginFailedException as ex: logger.error("one or more postbuild plugins failed: %s", ex) raise return self.build_result except Exception as ex: logger.debug("caught exception (%r) so running exit plugins", ex) raise finally: # We need to make sure all exit plugins are executed signal.signal(signal.SIGTERM, lambda *args: None) exit_runner = ExitPluginsRunner(self.builder.tasker, self, self.exit_plugins_conf, plugin_files=self.plugin_files) try: exit_runner.run(keep_going=True) except PluginFailedException as ex: logger.error("one or more exit plugins failed: %s", ex) raise finally: self.source.remove_tmpdir() self.fs_watcher.finish() signal.signal(signal.SIGTERM, signal.SIG_DFL)
def prepare( tmpdir, insecure_registry=None, namespace=None, primary_images_tag_conf=DEFAULT_TAGS_AMOUNT, primary_images_annotations=DEFAULT_TAGS_AMOUNT, build_process_failed=False, organization=None, reactor_config_map=False, ): """ Boiler-plate test set-up """ if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow(SOURCE, "test-image") setattr(workflow, 'builder', X()) flexmock(workflow, build_process_failed=build_process_failed) setattr(workflow.builder, 'image_id', 'asd123') setattr(workflow.builder, 'source', X()) setattr(workflow.builder.source, 'dockerfile_path', None) setattr(workflow.builder.source, 'path', None) df = tmpdir.join('Dockerfile') df.write('FROM base\n') df.write('LABEL name={}'.format(TEST_NAME_LABEL)) setattr(workflow.builder, 'df_path', str(df)) annotations = None if primary_images_annotations: primary_images = [ '{}:annotation_{}'.format(TEST_REPO_WITH_REGISTRY, x) for x in range(primary_images_annotations) ] primary_images.append( '{}:version-release'.format(TEST_REPO_WITH_REGISTRY)) annotations = {'repositories': {'primary': primary_images}} annotations build_result = BuildResult(annotations=annotations, image_id='foo') setattr(workflow, 'build_result', build_result) if primary_images_tag_conf: primary_images = [ '{}:tag_conf_{}'.format(TEST_REPO, x) for x in range(primary_images_tag_conf) ] primary_images.insert(0, '{}:version-release'.format(TEST_REPO)) workflow.tag_conf.add_primary_images(primary_images) fake_conf = osbs.conf.Configuration(conf_file=None, openshift_url='/') expectation = flexmock( osbs.conf).should_receive('Configuration').and_return(fake_conf) if namespace: expectation.with_args(conf_file=None, namespace=namespace, verify_ssl=not insecure_registry, openshift_url="/", use_auth=False, build_json_dir="/var/json_dir") plugin_args = {'imagestream': TEST_IMAGESTREAM} if reactor_config_map: openshift_map = { 'url': '/', 'auth': { 'enable': False }, 'insecure': insecure_registry, 'build_json_dir': '/var/json_dir', } source_registry_map = { 'url': TEST_REGISTRY, 'insecure': insecure_registry } workflow.plugin_workspace[ReactorConfigPlugin.key] = {} workflow.plugin_workspace[ReactorConfigPlugin.key][WORKSPACE_CONF_KEY] =\ ReactorConfig({ 'version': 1, 'openshift': openshift_map, 'source_registry': source_registry_map, 'registries_organization': organization, }) else: plugin_args.update({ 'docker_image_repo': TEST_REPO_WITH_REGISTRY, 'url': '/', 'build_json_dir': "/var/json_dir", 'verify_ssl': not insecure_registry, 'use_auth': False, 'insecure_registry': insecure_registry, }) runner = PostBuildPluginsRunner(tasker, workflow, [{ 'name': ImportImagePlugin.key, 'args': plugin_args }]) def mocked_import_image_tags(**kwargs): return if not hasattr(OSBS, 'import_image_tags'): setattr(OSBS, 'import_image_tags', mocked_import_image_tags) return runner
class WF(object): build_result = BuildResult(fail_reason="not built") autorebuild_canceled = False prebuild_results = {CheckAndSetRebuildPlugin.key: True} image = ImageName.parse('repo/name')
from tests.docker_mock import mock_docker from tests.util import requires_internet, is_string_type import inspect import signal import threading from time import sleep from atomic_reactor.inner import BuildResults, BuildResultsEncoder, BuildResultsJSONDecoder from atomic_reactor.inner import DockerBuildWorkflow BUILD_RESULTS_ATTRS = [ 'build_logs', 'built_img_inspect', 'built_img_info', 'base_img_info', 'base_plugins_output', 'built_img_plugins_output' ] DUMMY_BUILD_RESULT = BuildResult(image_id="image_id") DUMMY_FAILED_BUILD_RESULT = BuildResult(fail_reason='it happens') DUMMY_REMOTE_BUILD_RESULT = BuildResult.make_remote_image_result() def test_build_results_encoder(): results = BuildResults() expected_data = {} for attr in BUILD_RESULTS_ATTRS: setattr(results, attr, attr) expected_data[attr] = attr data = json.loads(json.dumps(results, cls=BuildResultsEncoder)) assert data == expected_data
from atomic_reactor.inner import BuildResults, BuildResultsEncoder, BuildResultsJSONDecoder from atomic_reactor.inner import DockerBuildWorkflow from atomic_reactor.inner import FSWatcher from atomic_reactor.constants import INSPECT_ROOTFS, INSPECT_ROOTFS_LAYERS BUILD_RESULTS_ATTRS = ['build_logs', 'built_img_inspect', 'built_img_info', 'base_img_info', 'base_plugins_output', 'built_img_plugins_output'] DUMMY_BUILD_RESULT = BuildResult(image_id="image_id") DUMMY_FAILED_BUILD_RESULT = BuildResult(fail_reason='it happens') DUMMY_REMOTE_BUILD_RESULT = BuildResult.make_remote_image_result() def test_build_results_encoder(): results = BuildResults() expected_data = {} for attr in BUILD_RESULTS_ATTRS: setattr(results, attr, attr) expected_data[attr] = attr data = json.loads(json.dumps(results, cls=BuildResultsEncoder)) assert data == expected_data def test_build_results_decoder(): data = {}
def run(self): """ Build image inside current environment using imagebuilder; It's expected this may run within (privileged) docker container. Returns: BuildResult """ builder = self.workflow.builder image = builder.image.to_str() # TODO: directly invoke go imagebuilder library in shared object via python module kwargs = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE) if not PY2: kwargs['encoding'] = 'utf-8' ib_process = subprocess.Popen( ['imagebuilder', '-t', image, builder.df_dir], **kwargs) self.log.debug( 'imagebuilder build has begun; waiting for it to finish') (output, last_error) = ([], None) while True: poll = ib_process.poll() out = sixdecode(ib_process.stdout.readline()) if out: self.log.info(out.strip()) output.append(out) err = sixdecode(ib_process.stderr.readline()) if err: self.log.error(err.strip()) output.append(err) # include stderr with stdout last_error = err # while noting the final line if out == '' and err == '': if poll is not None: break time.sleep(0.1) # don't busy-wait when there's no output if ib_process.returncode != 0: # imagebuilder uses stderr for normal output too; so in the case of an apparent # failure, single out the last line to include in the failure summary. err = last_error or "<imagebuilder had bad exit code but no error output>" return BuildResult( logs=output, fail_reason="image build failed (rc={}): {}".format( ib_process.returncode, err), ) image_id = builder.get_built_image_info()['Id'] if ':' not in image_id: # Older versions of the daemon do not include the prefix image_id = 'sha256:{}'.format(image_id) # since we need no squash, export the image for local operations like squash would have self.log.info("fetching image %s from docker", image) output_path = os.path.join(self.workflow.source.workdir, EXPORTED_SQUASHED_IMAGE_NAME) with open(output_path, "w") as image_file: image_file.write(self.tasker.d.get_image(image).data) img_metadata = get_exported_image_metadata(output_path, IMAGE_TYPE_DOCKER_ARCHIVE) self.workflow.exported_image_sequence.append(img_metadata) return BuildResult(logs=output, image_id=image_id, skip_layer_squash=True)
def __init__(self, source, image, prebuild_plugins=None, prepublish_plugins=None, postbuild_plugins=None, exit_plugins=None, plugin_files=None, openshift_build_selflink=None, client_version=None, buildstep_plugins=None, **kwargs): """ :param source: dict, where/how to get source code to put in image :param image: str, tag for built image ([registry/]image_name[:tag]) :param prebuild_plugins: dict, arguments for pre-build plugins :param prepublish_plugins: dict, arguments for test-build plugins :param postbuild_plugins: dict, arguments for post-build plugins :param plugin_files: list of str, load plugins also from these files :param openshift_build_selflink: str, link to openshift build (if we're actually running on openshift) without the actual hostname/IP address :param client_version: str, osbs-client version used to render build json :param buildstep_plugins: dict, arguments for build-step plugins """ self.source = get_source_instance_for(source, tmpdir=tempfile.mkdtemp()) self.image = image self.prebuild_plugins_conf = prebuild_plugins self.buildstep_plugins_conf = buildstep_plugins self.prepublish_plugins_conf = prepublish_plugins self.postbuild_plugins_conf = postbuild_plugins self.exit_plugins_conf = exit_plugins self.prebuild_results = {} self.buildstep_result = {} self.postbuild_results = {} self.prepub_results = {} self.exit_results = {} self.build_result = BuildResult(fail_reason="not built") self.plugin_workspace = {} self.plugins_timestamps = {} self.plugins_durations = {} self.plugins_errors = {} self.autorebuild_canceled = False self.build_canceled = False self.plugin_failed = False self.plugin_files = plugin_files self.kwargs = kwargs self.builder = None self.built_image_inspect = None self.layer_sizes = [] self._base_image_inspect = None self.pulled_base_images = set() # When an image is exported into tarball, it can then be processed by various plugins. # Each plugin that transforms the image should save it as a new file and append it to # the end of exported_image_sequence. Other plugins should then operate with last # member of this structure. Example: # [{'path': '/tmp/foo.tar', 'size': 12345678, 'md5sum': '<md5>', 'sha256sum': '<sha256>'}] # You can use util.get_exported_image_metadata to create a dict to append to this list. self.exported_image_sequence = [] self.tag_conf = TagConf() self.push_conf = PushConf() # mapping of downloaded files; DON'T PUT ANYTHING BIG HERE! # "path/to/file" -> "content" self.files = {} self.openshift_build_selflink = openshift_build_selflink # List of RPMs that go into the final result, as per rpm_util.parse_rpm_output self.image_components = None if client_version: logger.debug("build json was built by osbs-client %s", client_version) if kwargs: logger.warning("unprocessed keyword arguments: %s", kwargs)