Esempio n. 1
0
    def build_list(self, manifests):
        """
        Builds a manifest list or OCI image out of the given manifests
        """

        media_type = manifests[0]['media_type']
        if (not all(m['media_type'] == media_type for m in manifests)):
            raise PluginFailedException('worker manifests have inconsistent types: {}'
                                        .format(manifests))

        if media_type == MEDIA_TYPE_DOCKER_V2_SCHEMA2:
            list_type = MEDIA_TYPE_DOCKER_V2_MANIFEST_LIST
        elif media_type == MEDIA_TYPE_OCI_V1:
            list_type = MEDIA_TYPE_OCI_V1_INDEX
        else:
            raise PluginFailedException('worker manifests have unsupported type: {}'
                                        .format(media_type))

        return list_type, json.dumps({
                "schemaVersion": 2,
                "mediaType": list_type,
                "manifests": sorted([
                    {
                        "mediaType": media_type,
                        "size": m['size'],
                        "digest": m['digest'],
                        "platform": {
                            "architecture": m['architecture'],
                            "os": "linux"
                        }
                    } for m in manifests
                ], key=lambda entry: entry['platform']['architecture']),
        }, indent=4, sort_keys=True, separators=(',', ': '))
Esempio n. 2
0
    def run(self):
        if not self.platforms:
            raise RuntimeError("No enabled platform to build on")

        thread_pool = ThreadPool(len(self.platforms))
        result = thread_pool.map_async(self.select_and_start_cluster, self.platforms)

        try:
            result.get()
        # Always clean up worker builds on any error to avoid
        # runaway worker builds (includes orchestrator build cancellation)
        except Exception:
            thread_pool.terminate()
            self.log.info('build cancelled, cancelling worker builds')
            if self.worker_builds:
                ThreadPool(len(self.worker_builds)).map(
                    lambda bi: bi.cancel_build(), self.worker_builds)
            while not result.ready():
                result.wait(1)
            raise
        else:
            thread_pool.close()
            thread_pool.join()

        fail_reasons = {
            build_info.platform: build_info.get_fail_reason()
            for build_info in self.worker_builds
            if not build_info.build or not build_info.build.is_succeeded()
        }

        if fail_reasons:
            raise PluginFailedException(json.dumps(fail_reasons))
Esempio n. 3
0
    def get_grouped_manifests(self):
        grouped_manifests = []
        for registry, registry_conf in self.registries.items():
            if registry_conf.get('version') == 'v1':
                continue

            manifest_list_spec = {}
            manifest_list_spec['manifests'] = []
            all_annotations = self.workflow.build_result.annotations[
                'worker-builds']
            for platform in all_annotations:
                worker_image = all_annotations[platform]['digests'][0]
                tag = worker_image['tag']
                repository = worker_image['repository']
                arch_entry = {
                    'image': '{0}/{1}:{2}'.format(registry, repository, tag),
                    'platform': {
                        'os': 'linux',
                        'architecture': self.goarch.get(platform, platform)
                    }
                }
                manifest_list_spec['manifests'].append(arch_entry)

            manifest_list_spec['tags'] = [
                image.tag for image in self.workflow.tag_conf.images
            ]
            # use a unique image tag because manifest-tool can't accept a digest that
            # isn't in the respository yet
            registry_image = self.workflow.tag_conf.unique_images[0]
            registry_image.registry = registry
            manifest_list_spec['image'] = registry_image.to_str()
            self.log.info("Submitting manifest-list spec %s",
                          manifest_list_spec)
            self.submit_manifest_list(registry, registry_conf,
                                      manifest_list_spec)
            insecure = registry_conf.get('insecure', False)
            secret_path = registry_conf.get('secret')

            self.log.debug('attempting get_manifest_digests from %s for %s',
                           registry, registry_image)
            manifest_list_digest = get_manifest_digests(
                registry_image,
                registry=registry,
                insecure=insecure,
                dockercfg_path=secret_path,
                versions=('v2_list', ))
            if not manifest_list_digest.v2_list:
                raise PluginFailedException('no manifest list digest for %s',
                                            registry)
            self.log.debug('Digest for registry %s is %s', registry,
                           manifest_list_digest.v2_list)
            push_conf_registry = self.workflow.push_conf.add_docker_registry(
                registry, insecure=insecure)
            tag = registry_image.to_str(registry=False)
            push_conf_registry.digests[tag] = manifest_list_digest
            grouped_manifests.append(manifest_list_digest)

        self.log.info(
            "Manifest lists created and collected for all repositories")
        return grouped_manifests
Esempio n. 4
0
    def run(self):
        # verify that given states are subset of allowed states
        unknown_states = set(self.send_on) - self.allowed_states
        if len(unknown_states) > 0:
            raise PluginFailedException('Unknown state(s) "%s" for sendmail plugin' %
                                        '", "'.join(sorted(unknown_states)))

        rebuild = is_rebuild(self.workflow)
        success = not self.workflow.build_failed
        canceled = self.workflow.autorebuild_canceled

        self.log.info('checking conditions for sending notification ...')
        if self._should_send(rebuild, success, canceled):
            self.log.info('notification about build result will be sent')
            subject, body = self._render_mail(rebuild, success, canceled)
            try:
                self.log.debug('getting list of receivers for this component ...')
                receivers = self._get_receivers_list()
            except RuntimeError as e:
                self.log.error('couldn\'t get list of receivers, sending error message ...')
                # TODO: maybe improve the error message/subject
                body = '\n'.join([
                    'Failed to get contact for %s, error: %s' % (str(self.workflow.image), str(e)),
                    'Since your address is in "error_addresses", this email was sent to you to '
                    'take action on this.',
                    'Wanted to send following mail:',
                    '',
                    body
                ])
                receivers = self.error_addresses
            self.log.info('sending notification to %s ...', receivers)
            self._send_mail(receivers, subject, body)
        else:
            self.log.info('conditions for sending notification not met, doing nothing')
Esempio n. 5
0
 def _get_component_label(self):
     """Get value of Dockerfile label that is to be used as `global_component` to query
     PDC release-components API endpoint.
     """
     labels = DockerfileParser(self.workflow.builder.df_path).labels
     if self.pdc_component_df_label not in labels:
         raise PluginFailedException('No %s label in Dockerfile, can\'t get PDC component',
                                     self.pdc_component_df_label)
     return labels[self.pdc_component_df_label]
Esempio n. 6
0
    def test_execute_returns_failure(self, task_with_mocked_deps, caplog):
        task, mocked_workflow = task_with_mocked_deps

        err = PluginFailedException("something is wrong")
        mocked_workflow.should_receive("build_docker_image").and_raise(err)

        with pytest.raises(PluginFailedException, match="something is wrong"):
            task.execute()

        assert "task failed: something is wrong" in caplog.text
Esempio n. 7
0
    def _get_pdc_token(self):
        # we want to allow pdc_secret_path to be None in __init__ - I'm assuming that in future
        #  we'll want different sources of contact info, so we only want to raise when
        #  the plugin actually tries to authenticate against PDC and doesn't have pdc_secret_path
        if self.pdc_secret_path is None:
            raise PluginFailedException('Getting PDC token, but pdc_secret_path is unspecified')
        token_file = os.path.join(self.pdc_secret_path, self.PDC_TOKEN_FILE)

        self.log.debug('getting PDC token from file %s', token_file)

        with open(token_file, 'r') as f:
            return f.read().strip()
Esempio n. 8
0
    def _get_receivers_list(self):
        """Return list of receivers of the notification.

        :raises RuntimeError: if PDC can't be contacted or doesn't provide sufficient data
        :raises PluginFailedException: if there's a critical error while getting PDC data
        """

        # TODO: document what this plugin expects to be in Dockerfile/where it gets info from
        global_component = self._get_component_label()
        # this relies on bump_release plugin configuring source.git_commit to actually be
        #  branch name, not a commit
        if not isinstance(self.workflow.source, GitSource):
            raise PluginFailedException(
                'Source is not of type "GitSource", panic!')
        git_branch = self.workflow.source.git_commit
        try:
            r = requests.get(
                urljoin(self.pdc_url,
                        'rest_api/v1/release-component-contacts/'),
                headers={'Authorization': 'Token %s' % self._get_pdc_token()},
                params={
                    'global_component': global_component,
                    'dist_git_branch': git_branch,
                    'role': self.pdc_contact_role
                },
                verify=self.pdc_verify_cert)
        except requests.RequestException as e:
            self.log.error('failed to connect to PDC: %s', str(e))
            raise RuntimeError(e)

        if r.status_code != 200:
            self.log.error('PDC returned status code %s, full response: %s',
                           r.status_code, r.text)
            raise RuntimeError(
                'PDC returned non-200 status code (%s), see referenced build log'
                % r.status_code)

        contacts = r.json()

        if contacts['count'] == 0:
            self.log.error('no %s role for the component',
                           self.pdc_contact_role)
            raise RuntimeError('no %s role for the component' %
                               self.pdc_contact_role)

        send_to = []
        for contact in contacts['results']:
            send_to.append(contact['contact']['email'])

        return send_to
    def run(self):
        for registry, registry_conf in self.registries.items():
            if not registry.startswith('http://') and not registry.startswith('https://'):
                registry = 'https://' + registry

            registry_noschema = urlparse(registry).netloc

            auth = None
            secret_path = registry_conf.get('secret')
            if secret_path:
                self.log.debug("registry %s secret %s", registry_noschema, secret_path)
                dockercfg = Dockercfg(secret_path).get_credentials(registry_noschema)
                try:
                    username = dockercfg['username']
                    password = dockercfg['password']
                except KeyError:
                    self.log.error("credentials for registry %s not found in %s",
                                   registry_noschema, secret_path)
                else:
                    self.log.debug("found user %s for registry %s", username, registry_noschema)
                    auth = requests.auth.HTTPBasicAuth(username, password)

            for push_conf_registry in self.workflow.push_conf.docker_registries:
                if push_conf_registry.uri == registry_noschema:
                    break
            else:
                self.log.warning("requested deleting image from %s but we haven't pushed there",
                                 registry_noschema)
                continue

            for tag, digest in push_conf_registry.digests.items():
                repo = tag.split(':')[0]
                url = registry + "/v2/" + repo + "/manifests/" + digest
                insecure = push_conf_registry.insecure
                response = requests.delete(url, verify=not insecure, auth=auth)

                if response.status_code == requests.codes.ACCEPTED:
                    self.log.info("deleted manifest %s/%s@%s", registry_noschema, repo, digest)
                elif response.status_code == requests.codes.NOT_FOUND:
                    self.log.warning("cannot delete %s/%s@%s: not found",
                                     registry_noschema, repo, digest)
                elif response.status_code == requests.codes.METHOD_NOT_ALLOWED:
                    self.log.warning("cannot delete %s/%s@%s: image deletion disabled on registry",
                                     registry_noschema, repo, digest)
                else:
                    msg = "failed to delete %s/%s@%s: %s" % (registry_noschema, repo, digest,
                                                             response.reason)
                    self.log.error("%s\n%s", msg, response.text)
                    raise PluginFailedException(msg)
Esempio n. 10
0
    def check_for_errors(self, logs, insecure=False):
        for message in logs:
            if 'error' in message:
                msg = "while pushing: {0}".format(message['error'])
                self.log.error(msg)
                if 'errorDetail' in message:
                    detail = message['errorDetail'].get('message')
                    if detail:
                        self.log.error("error detail: %s", detail)

                if insecure:
                    self.log.error("perhaps you need to provide "
                                   "--insecure-registry to docker daemon?")

                raise PluginFailedException(msg)
Esempio n. 11
0
    def _send_mail(self, receivers_list, subject, body):
        """Actually sends the mail with `subject` and `body` to all members of `receivers_list`."""
        msg = MIMEText(body)
        msg['Subject'] = subject
        msg['From'] = self.from_address
        msg['To'] = ', '.join(receivers_list)

        s = None
        try:
            s = smtplib.SMTP(self.smtp_uri)
            s.sendmail(self.from_address, receivers_list, msg.as_string())
        except (socket.gaierror, smtplib.SMTPException) as e:
            raise PluginFailedException('Error communicating with SMTP server: %s' % str(e))
        finally:
            if s is not None:
                s.quit()
Esempio n. 12
0
    def run(self):
        # verify that given states are subset of allowed states
        unknown_states = self.send_on - self.allowed_states
        if len(unknown_states) > 0:
            raise PluginFailedException(
                'Unknown state(s) "%s" for sendmail plugin' %
                '", "'.join(sorted(unknown_states)))

        if not self.smtp:
            self.log.info('no smtp configuration, skipping plugin')
            return

        success = not self.workflow.build_process_failed
        manual_canceled = self.workflow.data.build_canceled

        self.log.info('checking conditions for sending notification ...')
        if self._should_send(success, manual_canceled):
            self.log.info('notification about build result will be sent')
            subject, body, full_logs = self._render_mail(
                success, manual_canceled)
            try:
                self.log.debug(
                    'getting list of receivers for this component ...')
                receivers = self._get_receivers_list()
            except RuntimeError as e:
                self.log.error(
                    'couldn\'t get list of receivers, sending error message ...'
                )
                # Render the body although the receivers cannot be fetched for error message
                body = '\n'.join([
                    'Failed to get contact for %s, error: %s' %
                    (str(self.workflow.image), str(e)),
                    'Since your address is in "error_addresses", this email was sent to you to '
                    'take action on this.', 'Wanted to send following mail:',
                    '', body
                ])
                receivers = self.error_addresses

            self._send_mail(receivers, subject, body, full_logs)
        else:
            self.log.info(
                'conditions for sending notification not met, doing nothing')
    def request_delete(self, session, url, manifest):
        try:
            response = session.delete(url)
            response.raise_for_status()
            self.log.info("deleted manifest %s", manifest)
            return True

        except (HTTPError, RetryError, Timeout) as ex:

            if ex.response.status_code == requests.codes.NOT_FOUND:
                self.log.warning("cannot delete %s: not found", manifest)
            elif ex.response.status_code == requests.codes.METHOD_NOT_ALLOWED:
                self.log.warning(
                    "cannot delete %s: image deletion disabled on registry",
                    manifest)
            else:
                msg = "failed to delete %s: %s" % (manifest,
                                                   ex.response.reason)
                self.log.error("%s\n%s", msg, ex.response.text)
                raise PluginFailedException(msg)

        return False
Esempio n. 14
0
    def get_worker_manifest(self, worker_data):
        worker_digests = worker_data['digests']

        msg = "worker_registries {0}".format(self.worker_registries)
        self.log.debug(msg)

        for registry, registry_conf in self.registries.items():
            if registry_conf.get('version') == 'v1':
                continue

            if not registry.startswith('http://') and not registry.startswith(
                    'https://'):
                registry = 'https://' + registry

            registry_noschema = urlparse(registry).netloc
            self.log.debug("evaluating registry %s", registry_noschema)

            insecure = registry_conf.get('insecure', False)
            auth = None
            secret_path = registry_conf.get('secret')
            if secret_path:
                self.log.debug("registry %s secret %s", registry_noschema,
                               secret_path)
                dockercfg = Dockercfg(secret_path).get_credentials(
                    registry_noschema)
                try:
                    username = dockercfg['username']
                    password = dockercfg['password']
                except KeyError:
                    self.log.error(
                        "credentials for registry %s not found in %s",
                        registry_noschema, secret_path)
                else:
                    self.log.debug("found user %s for registry %s", username,
                                   registry_noschema)
                    auth = requests.auth.HTTPBasicAuth(username, password)

            if registry_noschema in self.worker_registries:
                self.log.debug("getting manifests from %s", registry_noschema)
                digest = worker_digests[0]['digest']
                repo = worker_digests[0]['repository']

                # get a v2 schemav2 response for now
                v2schema2 = 'application/vnd.docker.distribution.manifest.v2+json'
                headers = {'accept': v2schema2}
                kwargs = {
                    'verify': not insecure,
                    'headers': headers,
                    'auth': auth
                }

                url = '{0}/v2/{1}/manifests/{2}'.format(registry, repo, digest)
                self.log.debug("attempting get from %s", url)
                response = requests.get(url, **kwargs)

                if response.json()['schemaVersion'] == '1':
                    msg = 'invalid schema from {0}'.format(url)
                    raise PluginFailedException(msg)

                image_manifest = response.content
                headers = {'Content-Type': v2schema2}
                kwargs = {
                    'verify': not insecure,
                    'headers': headers,
                    'auth': auth
                }

                push_conf_registry = self.workflow.push_conf.add_docker_registry(
                    registry, insecure=insecure)
                for image in self.workflow.tag_conf.images:
                    url = '{0}/v2/{1}/manifests/{2}'.format(
                        registry, repo, image.tag)
                    self.log.debug("for image_tag %s, putting at %s",
                                   image.tag, url)
                    response = requests.put(url, data=image_manifest, **kwargs)

                    if not response.ok:
                        msg = "PUT failed: {0},\n manifest was: {1}".format(
                            response.json(), image_manifest)
                        self.log.error(msg)
                    response.raise_for_status()

                    # add a tag for any plugins running later that expect it
                    push_conf_registry.digests[image.tag] = digest
                break
    source = Y()
    source.dockerfile_path = None
    source.path = None
    base_image = ImageName(repo="qwe", tag="asd")


DF_CONTENT = """
FROM fedora
RUN yum install -y python-django
CMD blabla"""
DF_CONTENT_LABELS = DF_CONTENT + '\nLABEL "Name"="rainbow" "Version"="123" "Release"="1"'


@pytest.mark.parametrize(
    'df_content, req_labels, expected',
    [(DF_CONTENT, None, PluginFailedException()),
     (DF_CONTENT_LABELS, None, None),
     (DF_CONTENT_LABELS, ['xyz'], PluginFailedException())])
def test_assertlabels_plugin(tmpdir, docker_tasker, df_content, req_labels,
                             expected):
    df = DockerfileParser(str(tmpdir))
    df.content = df_content

    workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image')
    workflow.builder = X
    workflow.builder.df_path = df.dockerfile_path
    workflow.builder.df_dir = str(tmpdir)

    runner = PreBuildPluginsRunner(docker_tasker, workflow,
                                   [{
                                       'name': AssertLabelsPlugin.key,
Esempio n. 16
0
    def build_docker_image(self):
        """
        build docker image

        :return: BuildResult
        """
        self.builder = InsideBuilder(self.source, self.image)
        try:
            self.fs_watcher.start()
            signal.signal(signal.SIGTERM, self.throw_canceled_build_exception)
            # time to run pre-build plugins, so they can access cloned repo
            logger.info("running pre-build plugins")
            prebuild_runner = PreBuildPluginsRunner(self.builder.tasker, self,
                                                    self.prebuild_plugins_conf,
                                                    plugin_files=self.plugin_files)
            try:
                prebuild_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more prebuild plugins failed: %s", ex)
                raise
            except AutoRebuildCanceledException as ex:
                logger.info(str(ex))
                self.autorebuild_canceled = True
                raise

            logger.info("running buildstep plugins")
            buildstep_runner = BuildStepPluginsRunner(self.builder.tasker, self,
                                                      self.buildstep_plugins_conf,
                                                      plugin_files=self.plugin_files)
            try:
                self.build_result = buildstep_runner.run()

                if self.build_result.is_failed():
                    raise PluginFailedException(self.build_result.fail_reason)
            except PluginFailedException as ex:
                self.builder.is_built = False
                logger.error('buildstep plugin failed: %s', ex)
                raise

            self.builder.is_built = True
            if self.build_result.is_image_available():
                self.builder.image_id = self.build_result.image_id

            # run prepublish plugins
            prepublish_runner = PrePublishPluginsRunner(self.builder.tasker, self,
                                                        self.prepublish_plugins_conf,
                                                        plugin_files=self.plugin_files)
            try:
                prepublish_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more prepublish plugins failed: %s", ex)
                raise

            if self.build_result.is_image_available():
                self.built_image_inspect = self.builder.inspect_built_image()
                history = self.builder.tasker.d.history(self.builder.image_id)
                diff_ids = self.built_image_inspect[INSPECT_ROOTFS][INSPECT_ROOTFS_LAYERS]

                # diff_ids is ordered oldest first
                # history is ordered newest first
                # We want layer_sizes to be ordered oldest first
                self.layer_sizes = [{"diff_id": diff_id, "size": layer['Size']}
                                    for (diff_id, layer) in zip(diff_ids, reversed(history))]

            postbuild_runner = PostBuildPluginsRunner(self.builder.tasker, self,
                                                      self.postbuild_plugins_conf,
                                                      plugin_files=self.plugin_files)
            try:
                postbuild_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more postbuild plugins failed: %s", ex)
                raise

            return self.build_result
        except Exception as ex:
            logger.debug("caught exception (%r) so running exit plugins", ex)
            raise
        finally:
            # We need to make sure all exit plugins are executed
            signal.signal(signal.SIGTERM, lambda *args: None)
            exit_runner = ExitPluginsRunner(self.builder.tasker, self,
                                            self.exit_plugins_conf,
                                            plugin_files=self.plugin_files)
            try:
                exit_runner.run(keep_going=True)
            except PluginFailedException as ex:
                logger.error("one or more exit plugins failed: %s", ex)
                raise
            finally:
                self.source.remove_tmpdir()
                self.fs_watcher.finish()

            signal.signal(signal.SIGTERM, signal.SIG_DFL)
    def run(self) -> Dict[str, Any]:
        """Build image inside current environment.

        :return: a mapping containing build results. If the build fails, key
            ``fail_reason`` must be included with a meaningful message.
        :rtype: Dict[str, Any]
        """
        fetch_sources_result = self.workflow.data.prebuild_results.get(PLUGIN_FETCH_SOURCES_KEY, {})
        source_data_dir = fetch_sources_result.get('image_sources_dir')
        remote_source_data_dir = fetch_sources_result.get('remote_sources_dir')
        maven_source_data_dir = fetch_sources_result.get('maven_sources_dir')

        source_exists = source_data_dir and os.path.isdir(source_data_dir)
        remote_source_exists = remote_source_data_dir and os.path.isdir(remote_source_data_dir)
        maven_source_exists = maven_source_data_dir and os.path.isdir(maven_source_data_dir)

        if not any([source_exists, remote_source_exists, maven_source_exists]):
            err_msg = "No SRPMs directory '{}' available".format(source_data_dir)
            err_msg += "\nNo Remote source directory '{}' available".format(remote_source_data_dir)
            err_msg += "\nNo Maven source directory '{}' available".format(maven_source_data_dir)
            self.log.error(err_msg)
            raise PluginFailedException(err_msg)

        if source_exists and not os.listdir(source_data_dir):
            self.log.warning("SRPMs directory '%s' is empty", source_data_dir)
        if remote_source_exists and not os.listdir(remote_source_data_dir):
            self.log.warning("Remote source directory '%s' is empty", remote_source_data_dir)
        if maven_source_exists and not os.listdir(maven_source_data_dir):
            self.log.warning("Maven source directory '%s' is empty", maven_source_data_dir)

        image_output_dir: Path = self.workflow.build_dir.source_container_output_dir
        image_output_dir.mkdir(exist_ok=True)
        cmd = ['bsi', '-d']
        drivers = set()

        if source_exists:
            drivers.add('sourcedriver_rpm_dir')
            cmd.append('-s')
            cmd.append('{}'.format(source_data_dir))

        if remote_source_exists:
            sources_subdirs = self.split_remote_sources_to_subdirs(remote_source_data_dir)
            drivers.add('sourcedriver_extra_src_dir')

            for source_subdir in sources_subdirs:
                cmd.append('-e')
                cmd.append(source_subdir)

        if maven_source_exists:
            drivers.add('sourcedriver_extra_src_dir')
            for maven_source_subdir in os.listdir(maven_source_data_dir):
                cmd.append('-e')
                cmd.append('{}'.format(os.path.join(maven_source_data_dir, maven_source_subdir)))

        driver_str = ','.join(drivers)
        cmd.insert(2, driver_str)
        cmd.append('-o')
        cmd.append('{}'.format(image_output_dir))

        try:
            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, text=True)
        except subprocess.CalledProcessError as e:
            self.log.error("BSI failed with output:\n%s", e.output)
            raise PluginFailedException('BSI utility failed build source image') from e

        self.log.debug("Build log:\n%s\n", output)

        # clean bsi temp directory
        bsi_temp_dir = os.path.join(os.getcwd(), 'SrcImg')
        if os.path.isdir(bsi_temp_dir):
            self.log.info('Will remove BSI temporary directory: %s', bsi_temp_dir)
            shutil.rmtree(bsi_temp_dir)

        # clean all downloaded sources
        if source_exists:
            self.log.info('Will remove directory with downloaded srpms: %s', source_data_dir)
            shutil.rmtree(source_data_dir)

        if remote_source_exists:
            self.log.info('Will remove directory with downloaded remote sources: %s',
                          remote_source_data_dir)
            shutil.rmtree(remote_source_data_dir)

        if maven_source_exists:
            self.log.info('Will remove directory with downloaded maven sources: %s',
                          maven_source_data_dir)
            shutil.rmtree(maven_source_data_dir)

        image_metadata = self.export_image(image_output_dir)

        self.log.info('Will remove unpacked image directory: %s', image_output_dir)
        shutil.rmtree(image_output_dir)

        return {
            'image_metadata': image_metadata,
            'logs': [output],
            'skip_layer_squash': True,
        }
Esempio n. 18
0
    def build_docker_image(self):
        """
        build docker image

        :return: BuildResult
        """
        exception_being_handled = False
        self.builder = InsideBuilder(self.source, self.image)
        # Make sure exit_runner is defined for finally block
        exit_runner = None
        try:
            self.fs_watcher.start()
            signal.signal(signal.SIGTERM, self.throw_canceled_build_exception)
            prebuild_runner = PreBuildPluginsRunner(
                self.builder.tasker,
                self,
                self.prebuild_plugins_conf,
                plugin_files=self.plugin_files)
            prepublish_runner = PrePublishPluginsRunner(
                self.builder.tasker,
                self,
                self.prepublish_plugins_conf,
                plugin_files=self.plugin_files)
            postbuild_runner = PostBuildPluginsRunner(
                self.builder.tasker,
                self,
                self.postbuild_plugins_conf,
                plugin_files=self.plugin_files)
            # time to run pre-build plugins, so they can access cloned repo
            logger.info("running pre-build plugins")
            try:
                prebuild_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more prebuild plugins failed: %s", ex)
                raise
            except AutoRebuildCanceledException as ex:
                logger.info(str(ex))
                self.autorebuild_canceled = True
                raise

            # we are delaying initialization, because prebuild plugin reactor_config
            # might change build method
            buildstep_runner = BuildStepPluginsRunner(
                self.builder.tasker,
                self,
                self.buildstep_plugins_conf,
                plugin_files=self.plugin_files)

            logger.info("running buildstep plugins")
            try:
                self.build_result = buildstep_runner.run()

                if self.build_result.is_failed():
                    raise PluginFailedException(self.build_result.fail_reason)
            except PluginFailedException as ex:
                self.builder.is_built = False
                logger.error('buildstep plugin failed: %s', ex)
                raise

            self.builder.is_built = True
            if self.build_result.is_image_available():
                self.builder.image_id = self.build_result.image_id

            # run prepublish plugins
            try:
                prepublish_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more prepublish plugins failed: %s", ex)
                raise

            if self.build_result.is_image_available():
                self.built_image_inspect = self.builder.inspect_built_image()
                history = self.builder.tasker.get_image_history(
                    self.builder.image_id)
                diff_ids = self.built_image_inspect[INSPECT_ROOTFS][
                    INSPECT_ROOTFS_LAYERS]

                # diff_ids is ordered oldest first
                # history is ordered newest first
                # We want layer_sizes to be ordered oldest first
                self.layer_sizes = [{
                    "diff_id": diff_id,
                    "size": layer['Size']
                } for (diff_id, layer) in zip(diff_ids, reversed(history))]

            try:
                postbuild_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more postbuild plugins failed: %s", ex)
                raise

            return self.build_result
        except Exception as ex:
            logger.debug("caught exception (%s) so running exit plugins",
                         exception_message(ex))
            exception_being_handled = True
            raise
        finally:
            # We need to make sure all exit plugins are executed
            signal.signal(signal.SIGTERM, lambda *args: None)

            exit_runner = ExitPluginsRunner(self.builder.tasker,
                                            self,
                                            self.exit_plugins_conf,
                                            keep_going=True,
                                            plugin_files=self.plugin_files)
            try:
                exit_runner.run(keep_going=True)
            except PluginFailedException as ex:
                logger.error("one or more exit plugins failed: %s", ex)

                # raise exception only in case that there is no previous exception being already
                # handled to prevent replacing original exceptions (root cause) with exceptions
                # from exit plugins
                if not exception_being_handled:
                    raise ex
            finally:
                self.source.remove_tmpdir()
                self.fs_watcher.finish()

            signal.signal(signal.SIGTERM, signal.SIG_DFL)
Esempio n. 19
0
    def build_docker_image(self):
        """
        build docker image

        :return: BuildResult
        """
        self.builder = InsideBuilder(self.source, self.image)
        try:
            signal.signal(signal.SIGTERM, self.throw_canceled_build_exception)
            # time to run pre-build plugins, so they can access cloned repo
            logger.info("running pre-build plugins")
            prebuild_runner = PreBuildPluginsRunner(
                self.builder.tasker,
                self,
                self.prebuild_plugins_conf,
                plugin_files=self.plugin_files)
            try:
                prebuild_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more prebuild plugins failed: %s", ex)
                raise
            except AutoRebuildCanceledException as ex:
                logger.info(str(ex))
                self.autorebuild_canceled = True
                raise

            logger.info("running buildstep plugins")
            buildstep_runner = BuildStepPluginsRunner(
                self.builder.tasker,
                self,
                self.buildstep_plugins_conf,
                plugin_files=self.plugin_files)
            try:
                self.build_result = buildstep_runner.run()

                if self.build_result.is_failed():
                    raise PluginFailedException(self.build_result.fail_reason)
            except PluginFailedException as ex:
                self.builder.is_built = False
                logger.error('buildstep plugin failed: %s', ex)
                raise

            self.builder.is_built = True
            if self.build_result.is_image_available():
                self.builder.image_id = self.build_result.image_id
                self.built_image_inspect = self.builder.inspect_built_image()

            # run prepublish plugins
            prepublish_runner = PrePublishPluginsRunner(
                self.builder.tasker,
                self,
                self.prepublish_plugins_conf,
                plugin_files=self.plugin_files)
            try:
                prepublish_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more prepublish plugins failed: %s", ex)
                raise

            postbuild_runner = PostBuildPluginsRunner(
                self.builder.tasker,
                self,
                self.postbuild_plugins_conf,
                plugin_files=self.plugin_files)
            try:
                postbuild_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more postbuild plugins failed: %s", ex)
                raise

            return self.build_result
        except Exception as ex:
            logger.debug("caught exception (%r) so running exit plugins", ex)
            raise
        finally:
            # We need to make sure all exit plugins are executed
            signal.signal(signal.SIGTERM, lambda *args: None)
            exit_runner = ExitPluginsRunner(self.builder.tasker,
                                            self,
                                            self.exit_plugins_conf,
                                            plugin_files=self.plugin_files)
            try:
                exit_runner.run(keep_going=True)
            except PluginFailedException as ex:
                logger.error("one or more exit plugins failed: %s", ex)
                raise
            finally:
                self.source.remove_tmpdir()

            signal.signal(signal.SIGTERM, signal.SIG_DFL)