示例#1
0
def test_build_bad_git_commit_dockerfile(tmpdir, source_params):
    provided_image = "test-build:test_tag"
    if MOCK:
        mock_docker(build_should_fail=True, provided_image_repotags=provided_image)

    source_params.update({'tmpdir': str(tmpdir)})
    s = get_source_instance_for(source_params)
    b = InsideBuilder(s, provided_image)
    build_result = b.build()
    assert build_result.is_failed()
示例#2
0
def test_build_bad_git_commit_dockerfile(tmpdir, source_params):
    provided_image = "test-build:test_tag"
    if MOCK:
        mock_docker(build_should_fail=True,
                    provided_image_repotags=provided_image)

    source_params.update({'tmpdir': str(tmpdir)})
    s = get_source_instance_for(source_params)
    b = InsideBuilder(s, provided_image)
    build_result = b.build()
    assert build_result.is_failed()
示例#3
0
def test_build_image(tmpdir, source_params):
    provided_image = "test-build:test_tag"
    if MOCK:
        mock_docker(provided_image_repotags=provided_image)

    source_params.update({'tmpdir': str(tmpdir)})
    s = get_source_instance_for(source_params)
    t = DockerTasker()
    b = InsideBuilder(s, provided_image)
    build_result = b.build()
    assert t.inspect_image(build_result.image_id)
    # clean
    t.remove_image(build_result.image_id)
示例#4
0
def test_build_image(tmpdir, source_params):
    provided_image = "test-build:test_tag"
    if MOCK:
        mock_docker(provided_image_repotags=provided_image)

    source_params.update({'tmpdir': str(tmpdir)})
    s = get_source_instance_for(source_params)
    t = DockerTasker()
    b = InsideBuilder(s, provided_image)
    build_result = b.build()
    assert t.inspect_image(build_result.image_id)
    # clean
    t.remove_image(build_result.image_id)
示例#5
0
def test_build_generator_raises(tmpdir):
    provided_image = "test-build:test_tag"
    if MOCK:
        mock_docker(provided_image_repotags=provided_image, build_should_fail=True, build_should_fail_generator=True)

    source_params = SOURCE.copy()
    source_params.update({"tmpdir": str(tmpdir)})
    s = get_source_instance_for(source_params)
    t = DockerTasker()
    b = InsideBuilder(s, provided_image)
    build_result = b.build()

    assert isinstance(build_result, ExceptionBuildResult)
    assert build_result.is_failed()
    assert "build generator failure" in build_result.logs
示例#6
0
def test_build_generator_raises(tmpdir):
    provided_image = "test-build:test_tag"
    if MOCK:
        mock_docker(provided_image_repotags=provided_image,
                    build_should_fail=True,
                    build_should_fail_generator=True)

    source_params = SOURCE.copy()
    source_params.update({'tmpdir': str(tmpdir)})
    s = get_source_instance_for(source_params)
    t = DockerTasker()
    b = InsideBuilder(s, provided_image)
    build_result = b.build()

    assert isinstance(build_result, ExceptionBuildResult)
    assert build_result.is_failed()
    assert 'build generator failure' in build_result.logs
示例#7
0
class DockerBuildWorkflow(object):
    """
    This class defines a workflow for building images:

    1. pull image from registry
    2. tag it properly if needed
    3. obtain source
    4. build image
    5. tag it
    6. push it to registries
    """

    def __init__(self, source, image, prebuild_plugins=None, prepublish_plugins=None,
                 postbuild_plugins=None, exit_plugins=None, plugin_files=None,
                 openshift_build_selflink=None, **kwargs):
        """
        :param source: dict, where/how to get source code to put in image
        :param image: str, tag for built image ([registry/]image_name[:tag])
        :param prebuild_plugins: dict, arguments for pre-build plugins
        :param prepublish_plugins: dict, arguments for test-build plugins
        :param postbuild_plugins: dict, arguments for post-build plugins
        :param plugin_files: list of str, load plugins also from these files
        :param openshift_build_selflink: str, link to openshift build (if we're actually running
            on openshift) without the actual hostname/IP address
        """
        self.source = get_source_instance_for(source, tmpdir=tempfile.mkdtemp())
        self.image = image

        self.prebuild_plugins_conf = prebuild_plugins
        self.prepublish_plugins_conf = prepublish_plugins
        self.postbuild_plugins_conf = postbuild_plugins
        self.exit_plugins_conf = exit_plugins
        self.prebuild_results = {}
        self.postbuild_results = {}
        self.autorebuild_canceled = False
        self.build_failed = False
        self.plugin_failed = False
        self.plugin_files = plugin_files

        self.kwargs = kwargs

        self.builder = None
        self.build_logs = []
        self.built_image_inspect = None
        self._base_image_inspect = None

        self.pulled_base_images = set()

        # When an image is exported into tarball, it can then be processed by various plugins.
        #  Each plugin that transforms the image should save it as a new file and append it to
        #  the end of exported_image_sequence. Other plugins should then operate with last
        #  member of this structure. Example:
        #  [{'path': '/tmp/foo.tar', 'size': 12345678, 'md5sum': '<md5>', 'sha256sum': '<sha256>'}]
        #  You can use util.get_exported_image_metadata to create a dict to append to this list.
        self.exported_image_sequence = []

        self.tag_conf = TagConf()
        self.push_conf = PushConf()

        # mapping of downloaded files; DON'T PUT ANYTHING BIG HERE!
        # "path/to/file" -> "content"
        self.files = {}

        self.openshift_build_selflink = openshift_build_selflink

        if kwargs:
            logger.warning("unprocessed keyword arguments: %s", kwargs)

    @property
    def build_process_failed(self):
        """
        Has any aspect of the build process failed?
        """
        return self.build_failed or self.plugin_failed

    # inspect base image lazily just before it's needed - pre plugins may change the base image
    @property
    def base_image_inspect(self):
        if self._base_image_inspect is None:
            self._base_image_inspect = self.builder.tasker.inspect_image(self.builder.base_image)
        return self._base_image_inspect

    def build_docker_image(self):
        """
        build docker image

        :return: BuildResults
        """
        self.builder = InsideBuilder(self.source, self.image)
        try:
            # time to run pre-build plugins, so they can access cloned repo
            logger.info("running pre-build plugins")
            prebuild_runner = PreBuildPluginsRunner(self.builder.tasker, self, self.prebuild_plugins_conf,
                                                    plugin_files=self.plugin_files)
            try:
                prebuild_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more prebuild plugins failed: %s", ex)
                raise
            except AutoRebuildCanceledException as ex:
                logger.info(str(ex))
                self.autorebuild_canceled = True
                raise

            build_result = self.builder.build()
            self.build_logs = build_result.logs

            self.build_failed = build_result.is_failed()

            if build_result.is_failed():
                # The docker build failed. Finish here, just run the
                # exit plugins (from the 'finally:' block below).
                return build_result

            self.built_image_inspect = self.builder.inspect_built_image()

            # run prepublish plugins
            prepublish_runner = PrePublishPluginsRunner(self.builder.tasker, self, self.prepublish_plugins_conf,
                                                        plugin_files=self.plugin_files)
            try:
                prepublish_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more prepublish plugins failed: %s", ex)
                raise

            postbuild_runner = PostBuildPluginsRunner(self.builder.tasker, self, self.postbuild_plugins_conf,
                                                      plugin_files=self.plugin_files)
            try:
                postbuild_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more postbuild plugins failed: %s", ex)
                raise

            return build_result
        finally:
            exit_runner = ExitPluginsRunner(self.builder.tasker, self,
                                            self.exit_plugins_conf,
                                            plugin_files=self.plugin_files)
            try:
                exit_runner.run(keep_going=True)
            except PluginFailedException as ex:
                logger.error("one or more exit plugins failed: %s", ex)
            finally:
                self.source.remove_tmpdir()
示例#8
0
class DockerBuildWorkflow(object):
    """
    This class defines a workflow for building images:

    1. pull image from registry
    2. tag it properly if needed
    3. obtain source
    4. build image
    5. tag it
    6. push it to registries
    """
    def __init__(self,
                 source,
                 image,
                 target_registries=None,
                 prebuild_plugins=None,
                 prepublish_plugins=None,
                 postbuild_plugins=None,
                 exit_plugins=None,
                 plugin_files=None,
                 target_registries_insecure=False,
                 **kwargs):
        """
        :param source: dict, where/how to get source code to put in image
        :param image: str, tag for built image ([registry/]image_name[:tag])
        :param target_registries: list of str, list of registries to push image to (might change in future)
        :param prebuild_plugins: dict, arguments for pre-build plugins
        :param prepublish_plugins: dict, arguments for test-build plugins
        :param postbuild_plugins: dict, arguments for post-build plugins
        :param plugin_files: list of str, load plugins also from these files
        :param target_registries_insecure: bool, allow connecting to target registries over plain http
        """
        self.source = get_source_instance_for(source,
                                              tmpdir=tempfile.mkdtemp())
        self.image = image

        self.prebuild_plugins_conf = prebuild_plugins
        self.prepublish_plugins_conf = prepublish_plugins
        self.postbuild_plugins_conf = postbuild_plugins
        self.exit_plugins_conf = exit_plugins
        self.prebuild_results = {}
        self.postbuild_results = {}
        self.build_failed = False
        self.plugin_failed = False
        self.plugin_files = plugin_files

        self.kwargs = kwargs

        self.builder = None
        self.build_logs = None
        self.built_image_inspect = None
        self._base_image_inspect = None

        self.pulled_base_images = set()

        # When an image is exported into tarball, it can then be processed by various plugins.
        #  Each plugin that transforms the image should save it as a new file and append it to
        #  the end of exported_image_sequence. Other plugins should then operate with last
        #  member of this structure. Example:
        #  [{'path': '/tmp/foo.tar', 'size': 12345678, 'md5sum': '<md5>', 'sha256sum': '<sha256>'}]
        #  You can use util.get_exported_image_metadata to create a dict to append to this list.
        self.exported_image_sequence = []

        self.tag_conf = TagConf()
        self.push_conf = PushConf()
        if target_registries:
            self.push_conf.add_docker_registries(
                target_registries, insecure=target_registries_insecure)

        # mapping of downloaded files; DON'T PUT ANYTHING BIG HERE!
        # "path/to/file" -> "content"
        self.files = {}

        if kwargs:
            logger.warning("unprocessed keyword arguments: %s", kwargs)

    @property
    def build_process_failed(self):
        """
        Has any aspect of the build process failed?
        """
        return self.build_failed or self.plugin_failed

    # inspect base image lazily just before it's needed - pre plugins may change the base image
    @property
    def base_image_inspect(self):
        if self._base_image_inspect is None:
            self._base_image_inspect = self.builder.tasker.inspect_image(
                self.builder.base_image)
        return self._base_image_inspect

    def build_docker_image(self):
        """
        build docker image

        :return: BuildResults
        """
        self.builder = InsideBuilder(self.source, self.image)
        try:
            # time to run pre-build plugins, so they can access cloned repo
            logger.info("running pre-build plugins")
            prebuild_runner = PreBuildPluginsRunner(
                self.builder.tasker,
                self,
                self.prebuild_plugins_conf,
                plugin_files=self.plugin_files)
            try:
                prebuild_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more prebuild plugins failed: %s", ex)
                raise

            build_result = self.builder.build()
            self.build_logs = build_result.logs

            self.build_failed = build_result.is_failed()

            if not build_result.is_failed():
                self.built_image_inspect = self.builder.inspect_built_image()

            # run prepublish plugins
            prepublish_runner = PrePublishPluginsRunner(
                self.builder.tasker,
                self,
                self.prepublish_plugins_conf,
                plugin_files=self.plugin_files)
            try:
                prepublish_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more prepublish plugins failed: %s", ex)
                raise

            if not build_result.is_failed():
                for registry in self.push_conf.docker_registries:
                    self.builder.push_built_image(registry.uri,
                                                  insecure=registry.insecure)

            postbuild_runner = PostBuildPluginsRunner(
                self.builder.tasker,
                self,
                self.postbuild_plugins_conf,
                plugin_files=self.plugin_files)
            try:
                postbuild_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more postbuild plugins failed: %s", ex)
                raise

            return build_result
        finally:
            self.source.remove_tmpdir()

            exit_runner = ExitPluginsRunner(self.builder.tasker,
                                            self,
                                            self.exit_plugins_conf,
                                            plugin_files=self.plugin_files)
            try:
                exit_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more exit plugins failed: %s", ex)
示例#9
0
class DockerBuildWorkflow(object):
    """
    This class defines a workflow for building images:

    1. pull image from registry
    2. tag it properly if needed
    3. obtain source
    4. build image
    5. tag it
    6. push it to registries
    """
    def __init__(self,
                 source,
                 image,
                 prebuild_plugins=None,
                 prepublish_plugins=None,
                 postbuild_plugins=None,
                 exit_plugins=None,
                 plugin_files=None,
                 openshift_build_selflink=None,
                 client_version=None,
                 **kwargs):
        """
        :param source: dict, where/how to get source code to put in image
        :param image: str, tag for built image ([registry/]image_name[:tag])
        :param prebuild_plugins: dict, arguments for pre-build plugins
        :param prepublish_plugins: dict, arguments for test-build plugins
        :param postbuild_plugins: dict, arguments for post-build plugins
        :param plugin_files: list of str, load plugins also from these files
        :param openshift_build_selflink: str, link to openshift build (if we're actually running
            on openshift) without the actual hostname/IP address
        """
        self.source = get_source_instance_for(source,
                                              tmpdir=tempfile.mkdtemp())
        self.image = image

        self.prebuild_plugins_conf = prebuild_plugins
        self.prepublish_plugins_conf = prepublish_plugins
        self.postbuild_plugins_conf = postbuild_plugins
        self.exit_plugins_conf = exit_plugins
        self.prebuild_results = {}
        self.postbuild_results = {}
        self.prepub_results = {}
        self.exit_results = {}
        self.plugin_workspace = {}
        self.plugins_timestamps = {}
        self.plugins_durations = {}
        self.plugins_errors = {}
        self.autorebuild_canceled = False
        self.build_failed = False
        self.plugin_failed = False
        self.plugin_files = plugin_files

        self.kwargs = kwargs

        self.builder = None
        self.build_logs = []
        self.built_image_inspect = None
        self._base_image_inspect = None

        self.pulled_base_images = set()

        # When an image is exported into tarball, it can then be processed by various plugins.
        #  Each plugin that transforms the image should save it as a new file and append it to
        #  the end of exported_image_sequence. Other plugins should then operate with last
        #  member of this structure. Example:
        #  [{'path': '/tmp/foo.tar', 'size': 12345678, 'md5sum': '<md5>', 'sha256sum': '<sha256>'}]
        #  You can use util.get_exported_image_metadata to create a dict to append to this list.
        self.exported_image_sequence = []

        self.tag_conf = TagConf()
        self.push_conf = PushConf()

        # mapping of downloaded files; DON'T PUT ANYTHING BIG HERE!
        # "path/to/file" -> "content"
        self.files = {}

        self.openshift_build_selflink = openshift_build_selflink

        if client_version:
            logger.debug("build json was built by osbs-client %s",
                         client_version)

        if kwargs:
            logger.warning("unprocessed keyword arguments: %s", kwargs)

    @property
    def build_process_failed(self):
        """
        Has any aspect of the build process failed?
        """
        return self.build_failed or self.plugin_failed

    # inspect base image lazily just before it's needed - pre plugins may change the base image
    @property
    def base_image_inspect(self):
        if self._base_image_inspect is None:
            self._base_image_inspect = self.builder.tasker.inspect_image(
                self.builder.base_image)
        return self._base_image_inspect

    def throw_canceled_build_exception(self, *args, **kwargs):
        raise BuildCanceledException("Build was canceled")

    def build_docker_image(self):
        """
        build docker image

        :return: BuildResults
        """
        self.builder = InsideBuilder(self.source, self.image)
        try:
            signal.signal(signal.SIGTERM, self.throw_canceled_build_exception)
            # time to run pre-build plugins, so they can access cloned repo
            logger.info("running pre-build plugins")
            prebuild_runner = PreBuildPluginsRunner(
                self.builder.tasker,
                self,
                self.prebuild_plugins_conf,
                plugin_files=self.plugin_files)
            try:
                prebuild_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more prebuild plugins failed: %s", ex)
                raise
            except AutoRebuildCanceledException as ex:
                logger.info(str(ex))
                self.autorebuild_canceled = True
                raise

            start_time = datetime.datetime.now()
            self.plugins_timestamps['dockerbuild'] = start_time.isoformat()

            build_result = self.builder.build()

            try:
                finish_time = datetime.datetime.now()
                duration = finish_time - start_time
                seconds = duration.total_seconds()
                logger.debug("build finished in %ds", seconds)
                self.plugins_durations['dockerbuild'] = seconds
            except Exception:
                logger.exception("failed to save build duration")

            self.build_logs = build_result.logs

            self.build_failed = build_result.is_failed()

            if build_result.is_failed():
                # The docker build failed. Finish here, just run the
                # exit plugins (from the 'finally:' block below).
                self.plugins_errors['dockerbuild'] = ''
                return build_result

            self.built_image_inspect = self.builder.inspect_built_image()

            # run prepublish plugins
            prepublish_runner = PrePublishPluginsRunner(
                self.builder.tasker,
                self,
                self.prepublish_plugins_conf,
                plugin_files=self.plugin_files)
            try:
                prepublish_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more prepublish plugins failed: %s", ex)
                raise

            postbuild_runner = PostBuildPluginsRunner(
                self.builder.tasker,
                self,
                self.postbuild_plugins_conf,
                plugin_files=self.plugin_files)
            try:
                postbuild_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more postbuild plugins failed: %s", ex)
                raise

            return build_result
        finally:
            # We need to make sure all exit plugins are executed
            signal.signal(signal.SIGTERM, lambda *args: None)
            exit_runner = ExitPluginsRunner(self.builder.tasker,
                                            self,
                                            self.exit_plugins_conf,
                                            plugin_files=self.plugin_files)
            try:
                exit_runner.run(keep_going=True)
            except PluginFailedException as ex:
                logger.error("one or more exit plugins failed: %s", ex)
                raise
            finally:
                self.source.remove_tmpdir()

            signal.signal(signal.SIGTERM, signal.SIG_DFL)
示例#10
0
class DockerBuildWorkflow(object):
    """
    This class defines a workflow for building images:

    1. pull image from registry
    2. tag it properly if needed
    3. obtain source
    4. build image
    5. tag it
    6. push it to registries
    """

    def __init__(self, source, image, parent_registry=None, target_registries=None,
                 prebuild_plugins=None, prepublish_plugins=None, postbuild_plugins=None,
                 plugin_files=None, parent_registry_insecure=False,
                 target_registries_insecure=False, dont_pull_base_image=False, **kwargs):
        """
        :param source: dict, where/how to get source code to put in image
        :param image: str, tag for built image ([registry/]image_name[:tag])
        :param parent_registry: str, registry to pull base image from
        :param target_registries: list of str, list of registries to push image to (might change in future)
        :param prebuild_plugins: dict, arguments for pre-build plugins
        :param prepublish_plugins: dict, arguments for test-build plugins
        :param postbuild_plugins: dict, arguments for post-build plugins
        :param plugin_files: list of str, load plugins also from these files
        :param parent_registry_insecure: bool, allow connecting to parent registry over plain http
        :param target_registries_insecure: bool, allow connecting to target registries over plain http
        :param dont_pull_base_image: bool, don't pull or update base image specified in dockerfile
        """
        self.source = get_source_instance_for(source, tmpdir=tempfile.mkdtemp())
        self.image = image

        self.parent_registry = parent_registry
        self.parent_registry_insecure = parent_registry_insecure

        self.prebuild_plugins_conf = prebuild_plugins
        self.prepublish_plugins_conf = prepublish_plugins
        self.postbuild_plugins_conf = postbuild_plugins
        self.prebuild_results = {}
        self.postbuild_results = {}
        self.plugin_files = plugin_files

        self.kwargs = kwargs

        self.builder = None
        self.build_logs = None
        self.built_image_inspect = None
        self.base_image_inspect = None

        self.dont_pull_base_image = dont_pull_base_image
        self.pulled_base_images = set()

        # squashed image tarball
        # set by squash plugin
        self.exported_squashed_image = {}

        self.tag_conf = TagConf()
        self.push_conf = PushConf()
        if target_registries:
            self.push_conf.add_docker_registries(target_registries, insecure=target_registries_insecure)

        # mapping of downloaded files; DON'T PUT ANYTHING BIG HERE!
        # "path/to/file" -> "content"
        self.files = {}

        if kwargs:
            logger.warning("unprocessed keyword arguments: %s", kwargs)

    def build_docker_image(self):
        """
        build docker image

        :return: BuildResults
        """
        self.builder = InsideBuilder(self.source, self.image)
        try:
            if not self.dont_pull_base_image:
                self.pulled_base_images = self.builder.pull_base_image(self.parent_registry,
                                                                       insecure=self.parent_registry_insecure)

            self.base_image_inspect = self.builder.tasker.inspect_image(self.builder.base_image)

            # time to run pre-build plugins, so they can access cloned repo,
            # base image
            logger.info("running pre-build plugins")
            prebuild_runner = PreBuildPluginsRunner(self.builder.tasker, self, self.prebuild_plugins_conf,
                                                    plugin_files=self.plugin_files)
            try:
                prebuild_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more prebuild plugins failed: %s", ex)
                return

            build_result = self.builder.build()
            self.build_logs = build_result.logs

            if not build_result.is_failed():
                self.built_image_inspect = self.builder.inspect_built_image()

            # run prepublish plugins
            prepublish_runner = PrePublishPluginsRunner(self.builder.tasker, self, self.prepublish_plugins_conf,
                                                        plugin_files=self.plugin_files)
            try:
                prepublish_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more prepublish plugins failed: %s", ex)
                return

            if not build_result.is_failed():
                for registry in self.push_conf.docker_registries:
                    self.builder.push_built_image(registry.uri,
                                                  insecure=registry.insecure)

            postbuild_runner = PostBuildPluginsRunner(self.builder.tasker, self, self.postbuild_plugins_conf,
                                                      plugin_files=self.plugin_files)
            try:
                postbuild_runner.run()
            except PluginFailedException as ex:
                logger.error("one or more postbuild plugins failed: %s", ex)
                return

            return build_result
        finally:
            self.source.remove_tmpdir()