Ejemplo n.º 1
0
def create_built_program_from_source_cached(ctx, src, options_bytes, devices=None,
        cache_dir=None, include_path=None):
    try:
        if cache_dir is not False:
            prg, already_built, was_cached = \
                    _create_built_program_from_source_cached(
                            ctx, src, options_bytes, devices, cache_dir,
                            include_path=include_path)
        else:
            prg = _cl._Program(ctx, src)
            was_cached = False
            already_built = False

    except Exception as e:
        from pyopencl import Error
        if (isinstance(e, Error)
                and e.code == _cl.status_code.BUILD_PROGRAM_FAILURE):  # noqa pylint:disable=no-member
            # no need to try again
            raise

        from warnings import warn
        from traceback import format_exc
        warn("PyOpenCL compiler caching failed with an exception:\n"
                "[begin exception]\n%s[end exception]"
                % format_exc())

        prg = _cl._Program(ctx, src)
        was_cached = False
        already_built = False

    if not already_built:
        prg.build(options_bytes, devices)

    return prg, was_cached
Ejemplo n.º 2
0
def create_built_program_from_source_cached(ctx, src, options=[], devices=None,
        cache_dir=None):
    try:
        if cache_dir != False:
            prg, already_built = _create_built_program_from_source_cached(
                    ctx, src, options, devices, cache_dir)
        else:
            prg = _cl._Program(ctx, src)
            already_built = False

    except Exception, e:
        raise
        from pyopencl import Error
        if (isinstance(e, Error) 
                and e.code == _cl.status_code.BUILD_PROGRAM_FAILURE):
            # no need to try again
            raise

        from warnings import warn
        from traceback import format_exc
        warn("PyOpenCL compiler caching failed with an exception:\n"
                "[begin exception]\n%s[end exception]"
                % format_exc())

        prg = _cl._Program(ctx, src)
        already_built = False
Ejemplo n.º 3
0
def create_built_program_from_source_cached(ctx,
                                            src,
                                            options=[],
                                            devices=None,
                                            cache_dir=None):
    try:
        if cache_dir is not False:
            prg, already_built = _create_built_program_from_source_cached(
                ctx, src, options, devices, cache_dir)
        else:
            prg = _cl._Program(ctx, src)
            already_built = False

    except Exception, e:
        raise
        from pyopencl import Error
        if (isinstance(e, Error)
                and e.code == _cl.status_code.BUILD_PROGRAM_FAILURE):
            # no need to try again
            raise

        from warnings import warn
        from traceback import format_exc
        warn("PyOpenCL compiler caching failed with an exception:\n"
             "[begin exception]\n%s[end exception]" % format_exc())

        prg = _cl._Program(ctx, src)
        already_built = False
Ejemplo n.º 4
0
    def build(self, options=[], devices=None, cache_dir=None):
        if isinstance(options, str):
            options = [options]

        options = options + ["-I", _find_pyopencl_include_path()]

        import os
        forced_options = os.environ.get("PYOPENCL_BUILD_OPTIONS")
        if forced_options:
            options = options + forced_options.split()

        if os.environ.get("PYOPENCL_NO_CACHE") and self._prg is None:
            self._prg = _cl._Program(self._context, self._source)

        if self._prg is not None:
            # uncached

            self._build_and_catch_errors(
                    lambda: self._prg.build(" ".join(options), devices),
                    options=options)

        else:
            # cached

            from pyopencl.cache import create_built_program_from_source_cached
            self._prg = self._build_and_catch_errors(
                    lambda: create_built_program_from_source_cached(
                        self._context, self._source, options, devices,
                        cache_dir=cache_dir),
                    options=options, source=self._source)

            del self._context

        return self
Ejemplo n.º 5
0
    def __init__(self, arg1, arg2=None, arg3=None):
        if arg2 is None:
            # 1-argument form: program
            self._prg = arg1

        elif arg3 is None:
            # 2-argument form: context, source
            context, source = arg1, arg2

            import sys
            if isinstance(source, unicode) and sys.version_info < (3, ):
                from warnings import warn
                warn(
                    "Received OpenCL source code in Unicode, "
                    "should be ASCII string. Attempting conversion.",
                    stacklevel=2)
                source = str(source)

            self._context = context
            self._source = source
            self._prg = None

        else:
            context, device, binaries = arg1, arg2, arg3
            self._context = context
            self._prg = _cl._Program(context, device, binaries)
Ejemplo n.º 6
0
    def __init__(self, arg1, arg2=None, arg3=None):
        if arg2 is None:
            # 1-argument form: program
            self._prg = arg1

        elif arg3 is None:
            # 2-argument form: context, source
            context, source = arg1, arg2

            import sys
            if isinstance(source, unicode) and sys.version_info < (3,):
                from warnings import warn
                warn("Received OpenCL source code in Unicode, "
                     "should be ASCII string. Attempting conversion.",
                     stacklevel=2)
                source = source.encode()

            self._context = context
            self._source = source
            self._prg = None

        else:
            context, device, binaries = arg1, arg2, arg3
            self._context = context
            self._prg = _cl._Program(context, device, binaries)
Ejemplo n.º 7
0
def create_built_program_from_source_cached(ctx,
                                            src,
                                            options_bytes,
                                            devices=None,
                                            cache_dir=None,
                                            include_path=None):
    try:
        was_cached = False
        already_built = False
        if cache_dir is not False:
            prg, already_built, was_cached = \
                    _create_built_program_from_source_cached(
                            ctx, src, options_bytes, devices, cache_dir,
                            include_path=include_path)
            if was_cached and not already_built:
                prg.build(options_bytes, devices)
                already_built = True
        else:
            prg = _cl._Program(ctx, src)

    except Exception as e:
        from pyopencl import Error
        build_program_failure = (
            isinstance(e, Error)
            and e.code == _cl.status_code.BUILD_PROGRAM_FAILURE)  # noqa pylint:disable=no-member

        # Mac error on intel CPU driver: can't build from cached version.
        # If we get a build_program_failure from the cached version then
        # build from source instead, otherwise report the failure.
        if build_program_failure and not was_cached:
            raise

        if not build_program_failure:
            from warnings import warn
            from traceback import format_exc
            warn("PyOpenCL compiler caching failed with an exception:\n"
                 "[begin exception]\n%s[end exception]" % format_exc())

        prg = _cl._Program(ctx, src)
        was_cached = False
        already_built = False

    if not already_built:
        prg.build(options_bytes, devices)

    return prg, was_cached
Ejemplo n.º 8
0
    def build(self, options=[], devices=None, cache_dir=None):
        if isinstance(options, str):
            options = [options]

        options = options + ["-I", _find_pyopencl_include_path()]

        import os
        if os.environ.get("PYOPENCL_NO_CACHE") and self._prg is None:
            self._prg = _cl._Program(self._context, self._source)

        if self._prg is not None:
            if isinstance(options, list):
                options = " ".join(options)

            self._prg._build(options, devices)
        else:
            from pyopencl.cache import create_built_program_from_source_cached

            err = None
            try:
                self._prg = create_built_program_from_source_cached(
                    self._context,
                    self._source,
                    options,
                    devices,
                    cache_dir=cache_dir)
            except _cl.RuntimeError, e:
                from pytools import Record

                class ErrorRecord(Record):
                    pass

                from tempfile import NamedTemporaryFile
                srcfile = NamedTemporaryFile(mode="wt",
                                             delete=False,
                                             suffix=".cl")
                try:
                    srcfile.write(self._source)
                finally:
                    srcfile.close()

                what = e.what + "\n(source saved as %s)" % srcfile.name
                code = e.code
                routine = e.routine

                err = _cl.RuntimeError(
                    ErrorRecord(what=lambda: what,
                                code=lambda: code,
                                routine=lambda: routine))

            if err is not None:
                # Python 3.2 outputs the whole list of currently active exceptions
                # This serves to remove one (redundant) level from that nesting.
                raise err

            del self._context
            del self._source
Ejemplo n.º 9
0
    def _get_prg(self):
        if self._prg is not None:
            return self._prg
        else:
            # "no program" can only happen in from-source case.
            from warnings import warn
            warn("Pre-build attribute access defeats compiler caching.", stacklevel=3)

            self._prg = _cl._Program(self._context, self._source)
            del self._context
            del self._source
            return self._prg
Ejemplo n.º 10
0
    def _get_prg(self):
        if self._prg is not None:
            return self._prg
        else:
            # "no program" can only happen in from-source case.
            from warnings import warn
            warn("Pre-build attribute access defeats compiler caching.",
                 stacklevel=3)

            self._prg = _cl._Program(self._context, self._source)
            del self._context
            return self._prg
Ejemplo n.º 11
0
    def build(self, options=[], devices=None, cache_dir=None):
        if isinstance(options, str):
            options = [options]

        options = options + ["-I", _find_pyopencl_include_path()]

        import os
        if os.environ.get("PYOPENCL_NO_CACHE") and self._prg is None:
            self._prg = _cl._Program(self._context, self._source)

        if self._prg is not None:
            if isinstance(options, list):
                options = " ".join(options)

            self._prg._build(options, devices)
        else:
            from pyopencl.cache import create_built_program_from_source_cached

            err = None
            try:
                self._prg = create_built_program_from_source_cached(
                        self._context, self._source, options, devices,
                        cache_dir=cache_dir)
            except _cl.RuntimeError, e:
                from pytools import Record
                class ErrorRecord(Record):
                    pass

                from tempfile import NamedTemporaryFile
                srcfile = NamedTemporaryFile(mode="wt", delete=False, suffix=".cl")
                try:
                    srcfile.write(self._source)
                finally:
                    srcfile.close()

                what = e.what + "\n(source saved as %s)" % srcfile.name
                code = e.code
                routine = e.routine

                err = _cl.RuntimeError(
                        ErrorRecord(
                            what=lambda : what,
                            code=lambda : code,
                            routine=lambda : routine))

            if err is not None:
                # Python 3.2 outputs the whole list of currently active exceptions
                # This serves to remove one (redundant) level from that nesting.
                raise err

            del self._context
            del self._source
Ejemplo n.º 12
0
def create_built_program_from_source_cached(ctx,
                                            src,
                                            options_bytes,
                                            devices=None,
                                            cache_dir=None,
                                            include_path=None):
    try:
        if cache_dir is not False:
            prg, already_built, was_cached = \
                    _create_built_program_from_source_cached(
                            ctx, src, options_bytes, devices, cache_dir,
                            include_path=include_path)
        else:
            prg = _cl._Program(ctx, src)
            was_cached = False
            already_built = False

    except Exception as e:
        from pyopencl import Error
        if (isinstance(e, Error)
                and e.code == _cl.status_code.BUILD_PROGRAM_FAILURE):  # noqa pylint:disable=no-member
            # no need to try again
            raise

        from warnings import warn
        from traceback import format_exc
        warn("PyOpenCL compiler caching failed with an exception:\n"
             "[begin exception]\n%s[end exception]" % format_exc())

        prg = _cl._Program(ctx, src)
        was_cached = False
        already_built = False

    if not already_built:
        prg.build(options_bytes, devices)

    return prg, was_cached
Ejemplo n.º 13
0
    def build(self, options=[], devices=None, cache_dir=None):
        if isinstance(options, str):
            options = [options]

        options = options + ["-I", _find_pyopencl_include_path()]

        import os
        forced_options = os.environ.get("PYOPENCL_BUILD_OPTIONS")
        if forced_options:
            options = options + forced_options.split()

        do_del_source = False
        if os.environ.get("PYOPENCL_NO_CACHE") and self._prg is None:
            self._prg = _cl._Program(self._context, self._source)

            do_del_source = True

        if self._prg is not None:
            # uncached

            self._build_and_catch_errors(
                lambda: self._prg.build(" ".join(options), devices),
                options=options)

        else:
            # cached

            from pyopencl.cache import create_built_program_from_source_cached
            self._prg = self._build_and_catch_errors(
                lambda: create_built_program_from_source_cached(self._context,
                                                                self._source,
                                                                options,
                                                                devices,
                                                                cache_dir=
                                                                cache_dir),
                options=options)

            del self._context
            do_del_source = True

        if do_del_source:
            del self._source

        return self
Ejemplo n.º 14
0
    def build(self, options=[], devices=None, cache_dir=None):
        if isinstance(options, str):
            options = [options]
        elif isinstance(options, unicode):
            options = [options.encode("utf8")]

        options = (
            options
            + _DEFAULT_BUILD_OPTIONS
            + _DEFAULT_INCLUDE_OPTIONS
            + _PLAT_BUILD_OPTIONS.get(self._context.devices[0].platform.name, [])
        )

        import os

        forced_options = os.environ.get("PYOPENCL_BUILD_OPTIONS")
        if forced_options:
            options = options + forced_options.split()

        if os.environ.get("PYOPENCL_NO_CACHE") and self._prg is None:
            self._prg = _cl._Program(self._context, self._source)

        if self._prg is not None:
            # uncached

            self._build_and_catch_errors(lambda: self._prg.build(" ".join(options), devices), options=options)

        else:
            # cached

            from pyopencl.cache import create_built_program_from_source_cached

            self._prg = self._build_and_catch_errors(
                lambda: create_built_program_from_source_cached(
                    self._context, self._source, options, devices, cache_dir=cache_dir
                ),
                options=options,
                source=self._source,
            )

            del self._context

        return self
Ejemplo n.º 15
0
    def build(self, options=[], devices=None, cache_dir=None):
        if isinstance(options, str):
            options = [options]
        elif isinstance(options, unicode):
            options = [options.encode("utf8")]

        options = (options + _DEFAULT_BUILD_OPTIONS +
                   _DEFAULT_INCLUDE_OPTIONS + _PLAT_BUILD_OPTIONS.get(
                       self._context.devices[0].platform.name, []))

        import os
        forced_options = os.environ.get("PYOPENCL_BUILD_OPTIONS")
        if forced_options:
            options = options + forced_options.split()

        if os.environ.get("PYOPENCL_NO_CACHE") and self._prg is None:
            self._prg = _cl._Program(self._context, self._source)

        if self._prg is not None:
            # uncached

            self._build_and_catch_errors(
                lambda: self._prg.build(" ".join(options), devices),
                options=options)

        else:
            # cached

            from pyopencl.cache import create_built_program_from_source_cached
            self._prg = self._build_and_catch_errors(
                lambda: create_built_program_from_source_cached(self._context,
                                                                self._source,
                                                                options,
                                                                devices,
                                                                cache_dir=
                                                                cache_dir),
                options=options,
                source=self._source)

            del self._context

        return self
Ejemplo n.º 16
0
        compiler_output(
            "Built kernel retrieved from cache. Original from-source "
            "build had warnings:\n" + message)

    # {{{ build on the build-needing devices, in one go

    result = None
    already_built = False

    if to_be_built_indices:
        # defeat implementation caches:
        from uuid import uuid4
        src = src + "\n\n__constant int pyopencl_defeat_cache_%s = 0;" % (
            uuid4().hex)

        prg = _cl._Program(ctx, src)
        prg.build(options, [devices[i] for i in to_be_built_indices])

        prg_devs = prg.get_info(_cl.program_info.DEVICES)
        prg_bins = prg.get_info(_cl.program_info.BINARIES)
        prg_logs = prg._get_build_logs()

        for dest_index in to_be_built_indices:
            dev = devices[dest_index]
            src_index = prg_devs.index(dev)
            binaries[dest_index] = prg_bins[src_index]
            _, logs[dest_index] = prg_logs[src_index]

        if len(to_be_built_indices) == len(devices):
            # Important special case: if code for all devices was built,
            # then we may simply use the program that we just built as the
Ejemplo n.º 17
0
def _create_built_program_from_source_cached(ctx, src, options_bytes,
        devices, cache_dir, include_path):
    from os.path import join

    if cache_dir is None:
        import appdirs
        cache_dir = join(appdirs.user_cache_dir("pyopencl", "pyopencl"),
                "pyopencl-compiler-cache-v2-py%s" % (
                    ".".join(str(i) for i in sys.version_info),))

    # {{{ ensure cache directory exists

    try:
        os.makedirs(cache_dir)
    except OSError as e:
        from errno import EEXIST
        if e.errno != EEXIST:
            raise

    # }}}

    if devices is None:
        devices = ctx.devices

    cache_keys = [get_cache_key(device, options_bytes, src) for device in devices]

    binaries = []
    to_be_built_indices = []
    logs = []
    for i, (device, cache_key) in enumerate(zip(devices, cache_keys)):
        cache_result = retrieve_from_cache(cache_dir, cache_key)

        if cache_result is None:
            logger.debug("build program: binary cache miss (key: %s)" % cache_key)

            to_be_built_indices.append(i)
            binaries.append(None)
            logs.append(None)
        else:
            logger.debug("build program: binary cache hit (key: %s)" % cache_key)

            binary, log = cache_result
            binaries.append(binary)
            logs.append(log)

    message = (75*"="+"\n").join(
            "Build on %s succeeded, but said:\n\n%s" % (dev, log)
            for dev, log in zip(devices, logs)
            if log is not None and log.strip())

    if message:
        from pyopencl import compiler_output
        compiler_output(
                "Built kernel retrieved from cache. Original from-source "
                "build had warnings:\n"+message)

    # {{{ build on the build-needing devices, in one go

    result = None
    already_built = False
    was_cached = not to_be_built_indices

    if to_be_built_indices:
        # defeat implementation caches:
        from uuid import uuid4
        src = src + "\n\n__constant int pyopencl_defeat_cache_%s = 0;" % (
                uuid4().hex)

        logger.debug("build program: start building program from source on %s"
                % ", ".join(str(devices[i]) for i in to_be_built_indices))

        prg = _cl._Program(ctx, src)
        prg.build(options_bytes, [devices[i] for i in to_be_built_indices])

        logger.debug("build program: from-source build complete")

        prg_devs = prg.get_info(_cl.program_info.DEVICES)
        prg_bins = prg.get_info(_cl.program_info.BINARIES)
        prg_logs = prg._get_build_logs()

        for dest_index in to_be_built_indices:
            dev = devices[dest_index]
            src_index = prg_devs.index(dev)
            binaries[dest_index] = prg_bins[src_index]
            _, logs[dest_index] = prg_logs[src_index]

        if len(to_be_built_indices) == len(devices):
            # Important special case: if code for all devices was built,
            # then we may simply use the program that we just built as the
            # final result.

            result = prg
            already_built = True

    if result is None:
        result = _cl._Program(ctx, devices, binaries)

    # }}}

    # {{{ save binaries to cache

    if to_be_built_indices:
        cleanup_m = CleanupManager()
        try:
            try:
                CacheLockManager(cleanup_m, cache_dir)

                for i in to_be_built_indices:
                    cache_key = cache_keys[i]
                    binary = binaries[i]

                    mod_cache_dir_m = ModuleCacheDirManager(cleanup_m,
                            join(cache_dir, cache_key))
                    info_path = mod_cache_dir_m.sub("info")
                    binary_path = mod_cache_dir_m.sub("binary")
                    source_path = mod_cache_dir_m.sub("source.cl")

                    outf = open(source_path, "wt")
                    outf.write(src)
                    outf.close()

                    outf = open(binary_path, "wb")
                    outf.write(binary)
                    outf.close()

                    from six.moves.cPickle import dump
                    info_file = open(info_path, "wb")
                    dump(_SourceInfo(
                        dependencies=get_dependencies(src, include_path),
                        log=logs[i]), info_file)
                    info_file.close()

            except Exception:
                cleanup_m.error_clean_up()
                raise
        finally:
            cleanup_m.clean_up()

    # }}}

    return result, already_built, was_cached
Ejemplo n.º 18
0
def _create_built_program_from_source_cached(ctx, src, options, devices, cache_dir):
    include_path = ["."] + [
            option[2:]
            for option in options
            if option.startswith("-I") or option.startswith("/I")]

    if cache_dir is None:
        from os.path import join
        from tempfile import gettempdir
        import getpass
        cache_dir = join(gettempdir(),
                "pyopencl-compiler-cache-v2-uid%s-py%s" % (
                    getpass.getuser(), ".".join(str(i) for i in sys.version_info)))

    # {{{ ensure cache directory exists

    try:
        os.mkdir(cache_dir)
    except OSError as e:
        from errno import EEXIST
        if e.errno != EEXIST:
            raise

    # }}}

    if devices is None:
        devices = ctx.devices

    cache_keys = [get_cache_key(device, options, src) for device in devices]

    binaries = []
    to_be_built_indices = []
    logs = []
    for i, (device, cache_key) in enumerate(zip(devices, cache_keys)):
        cache_result = retrieve_from_cache(cache_dir, cache_key)

        if cache_result is None:
            to_be_built_indices.append(i)
            binaries.append(None)
            logs.append(None)
        else:
            binary, log = cache_result
            binaries.append(binary)
            logs.append(log)

    message = (75*"="+"\n").join(
            "Build on %s succeeded, but said:\n\n%s" % (dev, log) 
            for dev, log in zip(devices, logs)
            if log is not None and log.strip())

    if message:
        from warnings import warn
        warn("Build succeeded, but resulted in non-empty logs:\n"+message)
    # {{{ build on the build-needing devices, in one go

    result = None
    already_built = False

    if to_be_built_indices:
        prg = _cl._Program(ctx, src)
        prg.build(options, [devices[i] for i in to_be_built_indices])

        prg_devs = prg.get_info(_cl.program_info.DEVICES)
        prg_bins = prg.get_info(_cl.program_info.BINARIES)
        prg_logs = prg._get_build_logs()

        for i, dest_index in enumerate(to_be_built_indices):
            assert prg_devs[i] == devices[dest_index]
            binaries[dest_index] = prg_bins[i]
            _, logs[dest_index] = prg_logs[i]

        if len(to_be_built_indices) == len(devices):
            # Important special case: if code for all devices was built,
            # then we may simply use the program that we just built as the
            # final result.

            result = prg
            already_built = True

    if result is None:
        result = _cl._Program(ctx, devices, binaries)

    # }}}

    # {{{ save binaries to cache

    if to_be_built_indices:
        cleanup_m = CleanupManager()
        try:
            lock_m = CacheLockManager(cleanup_m, cache_dir)

            for i in to_be_built_indices:
                cache_key = cache_keys[i]
                device = devices[i]
                binary = binaries[i]

                mod_cache_dir_m = ModuleCacheDirManager(cleanup_m,
                        join(cache_dir, cache_key))
                info_path = mod_cache_dir_m.sub("info")
                binary_path = mod_cache_dir_m.sub("binary")
                source_path = mod_cache_dir_m.sub("source.cl")

                outf = open(source_path, "wt")
                outf.write(src)
                outf.close()

                outf = open(binary_path, "wb")
                outf.write(binary)
                outf.close()

                from cPickle import dump
                info_file = open(info_path, "wb")
                dump(_SourceInfo(
                    dependencies=get_dependencies(src, include_path),
                    log=logs[i]), info_file)
                info_file.close()

        except:
            cleanup_m.error_clean_up()
            raise
        finally:
            cleanup_m.clean_up()

    # }}}

    return result, already_built
Ejemplo n.º 19
0
def _create_built_program_from_source_cached(ctx, src, options_bytes, devices,
                                             cache_dir, include_path):
    from os.path import join

    if cache_dir is None:
        import appdirs
        cache_dir = join(
            appdirs.user_cache_dir("pyopencl", "pyopencl"),
            "pyopencl-compiler-cache-v2-py%s" %
            (".".join(str(i) for i in sys.version_info), ))

    # {{{ ensure cache directory exists

    try:
        os.makedirs(cache_dir)
    except OSError as e:
        from errno import EEXIST
        if e.errno != EEXIST:
            raise

    # }}}

    if devices is None:
        devices = ctx.devices

    cache_keys = [
        get_cache_key(device, options_bytes, src) for device in devices
    ]

    binaries = []
    to_be_built_indices = []
    logs = []
    for i, (device, cache_key) in enumerate(zip(devices, cache_keys)):
        cache_result = retrieve_from_cache(cache_dir, cache_key)

        if cache_result is None:
            logger.debug("build program: binary cache miss (key: %s)" %
                         cache_key)

            to_be_built_indices.append(i)
            binaries.append(None)
            logs.append(None)
        else:
            logger.debug("build program: binary cache hit (key: %s)" %
                         cache_key)

            binary, log = cache_result
            binaries.append(binary)
            logs.append(log)

    message = (75 * "=" + "\n").join(
        "Build on %s succeeded, but said:\n\n%s" % (dev, log)
        for dev, log in zip(devices, logs) if log is not None and log.strip())

    if message:
        from pyopencl import compiler_output
        compiler_output(
            "Built kernel retrieved from cache. Original from-source "
            "build had warnings:\n" + message)

    # {{{ build on the build-needing devices, in one go

    result = None
    already_built = False
    was_cached = not to_be_built_indices

    if to_be_built_indices:
        # defeat implementation caches:
        from uuid import uuid4
        src = src + "\n\n__constant int pyopencl_defeat_cache_%s = 0;" % (
            uuid4().hex)

        logger.debug(
            "build program: start building program from source on %s" %
            ", ".join(str(devices[i]) for i in to_be_built_indices))

        prg = _cl._Program(ctx, src)
        prg.build(options_bytes, [devices[i] for i in to_be_built_indices])

        logger.debug("build program: from-source build complete")

        prg_devs = prg.get_info(_cl.program_info.DEVICES)
        prg_bins = prg.get_info(_cl.program_info.BINARIES)
        prg_logs = prg._get_build_logs()

        for dest_index in to_be_built_indices:
            dev = devices[dest_index]
            src_index = prg_devs.index(dev)
            binaries[dest_index] = prg_bins[src_index]
            _, logs[dest_index] = prg_logs[src_index]

        if len(to_be_built_indices) == len(devices):
            # Important special case: if code for all devices was built,
            # then we may simply use the program that we just built as the
            # final result.

            result = prg
            already_built = True

    if result is None:
        result = _cl._Program(ctx, devices, binaries)

    # }}}

    # {{{ save binaries to cache

    if to_be_built_indices:
        cleanup_m = CleanupManager()
        try:
            try:
                CacheLockManager(cleanup_m, cache_dir)

                for i in to_be_built_indices:
                    cache_key = cache_keys[i]
                    binary = binaries[i]

                    mod_cache_dir_m = ModuleCacheDirManager(
                        cleanup_m, join(cache_dir, cache_key))
                    info_path = mod_cache_dir_m.sub("info")
                    binary_path = mod_cache_dir_m.sub("binary")
                    source_path = mod_cache_dir_m.sub("source.cl")

                    outf = open(source_path, "wt")
                    outf.write(src)
                    outf.close()

                    outf = open(binary_path, "wb")
                    outf.write(binary)
                    outf.close()

                    from six.moves.cPickle import dump
                    info_file = open(info_path, "wb")
                    dump(
                        _SourceInfo(dependencies=get_dependencies(
                            src, include_path),
                                    log=logs[i]), info_file)
                    info_file.close()

            except Exception:
                cleanup_m.error_clean_up()
                raise
        finally:
            cleanup_m.clean_up()

    # }}}

    return result, already_built, was_cached
Ejemplo n.º 20
0
        compiler_output(
                "Built kernel retrieved from cache. Original from-source "
                "build had warnings:\n"+message)

    # {{{ build on the build-needing devices, in one go

    result = None
    already_built = False

    if to_be_built_indices:
        # defeat implementation caches:
        from uuid import uuid4
        src = src + "\n\n__constant int pyopencl_defeat_cache_%s = 0;" % (
                uuid4().hex)

        prg = _cl._Program(ctx, src)
        prg.build(options, [devices[i] for i in to_be_built_indices])

        prg_devs = prg.get_info(_cl.program_info.DEVICES)
        prg_bins = prg.get_info(_cl.program_info.BINARIES)
        prg_logs = prg._get_build_logs()

        for i, dest_index in enumerate(to_be_built_indices):
            assert prg_devs[i] == devices[dest_index]
            binaries[dest_index] = prg_bins[i]
            _, logs[dest_index] = prg_logs[i]

        if len(to_be_built_indices) == len(devices):
            # Important special case: if code for all devices was built,
            # then we may simply use the program that we just built as the
            # final result.