def create_built_program_from_source_cached(ctx, src, options_bytes, devices=None, cache_dir=None, include_path=None): try: if cache_dir is not False: prg, already_built, was_cached = \ _create_built_program_from_source_cached( ctx, src, options_bytes, devices, cache_dir, include_path=include_path) else: prg = _cl._Program(ctx, src) was_cached = False already_built = False except Exception as e: from pyopencl import Error if (isinstance(e, Error) and e.code == _cl.status_code.BUILD_PROGRAM_FAILURE): # noqa pylint:disable=no-member # no need to try again raise from warnings import warn from traceback import format_exc warn("PyOpenCL compiler caching failed with an exception:\n" "[begin exception]\n%s[end exception]" % format_exc()) prg = _cl._Program(ctx, src) was_cached = False already_built = False if not already_built: prg.build(options_bytes, devices) return prg, was_cached
def create_built_program_from_source_cached(ctx, src, options=[], devices=None, cache_dir=None): try: if cache_dir != False: prg, already_built = _create_built_program_from_source_cached( ctx, src, options, devices, cache_dir) else: prg = _cl._Program(ctx, src) already_built = False except Exception, e: raise from pyopencl import Error if (isinstance(e, Error) and e.code == _cl.status_code.BUILD_PROGRAM_FAILURE): # no need to try again raise from warnings import warn from traceback import format_exc warn("PyOpenCL compiler caching failed with an exception:\n" "[begin exception]\n%s[end exception]" % format_exc()) prg = _cl._Program(ctx, src) already_built = False
def create_built_program_from_source_cached(ctx, src, options=[], devices=None, cache_dir=None): try: if cache_dir is not False: prg, already_built = _create_built_program_from_source_cached( ctx, src, options, devices, cache_dir) else: prg = _cl._Program(ctx, src) already_built = False except Exception, e: raise from pyopencl import Error if (isinstance(e, Error) and e.code == _cl.status_code.BUILD_PROGRAM_FAILURE): # no need to try again raise from warnings import warn from traceback import format_exc warn("PyOpenCL compiler caching failed with an exception:\n" "[begin exception]\n%s[end exception]" % format_exc()) prg = _cl._Program(ctx, src) already_built = False
def build(self, options=[], devices=None, cache_dir=None): if isinstance(options, str): options = [options] options = options + ["-I", _find_pyopencl_include_path()] import os forced_options = os.environ.get("PYOPENCL_BUILD_OPTIONS") if forced_options: options = options + forced_options.split() if os.environ.get("PYOPENCL_NO_CACHE") and self._prg is None: self._prg = _cl._Program(self._context, self._source) if self._prg is not None: # uncached self._build_and_catch_errors( lambda: self._prg.build(" ".join(options), devices), options=options) else: # cached from pyopencl.cache import create_built_program_from_source_cached self._prg = self._build_and_catch_errors( lambda: create_built_program_from_source_cached( self._context, self._source, options, devices, cache_dir=cache_dir), options=options, source=self._source) del self._context return self
def __init__(self, arg1, arg2=None, arg3=None): if arg2 is None: # 1-argument form: program self._prg = arg1 elif arg3 is None: # 2-argument form: context, source context, source = arg1, arg2 import sys if isinstance(source, unicode) and sys.version_info < (3, ): from warnings import warn warn( "Received OpenCL source code in Unicode, " "should be ASCII string. Attempting conversion.", stacklevel=2) source = str(source) self._context = context self._source = source self._prg = None else: context, device, binaries = arg1, arg2, arg3 self._context = context self._prg = _cl._Program(context, device, binaries)
def __init__(self, arg1, arg2=None, arg3=None): if arg2 is None: # 1-argument form: program self._prg = arg1 elif arg3 is None: # 2-argument form: context, source context, source = arg1, arg2 import sys if isinstance(source, unicode) and sys.version_info < (3,): from warnings import warn warn("Received OpenCL source code in Unicode, " "should be ASCII string. Attempting conversion.", stacklevel=2) source = source.encode() self._context = context self._source = source self._prg = None else: context, device, binaries = arg1, arg2, arg3 self._context = context self._prg = _cl._Program(context, device, binaries)
def create_built_program_from_source_cached(ctx, src, options_bytes, devices=None, cache_dir=None, include_path=None): try: was_cached = False already_built = False if cache_dir is not False: prg, already_built, was_cached = \ _create_built_program_from_source_cached( ctx, src, options_bytes, devices, cache_dir, include_path=include_path) if was_cached and not already_built: prg.build(options_bytes, devices) already_built = True else: prg = _cl._Program(ctx, src) except Exception as e: from pyopencl import Error build_program_failure = ( isinstance(e, Error) and e.code == _cl.status_code.BUILD_PROGRAM_FAILURE) # noqa pylint:disable=no-member # Mac error on intel CPU driver: can't build from cached version. # If we get a build_program_failure from the cached version then # build from source instead, otherwise report the failure. if build_program_failure and not was_cached: raise if not build_program_failure: from warnings import warn from traceback import format_exc warn("PyOpenCL compiler caching failed with an exception:\n" "[begin exception]\n%s[end exception]" % format_exc()) prg = _cl._Program(ctx, src) was_cached = False already_built = False if not already_built: prg.build(options_bytes, devices) return prg, was_cached
def build(self, options=[], devices=None, cache_dir=None): if isinstance(options, str): options = [options] options = options + ["-I", _find_pyopencl_include_path()] import os if os.environ.get("PYOPENCL_NO_CACHE") and self._prg is None: self._prg = _cl._Program(self._context, self._source) if self._prg is not None: if isinstance(options, list): options = " ".join(options) self._prg._build(options, devices) else: from pyopencl.cache import create_built_program_from_source_cached err = None try: self._prg = create_built_program_from_source_cached( self._context, self._source, options, devices, cache_dir=cache_dir) except _cl.RuntimeError, e: from pytools import Record class ErrorRecord(Record): pass from tempfile import NamedTemporaryFile srcfile = NamedTemporaryFile(mode="wt", delete=False, suffix=".cl") try: srcfile.write(self._source) finally: srcfile.close() what = e.what + "\n(source saved as %s)" % srcfile.name code = e.code routine = e.routine err = _cl.RuntimeError( ErrorRecord(what=lambda: what, code=lambda: code, routine=lambda: routine)) if err is not None: # Python 3.2 outputs the whole list of currently active exceptions # This serves to remove one (redundant) level from that nesting. raise err del self._context del self._source
def _get_prg(self): if self._prg is not None: return self._prg else: # "no program" can only happen in from-source case. from warnings import warn warn("Pre-build attribute access defeats compiler caching.", stacklevel=3) self._prg = _cl._Program(self._context, self._source) del self._context del self._source return self._prg
def _get_prg(self): if self._prg is not None: return self._prg else: # "no program" can only happen in from-source case. from warnings import warn warn("Pre-build attribute access defeats compiler caching.", stacklevel=3) self._prg = _cl._Program(self._context, self._source) del self._context return self._prg
def build(self, options=[], devices=None, cache_dir=None): if isinstance(options, str): options = [options] options = options + ["-I", _find_pyopencl_include_path()] import os if os.environ.get("PYOPENCL_NO_CACHE") and self._prg is None: self._prg = _cl._Program(self._context, self._source) if self._prg is not None: if isinstance(options, list): options = " ".join(options) self._prg._build(options, devices) else: from pyopencl.cache import create_built_program_from_source_cached err = None try: self._prg = create_built_program_from_source_cached( self._context, self._source, options, devices, cache_dir=cache_dir) except _cl.RuntimeError, e: from pytools import Record class ErrorRecord(Record): pass from tempfile import NamedTemporaryFile srcfile = NamedTemporaryFile(mode="wt", delete=False, suffix=".cl") try: srcfile.write(self._source) finally: srcfile.close() what = e.what + "\n(source saved as %s)" % srcfile.name code = e.code routine = e.routine err = _cl.RuntimeError( ErrorRecord( what=lambda : what, code=lambda : code, routine=lambda : routine)) if err is not None: # Python 3.2 outputs the whole list of currently active exceptions # This serves to remove one (redundant) level from that nesting. raise err del self._context del self._source
def build(self, options=[], devices=None, cache_dir=None): if isinstance(options, str): options = [options] options = options + ["-I", _find_pyopencl_include_path()] import os forced_options = os.environ.get("PYOPENCL_BUILD_OPTIONS") if forced_options: options = options + forced_options.split() do_del_source = False if os.environ.get("PYOPENCL_NO_CACHE") and self._prg is None: self._prg = _cl._Program(self._context, self._source) do_del_source = True if self._prg is not None: # uncached self._build_and_catch_errors( lambda: self._prg.build(" ".join(options), devices), options=options) else: # cached from pyopencl.cache import create_built_program_from_source_cached self._prg = self._build_and_catch_errors( lambda: create_built_program_from_source_cached(self._context, self._source, options, devices, cache_dir= cache_dir), options=options) del self._context do_del_source = True if do_del_source: del self._source return self
def build(self, options=[], devices=None, cache_dir=None): if isinstance(options, str): options = [options] elif isinstance(options, unicode): options = [options.encode("utf8")] options = ( options + _DEFAULT_BUILD_OPTIONS + _DEFAULT_INCLUDE_OPTIONS + _PLAT_BUILD_OPTIONS.get(self._context.devices[0].platform.name, []) ) import os forced_options = os.environ.get("PYOPENCL_BUILD_OPTIONS") if forced_options: options = options + forced_options.split() if os.environ.get("PYOPENCL_NO_CACHE") and self._prg is None: self._prg = _cl._Program(self._context, self._source) if self._prg is not None: # uncached self._build_and_catch_errors(lambda: self._prg.build(" ".join(options), devices), options=options) else: # cached from pyopencl.cache import create_built_program_from_source_cached self._prg = self._build_and_catch_errors( lambda: create_built_program_from_source_cached( self._context, self._source, options, devices, cache_dir=cache_dir ), options=options, source=self._source, ) del self._context return self
def build(self, options=[], devices=None, cache_dir=None): if isinstance(options, str): options = [options] elif isinstance(options, unicode): options = [options.encode("utf8")] options = (options + _DEFAULT_BUILD_OPTIONS + _DEFAULT_INCLUDE_OPTIONS + _PLAT_BUILD_OPTIONS.get( self._context.devices[0].platform.name, [])) import os forced_options = os.environ.get("PYOPENCL_BUILD_OPTIONS") if forced_options: options = options + forced_options.split() if os.environ.get("PYOPENCL_NO_CACHE") and self._prg is None: self._prg = _cl._Program(self._context, self._source) if self._prg is not None: # uncached self._build_and_catch_errors( lambda: self._prg.build(" ".join(options), devices), options=options) else: # cached from pyopencl.cache import create_built_program_from_source_cached self._prg = self._build_and_catch_errors( lambda: create_built_program_from_source_cached(self._context, self._source, options, devices, cache_dir= cache_dir), options=options, source=self._source) del self._context return self
compiler_output( "Built kernel retrieved from cache. Original from-source " "build had warnings:\n" + message) # {{{ build on the build-needing devices, in one go result = None already_built = False if to_be_built_indices: # defeat implementation caches: from uuid import uuid4 src = src + "\n\n__constant int pyopencl_defeat_cache_%s = 0;" % ( uuid4().hex) prg = _cl._Program(ctx, src) prg.build(options, [devices[i] for i in to_be_built_indices]) prg_devs = prg.get_info(_cl.program_info.DEVICES) prg_bins = prg.get_info(_cl.program_info.BINARIES) prg_logs = prg._get_build_logs() for dest_index in to_be_built_indices: dev = devices[dest_index] src_index = prg_devs.index(dev) binaries[dest_index] = prg_bins[src_index] _, logs[dest_index] = prg_logs[src_index] if len(to_be_built_indices) == len(devices): # Important special case: if code for all devices was built, # then we may simply use the program that we just built as the
def _create_built_program_from_source_cached(ctx, src, options_bytes, devices, cache_dir, include_path): from os.path import join if cache_dir is None: import appdirs cache_dir = join(appdirs.user_cache_dir("pyopencl", "pyopencl"), "pyopencl-compiler-cache-v2-py%s" % ( ".".join(str(i) for i in sys.version_info),)) # {{{ ensure cache directory exists try: os.makedirs(cache_dir) except OSError as e: from errno import EEXIST if e.errno != EEXIST: raise # }}} if devices is None: devices = ctx.devices cache_keys = [get_cache_key(device, options_bytes, src) for device in devices] binaries = [] to_be_built_indices = [] logs = [] for i, (device, cache_key) in enumerate(zip(devices, cache_keys)): cache_result = retrieve_from_cache(cache_dir, cache_key) if cache_result is None: logger.debug("build program: binary cache miss (key: %s)" % cache_key) to_be_built_indices.append(i) binaries.append(None) logs.append(None) else: logger.debug("build program: binary cache hit (key: %s)" % cache_key) binary, log = cache_result binaries.append(binary) logs.append(log) message = (75*"="+"\n").join( "Build on %s succeeded, but said:\n\n%s" % (dev, log) for dev, log in zip(devices, logs) if log is not None and log.strip()) if message: from pyopencl import compiler_output compiler_output( "Built kernel retrieved from cache. Original from-source " "build had warnings:\n"+message) # {{{ build on the build-needing devices, in one go result = None already_built = False was_cached = not to_be_built_indices if to_be_built_indices: # defeat implementation caches: from uuid import uuid4 src = src + "\n\n__constant int pyopencl_defeat_cache_%s = 0;" % ( uuid4().hex) logger.debug("build program: start building program from source on %s" % ", ".join(str(devices[i]) for i in to_be_built_indices)) prg = _cl._Program(ctx, src) prg.build(options_bytes, [devices[i] for i in to_be_built_indices]) logger.debug("build program: from-source build complete") prg_devs = prg.get_info(_cl.program_info.DEVICES) prg_bins = prg.get_info(_cl.program_info.BINARIES) prg_logs = prg._get_build_logs() for dest_index in to_be_built_indices: dev = devices[dest_index] src_index = prg_devs.index(dev) binaries[dest_index] = prg_bins[src_index] _, logs[dest_index] = prg_logs[src_index] if len(to_be_built_indices) == len(devices): # Important special case: if code for all devices was built, # then we may simply use the program that we just built as the # final result. result = prg already_built = True if result is None: result = _cl._Program(ctx, devices, binaries) # }}} # {{{ save binaries to cache if to_be_built_indices: cleanup_m = CleanupManager() try: try: CacheLockManager(cleanup_m, cache_dir) for i in to_be_built_indices: cache_key = cache_keys[i] binary = binaries[i] mod_cache_dir_m = ModuleCacheDirManager(cleanup_m, join(cache_dir, cache_key)) info_path = mod_cache_dir_m.sub("info") binary_path = mod_cache_dir_m.sub("binary") source_path = mod_cache_dir_m.sub("source.cl") outf = open(source_path, "wt") outf.write(src) outf.close() outf = open(binary_path, "wb") outf.write(binary) outf.close() from six.moves.cPickle import dump info_file = open(info_path, "wb") dump(_SourceInfo( dependencies=get_dependencies(src, include_path), log=logs[i]), info_file) info_file.close() except Exception: cleanup_m.error_clean_up() raise finally: cleanup_m.clean_up() # }}} return result, already_built, was_cached
def _create_built_program_from_source_cached(ctx, src, options, devices, cache_dir): include_path = ["."] + [ option[2:] for option in options if option.startswith("-I") or option.startswith("/I")] if cache_dir is None: from os.path import join from tempfile import gettempdir import getpass cache_dir = join(gettempdir(), "pyopencl-compiler-cache-v2-uid%s-py%s" % ( getpass.getuser(), ".".join(str(i) for i in sys.version_info))) # {{{ ensure cache directory exists try: os.mkdir(cache_dir) except OSError as e: from errno import EEXIST if e.errno != EEXIST: raise # }}} if devices is None: devices = ctx.devices cache_keys = [get_cache_key(device, options, src) for device in devices] binaries = [] to_be_built_indices = [] logs = [] for i, (device, cache_key) in enumerate(zip(devices, cache_keys)): cache_result = retrieve_from_cache(cache_dir, cache_key) if cache_result is None: to_be_built_indices.append(i) binaries.append(None) logs.append(None) else: binary, log = cache_result binaries.append(binary) logs.append(log) message = (75*"="+"\n").join( "Build on %s succeeded, but said:\n\n%s" % (dev, log) for dev, log in zip(devices, logs) if log is not None and log.strip()) if message: from warnings import warn warn("Build succeeded, but resulted in non-empty logs:\n"+message) # {{{ build on the build-needing devices, in one go result = None already_built = False if to_be_built_indices: prg = _cl._Program(ctx, src) prg.build(options, [devices[i] for i in to_be_built_indices]) prg_devs = prg.get_info(_cl.program_info.DEVICES) prg_bins = prg.get_info(_cl.program_info.BINARIES) prg_logs = prg._get_build_logs() for i, dest_index in enumerate(to_be_built_indices): assert prg_devs[i] == devices[dest_index] binaries[dest_index] = prg_bins[i] _, logs[dest_index] = prg_logs[i] if len(to_be_built_indices) == len(devices): # Important special case: if code for all devices was built, # then we may simply use the program that we just built as the # final result. result = prg already_built = True if result is None: result = _cl._Program(ctx, devices, binaries) # }}} # {{{ save binaries to cache if to_be_built_indices: cleanup_m = CleanupManager() try: lock_m = CacheLockManager(cleanup_m, cache_dir) for i in to_be_built_indices: cache_key = cache_keys[i] device = devices[i] binary = binaries[i] mod_cache_dir_m = ModuleCacheDirManager(cleanup_m, join(cache_dir, cache_key)) info_path = mod_cache_dir_m.sub("info") binary_path = mod_cache_dir_m.sub("binary") source_path = mod_cache_dir_m.sub("source.cl") outf = open(source_path, "wt") outf.write(src) outf.close() outf = open(binary_path, "wb") outf.write(binary) outf.close() from cPickle import dump info_file = open(info_path, "wb") dump(_SourceInfo( dependencies=get_dependencies(src, include_path), log=logs[i]), info_file) info_file.close() except: cleanup_m.error_clean_up() raise finally: cleanup_m.clean_up() # }}} return result, already_built
def _create_built_program_from_source_cached(ctx, src, options_bytes, devices, cache_dir, include_path): from os.path import join if cache_dir is None: import appdirs cache_dir = join( appdirs.user_cache_dir("pyopencl", "pyopencl"), "pyopencl-compiler-cache-v2-py%s" % (".".join(str(i) for i in sys.version_info), )) # {{{ ensure cache directory exists try: os.makedirs(cache_dir) except OSError as e: from errno import EEXIST if e.errno != EEXIST: raise # }}} if devices is None: devices = ctx.devices cache_keys = [ get_cache_key(device, options_bytes, src) for device in devices ] binaries = [] to_be_built_indices = [] logs = [] for i, (device, cache_key) in enumerate(zip(devices, cache_keys)): cache_result = retrieve_from_cache(cache_dir, cache_key) if cache_result is None: logger.debug("build program: binary cache miss (key: %s)" % cache_key) to_be_built_indices.append(i) binaries.append(None) logs.append(None) else: logger.debug("build program: binary cache hit (key: %s)" % cache_key) binary, log = cache_result binaries.append(binary) logs.append(log) message = (75 * "=" + "\n").join( "Build on %s succeeded, but said:\n\n%s" % (dev, log) for dev, log in zip(devices, logs) if log is not None and log.strip()) if message: from pyopencl import compiler_output compiler_output( "Built kernel retrieved from cache. Original from-source " "build had warnings:\n" + message) # {{{ build on the build-needing devices, in one go result = None already_built = False was_cached = not to_be_built_indices if to_be_built_indices: # defeat implementation caches: from uuid import uuid4 src = src + "\n\n__constant int pyopencl_defeat_cache_%s = 0;" % ( uuid4().hex) logger.debug( "build program: start building program from source on %s" % ", ".join(str(devices[i]) for i in to_be_built_indices)) prg = _cl._Program(ctx, src) prg.build(options_bytes, [devices[i] for i in to_be_built_indices]) logger.debug("build program: from-source build complete") prg_devs = prg.get_info(_cl.program_info.DEVICES) prg_bins = prg.get_info(_cl.program_info.BINARIES) prg_logs = prg._get_build_logs() for dest_index in to_be_built_indices: dev = devices[dest_index] src_index = prg_devs.index(dev) binaries[dest_index] = prg_bins[src_index] _, logs[dest_index] = prg_logs[src_index] if len(to_be_built_indices) == len(devices): # Important special case: if code for all devices was built, # then we may simply use the program that we just built as the # final result. result = prg already_built = True if result is None: result = _cl._Program(ctx, devices, binaries) # }}} # {{{ save binaries to cache if to_be_built_indices: cleanup_m = CleanupManager() try: try: CacheLockManager(cleanup_m, cache_dir) for i in to_be_built_indices: cache_key = cache_keys[i] binary = binaries[i] mod_cache_dir_m = ModuleCacheDirManager( cleanup_m, join(cache_dir, cache_key)) info_path = mod_cache_dir_m.sub("info") binary_path = mod_cache_dir_m.sub("binary") source_path = mod_cache_dir_m.sub("source.cl") outf = open(source_path, "wt") outf.write(src) outf.close() outf = open(binary_path, "wb") outf.write(binary) outf.close() from six.moves.cPickle import dump info_file = open(info_path, "wb") dump( _SourceInfo(dependencies=get_dependencies( src, include_path), log=logs[i]), info_file) info_file.close() except Exception: cleanup_m.error_clean_up() raise finally: cleanup_m.clean_up() # }}} return result, already_built, was_cached
compiler_output( "Built kernel retrieved from cache. Original from-source " "build had warnings:\n"+message) # {{{ build on the build-needing devices, in one go result = None already_built = False if to_be_built_indices: # defeat implementation caches: from uuid import uuid4 src = src + "\n\n__constant int pyopencl_defeat_cache_%s = 0;" % ( uuid4().hex) prg = _cl._Program(ctx, src) prg.build(options, [devices[i] for i in to_be_built_indices]) prg_devs = prg.get_info(_cl.program_info.DEVICES) prg_bins = prg.get_info(_cl.program_info.BINARIES) prg_logs = prg._get_build_logs() for i, dest_index in enumerate(to_be_built_indices): assert prg_devs[i] == devices[dest_index] binaries[dest_index] = prg_bins[i] _, logs[dest_index] = prg_logs[i] if len(to_be_built_indices) == len(devices): # Important special case: if code for all devices was built, # then we may simply use the program that we just built as the # final result.