def create_built_program_from_source_cached(ctx, src, options_bytes, devices=None,
cache_dir=None, include_path=None):
try:
if cache_dir is not False:
prg, already_built = _create_built_program_from_source_cached(
ctx, src, options_bytes, devices, cache_dir,
include_path=include_path)
else:
prg = _cl._Program(ctx, src)
already_built = False
except Exception as e:
raise
from pyopencl import Error
if (isinstance(e, Error)
and e.code == _cl.status_code.BUILD_PROGRAM_FAILURE):
# no need to try again
raise
from warnings import warn
from traceback import format_exc
warn("PyOpenCL compiler caching failed with an exception:\n"
"[begin exception]\n%s[end exception]"
% format_exc())
prg = _cl._Program(ctx, src)
already_built = False
if not already_built:
prg.build(options_bytes, devices)
return prg
def __init__(self, arg1, arg2=None, arg3=None):
if arg2 is None:
# 1-argument form: program
self._prg = arg1
elif arg3 is None:
# 2-argument form: context, source
context, source = arg1, arg2
from pyopencl.tools import is_spirv
if is_spirv(source):
# no caching in SPIR-V case
self._context = context
self._prg = _cl._Program(context, source)
return
import sys
if isinstance(source, six.text_type) and sys.version_info < (3,):
from warnings import warn
warn("Received OpenCL source code in Unicode, "
"should be ASCII string. Attempting conversion.",
stacklevel=2)
source = source.encode()
self._context = context
self._source = source
self._prg = None
else:
context, device, binaries = arg1, arg2, arg3
self._context = context
self._prg = _cl._Program(context, device, binaries)
def create_built_program_from_source_cached(ctx, src, options_bytes, devices=None,
cache_dir=None, include_path=None):
try:
if cache_dir is not False:
prg, already_built = _create_built_program_from_source_cached(
ctx, src, options_bytes, devices, cache_dir,
include_path=include_path)
else:
prg = _cl._Program(ctx, src)
already_built = False
except Exception as e:
raise
from pyopencl import Error
if (isinstance(e, Error)
and e.code == _cl.status_code.BUILD_PROGRAM_FAILURE):
# no need to try again
raise
from warnings import warn
from traceback import format_exc
warn("PyOpenCL compiler caching failed with an exception:\n"
"[begin exception]\n%s[end exception]"
% format_exc())
prg = _cl._Program(ctx, src)
already_built = False
if not already_built:
prg.build(options_bytes, devices)
return prg
def build(self, options=[], devices=None, cache_dir=None):
options_bytes, include_path = self._process_build_options(
self._context, options)
if cache_dir is None:
cache_dir = getattr(self._context, 'cache_dir', None)
import os
if os.environ.get("PYOPENCL_NO_CACHE") and self._prg is None:
self._prg = _cl._Program(self._context, self._source)
if self._prg is not None:
# uncached
self._build_and_catch_errors(
lambda: self._prg.build(options_bytes, devices),
options_bytes=options_bytes)
else:
# cached
from pyopencl.cache import create_built_program_from_source_cached
self._prg = self._build_and_catch_errors(
lambda: create_built_program_from_source_cached(
self._context, self._source, options_bytes, devices,
cache_dir=cache_dir, include_path=include_path),
options_bytes=options_bytes, source=self._source)
del self._context
return self
def _get_prg(self):
if self._prg is not None:
return self._prg
else:
# "no program" can only happen in from-source case.
from warnings import warn
warn("Pre-build attribute access defeats compiler caching.",
stacklevel=3)
self._prg = _cl._Program(self._context, self._source)
del self._context
return self._prg
def build(self, options=[], devices=None, cache_dir=None):
options_bytes, include_path = self._process_build_options(
self._context, options)
if cache_dir is None:
cache_dir = getattr(self._context, 'cache_dir', None)
import os
build_descr = None
if os.environ.get("PYOPENCL_NO_CACHE") and self._prg is None:
build_descr = "uncached source build (cache disabled by user)"
self._prg = _cl._Program(self._context, self._source)
from time import time
start_time = time()
was_cached = False
if self._prg is not None:
# uncached
if build_descr is None:
build_descr = "uncached source build"
self._build_and_catch_errors(
lambda: self._prg.build(options_bytes, devices),
options_bytes=options_bytes)
else:
# cached
from pyopencl.cache import create_built_program_from_source_cached
self._prg, was_cached = self._build_and_catch_errors(
lambda: create_built_program_from_source_cached(
self._context, self._source, options_bytes, devices,
cache_dir=cache_dir, include_path=include_path),
options_bytes=options_bytes, source=self._source)
if was_cached:
build_descr = "cache retrieval"
else:
build_descr = "source build resulting from a binary cache miss"
del self._context
end_time = time()
self._build_duration_info = (build_descr, was_cached, end_time-start_time)
return self
def _create_built_program_from_source_cached(ctx, src, options_bytes, devices,
cache_dir, include_path):
from os.path import join
if cache_dir is None:
import appdirs
cache_dir = join(
appdirs.user_cache_dir("pyopencl", "pyopencl"),
"pyopencl-compiler-cache-v2-py%s" %
(".".join(str(i) for i in sys.version_info), ))
# {{{ ensure cache directory exists
try:
os.makedirs(cache_dir)
except OSError as e:
from errno import EEXIST
if e.errno != EEXIST:
raise
# }}}
if devices is None:
devices = ctx.devices
cache_keys = [
get_cache_key(device, options_bytes, src) for device in devices
]
binaries = []
to_be_built_indices = []
logs = []
for i, (device, cache_key) in enumerate(zip(devices, cache_keys)):
cache_result = retrieve_from_cache(cache_dir, cache_key)
if cache_result is None:
to_be_built_indices.append(i)
binaries.append(None)
logs.append(None)
else:
binary, log = cache_result
binaries.append(binary)
logs.append(log)
message = (75 * "=" + "\n").join(
"Build on %s succeeded, but said:\n\n%s" % (dev, log)
for dev, log in zip(devices, logs) if log is not None and log.strip())
if message:
from pyopencl.cffi_cl import compiler_output
compiler_output(
"Built kernel retrieved from cache. Original from-source "
"build had warnings:\n" + message)
# {{{ build on the build-needing devices, in one go
result = None
already_built = False
if to_be_built_indices:
# defeat implementation caches:
from uuid import uuid4
src = src + "\n\n__constant int pyopencl_defeat_cache_%s = 0;" % (
uuid4().hex)
prg = _cl._Program(ctx, src)
prg.build(options_bytes, [devices[i] for i in to_be_built_indices])
prg_devs = prg.get_info(_cl.program_info.DEVICES)
prg_bins = prg.get_info(_cl.program_info.BINARIES)
prg_logs = prg._get_build_logs()
for dest_index in to_be_built_indices:
dev = devices[dest_index]
src_index = prg_devs.index(dev)
binaries[dest_index] = prg_bins[src_index]
_, logs[dest_index] = prg_logs[src_index]
if len(to_be_built_indices) == len(devices):
# Important special case: if code for all devices was built,
# then we may simply use the program that we just built as the
# final result.
result = prg
already_built = True
if result is None:
result = _cl._Program(ctx, devices, binaries)
# }}}
# {{{ save binaries to cache
if to_be_built_indices:
cleanup_m = CleanupManager()
try:
try:
CacheLockManager(cleanup_m, cache_dir)
for i in to_be_built_indices:
cache_key = cache_keys[i]
binary = binaries[i]
mod_cache_dir_m = ModuleCacheDirManager(
cleanup_m, join(cache_dir, cache_key))
info_path = mod_cache_dir_m.sub("info")
binary_path = mod_cache_dir_m.sub("binary")
source_path = mod_cache_dir_m.sub("source.cl")
outf = open(source_path, "wt")
outf.write(src)
outf.close()
outf = open(binary_path, "wb")
outf.write(binary)
outf.close()
from six.moves.cPickle import dump
info_file = open(info_path, "wb")
dump(
_SourceInfo(dependencies=get_dependencies(
src, include_path),
log=logs[i]), info_file)
info_file.close()
except:
cleanup_m.error_clean_up()
raise
finally:
cleanup_m.clean_up()
# }}}
return result, already_built
def _create_built_program_from_source_cached(ctx, src, options_bytes,
devices, cache_dir, include_path):
from os.path import join
if cache_dir is None:
import appdirs
cache_dir = join(appdirs.user_cache_dir("pyopencl", "pyopencl"),
"pyopencl-compiler-cache-v2-py%s" % (
".".join(str(i) for i in sys.version_info),))
# {{{ ensure cache directory exists
try:
os.makedirs(cache_dir)
except OSError as e:
from errno import EEXIST
if e.errno != EEXIST:
raise
# }}}
if devices is None:
devices = ctx.devices
cache_keys = [get_cache_key(device, options_bytes, src) for device in devices]
binaries = []
to_be_built_indices = []
logs = []
for i, (device, cache_key) in enumerate(zip(devices, cache_keys)):
cache_result = retrieve_from_cache(cache_dir, cache_key)
if cache_result is None:
to_be_built_indices.append(i)
binaries.append(None)
logs.append(None)
else:
binary, log = cache_result
binaries.append(binary)
logs.append(log)
message = (75*"="+"\n").join(
"Build on %s succeeded, but said:\n\n%s" % (dev, log)
for dev, log in zip(devices, logs)
if log is not None and log.strip())
if message:
from pyopencl.cffi_cl import compiler_output
compiler_output(
"Built kernel retrieved from cache. Original from-source "
"build had warnings:\n"+message)
# {{{ build on the build-needing devices, in one go
result = None
already_built = False
if to_be_built_indices:
# defeat implementation caches:
from uuid import uuid4
src = src + "\n\n__constant int pyopencl_defeat_cache_%s = 0;" % (
uuid4().hex)
prg = _cl._Program(ctx, src)
prg.build(options_bytes, [devices[i] for i in to_be_built_indices])
prg_devs = prg.get_info(_cl.program_info.DEVICES)
prg_bins = prg.get_info(_cl.program_info.BINARIES)
prg_logs = prg._get_build_logs()
for dest_index in to_be_built_indices:
dev = devices[dest_index]
src_index = prg_devs.index(dev)
binaries[dest_index] = prg_bins[src_index]
_, logs[dest_index] = prg_logs[src_index]
if len(to_be_built_indices) == len(devices):
# Important special case: if code for all devices was built,
# then we may simply use the program that we just built as the
# final result.
result = prg
already_built = True
if result is None:
result = _cl._Program(ctx, devices, binaries)
# }}}
# {{{ save binaries to cache
if to_be_built_indices:
cleanup_m = CleanupManager()
try:
try:
CacheLockManager(cleanup_m, cache_dir)
for i in to_be_built_indices:
cache_key = cache_keys[i]
binary = binaries[i]
mod_cache_dir_m = ModuleCacheDirManager(cleanup_m,
join(cache_dir, cache_key))
info_path = mod_cache_dir_m.sub("info")
binary_path = mod_cache_dir_m.sub("binary")
source_path = mod_cache_dir_m.sub("source.cl")
outf = open(source_path, "wt")
outf.write(src)
outf.close()
outf = open(binary_path, "wb")
outf.write(binary)
outf.close()
from six.moves.cPickle import dump
info_file = open(info_path, "wb")
dump(_SourceInfo(
dependencies=get_dependencies(src, include_path),
log=logs[i]), info_file)
info_file.close()
except:
cleanup_m.error_clean_up()
raise
finally:
cleanup_m.clean_up()
# }}}
return result, already_built
def __init__(self, arg1, arg2=None, arg3=None):
super(Program, self).__init__(arg1, arg2, arg3)
if self._prg is None:
import pyopencl.cffi_cl as _cl
self._prg = _cl._Program(self._context, self._source)
