to_be_built_indices.append(i)
binaries.append(None)
logs.append(None)
else:
binary, log = cache_result
binaries.append(binary)
logs.append(log)
message = (75 * "=" + "\n").join(
"Build on %s succeeded, but said:\n\n%s" % (dev, log)
for dev, log in zip(devices, logs) if log is not None and log.strip())
if message:
from pyopencl import compiler_output
compiler_output(
"Built kernel retrieved from cache. Original from-source "
"build had warnings:\n" + message)
# {{{ build on the build-needing devices, in one go
result = None
already_built = False
if to_be_built_indices:
# defeat implementation caches:
from uuid import uuid4
src = src + "\n\n__constant int pyopencl_defeat_cache_%s = 0;" % (
uuid4().hex)
prg = _cl._Program(ctx, src)
prg.build(options, [devices[i] for i in to_be_built_indices])
def _create_built_program_from_source_cached(ctx, src, options_bytes, devices,
cache_dir, include_path):
from os.path import join
if cache_dir is None:
import appdirs
cache_dir = join(
appdirs.user_cache_dir("pyopencl", "pyopencl"),
"pyopencl-compiler-cache-v2-py%s" %
(".".join(str(i) for i in sys.version_info), ))
# {{{ ensure cache directory exists
try:
os.makedirs(cache_dir)
except OSError as e:
from errno import EEXIST
if e.errno != EEXIST:
raise
# }}}
if devices is None:
devices = ctx.devices
cache_keys = [
get_cache_key(device, options_bytes, src) for device in devices
]
binaries = []
to_be_built_indices = []
logs = []
for i, (device, cache_key) in enumerate(zip(devices, cache_keys)):
cache_result = retrieve_from_cache(cache_dir, cache_key)
if cache_result is None:
logger.debug("build program: binary cache miss (key: %s)" %
cache_key)
to_be_built_indices.append(i)
binaries.append(None)
logs.append(None)
else:
logger.debug("build program: binary cache hit (key: %s)" %
cache_key)
binary, log = cache_result
binaries.append(binary)
logs.append(log)
message = (75 * "=" + "\n").join(
"Build on %s succeeded, but said:\n\n%s" % (dev, log)
for dev, log in zip(devices, logs) if log is not None and log.strip())
if message:
from pyopencl import compiler_output
compiler_output(
"Built kernel retrieved from cache. Original from-source "
"build had warnings:\n" + message)
# {{{ build on the build-needing devices, in one go
result = None
already_built = False
was_cached = not to_be_built_indices
if to_be_built_indices:
# defeat implementation caches:
from uuid import uuid4
src = src + "\n\n__constant int pyopencl_defeat_cache_%s = 0;" % (
uuid4().hex)
logger.debug(
"build program: start building program from source on %s" %
", ".join(str(devices[i]) for i in to_be_built_indices))
prg = _cl._Program(ctx, src)
prg.build(options_bytes, [devices[i] for i in to_be_built_indices])
logger.debug("build program: from-source build complete")
prg_devs = prg.get_info(_cl.program_info.DEVICES)
prg_bins = prg.get_info(_cl.program_info.BINARIES)
prg_logs = prg._get_build_logs()
for dest_index in to_be_built_indices:
dev = devices[dest_index]
src_index = prg_devs.index(dev)
binaries[dest_index] = prg_bins[src_index]
_, logs[dest_index] = prg_logs[src_index]
if len(to_be_built_indices) == len(devices):
# Important special case: if code for all devices was built,
# then we may simply use the program that we just built as the
# final result.
result = prg
already_built = True
if result is None:
result = _cl._Program(ctx, devices, binaries)
# }}}
# {{{ save binaries to cache
if to_be_built_indices:
cleanup_m = CleanupManager()
try:
try:
CacheLockManager(cleanup_m, cache_dir)
for i in to_be_built_indices:
cache_key = cache_keys[i]
binary = binaries[i]
mod_cache_dir_m = ModuleCacheDirManager(
cleanup_m, join(cache_dir, cache_key))
info_path = mod_cache_dir_m.sub("info")
binary_path = mod_cache_dir_m.sub("binary")
source_path = mod_cache_dir_m.sub("source.cl")
outf = open(source_path, "wt")
outf.write(src)
outf.close()
outf = open(binary_path, "wb")
outf.write(binary)
outf.close()
from six.moves.cPickle import dump
info_file = open(info_path, "wb")
dump(
_SourceInfo(dependencies=get_dependencies(
src, include_path),
log=logs[i]), info_file)
info_file.close()
except Exception:
cleanup_m.error_clean_up()
raise
finally:
cleanup_m.clean_up()
# }}}
return result, already_built, was_cached
def _create_built_program_from_source_cached(ctx, src, options_bytes,
devices, cache_dir, include_path):
from os.path import join
if cache_dir is None:
import appdirs
cache_dir = join(appdirs.user_cache_dir("pyopencl", "pyopencl"),
"pyopencl-compiler-cache-v2-py%s" % (
".".join(str(i) for i in sys.version_info),))
# {{{ ensure cache directory exists
try:
os.makedirs(cache_dir)
except OSError as e:
from errno import EEXIST
if e.errno != EEXIST:
raise
# }}}
if devices is None:
devices = ctx.devices
cache_keys = [get_cache_key(device, options_bytes, src) for device in devices]
binaries = []
to_be_built_indices = []
logs = []
for i, (device, cache_key) in enumerate(zip(devices, cache_keys)):
cache_result = retrieve_from_cache(cache_dir, cache_key)
if cache_result is None:
logger.debug("build program: binary cache miss (key: %s)" % cache_key)
to_be_built_indices.append(i)
binaries.append(None)
logs.append(None)
else:
logger.debug("build program: binary cache hit (key: %s)" % cache_key)
binary, log = cache_result
binaries.append(binary)
logs.append(log)
message = (75*"="+"\n").join(
"Build on %s succeeded, but said:\n\n%s" % (dev, log)
for dev, log in zip(devices, logs)
if log is not None and log.strip())
if message:
from pyopencl import compiler_output
compiler_output(
"Built kernel retrieved from cache. Original from-source "
"build had warnings:\n"+message)
# {{{ build on the build-needing devices, in one go
result = None
already_built = False
was_cached = not to_be_built_indices
if to_be_built_indices:
# defeat implementation caches:
from uuid import uuid4
src = src + "\n\n__constant int pyopencl_defeat_cache_%s = 0;" % (
uuid4().hex)
logger.debug("build program: start building program from source on %s"
% ", ".join(str(devices[i]) for i in to_be_built_indices))
prg = _cl._Program(ctx, src)
prg.build(options_bytes, [devices[i] for i in to_be_built_indices])
logger.debug("build program: from-source build complete")
prg_devs = prg.get_info(_cl.program_info.DEVICES)
prg_bins = prg.get_info(_cl.program_info.BINARIES)
prg_logs = prg._get_build_logs()
for dest_index in to_be_built_indices:
dev = devices[dest_index]
src_index = prg_devs.index(dev)
binaries[dest_index] = prg_bins[src_index]
_, logs[dest_index] = prg_logs[src_index]
if len(to_be_built_indices) == len(devices):
# Important special case: if code for all devices was built,
# then we may simply use the program that we just built as the
# final result.
result = prg
already_built = True
if result is None:
result = _cl._Program(ctx, devices, binaries)
# }}}
# {{{ save binaries to cache
if to_be_built_indices:
cleanup_m = CleanupManager()
try:
try:
CacheLockManager(cleanup_m, cache_dir)
for i in to_be_built_indices:
cache_key = cache_keys[i]
binary = binaries[i]
mod_cache_dir_m = ModuleCacheDirManager(cleanup_m,
join(cache_dir, cache_key))
info_path = mod_cache_dir_m.sub("info")
binary_path = mod_cache_dir_m.sub("binary")
source_path = mod_cache_dir_m.sub("source.cl")
outf = open(source_path, "wt")
outf.write(src)
outf.close()
outf = open(binary_path, "wb")
outf.write(binary)
outf.close()
from six.moves.cPickle import dump
info_file = open(info_path, "wb")
dump(_SourceInfo(
dependencies=get_dependencies(src, include_path),
log=logs[i]), info_file)
info_file.close()
except Exception:
cleanup_m.error_clean_up()
raise
finally:
cleanup_m.clean_up()
# }}}
return result, already_built, was_cached
binaries.append(None)
logs.append(None)
else:
binary, log = cache_result
binaries.append(binary)
logs.append(log)
message = (75*"="+"\n").join(
"Build on %s succeeded, but said:\n\n%s" % (dev, log)
for dev, log in zip(devices, logs)
if log is not None and log.strip())
if message:
from pyopencl import compiler_output
compiler_output(
"Built kernel retrieved from cache. Original from-source "
"build had warnings:\n"+message)
# {{{ build on the build-needing devices, in one go
result = None
already_built = False
if to_be_built_indices:
# defeat implementation caches:
from uuid import uuid4
src = src + "\n\n__constant int pyopencl_defeat_cache_%s = 0;" % (
uuid4().hex)
prg = _cl._Program(ctx, src)
prg.build(options, [devices[i] for i in to_be_built_indices])
