def import_c(fname):
# import logging
# logging.getLogger("theano.gof.cmodule").setLevel(logging.DEBUG)
#compile code
import theano
import theano.gof.cmodule as cmodule
import theano.gof.compilelock as compilelock
from theano.gof.utils import hash_from_code
compilelock.get_lock()
try:
compile_args = [
"-std=c++11", "-fPIC", "-O3", "-fno-math-errno",
"-Wno-unused-label", "-Wno-unused-variable", "-Wno-write-strings"
]
compile_dir = cmodule.dlimport_workdir(theano.config.compiledir)
with open(fname, "r") as f:
compile_code = f.read()
compile_hash = os.path.basename(os.path.splitext(fname)[0])
cmodule_import = cmodule.GCC_compiler.compile_str(compile_hash,
compile_code,
compile_dir,
preargs=compile_args)
except Exception as e:
print("Warning: Failed to compile cmodule:", e)
cmodule_import = None
finally:
compilelock.release_lock()
return cmodule_import
os.makedirs(tmpdir)
compiler = nvcc_compiler.NVCC_compiler()
compiler.compile_str('cuda_ndarray',
code,
location=cuda_ndarray_loc,
include_dirs=[cuda_path],
libs=[config.cublas.lib],
preargs=['-O3'] +
compiler.compile_args())
from cuda_ndarray.cuda_ndarray import *
except Exception as e:
_logger.error("Failed to compile cuda_ndarray.cu: %s", str(e))
set_cuda_disabled()
finally:
release_lock()
del compile_cuda_ndarray
if cuda_available:
global cuda_initialization_error_message
# The module should be compiled.
from cuda_ndarray.cuda_ndarray import *
# If necessary,
# create a symlink called libcuda_ndarray.so
# which nvcc_compiler.NVCC_compiler uses when linking
# any module except "cuda_ndarray" itself.
def ok():
"""
Check if an existing library exists and can be read.
except OSError as e:
assert e.errno == errno.EEXIST
assert os.path.exists(location), location
if not os.path.exists(os.path.join(location, '__init__.py')):
open(os.path.join(location, '__init__.py'), 'w').close()
try:
from cutils_ext.cutils_ext import * # noqa
except ImportError:
get_lock()
# Ensure no-one else is currently modifying the content of the compilation
# directory. This is important to prevent multiple processes from trying to
# compile the cutils_ext module simultaneously.
try:
try:
# We must retry to import it as some other process could
# have been compiling it between the first failed import
# and when we receive the lock
from cutils_ext.cutils_ext import * # noqa
except ImportError:
compile_cutils()
from cutils_ext.cutils_ext import * # noqa
finally:
# Release lock on compilation directory.
release_lock()
finally:
if sys.path[0] == config.compiledir:
del sys.path[0]
def release_writelock(self):
"""
Release the previously obtained writelock
"""
compilelock.release_lock()
def release_writelock(self):
"""
Release the previously obtained writelock
"""
compilelock.release_lock()
def abll_compile():
# Compile .cu files in abll
_logger.debug('nvcc_compiler.rpath_defaults: %s',
str(nvcc_compiler.rpath_defaults))
import time
t1 = time.time()
if should_recompile():
_logger.debug('should recompile')
# Concatenate all .cu files into one big mod.cu
code = []
for source_file in srcs:
code.append(open(os.path.join(src_dir, source_file)).read())
code = '\n'.join(code)
get_lock()
try:
# Check if the compilation has already been done by another process
# while we were waiting for the lock
if should_recompile():
_logger.debug('recompiling')
try:
compiler = nvcc_compiler.NVCC_compiler()
args = compiler.compile_args()
# compiler.compile_args() can execute a
# compilation This currently will remove empty
# directory in the compile dir. So we must make
# destination directory after calling it.
if not os.path.exists(loc):
os.makedirs(loc)
compiler.compile_str(
'abll',
code,
location=loc,
include_dirs=[include_dir],
lib_dirs=nvcc_compiler.rpath_defaults + [loc],
libs=libs,
preargs=['-O3'] + args,
py_module=False,)
except Exception as e:
_logger.error('Failed to compile %s %s: %s',
os.path.join(loc, 'mod.cu'),
srcs, str(e))
return False
else:
_logger.debug('already compiled by another process')
finally:
release_lock()
else:
_logger.debug('not recompiling')
# If necessary, create a symlink called libabll.so
if not symlink_ok():
if sys.platform == 'win32':
# The Python `os` module does not support symlinks on win32.
shutil.copyfile(abll_so, libabll_so)
else:
try:
os.symlink(abll_so, libabll_so)
except OSError as e:
# This may happen for instance when running multiple
# concurrent jobs, if two of them try to create the
# symlink simultaneously.
# If that happens, we verify that the existing symlink is
# indeed working.
if (getattr(e, 'errno', None) != errno.EEXIST
or not symlink_ok()):
raise
# Raise an error if libabll_so is still not available
open(libabll_so).close()
# Add abll to the list of places that are hard-coded into
# compiled modules' runtime library search list.
nvcc_compiler.add_standard_rpath(loc)
t2 = time.time()
_logger.debug('successfully imported. Compiled in %fs', t2 - t1)
return True
def convnet_compile():
# Compile .cu files in cuda_convnet
_logger.debug('nvcc_compiler.rpath_defaults: %s',
str(nvcc_compiler.rpath_defaults))
import time
t1 = time.time()
if should_recompile():
_logger.debug('should recompile')
# Concatenate all .cu files into one big mod.cu
code = []
for source_file in cuda_convnet_file_sources:
code.append(open(os.path.join(this_dir, source_file)).read())
code = '\n'.join(code)
get_lock()
try:
# Check if the compilation has already been done by another process
# while we were waiting for the lock
if should_recompile():
_logger.debug('recompiling')
try:
compiler = nvcc_compiler.NVCC_compiler()
args = compiler.compile_args()
# compiler.compile_args() can execute a
# compilation This currently will remove empty
# directory in the compile dir. So we must make
# destination directory after calling it.
if not os.path.exists(cuda_convnet_loc):
os.makedirs(cuda_convnet_loc)
compiler.compile_str(
'cuda_convnet',
code,
location=cuda_convnet_loc,
include_dirs=[this_dir, config.pthreads.inc_dir]
if config.pthreads.inc_dir else [this_dir],
lib_dirs=nvcc_compiler.rpath_defaults +
[cuda_convnet_loc] + ([config.pthreads.lib_dir] if
config.pthreads.lib_dir else []),
libs=['cublas', config.pthreads.lib]
if config.pthreads.lib else ['cublas'],
preargs=['-O3'] + args,
py_module=False)
except Exception as e:
_logger.error("Failed to compile %s %s: %s",
os.path.join(cuda_convnet_loc, 'mod.cu'),
cuda_convnet_file_sources, str(e))
return False
else:
_logger.debug('already compiled by another process')
finally:
release_lock()
else:
_logger.debug('not recompiling')
# If necessary, create a symlink called libcuda_convnet.so
if not symlink_ok():
if sys.platform == "win32":
# The Python `os` module does not support symlinks on win32.
shutil.copyfile(cuda_convnet_so, libcuda_convnet_so)
else:
try:
os.symlink(cuda_convnet_so, libcuda_convnet_so)
except OSError as e:
# This may happen for instance when running multiple
# concurrent jobs, if two of them try to create the
# symlink simultaneously.
# If that happens, we verify that the existing symlink is
# indeed working.
if (getattr(e, 'errno', None) != errno.EEXIST
or not symlink_ok()):
raise
# Raise an error if libcuda_convnet_so is still not available
open(libcuda_convnet_so).close()
# Add cuda_convnet to the list of places that are hard-coded into
# compiled modules' runtime library search list.
nvcc_compiler.add_standard_rpath(cuda_convnet_loc)
t2 = time.time()
_logger.debug('successfully imported. Compiled in %fs', t2 - t1)
return True
def convnet_compile():
# Compile .cu files in cuda_convnet
_logger.debug("nvcc_compiler.rpath_defaults: %s", str(nvcc_compiler.rpath_defaults))
import time
t1 = time.time()
if should_recompile():
_logger.debug("should recompile")
# Concatenate all .cu files into one big mod.cu
code = []
for source_file in cuda_convnet_file_sources:
code.append(open(os.path.join(this_dir, source_file)).read())
code = "\n".join(code)
get_lock()
try:
# Check if the compilation has already been done by another process
# while we were waiting for the lock
if should_recompile():
_logger.debug("recompiling")
try:
compiler = nvcc_compiler.NVCC_compiler()
args = compiler.compile_args()
# compiler.compile_args() can execute a
# compilation This currently will remove empty
# directory in the compile dir. So we must make
# destination directory after calling it.
if not os.path.exists(cuda_convnet_loc):
os.makedirs(cuda_convnet_loc)
compiler.compile_str(
"cuda_convnet",
code,
location=cuda_convnet_loc,
include_dirs=[this_dir, config.pthreads.inc_dir] if config.pthreads.inc_dir else [this_dir],
lib_dirs=nvcc_compiler.rpath_defaults
+ [cuda_convnet_loc]
+ ([config.pthreads.lib_dir] if config.pthreads.lib_dir else []),
libs=["cublas", config.pthreads.lib] if config.pthreads.lib else ["cublas"],
preargs=["-O3"] + args,
py_module=False,
)
except Exception as e:
_logger.error(
"Failed to compile %s %s: %s",
os.path.join(cuda_convnet_loc, "mod.cu"),
cuda_convnet_file_sources,
str(e),
)
return False
else:
_logger.debug("already compiled by another process")
finally:
release_lock()
else:
_logger.debug("not recompiling")
# If necessary, create a symlink called libcuda_convnet.so
if not symlink_ok():
if sys.platform == "win32":
# The Python `os` module does not support symlinks on win32.
shutil.copyfile(cuda_convnet_so, libcuda_convnet_so)
else:
try:
os.symlink(cuda_convnet_so, libcuda_convnet_so)
except OSError as e:
# This may happen for instance when running multiple
# concurrent jobs, if two of them try to create the
# symlink simultaneously.
# If that happens, we verify that the existing symlink is
# indeed working.
if getattr(e, "errno", None) != errno.EEXIST or not symlink_ok():
raise
# Raise an error if libcuda_convnet_so is still not available
open(libcuda_convnet_so).close()
# Add cuda_convnet to the list of places that are hard-coded into
# compiled modules' runtime library search list.
nvcc_compiler.add_standard_rpath(cuda_convnet_loc)
t2 = time.time()
_logger.debug("successfully imported. Compiled in %fs", t2 - t1)
return True