# If necessary, create a symlink called libcuda_convnet.so
if not symlink_ok():
if sys.platform == "win32":
# The Python `os` module does not support symlinks on win32.
shutil.copyfile(cuda_convnet_so, libcuda_convnet_so)
else:
try:
os.symlink(cuda_convnet_so, libcuda_convnet_so)
except OSError, e:
# This may happen for instance when running multiple
# concurrent jobs, if two of them try to create the
# symlink simultaneously.
# If that happens, we verify that the existing symlink is
# indeed working.
if (getattr(e, 'errno', None) != errno.EEXIST
or not symlink_ok()):
raise
# Raise an error if libcuda_convnet_so is still not available
open(libcuda_convnet_so).close()
# Add cuda_convnet to the list of places that are hard-coded into
# compiled modules' runtime library search list.
nvcc_compiler.add_standard_rpath(cuda_convnet_loc)
t2 = time.time()
_logger.debug('successfully imported. Compiled in %fs', t2 - t1)
return True
def abll_compile():
# Compile .cu files in abll
_logger.debug('nvcc_compiler.rpath_defaults: %s',
str(nvcc_compiler.rpath_defaults))
import time
t1 = time.time()
if should_recompile():
_logger.debug('should recompile')
# Concatenate all .cu files into one big mod.cu
code = []
for source_file in srcs:
code.append(open(os.path.join(src_dir, source_file)).read())
code = '\n'.join(code)
get_lock()
try:
# Check if the compilation has already been done by another process
# while we were waiting for the lock
if should_recompile():
_logger.debug('recompiling')
try:
compiler = nvcc_compiler.NVCC_compiler()
args = compiler.compile_args()
# compiler.compile_args() can execute a
# compilation This currently will remove empty
# directory in the compile dir. So we must make
# destination directory after calling it.
if not os.path.exists(loc):
os.makedirs(loc)
compiler.compile_str(
'abll',
code,
location=loc,
include_dirs=[include_dir],
lib_dirs=nvcc_compiler.rpath_defaults + [loc],
libs=libs,
preargs=['-O3'] + args,
py_module=False,
)
except Exception as e:
_logger.error('Failed to compile %s %s: %s',
os.path.join(loc, 'mod.cu'), srcs, str(e))
return False
else:
_logger.debug('already compiled by another process')
finally:
release_lock()
else:
_logger.debug('not recompiling')
# If necessary, create a symlink called libabll.so
if not symlink_ok():
if sys.platform == 'win32':
# The Python `os` module does not support symlinks on win32.
shutil.copyfile(abll_so, libabll_so)
else:
try:
os.symlink(abll_so, libabll_so)
except OSError as e:
# This may happen for instance when running multiple
# concurrent jobs, if two of them try to create the
# symlink simultaneously.
# If that happens, we verify that the existing symlink is
# indeed working.
if (getattr(e, 'errno', None) != errno.EEXIST
or not symlink_ok()):
raise
# Raise an error if libabll_so is still not available
open(libabll_so).close()
# Add abll to the list of places that are hard-coded into
# compiled modules' runtime library search list.
nvcc_compiler.add_standard_rpath(loc)
t2 = time.time()
_logger.debug('successfully imported. Compiled in %fs', t2 - t1)
return True
def abll_compile():
# Compile .cu files in abll
_logger.debug('nvcc_compiler.rpath_defaults: %s',
str(nvcc_compiler.rpath_defaults))
import time
t1 = time.time()
if should_recompile():
_logger.debug('should recompile')
# Concatenate all .cu files into one big mod.cu
code = []
for source_file in srcs:
code.append(open(os.path.join(src_dir, source_file)).read())
code = '\n'.join(code)
get_lock()
try:
# Check if the compilation has already been done by another process
# while we were waiting for the lock
if should_recompile():
_logger.debug('recompiling')
try:
compiler = nvcc_compiler.NVCC_compiler()
args = compiler.compile_args()
# compiler.compile_args() can execute a
# compilation This currently will remove empty
# directory in the compile dir. So we must make
# destination directory after calling it.
if not os.path.exists(loc):
os.makedirs(loc)
compiler.compile_str(
'abll',
code,
location=loc,
include_dirs=[include_dir],
lib_dirs=nvcc_compiler.rpath_defaults + [loc],
libs=libs,
preargs=['-O3'] + args,
py_module=False,)
except Exception as e:
_logger.error('Failed to compile %s %s: %s',
os.path.join(loc, 'mod.cu'),
srcs, str(e))
return False
else:
_logger.debug('already compiled by another process')
finally:
release_lock()
else:
_logger.debug('not recompiling')
# If necessary, create a symlink called libabll.so
if not symlink_ok():
if sys.platform == 'win32':
# The Python `os` module does not support symlinks on win32.
shutil.copyfile(abll_so, libabll_so)
else:
try:
os.symlink(abll_so, libabll_so)
except OSError as e:
# This may happen for instance when running multiple
# concurrent jobs, if two of them try to create the
# symlink simultaneously.
# If that happens, we verify that the existing symlink is
# indeed working.
if (getattr(e, 'errno', None) != errno.EEXIST
or not symlink_ok()):
raise
# Raise an error if libabll_so is still not available
open(libabll_so).close()
# Add abll to the list of places that are hard-coded into
# compiled modules' runtime library search list.
nvcc_compiler.add_standard_rpath(loc)
t2 = time.time()
_logger.debug('successfully imported. Compiled in %fs', t2 - t1)
return True
def convnet_compile():
# Compile .cu files in cuda_convnet
_logger.debug("nvcc_compiler.rpath_defaults: %s", str(nvcc_compiler.rpath_defaults))
import time
t1 = time.time()
if should_recompile():
_logger.debug("should recompile")
# Concatenate all .cu files into one big mod.cu
code = []
for source_file in cuda_convnet_file_sources:
code.append(open(os.path.join(this_dir, source_file)).read())
code = "\n".join(code)
get_lock()
try:
# Check if the compilation has already been done by another process
# while we were waiting for the lock
if should_recompile():
_logger.debug("recompiling")
try:
compiler = nvcc_compiler.NVCC_compiler()
args = compiler.compile_args()
# compiler.compile_args() can execute a
# compilation This currently will remove empty
# directory in the compile dir. So we must make
# destination directory after calling it.
if not os.path.exists(cuda_convnet_loc):
os.makedirs(cuda_convnet_loc)
compiler.compile_str(
"cuda_convnet",
code,
location=cuda_convnet_loc,
include_dirs=[this_dir, config.pthreads.inc_dir] if config.pthreads.inc_dir else [this_dir],
lib_dirs=nvcc_compiler.rpath_defaults
+ [cuda_convnet_loc]
+ ([config.pthreads.lib_dir] if config.pthreads.lib_dir else []),
libs=["cublas", config.pthreads.lib] if config.pthreads.lib else ["cublas"],
preargs=["-O3"] + args,
py_module=False,
)
except Exception as e:
_logger.error(
"Failed to compile %s %s: %s",
os.path.join(cuda_convnet_loc, "mod.cu"),
cuda_convnet_file_sources,
str(e),
)
return False
else:
_logger.debug("already compiled by another process")
finally:
release_lock()
else:
_logger.debug("not recompiling")
# If necessary, create a symlink called libcuda_convnet.so
if not symlink_ok():
if sys.platform == "win32":
# The Python `os` module does not support symlinks on win32.
shutil.copyfile(cuda_convnet_so, libcuda_convnet_so)
else:
try:
os.symlink(cuda_convnet_so, libcuda_convnet_so)
except OSError as e:
# This may happen for instance when running multiple
# concurrent jobs, if two of them try to create the
# symlink simultaneously.
# If that happens, we verify that the existing symlink is
# indeed working.
if getattr(e, "errno", None) != errno.EEXIST or not symlink_ok():
raise
# Raise an error if libcuda_convnet_so is still not available
open(libcuda_convnet_so).close()
# Add cuda_convnet to the list of places that are hard-coded into
# compiled modules' runtime library search list.
nvcc_compiler.add_standard_rpath(cuda_convnet_loc)
t2 = time.time()
_logger.debug("successfully imported. Compiled in %fs", t2 - t1)
return True
