# If necessary, create a symlink called libcuda_convnet.so if not symlink_ok(): if sys.platform == "win32": # The Python `os` module does not support symlinks on win32. shutil.copyfile(cuda_convnet_so, libcuda_convnet_so) else: try: os.symlink(cuda_convnet_so, libcuda_convnet_so) except OSError, e: # This may happen for instance when running multiple # concurrent jobs, if two of them try to create the # symlink simultaneously. # If that happens, we verify that the existing symlink is # indeed working. if (getattr(e, 'errno', None) != errno.EEXIST or not symlink_ok()): raise # Raise an error if libcuda_convnet_so is still not available open(libcuda_convnet_so).close() # Add cuda_convnet to the list of places that are hard-coded into # compiled modules' runtime library search list. nvcc_compiler.add_standard_rpath(cuda_convnet_loc) t2 = time.time() _logger.debug('successfully imported. Compiled in %fs', t2 - t1) return True
def abll_compile(): # Compile .cu files in abll _logger.debug('nvcc_compiler.rpath_defaults: %s', str(nvcc_compiler.rpath_defaults)) import time t1 = time.time() if should_recompile(): _logger.debug('should recompile') # Concatenate all .cu files into one big mod.cu code = [] for source_file in srcs: code.append(open(os.path.join(src_dir, source_file)).read()) code = '\n'.join(code) get_lock() try: # Check if the compilation has already been done by another process # while we were waiting for the lock if should_recompile(): _logger.debug('recompiling') try: compiler = nvcc_compiler.NVCC_compiler() args = compiler.compile_args() # compiler.compile_args() can execute a # compilation This currently will remove empty # directory in the compile dir. So we must make # destination directory after calling it. if not os.path.exists(loc): os.makedirs(loc) compiler.compile_str( 'abll', code, location=loc, include_dirs=[include_dir], lib_dirs=nvcc_compiler.rpath_defaults + [loc], libs=libs, preargs=['-O3'] + args, py_module=False, ) except Exception as e: _logger.error('Failed to compile %s %s: %s', os.path.join(loc, 'mod.cu'), srcs, str(e)) return False else: _logger.debug('already compiled by another process') finally: release_lock() else: _logger.debug('not recompiling') # If necessary, create a symlink called libabll.so if not symlink_ok(): if sys.platform == 'win32': # The Python `os` module does not support symlinks on win32. shutil.copyfile(abll_so, libabll_so) else: try: os.symlink(abll_so, libabll_so) except OSError as e: # This may happen for instance when running multiple # concurrent jobs, if two of them try to create the # symlink simultaneously. # If that happens, we verify that the existing symlink is # indeed working. if (getattr(e, 'errno', None) != errno.EEXIST or not symlink_ok()): raise # Raise an error if libabll_so is still not available open(libabll_so).close() # Add abll to the list of places that are hard-coded into # compiled modules' runtime library search list. nvcc_compiler.add_standard_rpath(loc) t2 = time.time() _logger.debug('successfully imported. Compiled in %fs', t2 - t1) return True
def abll_compile(): # Compile .cu files in abll _logger.debug('nvcc_compiler.rpath_defaults: %s', str(nvcc_compiler.rpath_defaults)) import time t1 = time.time() if should_recompile(): _logger.debug('should recompile') # Concatenate all .cu files into one big mod.cu code = [] for source_file in srcs: code.append(open(os.path.join(src_dir, source_file)).read()) code = '\n'.join(code) get_lock() try: # Check if the compilation has already been done by another process # while we were waiting for the lock if should_recompile(): _logger.debug('recompiling') try: compiler = nvcc_compiler.NVCC_compiler() args = compiler.compile_args() # compiler.compile_args() can execute a # compilation This currently will remove empty # directory in the compile dir. So we must make # destination directory after calling it. if not os.path.exists(loc): os.makedirs(loc) compiler.compile_str( 'abll', code, location=loc, include_dirs=[include_dir], lib_dirs=nvcc_compiler.rpath_defaults + [loc], libs=libs, preargs=['-O3'] + args, py_module=False,) except Exception as e: _logger.error('Failed to compile %s %s: %s', os.path.join(loc, 'mod.cu'), srcs, str(e)) return False else: _logger.debug('already compiled by another process') finally: release_lock() else: _logger.debug('not recompiling') # If necessary, create a symlink called libabll.so if not symlink_ok(): if sys.platform == 'win32': # The Python `os` module does not support symlinks on win32. shutil.copyfile(abll_so, libabll_so) else: try: os.symlink(abll_so, libabll_so) except OSError as e: # This may happen for instance when running multiple # concurrent jobs, if two of them try to create the # symlink simultaneously. # If that happens, we verify that the existing symlink is # indeed working. if (getattr(e, 'errno', None) != errno.EEXIST or not symlink_ok()): raise # Raise an error if libabll_so is still not available open(libabll_so).close() # Add abll to the list of places that are hard-coded into # compiled modules' runtime library search list. nvcc_compiler.add_standard_rpath(loc) t2 = time.time() _logger.debug('successfully imported. Compiled in %fs', t2 - t1) return True
def convnet_compile(): # Compile .cu files in cuda_convnet _logger.debug("nvcc_compiler.rpath_defaults: %s", str(nvcc_compiler.rpath_defaults)) import time t1 = time.time() if should_recompile(): _logger.debug("should recompile") # Concatenate all .cu files into one big mod.cu code = [] for source_file in cuda_convnet_file_sources: code.append(open(os.path.join(this_dir, source_file)).read()) code = "\n".join(code) get_lock() try: # Check if the compilation has already been done by another process # while we were waiting for the lock if should_recompile(): _logger.debug("recompiling") try: compiler = nvcc_compiler.NVCC_compiler() args = compiler.compile_args() # compiler.compile_args() can execute a # compilation This currently will remove empty # directory in the compile dir. So we must make # destination directory after calling it. if not os.path.exists(cuda_convnet_loc): os.makedirs(cuda_convnet_loc) compiler.compile_str( "cuda_convnet", code, location=cuda_convnet_loc, include_dirs=[this_dir, config.pthreads.inc_dir] if config.pthreads.inc_dir else [this_dir], lib_dirs=nvcc_compiler.rpath_defaults + [cuda_convnet_loc] + ([config.pthreads.lib_dir] if config.pthreads.lib_dir else []), libs=["cublas", config.pthreads.lib] if config.pthreads.lib else ["cublas"], preargs=["-O3"] + args, py_module=False, ) except Exception as e: _logger.error( "Failed to compile %s %s: %s", os.path.join(cuda_convnet_loc, "mod.cu"), cuda_convnet_file_sources, str(e), ) return False else: _logger.debug("already compiled by another process") finally: release_lock() else: _logger.debug("not recompiling") # If necessary, create a symlink called libcuda_convnet.so if not symlink_ok(): if sys.platform == "win32": # The Python `os` module does not support symlinks on win32. shutil.copyfile(cuda_convnet_so, libcuda_convnet_so) else: try: os.symlink(cuda_convnet_so, libcuda_convnet_so) except OSError as e: # This may happen for instance when running multiple # concurrent jobs, if two of them try to create the # symlink simultaneously. # If that happens, we verify that the existing symlink is # indeed working. if getattr(e, "errno", None) != errno.EEXIST or not symlink_ok(): raise # Raise an error if libcuda_convnet_so is still not available open(libcuda_convnet_so).close() # Add cuda_convnet to the list of places that are hard-coded into # compiled modules' runtime library search list. nvcc_compiler.add_standard_rpath(cuda_convnet_loc) t2 = time.time() _logger.debug("successfully imported. Compiled in %fs", t2 - t1) return True