Beispiel #1
0
def import_c(fname):
    # import logging
    # logging.getLogger("theano.gof.cmodule").setLevel(logging.DEBUG)

    #compile code
    import theano
    import theano.gof.cmodule as cmodule
    import theano.gof.compilelock as compilelock
    from theano.gof.utils import hash_from_code
    compilelock.get_lock()
    try:
        compile_args = [
            "-std=c++11", "-fPIC", "-O3", "-fno-math-errno",
            "-Wno-unused-label", "-Wno-unused-variable", "-Wno-write-strings"
        ]
        compile_dir = cmodule.dlimport_workdir(theano.config.compiledir)
        with open(fname, "r") as f:
            compile_code = f.read()

        compile_hash = os.path.basename(os.path.splitext(fname)[0])
        cmodule_import = cmodule.GCC_compiler.compile_str(compile_hash,
                                                          compile_code,
                                                          compile_dir,
                                                          preargs=compile_args)
    except Exception as e:
        print("Warning: Failed to compile cmodule:", e)
        cmodule_import = None
    finally:
        compilelock.release_lock()

    return cmodule_import
Beispiel #2
0
                            os.makedirs(tmpdir)

                    compiler = nvcc_compiler.NVCC_compiler()
                    compiler.compile_str('cuda_ndarray',
                                         code,
                                         location=cuda_ndarray_loc,
                                         include_dirs=[cuda_path],
                                         libs=[config.cublas.lib],
                                         preargs=['-O3'] +
                                         compiler.compile_args())
                    from cuda_ndarray.cuda_ndarray import *
            except Exception as e:
                _logger.error("Failed to compile cuda_ndarray.cu: %s", str(e))
                set_cuda_disabled()
    finally:
        release_lock()

del compile_cuda_ndarray

if cuda_available:
    global cuda_initialization_error_message
    # The module should be compiled.
    from cuda_ndarray.cuda_ndarray import *

    # If necessary,
    # create a symlink called libcuda_ndarray.so
    # which nvcc_compiler.NVCC_compiler uses when linking
    # any module except "cuda_ndarray" itself.
    def ok():
        """
        Check if an existing library exists and can be read.
Beispiel #3
0
        except OSError as e:
            assert e.errno == errno.EEXIST
            assert os.path.exists(location), location
    if not os.path.exists(os.path.join(location, '__init__.py')):
        open(os.path.join(location, '__init__.py'), 'w').close()

    try:
        from cutils_ext.cutils_ext import *  # noqa
    except ImportError:
        get_lock()
    # Ensure no-one else is currently modifying the content of the compilation
    # directory. This is important to prevent multiple processes from trying to
    # compile the cutils_ext module simultaneously.
        try:
            try:
                # We must retry to import it as some other process could
                # have been compiling it between the first failed import
                # and when we receive the lock
                from cutils_ext.cutils_ext import *  # noqa
            except ImportError:

                compile_cutils()
                from cutils_ext.cutils_ext import *  # noqa

        finally:
            # Release lock on compilation directory.
            release_lock()
finally:
    if sys.path[0] == config.compiledir:
        del sys.path[0]
Beispiel #4
0
 def release_writelock(self):
     """
     Release the previously obtained writelock
     """
     compilelock.release_lock()
Beispiel #5
0
 def release_writelock(self):
     """
     Release the previously obtained writelock
     """
     compilelock.release_lock()
Beispiel #6
0
def abll_compile():
    # Compile .cu files in abll
    _logger.debug('nvcc_compiler.rpath_defaults: %s',
                  str(nvcc_compiler.rpath_defaults))
    import time
    t1 = time.time()
    if should_recompile():
        _logger.debug('should recompile')

        # Concatenate all .cu files into one big mod.cu
        code = []
        for source_file in srcs:
            code.append(open(os.path.join(src_dir, source_file)).read())
        code = '\n'.join(code)

        get_lock()
        try:
            # Check if the compilation has already been done by another process
            # while we were waiting for the lock
            if should_recompile():
                _logger.debug('recompiling')

                try:
                    compiler = nvcc_compiler.NVCC_compiler()
                    args = compiler.compile_args()

                    # compiler.compile_args() can execute a
                    # compilation This currently will remove empty
                    # directory in the compile dir.  So we must make
                    # destination directory after calling it.
                    if not os.path.exists(loc):
                        os.makedirs(loc)
                    compiler.compile_str(
                        'abll',
                        code,
                        location=loc,
                        include_dirs=[include_dir],
                        lib_dirs=nvcc_compiler.rpath_defaults + [loc],
                        libs=libs,
                        preargs=['-O3'] + args,
                        py_module=False,)
                except Exception as e:
                    _logger.error('Failed to compile %s %s: %s',
                                  os.path.join(loc, 'mod.cu'),
                                  srcs, str(e))
                    return False
            else:
                _logger.debug('already compiled by another process')

        finally:
            release_lock()
    else:
        _logger.debug('not recompiling')

    # If necessary, create a symlink called libabll.so
    if not symlink_ok():
        if sys.platform == 'win32':
            # The Python `os` module does not support symlinks on win32.
            shutil.copyfile(abll_so, libabll_so)
        else:
            try:
                os.symlink(abll_so, libabll_so)
            except OSError as e:
                # This may happen for instance when running multiple
                # concurrent jobs, if two of them try to create the
                # symlink simultaneously.
                # If that happens, we verify that the existing symlink is
                # indeed working.
                if (getattr(e, 'errno', None) != errno.EEXIST
                        or not symlink_ok()):
                    raise

    # Raise an error if libabll_so is still not available
    open(libabll_so).close()

    # Add abll to the list of places that are hard-coded into
    # compiled modules' runtime library search list.
    nvcc_compiler.add_standard_rpath(loc)

    t2 = time.time()
    _logger.debug('successfully imported. Compiled in %fs', t2 - t1)

    return True
Beispiel #7
0
def convnet_compile():
    # Compile .cu files in cuda_convnet
    _logger.debug('nvcc_compiler.rpath_defaults: %s',
                  str(nvcc_compiler.rpath_defaults))
    import time
    t1 = time.time()
    if should_recompile():
        _logger.debug('should recompile')

        # Concatenate all .cu files into one big mod.cu
        code = []
        for source_file in cuda_convnet_file_sources:
            code.append(open(os.path.join(this_dir, source_file)).read())
        code = '\n'.join(code)

        get_lock()
        try:
            # Check if the compilation has already been done by another process
            # while we were waiting for the lock
            if should_recompile():
                _logger.debug('recompiling')

                try:
                    compiler = nvcc_compiler.NVCC_compiler()
                    args = compiler.compile_args()

                    # compiler.compile_args() can execute a
                    # compilation This currently will remove empty
                    # directory in the compile dir.  So we must make
                    # destination directory after calling it.
                    if not os.path.exists(cuda_convnet_loc):
                        os.makedirs(cuda_convnet_loc)
                    compiler.compile_str(
                        'cuda_convnet',
                        code,
                        location=cuda_convnet_loc,
                        include_dirs=[this_dir, config.pthreads.inc_dir]
                        if config.pthreads.inc_dir else [this_dir],
                        lib_dirs=nvcc_compiler.rpath_defaults +
                        [cuda_convnet_loc] + ([config.pthreads.lib_dir] if
                                              config.pthreads.lib_dir else []),
                        libs=['cublas', config.pthreads.lib]
                        if config.pthreads.lib else ['cublas'],
                        preargs=['-O3'] + args,
                        py_module=False)
                except Exception as e:
                    _logger.error("Failed to compile %s %s: %s",
                                  os.path.join(cuda_convnet_loc, 'mod.cu'),
                                  cuda_convnet_file_sources, str(e))
                    return False
            else:
                _logger.debug('already compiled by another process')

        finally:
            release_lock()
    else:
        _logger.debug('not recompiling')

    # If necessary, create a symlink called libcuda_convnet.so
    if not symlink_ok():
        if sys.platform == "win32":
            # The Python `os` module does not support symlinks on win32.
            shutil.copyfile(cuda_convnet_so, libcuda_convnet_so)
        else:
            try:
                os.symlink(cuda_convnet_so, libcuda_convnet_so)
            except OSError as e:
                # This may happen for instance when running multiple
                # concurrent jobs, if two of them try to create the
                # symlink simultaneously.
                # If that happens, we verify that the existing symlink is
                # indeed working.
                if (getattr(e, 'errno', None) != errno.EEXIST
                        or not symlink_ok()):
                    raise

    # Raise an error if libcuda_convnet_so is still not available
    open(libcuda_convnet_so).close()

    # Add cuda_convnet to the list of places that are hard-coded into
    # compiled modules' runtime library search list.
    nvcc_compiler.add_standard_rpath(cuda_convnet_loc)

    t2 = time.time()
    _logger.debug('successfully imported. Compiled in %fs', t2 - t1)

    return True
def convnet_compile():
    # Compile .cu files in cuda_convnet
    _logger.debug("nvcc_compiler.rpath_defaults: %s", str(nvcc_compiler.rpath_defaults))
    import time

    t1 = time.time()
    if should_recompile():
        _logger.debug("should recompile")

        # Concatenate all .cu files into one big mod.cu
        code = []
        for source_file in cuda_convnet_file_sources:
            code.append(open(os.path.join(this_dir, source_file)).read())
        code = "\n".join(code)

        get_lock()
        try:
            # Check if the compilation has already been done by another process
            # while we were waiting for the lock
            if should_recompile():
                _logger.debug("recompiling")

                try:
                    compiler = nvcc_compiler.NVCC_compiler()
                    args = compiler.compile_args()

                    # compiler.compile_args() can execute a
                    # compilation This currently will remove empty
                    # directory in the compile dir.  So we must make
                    # destination directory after calling it.
                    if not os.path.exists(cuda_convnet_loc):
                        os.makedirs(cuda_convnet_loc)
                    compiler.compile_str(
                        "cuda_convnet",
                        code,
                        location=cuda_convnet_loc,
                        include_dirs=[this_dir, config.pthreads.inc_dir] if config.pthreads.inc_dir else [this_dir],
                        lib_dirs=nvcc_compiler.rpath_defaults
                        + [cuda_convnet_loc]
                        + ([config.pthreads.lib_dir] if config.pthreads.lib_dir else []),
                        libs=["cublas", config.pthreads.lib] if config.pthreads.lib else ["cublas"],
                        preargs=["-O3"] + args,
                        py_module=False,
                    )
                except Exception as e:
                    _logger.error(
                        "Failed to compile %s %s: %s",
                        os.path.join(cuda_convnet_loc, "mod.cu"),
                        cuda_convnet_file_sources,
                        str(e),
                    )
                    return False
            else:
                _logger.debug("already compiled by another process")

        finally:
            release_lock()
    else:
        _logger.debug("not recompiling")

    # If necessary, create a symlink called libcuda_convnet.so
    if not symlink_ok():
        if sys.platform == "win32":
            # The Python `os` module does not support symlinks on win32.
            shutil.copyfile(cuda_convnet_so, libcuda_convnet_so)
        else:
            try:
                os.symlink(cuda_convnet_so, libcuda_convnet_so)
            except OSError as e:
                # This may happen for instance when running multiple
                # concurrent jobs, if two of them try to create the
                # symlink simultaneously.
                # If that happens, we verify that the existing symlink is
                # indeed working.
                if getattr(e, "errno", None) != errno.EEXIST or not symlink_ok():
                    raise

    # Raise an error if libcuda_convnet_so is still not available
    open(libcuda_convnet_so).close()

    # Add cuda_convnet to the list of places that are hard-coded into
    # compiled modules' runtime library search list.
    nvcc_compiler.add_standard_rpath(cuda_convnet_loc)

    t2 = time.time()
    _logger.debug("successfully imported. Compiled in %fs", t2 - t1)

    return True