def build(self, src): # Compute a digest of the current processor, compiler, and source ckey = digest(self.proc, self.version, self.cmd, src) # Attempt to load the library from the cache mod = self._cache_loadlib(ckey) # Otherwise, we need to compile the kernel if not mod: # Create a scratch directory tmpidx = next(self._dir_seq) tmpdir = tempfile.mkdtemp(prefix=f'pyfr-{tmpidx}-') try: # Compile and link the source into a shared library cname, lname = 'tmp.c', platform_libname('tmp') # Write the source code out with open(os.path.join(tmpdir, cname), 'w') as f: f.write(src) # Invoke the compiler call_capture_output(self.cc_cmd(cname, lname), cwd=tmpdir) # Determine the fully qualified library name lpath = os.path.join(tmpdir, lname) # Add it to the cache and load mod = self._cache_set_and_loadlib(ckey, lpath) finally: # Unless we're debugging delete the scratch directory if 'PYFR_DEBUG_OMP_KEEP_LIBS' not in os.environ: rm(tmpdir) return OpenMPCompilerModule(mod)
def _build(self, tmpdir): # File names cn, on, ln = 'tmp.c', 'tmp.o', platform_libname('tmp') # Write the source code out with open(os.path.join(tmpdir, cn), 'w') as f: f.write(self._src) # Compile cmd = [self._cc, '-std=c99', # Enable C99 support '-Ofast', # Optimise, incl. -ffast-math '-march=native', # Use CPU-specific instructions '-fopenmp', # Enable OpenMP support '-fPIC', # Position-independent code for shared lib '-c', '-o', on, cn] call_capture_output(cmd, cwd=tmpdir) # Link cmd = [self._cc, '-shared', # Create a shared library '-fopenmp', # Required for OpenMP '-o', ln, on,] call_capture_output(cmd, cwd=tmpdir) return ln
def _build(self, tmpdir): # File names cn, on, ln = 'tmp.c', 'tmp.o', platform_libname('tmp') # Write the source code out with open(os.path.join(tmpdir, cn), 'w') as f: f.write(self.src) # Compile cmd = [ self._cc, '-std=c99', # Enable C99 support '-Ofast', # Optimise, incl. -ffast-math '-march=native', # Use CPU-specific instructions '-fopenmp', # Enable OpenMP support '-fPIC', # Position-independent code for shared lib '-c', '-o', on, cn ] call_capture_output(cmd, cwd=tmpdir) # Link cmd = [ self._cc, '-shared', # Create a shared library '-fopenmp', # Required for OpenMP '-o', ln, on, ] call_capture_output(cmd, cwd=tmpdir) return ln
def __init__(self, src, cfg): # Find GCC (or a compatible alternative) self.cc = cfg.getpath('backend-openmp', 'cc', 'cc') # User specified compiler flags self.cflags = shlex.split(cfg.get('backend-openmp', 'cflags', '')) # Get the processor string proc = platform.processor() # Get the compiler version string version = call_capture_output([self.cc, '-v']) # Get the base compiler command strig cmd = self.cc_cmd(None, None) # Compute a digest of the current processor, compiler, and source self.digest = digest(proc, version, cmd, src) # Attempt to load the library from the cache self.mod = self._cache_loadlib() # Otherwise, we need to compile the kernel if not self.mod: # Create a scratch directory tmpidx = next(self._dir_seq) tmpdir = tempfile.mkdtemp(prefix='pyfr-{0}-'.format(tmpidx)) try: # Compile and link the source into a shared library cname, lname = 'tmp.c', platform_libname('tmp') # Write the source code out with open(os.path.join(tmpdir, cname), 'w') as f: f.write(src) # Invoke the compiler call_capture_output(self.cc_cmd(cname, lname), cwd=tmpdir) # Determine the fully qualified library name lpath = os.path.join(tmpdir, lname) # Add it to the cache and load self.mod = self._cache_set_and_loadlib(lpath) finally: # Unless we're debugging delete the scratch directory if 'PYFR_DEBUG_OMP_KEEP_LIBS' not in os.environ: rm(tmpdir)
def guess_toolchain(): """Guess and return a :class:`Toolchain` instance. Raise :exc:`ToolchainGuessError` if no toolchain could be found. """ try: #This will fail on non-POSIX systems kwargs = _guess_toolchain_kwargs_from_python_config() except: #If it does, return a distutils toolchain return guess_distutils_toolchain() from pytools.prefork import call_capture_output result, version, stderr = call_capture_output([kwargs["cc"], "--version"]) if result != 0: raise ToolchainGuessError("compiler version query failed: " + stderr) if "Free Software Foundation" in version: if "-Wstrict-prototypes" in kwargs["cflags"]: kwargs["cflags"].remove("-Wstrict-prototypes") if "darwin" in version: # Are we running in 32-bit mode? # The python interpreter may have been compiled as a Fat binary # So we need to check explicitly how we're running # And update the cflags accordingly import sys if sys.maxint == 0x7fffffff: kwargs["cflags"].extend(['-arch', 'i386']) return GCCToolchain(**kwargs) else: raise ToolchainGuessError("unknown compiler")
def preprocess_source(source, options, nvcc): handle, source_path = mkstemp(suffix='.cu') outf = open(source_path, 'w') outf.write(source) outf.close() os.close(handle) cmdline = [nvcc, '--preprocess'] + options + [source_path] if 'win32' in sys.platform: cmdline.extend(['--compiler-options', '-EP']) else: cmdline.extend(['--compiler-options', '-P']) result, stdout, stderr = call_capture_output(cmdline, error_on_nonzero=False) if result != 0: from pycuda.driver import CompileError raise CompileError("nvcc preprocessing of %s failed" % source_path, cmdline, stderr=stderr) # sanity check if len(stdout) < 0.5*len(source): from pycuda.driver import CompileError raise CompileError("nvcc preprocessing of %s failed with ridiculously " "small code output - likely unsupported compiler." % source_path, cmdline, stderr=stderr.decode("utf-8", "replace")) unlink(source_path) return stdout.decode("utf-8", "replace")
def preprocess_source(source, options, nvcc): handle, source_path = mkstemp(suffix='.cu') outf = open(source_path, 'w') outf.write(source) outf.close() os.close(handle) cmdline = [nvcc, '--preprocess'] + options + [source_path] if 'win32' in sys.platform: cmdline.extend(['--compiler-options', '-EP']) else: cmdline.extend(['--compiler-options', '-P']) result, stdout, stderr = call_capture_output(cmdline, error_on_nonzero=False) if result != 0: from pycuda.driver import CompileError raise CompileError("nvcc preprocessing of %s failed" % source_path, cmdline, stderr=stderr) # sanity check if len(stdout) < 0.5 * len(source): from pycuda.driver import CompileError raise CompileError("nvcc preprocessing of %s failed with ridiculously " "small code output - likely unsupported compiler." % source_path, cmdline, stderr=stderr.decode("utf-8", "replace")) unlink(source_path) return stdout.decode("utf-8", "replace")
def build_object(self, ext_file, source_files, debug=False): cc_cmdline = ( self._cmdline(source_files, True) + ["-o", ext_file] ) from pytools.prefork import call_capture_output if debug: print " ".join(cc_cmdline) result, stdout, stderr = call_capture_output(cc_cmdline) print stderr print stdout if "error" in stderr: # work around a bug in nvcc, which doesn't provide a non-zero # return code even if it failed. result = 1 if result != 0: import sys print >> sys.stderr, "FAILED compiler invocation:", \ " ".join(cc_cmdline) raise CompileError, "module compilation failed"
def guess_toolchain(): """Guess and return a :class:`Toolchain` instance. Raise :exc:`ToolchainGuessError` if no toolchain could be found. """ kwargs = _guess_toolchain_kwargs_from_python_config() from pytools.prefork import call_capture_output result, version, stderr = call_capture_output([kwargs["cc"], "--version"]) if result != 0: raise ToolchainGuessError("compiler version query failed: "+stderr) if "Free Software Foundation" in version: if "-Wstrict-prototypes" in kwargs["cflags"]: kwargs["cflags"].remove("-Wstrict-prototypes") if "darwin" in version: # Are we running in 32-bit mode? # The python interpreter may have been compiled as a Fat binary # So we need to check explicitly how we're running # And update the cflags accordingly import sys if sys.maxint == 0x7fffffff: kwargs["cflags"].extend(['-arch', 'i386']) return GCCToolchain(**kwargs) elif "Intel Corporation" in version: return IntelToolchain(**kwargs) else: raise ToolchainGuessError("unknown compiler")
def call_capture_output(*args): from pytools.prefork import call_capture_output import sys encoding = sys.getdefaultencoding() result, stdout, stderr = call_capture_output(*args) return result, stdout.decode(encoding), stderr.decode(encoding)
def guess_toolchain(): """Guess and return a :class:`Toolchain` instance. Raise :exc:`ToolchainGuessError` if no toolchain could be found. """ kwargs = _guess_toolchain_kwargs_from_python_config() result, version, stderr = call_capture_output([kwargs["cc"], "--version"]) if result != 0: raise ToolchainGuessError(f"compiler version query failed: {stderr}") if "Free Software Foundation" in version: if "-Wstrict-prototypes" in kwargs["cflags"]: kwargs["cflags"].remove("-Wstrict-prototypes") if "darwin" in version: # Are we running in 32-bit mode? # The python interpreter may have been compiled as a Fat binary # So we need to check explicitly how we're running # And update the cflags accordingly import sys if sys.maxsize == 0x7fffffff: kwargs["cflags"].extend(["-arch", "i386"]) return GCCToolchain(**kwargs) else: raise ToolchainGuessError( "Unable to determine compiler. Tried running " f"'{kwargs['cc']} --version' and failed.")
def detect(self, compiler): from pytools.prefork import call_capture_output try: retcode, stdout, stderr = call_capture_output([compiler, "--version"]) except: return False return (retcode == 0)
def __enter__(self): self.temp_dir_mgr = None temp_dir_mgr = _TempDirManager() try: working_dir = temp_dir_mgr.path from os.path import join source_file_name = join(working_dir, "temp." + self.extension) source_file = open(source_file_name, "w") try: source_file.write(self.source) finally: source_file.close() output_file_name = join(working_dir, "output.msh") cmdline = [ self.gmsh_executable, "-%d" % self.dimensions, "-o", output_file_name, "-nopopup" ] if self.order is not None: cmdline.extend(["-order", str(self.order)]) if self.incomplete_elements is not None: cmdline.extend([ "-string", "Mesh.SecondOrderIncomplete = %d;" % int(self.incomplete_elements) ]) cmdline.extend(self.other_options) cmdline.append(source_file_name) from pytools.prefork import call_capture_output retcode, stdout, stderr = call_capture_output(cmdline, working_dir) if stderr and "error" in stderr.lower(): msg = "gmsh execution failed with message:\n\n" if stdout: msg += stdout + "\n" msg += stderr + "\n" raise GmshError(msg) if stderr: from warnings import warn msg = "gmsh issued the following messages:\n\n" if stdout: msg += stdout + "\n" msg += stderr + "\n" warn(msg) self.output_file = open(output_file_name, "r") self.temp_dir_mgr = temp_dir_mgr return self.output_file except: temp_dir_mgr.clean_up() raise
def version(self): from distutils.version import LooseVersion cmdline = [self.gmsh_executable, '-version'] from pytools.prefork import call_capture_output retcode, stdout, stderr = call_capture_output(cmdline) version = stderr.decode().strip() return LooseVersion(version)
def detect(self, compiler): from pytools.prefork import call_capture_output try: retcode, stdout, stderr = call_capture_output( [compiler, "--version"]) except: return False return (retcode == 0)
def get_nvcc_version(nvcc): cmdline = [nvcc, "--version"] result, stdout, stderr = call_capture_output(cmdline) if result != 0 or not stdout: from warnings import warn warn("NVCC version could not be determined.") stdout = "nvcc unknown version" return stdout.decode("utf-8", "replace")
def version(self): from distutils.version import LooseVersion cmdline = [self.gmsh_executable, '-version'] from pytools.prefork import call_capture_output retcode, stdout, stderr = call_capture_output(cmdline) # stderr can contain irregular info import re version = re.search(r'[0-9]+.[0-9]+.[0-9]+', stderr.decode().strip()).group() return LooseVersion(version)
def __enter__(self): self.temp_dir_mgr = None temp_dir_mgr = _TempDirManager() try: working_dir = temp_dir_mgr.path from os.path import join source_file_name = join(working_dir, "temp." + self.extension) source_file = open(source_file_name, "w") try: source_file.write(self.source) finally: source_file.close() output_file_name = join(working_dir, "output.msh") cmdline = [self.gmsh_executable, "-%d" % self.dimensions, "-o", output_file_name, "-nopopup"] if self.order is not None: cmdline.extend(["-order", str(self.order)]) if self.incomplete_elements is not None: cmdline.extend(["-string", "Mesh.SecondOrderIncomplete = %d;" % int(self.incomplete_elements)]) cmdline.extend(self.other_options) cmdline.append(source_file_name) from pytools.prefork import call_capture_output retcode, stdout, stderr = call_capture_output(cmdline, working_dir) if stderr and "error" in stderr.lower(): msg = "gmsh execution failed with message:\n\n" if stdout: msg += stdout + "\n" msg += stderr + "\n" raise GmshError(msg) if stderr: from warnings import warn msg = "gmsh issued the following messages:\n\n" if stdout: msg += stdout + "\n" msg += stderr + "\n" warn(msg) self.output_file = open(output_file_name, "r") self.temp_dir_mgr = temp_dir_mgr return self.output_file except: temp_dir_mgr.clean_up() raise
def build_lib(lib, extensions, source_dir, CC, dst_dir, inc_dirs): _build_timer.start() _lib_filename = os.path.join(dst_dir, lib + '.so') _lib_src_filename = os.path.join(source_dir, lib + extensions[1]) _c_cmd = CC.binary + [_lib_src_filename] + ['-o'] + \ [_lib_filename] + CC.c_flags + CC.l_flags + \ ['-I' + str(d) for d in inc_dirs] + \ ['-I' + str(source_dir)] if ppmd.runtime.DEBUG > 0: _c_cmd += CC.dbg_flags if ppmd.runtime.OPT > 0: _c_cmd += CC.opt_flags _c_cmd += CC.shared_lib_flag stdout_filename = os.path.join(dst_dir, lib + '.log') stderr_filename = os.path.join(dst_dir, lib + '.err') with open(stdout_filename, 'w') as stdout_fh: with open(stderr_filename, 'w') as stderr_fh: stdout_fh.write('# Compilation command:\n') stdout_fh.write(' '.join(_c_cmd)) stdout_fh.write('\n\n') stdout_fh.flush() result, stdout, stderr = call_capture_output( _c_cmd, error_on_nonzero=False) stdout = stdout.decode(sys.stdout.encoding) stderr = stderr.decode(sys.stdout.encoding) stderr_fh.write(str(result) + '\n') stderr_fh.write(stdout) stderr_fh.write(stderr) stderr_fh.flush() if result != 0: print("\n---- COMPILER OUTPUT START ----") print(stdout) print(stderr) print("----- COMPILER OUTPUT END -----") raise RuntimeError('PPMD build error: library not built.') # Check library exists in the file system if not os.path.exists(_lib_filename): print("Critical build Error: Library not found,\n" + \ _lib_filename + "\n rank:", _MPIRANK) raise RuntimeError('compiler call did not error, but no binary found') _build_timer.pause() opt.PROFILE['Build:' + CC.binary[0] + ':'] = (_build_timer.time()) return _lib_filename
def get_dependencies(self, source_files): from codepy.tools import join_continued_lines result, stdout, stderr = call_capture_output( [self.cc] + ["-M"] + ["-D%s" % define for define in self.defines] + ["-U%s" % undefine for undefine in self.undefines] + ["-I%s" % idir for idir in self.include_dirs] + self.cflags + source_files) if result != 0: raise CompileError("getting dependencies failed: " + stderr) lines = join_continued_lines(stdout.split("\n")) from pytools import flatten return set(flatten(line.split()[2:] for line in lines))
def get_nvcc_version(nvcc): cmdline = [nvcc, "--version"] try: try: from pytools.prefork import call_capture_output except ImportError: from pytools.prefork import call_capture_stdout return call_capture_stdout(cmdline) else: retcode, stdout, stderr = call_capture_output(cmdline) return stdout except OSError, e: raise OSError("%s was not found (is it on the PATH?) [%s]" % (nvcc, str(e)))
def llvmir_to_obj(source, llc="llc"): """ Returns the compiled object code for the LLVM code *source*. """ with tempfile.NamedTemporaryFile(mode="w", suffix=".ll") as llfp: llfp.write(source) llfp.file.flush() with tempfile.NamedTemporaryFile(suffix=".o", mode="rb") as objfp: cmdline = [llc, llfp.name, "-o", objfp.name, "-filetype=obj"] result, stdout, stderr = call_capture_output(cmdline) obj_code = objfp.read() return obj_code
def __init__(self, src, dev, cfg): # Create a scratch directory tmpidx = next(self._dir_seq) tmpdir = tempfile.mkdtemp(prefix='pyfr-{0}-'.format(tmpidx)) # Find MKL mklroot = cfg.get('backend-mic', 'mkl-root', '$MKLROOT') try: # File names cn, ln = 'tmp.c', 'libtmp.so' # Write the source code out with open(os.path.join(tmpdir, cn), 'w') as f: f.write(src) # Compile and link cmd = [ 'icc', '-shared', # Create a shared library '-std=c99', # Enable C99 support '-Ofast', # Optimise '-mmic', # Compile for the MIC '-fopenmp', # Enable OpenMP support '-L{0}/lib/mic'.format(mklroot), # MKL stuff '-lmkl_intel_lp64', # ... '-lmkl_core', # ... '-lmkl_intel_thread', # ... '-fPIC', # Position-independent code '-o', ln, cn ] call_capture_output(cmd, cwd=tmpdir) # Load self._mod = dev.load_library(os.path.join(tmpdir, ln)) finally: rm(tmpdir)
def __init__(self, cfg): # Find GCC (or a compatible alternative) self.cc = cfg.getpath('backend-openmp', 'cc', 'cc') # User specified compiler flags self.cflags = shlex.split(cfg.get('backend-openmp', 'cflags', '')) # Get the processor string self.proc = platform.processor() # Get the compiler version string self.version = call_capture_output([self.cc, '-v']) # Get the base compiler command strig self.cmd = self.cc_cmd(None, None)
def mlir_translate(source, options, mlir_translate="mlir-translate"): """ Calls ``mlir-translate`` on *source* with *options* as additional arguments. :arg source: The code to be passed to mlir-translate. :arg options: An instance of :class:`list`. :return: Transformed *source* as emitted by ``mlir-translate``. """ with tempfile.NamedTemporaryFile(mode="w", suffix=".mlir", delete=False) as fp: fp.write(source) fp.file.flush() cmdline = [mlir_translate, fp.name] + options result, stdout, stderr = call_capture_output(cmdline) return stdout.decode()
def mlir_opt(source: str, options: List[str], mlir_opt="mlir-opt"): """ Calls ``mlir-opt`` on *source* with *options* as additional arguments. :arg source: The code to be passed to mlir-opt. :arg options: An instance of :class:`list`. :return: Transformed *source* as emitted by ``mlir-opt``. """ assert "-o" not in options with tempfile.NamedTemporaryFile(mode="w", suffix=".mlir") as fp: fp.write(source) fp.file.flush() cmdline = [mlir_opt, fp.name] + options result, stdout, stderr = call_capture_output(cmdline) return stdout.decode()
def guess_toolchain(): """Guess and return a :class:`Toolchain` instance. Raise :exc:`ToolchainGuessError` if no toolchain could be found. """ kwargs = _guess_toolchain_kwargs_from_python_config() try: result, version, stderr = call_capture_output( [kwargs["cc"], "--version"]) except ExecError: raise ToolchainGuessError("System compiler {} not found".format( kwargs['cc'])) if result != 0: raise ToolchainGuessError("compiler version query failed: " + stderr) if "Free Software Foundation" in version: if "-Wstrict-prototypes" in kwargs["cflags"]: kwargs["cflags"].remove("-Wstrict-prototypes") if "darwin" in version: # Are we running in 32-bit mode? # The python interpreter may have been compiled as a Fat binary # So we need to check explicitly how we're running # And update the cflags accordingly import sys if sys.maxsize == 0x7fffffff: kwargs["cflags"].extend(['-arch', 'i386']) return GCCToolchain(**kwargs) elif "Apple LLVM" in version and "clang" in version: if "-Wstrict-prototypes" in kwargs["cflags"]: kwargs["cflags"].remove("-Wstrict-prototypes") if "darwin" in version: # Are we running in 32-bit mode? # The python interpreter may have been compiled as a Fat binary # So we need to check explicitly how we're running # And update the cflags accordingly import sys if sys.maxsize == 0x7fffffff: kwargs["cflags"].extend(['-arch', 'i386']) return GCCToolchain(**kwargs) else: raise ToolchainGuessError("unknown compiler")
def build_object(self, ext_file, source_files, debug=False): cc_cmdline = (self._cmdline(source_files, True) + ["-o", ext_file]) if debug: print(" ".join(cc_cmdline)) result, stdout, stderr = call_capture_output(cc_cmdline) print(stderr) print(stdout) if "error" in stderr: # work around a bug in nvcc, which doesn't provide a non-zero # return code even if it failed. result = 1 if result != 0: import sys print("FAILED compiler invocation:" + " ".join(cc_cmdline), file=sys.stderr) raise CompileError("module compilation failed")
def get_dependencies(self, source_files): from codepy.tools import join_continued_lines from pytools.prefork import call_capture_output result, stdout, stderr = call_capture_output( [self.cc] + ["-M"] + ["-D%s" % define for define in self.defines] + ["-U%s" % undefine for undefine in self.defines] + ["-I%s" % idir for idir in self.include_dirs] + source_files ) if result != 0: raise CompileError("getting dependencies failed: "+stderr) lines = join_continued_lines(stdout.split("\n")) from pytools import flatten return set(flatten( line.split()[2:] for line in lines))
def preprocess_source(source, options, nvcc): handle, source_path = mkstemp(suffix=".cu") outf = open(source_path, "w") outf.write(source) outf.close() os.close(handle) cmdline = [nvcc, "--preprocess"] + options + [source_path] if "win32" in sys.platform: cmdline.extend(["--compiler-options", "-EP"]) else: cmdline.extend(["--compiler-options", "-P"]) result, stdout, stderr = call_capture_output(cmdline, error_on_nonzero=False) if result != 0: from pycuda.driver import CompileError raise CompileError("nvcc preprocessing of %s failed" % source_path, cmdline, stderr=stderr) # sanity check if len(stdout) < 0.5 * len(source): from pycuda.driver import CompileError raise CompileError( "nvcc preprocessing of %s failed with ridiculously " "small code output - likely unsupported compiler." % source_path, cmdline, stderr=stderr.decode("utf-8", "replace"), ) unlink(source_path) preprocessed_str = stdout.decode("utf-8", "replace") # remove the temporary filename from the preprocessed source code to get reproducible hashes return preprocessed_str.replace(os.path.basename(source_path), "")
def get_dependencies(self, source_files): from codepy.tools import join_continued_lines result, stdout, stderr = call_capture_output( [self.cc] + ["-M"] + [f"-D{define}" for define in self.defines] + [f"-U{undefine}" for undefine in self.undefines] + [f"-I{idir}" for idir in self.include_dirs] + self.cflags + source_files ) if result != 0: raise CompileError(f"getting dependencies failed: {stderr}") lines = join_continued_lines(stdout.split("\n")) lines = [line for line in lines if not (line.strip() and line.strip()[0] == "#")] from pytools import flatten return set(flatten( line.split()[2:] for line in lines))
def get_nvcc_version(nvcc): cmdline = [nvcc, "--version"] try: try: from pytools.prefork import call_capture_output except ImportError: from pytools.prefork import call_capture_stdout result = call_capture_stdout(cmdline) else: retcode, stdout, stderr = call_capture_output(cmdline) result = stdout if result is None: from warnings import warn warn("NVCC version could not be determined.") result = "nvcc unknown version" return result except OSError, e: raise OSError("%s was not found (is it on the PATH?) [%s]" % (nvcc, str(e)))
def compile_plain(source, options, keep, nvcc, cache_dir, target="cubin"): from os.path import join assert target in ["cubin", "ptx", "fatbin"] if cache_dir: checksum = _new_md5() if '#include' in source: checksum.update(preprocess_source(source, options, nvcc).encode("utf-8")) else: checksum.update(source.encode("utf-8")) for option in options: checksum.update(option.encode("utf-8")) checksum.update(get_nvcc_version(nvcc).encode("utf-8")) from pycuda.characterize import platform_bits checksum.update(str(platform_bits()).encode("utf-8")) cache_file = checksum.hexdigest() cache_path = join(cache_dir, cache_file + "." + target) try: cache_file = open(cache_path, "rb") try: return cache_file.read() finally: cache_file.close() except: pass from tempfile import mkdtemp file_dir = mkdtemp() file_root = "kernel" cu_file_name = file_root + ".cu" cu_file_path = join(file_dir, cu_file_name) outf = open(cu_file_path, "w") outf.write(str(source)) outf.close() if keep: options = options[:] options.append("--keep") print("*** compiler output in %s" % file_dir) cmdline = [nvcc, "--" + target] + options + [cu_file_name] result, stdout, stderr = call_capture_output(cmdline, cwd=file_dir, error_on_nonzero=False) try: result_f = open(join(file_dir, file_root + "." + target), "rb") except IOError: no_output = True else: no_output = False if result != 0 or (no_output and (stdout or stderr)): if result == 0: from warnings import warn warn("PyCUDA: nvcc exited with status 0, but appears to have " "encountered an error") from pycuda.driver import CompileError raise CompileError("nvcc compilation of %s failed" % cu_file_path, cmdline, stdout=stdout.decode("utf-8", "replace"), stderr=stderr.decode("utf-8", "replace")) if stdout or stderr: lcase_err_text = (stdout+stderr).decode("utf-8", "replace").lower() from warnings import warn if "demoted" in lcase_err_text or "demoting" in lcase_err_text: warn("nvcc said it demoted types in source code it " "compiled--this is likely not what you want.", stacklevel=4) warn("The CUDA compiler succeeded, but said the following:\n" + (stdout+stderr).decode("utf-8", "replace"), stacklevel=4) result_data = result_f.read() result_f.close() if cache_dir: outf = open(cache_path, "wb") outf.write(result_data) outf.close() if not keep: from os import listdir, unlink, rmdir for name in listdir(file_dir): unlink(join(file_dir, name)) rmdir(file_dir) return result_data
def get_version(self): result, stdout, stderr = call_capture_output([self.cc, "--version"]) if result != 0: raise RuntimeError("version query failed: " + stderr) return stdout
def __enter__(self): self.temp_dir_mgr = None temp_dir_mgr = _TempDirManager() try: working_dir = temp_dir_mgr.path from os.path import join, abspath, exists if isinstance(self.source, ScriptSource): source_file_name = join(working_dir, "temp." + self.source.extension) with open(source_file_name, "w") as source_file: source_file.write(self.source.source) elif isinstance(self.source, FileSource): source_file_name = abspath(self.source.filename) if not exists(source_file_name): raise IOError("'%s' does not exist" % source_file_name) elif isinstance(self.source, ScriptWithFilesSource): source_file_name = join(working_dir, self.source.source_name) with open(source_file_name, "w") as source_file: source_file.write(self.source.source) from os.path import basename from shutil import copyfile for f in self.source.filenames: copyfile(f, join(working_dir, basename(f))) else: raise RuntimeError("'source' type unrecognized") output_file_name = join(working_dir, self.output_file_name) cmdline = [ self.gmsh_executable, "-o", self.output_file_name, "-nopopup", "-format", "msh2" ] # NOTE: handle unit incompatibility introduced in GMSH4 # https://gitlab.onelab.info/gmsh/gmsh/issues/397 if self.version < '4.0.0': if self.target_unit == 'M': cmdline.extend(["-string", "Geometry.OCCScaling=1000;"]) else: cmdline.extend([ "-string", "Geometry.OCCTargetUnit='{}';".format(self.target_unit) ]) if self.dimensions is not None: cmdline.append("-%d" % self.dimensions) if self.order is not None: cmdline.extend(["-order", str(self.order)]) if self.incomplete_elements is not None: cmdline.extend([ "-string", "Mesh.SecondOrderIncomplete = %d;" % int(self.incomplete_elements) ]) cmdline.extend(self.other_options) cmdline.append(source_file_name) if self.dimensions is None: cmdline.append("-") logger.info("invoking gmsh: '%s'" % " ".join(cmdline)) from pytools.prefork import call_capture_output retcode, stdout, stderr = call_capture_output(cmdline, working_dir) logger.info("return from gmsh") stdout = stdout.decode("utf-8") stderr = stderr.decode("utf-8") import re error_match = re.match(r"([0-9]+)\s+error", stdout) warning_match = re.match(r"([0-9]+)\s+warning", stdout) if error_match is not None or warning_match is not None: # if we have one, we expect to see both assert error_match is not None or warning_match is not None num_warnings = int(warning_match.group(1)) num_errors = int(error_match.group(1)) else: num_warnings = 0 num_errors = 0 if num_errors: msg = "gmsh execution failed with message:\n\n" if stdout: msg += stdout + "\n" msg += stderr + "\n" raise GmshError(msg) if num_warnings: from warnings import warn msg = "gmsh issued the following warning messages:\n\n" if stdout: msg += stdout + "\n" msg += stderr + "\n" warn(msg) self.output_file = open(output_file_name, "r") if self.save_tmp_files_in: import shutil import errno try: shutil.copytree(working_dir, self.save_tmp_files_in) except FileExistsError: import select import sys print( "%s exists! Overwrite? (Y/N, will default to Y in 10sec)." % self.save_tmp_files_in) decision = None while not decision: i, o, e = select.select([sys.stdin], [], [], 10) if i == "N" or i == "n": decision = 0 elif i == "Y" or i == "y" or not i: decision = 1 else: print("Illegal input %s, please retry." % i) if decision == 0: pass else: assert decision == 1 shutil.rmtree(self.save_tmp_files_in) shutil.copytree(working_dir, self.save_tmp_files_in) except OSError as exc: if exc.errno == errno.ENOTDIR: shutil.copy( output_file_name, '/'.join([ self.save_tmp_files_in, self.output_file_name ])) else: raise self.temp_dir_mgr = temp_dir_mgr return self except Exception: temp_dir_mgr.clean_up() raise
try: from pytools.prefork import call_capture_output except ImportError: from pytools.prefork import call try: result = call(cmdline, cwd=file_dir) except OSError, e: raise OSError("%s was not found (is it on the PATH?) [%s]" % (nvcc, str(e))) stdout = None stderr = None else: result, stdout, stderr = call_capture_output( cmdline, cwd=file_dir, error_on_nonzero=False) try: cubin_f = open(join(file_dir, file_root + ".cubin"), "rb") except IOError: no_output = True else: no_output = False if result != 0 or (no_output and (stdout or stderr)): if result == 0: from warnings import warn warn("PyCUDA: nvcc exited with status 0, but appears to have " "encountered an error") from pycuda.driver import CompileError
def get_version(self): from pytools.prefork import call_capture_output result, stdout, stderr = call_capture_output([self.cc, "--version"]) if result != 0: raise RuntimeError("version query failed: "+stderr) return stdout
def __enter__(self): self.temp_dir_mgr = None temp_dir_mgr = _TempDirManager() try: working_dir = temp_dir_mgr.path from os.path import join, abspath, exists if isinstance(self.source, ScriptSource): source_file_name = join( working_dir, "temp."+self.source.extension) with open(source_file_name, "w") as source_file: source_file.write(self.source.source) elif isinstance(self.source, FileSource): source_file_name = abspath(self.source.filename) if not exists(source_file_name): raise IOError("'%s' does not exist" % source_file_name) elif isinstance(self.source, ScriptWithFilesSource): source_file_name = join( working_dir, self.source.source_name) with open(source_file_name, "w") as source_file: source_file.write(self.source.source) from os.path import basename from shutil import copyfile for f in self.source.filenames: copyfile(f, join(working_dir, basename(f))) else: raise RuntimeError("'source' type unrecognized") output_file_name = join(working_dir, self.output_file_name) cmdline = [ self.gmsh_executable, "-o", self.output_file_name, "-nopopup"] if self.dimensions is not None: cmdline.append("-%d" % self.dimensions) if self.order is not None: cmdline.extend(["-order", str(self.order)]) if self.incomplete_elements is not None: cmdline.extend(["-string", "Mesh.SecondOrderIncomplete = %d;" % int(self.incomplete_elements)]) cmdline.extend(self.other_options) cmdline.append(source_file_name) if self.dimensions is None: cmdline.append("-") logger.info("invoking gmsh: '%s'" % " ".join(cmdline)) from pytools.prefork import call_capture_output retcode, stdout, stderr = call_capture_output( cmdline, working_dir) logger.info("return from gmsh") stdout = stdout.decode("utf-8") stderr = stderr.decode("utf-8") if stderr and "error" in stderr.lower(): msg = "gmsh execution failed with message:\n\n" if stdout: msg += stdout+"\n" msg += stderr+"\n" raise GmshError(msg) if stderr: from warnings import warn msg = "gmsh issued the following messages:\n\n" if stdout: msg += stdout+"\n" msg += stderr+"\n" warn(msg) self.output_file = open(output_file_name, "r") self.temp_dir_mgr = temp_dir_mgr return self except: temp_dir_mgr.clean_up() raise
def compile_plain(source, options, keep, nvcc, cache_dir, target="cubin"): from os.path import join assert target in ["cubin", "ptx", "fatbin"] if cache_dir: checksum = _new_md5() if '#include' in source: checksum.update( preprocess_source(source, options, nvcc).encode("utf-8")) else: checksum.update(source.encode("utf-8")) for option in options: checksum.update(option.encode("utf-8")) checksum.update(get_nvcc_version(nvcc).encode("utf-8")) from pycuda.characterize import platform_bits checksum.update(str(platform_bits()).encode("utf-8")) cache_file = checksum.hexdigest() cache_path = join(cache_dir, cache_file + "." + target) try: cache_file = open(cache_path, "rb") try: return cache_file.read() finally: cache_file.close() except: pass from tempfile import mkdtemp file_dir = mkdtemp() file_root = "kernel" cu_file_name = file_root + ".cu" cu_file_path = join(file_dir, cu_file_name) outf = open(cu_file_path, "w") outf.write(str(source)) outf.close() if keep: options = options[:] options.append("--keep") print("*** compiler output in %s" % file_dir) cmdline = [nvcc, "--" + target] + options + [cu_file_name] result, stdout, stderr = call_capture_output(cmdline, cwd=file_dir, error_on_nonzero=False) try: result_f = open(join(file_dir, file_root + "." + target), "rb") except IOError: no_output = True else: no_output = False if result != 0 or (no_output and (stdout or stderr)): if result == 0: from warnings import warn warn("PyCUDA: nvcc exited with status 0, but appears to have " "encountered an error") from pycuda.driver import CompileError raise CompileError("nvcc compilation of %s failed" % cu_file_path, cmdline, stdout=stdout.decode("utf-8", "replace"), stderr=stderr.decode("utf-8", "replace")) if stdout or stderr: lcase_err_text = (stdout + stderr).decode("utf-8", "replace").lower() from warnings import warn if "demoted" in lcase_err_text or "demoting" in lcase_err_text: warn( "nvcc said it demoted types in source code it " "compiled--this is likely not what you want.", stacklevel=4) warn("The CUDA compiler succeeded, but said the following:\n" + (stdout + stderr).decode("utf-8", "replace"), stacklevel=4) result_data = result_f.read() result_f.close() if cache_dir: outf = open(cache_path, "wb") outf.write(result_data) outf.close() if not keep: from os import listdir, unlink, rmdir for name in listdir(file_dir): unlink(join(file_dir, name)) rmdir(file_dir) return result_data