def build(self, src):
# Compute a digest of the current processor, compiler, and source
ckey = digest(self.proc, self.version, self.cmd, src)
# Attempt to load the library from the cache
mod = self._cache_loadlib(ckey)
# Otherwise, we need to compile the kernel
if not mod:
# Create a scratch directory
tmpidx = next(self._dir_seq)
tmpdir = tempfile.mkdtemp(prefix=f'pyfr-{tmpidx}-')
try:
# Compile and link the source into a shared library
cname, lname = 'tmp.c', platform_libname('tmp')
# Write the source code out
with open(os.path.join(tmpdir, cname), 'w') as f:
f.write(src)
# Invoke the compiler
call_capture_output(self.cc_cmd(cname, lname), cwd=tmpdir)
# Determine the fully qualified library name
lpath = os.path.join(tmpdir, lname)
# Add it to the cache and load
mod = self._cache_set_and_loadlib(ckey, lpath)
finally:
# Unless we're debugging delete the scratch directory
if 'PYFR_DEBUG_OMP_KEEP_LIBS' not in os.environ:
rm(tmpdir)
return OpenMPCompilerModule(mod)
def _build(self, tmpdir):
# File names
cn, on, ln = 'tmp.c', 'tmp.o', platform_libname('tmp')
# Write the source code out
with open(os.path.join(tmpdir, cn), 'w') as f:
f.write(self._src)
# Compile
cmd = [self._cc,
'-std=c99', # Enable C99 support
'-Ofast', # Optimise, incl. -ffast-math
'-march=native', # Use CPU-specific instructions
'-fopenmp', # Enable OpenMP support
'-fPIC', # Position-independent code for shared lib
'-c', '-o', on, cn]
call_capture_output(cmd, cwd=tmpdir)
# Link
cmd = [self._cc,
'-shared', # Create a shared library
'-fopenmp', # Required for OpenMP
'-o', ln, on,]
call_capture_output(cmd, cwd=tmpdir)
return ln
def _build(self, tmpdir):
# File names
cn, on, ln = 'tmp.c', 'tmp.o', platform_libname('tmp')
# Write the source code out
with open(os.path.join(tmpdir, cn), 'w') as f:
f.write(self.src)
# Compile
cmd = [
self._cc,
'-std=c99', # Enable C99 support
'-Ofast', # Optimise, incl. -ffast-math
'-march=native', # Use CPU-specific instructions
'-fopenmp', # Enable OpenMP support
'-fPIC', # Position-independent code for shared lib
'-c',
'-o',
on,
cn
]
call_capture_output(cmd, cwd=tmpdir)
# Link
cmd = [
self._cc,
'-shared', # Create a shared library
'-fopenmp', # Required for OpenMP
'-o',
ln,
on,
]
call_capture_output(cmd, cwd=tmpdir)
return ln
def __init__(self, src, cfg):
# Find GCC (or a compatible alternative)
self.cc = cfg.getpath('backend-openmp', 'cc', 'cc')
# User specified compiler flags
self.cflags = shlex.split(cfg.get('backend-openmp', 'cflags', ''))
# Get the processor string
proc = platform.processor()
# Get the compiler version string
version = call_capture_output([self.cc, '-v'])
# Get the base compiler command strig
cmd = self.cc_cmd(None, None)
# Compute a digest of the current processor, compiler, and source
self.digest = digest(proc, version, cmd, src)
# Attempt to load the library from the cache
self.mod = self._cache_loadlib()
# Otherwise, we need to compile the kernel
if not self.mod:
# Create a scratch directory
tmpidx = next(self._dir_seq)
tmpdir = tempfile.mkdtemp(prefix='pyfr-{0}-'.format(tmpidx))
try:
# Compile and link the source into a shared library
cname, lname = 'tmp.c', platform_libname('tmp')
# Write the source code out
with open(os.path.join(tmpdir, cname), 'w') as f:
f.write(src)
# Invoke the compiler
call_capture_output(self.cc_cmd(cname, lname), cwd=tmpdir)
# Determine the fully qualified library name
lpath = os.path.join(tmpdir, lname)
# Add it to the cache and load
self.mod = self._cache_set_and_loadlib(lpath)
finally:
# Unless we're debugging delete the scratch directory
if 'PYFR_DEBUG_OMP_KEEP_LIBS' not in os.environ:
rm(tmpdir)
def __init__(self, src, cfg):
# Find GCC (or a compatible alternative)
self.cc = cfg.getpath('backend-openmp', 'cc', 'cc')
# User specified compiler flags
self.cflags = shlex.split(cfg.get('backend-openmp', 'cflags', ''))
# Get the processor string
proc = platform.processor()
# Get the compiler version string
version = call_capture_output([self.cc, '-v'])
# Get the base compiler command strig
cmd = self.cc_cmd(None, None)
# Compute a digest of the current processor, compiler, and source
self.digest = digest(proc, version, cmd, src)
# Attempt to load the library from the cache
self.mod = self._cache_loadlib()
# Otherwise, we need to compile the kernel
if not self.mod:
# Create a scratch directory
tmpidx = next(self._dir_seq)
tmpdir = tempfile.mkdtemp(prefix='pyfr-{0}-'.format(tmpidx))
try:
# Compile and link the source into a shared library
cname, lname = 'tmp.c', platform_libname('tmp')
# Write the source code out
with open(os.path.join(tmpdir, cname), 'w') as f:
f.write(src)
# Invoke the compiler
call_capture_output(self.cc_cmd(cname, lname), cwd=tmpdir)
# Determine the fully qualified library name
lpath = os.path.join(tmpdir, lname)
# Add it to the cache and load
self.mod = self._cache_set_and_loadlib(lpath)
finally:
# Unless we're debugging delete the scratch directory
if 'PYFR_DEBUG_OMP_KEEP_LIBS' not in os.environ:
rm(tmpdir)
def guess_toolchain():
"""Guess and return a :class:`Toolchain` instance.
Raise :exc:`ToolchainGuessError` if no toolchain could be found.
"""
try:
#This will fail on non-POSIX systems
kwargs = _guess_toolchain_kwargs_from_python_config()
except:
#If it does, return a distutils toolchain
return guess_distutils_toolchain()
from pytools.prefork import call_capture_output
result, version, stderr = call_capture_output([kwargs["cc"], "--version"])
if result != 0:
raise ToolchainGuessError("compiler version query failed: " + stderr)
if "Free Software Foundation" in version:
if "-Wstrict-prototypes" in kwargs["cflags"]:
kwargs["cflags"].remove("-Wstrict-prototypes")
if "darwin" in version:
# Are we running in 32-bit mode?
# The python interpreter may have been compiled as a Fat binary
# So we need to check explicitly how we're running
# And update the cflags accordingly
import sys
if sys.maxint == 0x7fffffff:
kwargs["cflags"].extend(['-arch', 'i386'])
return GCCToolchain(**kwargs)
else:
raise ToolchainGuessError("unknown compiler")
def preprocess_source(source, options, nvcc):
handle, source_path = mkstemp(suffix='.cu')
outf = open(source_path, 'w')
outf.write(source)
outf.close()
os.close(handle)
cmdline = [nvcc, '--preprocess'] + options + [source_path]
if 'win32' in sys.platform:
cmdline.extend(['--compiler-options', '-EP'])
else:
cmdline.extend(['--compiler-options', '-P'])
result, stdout, stderr = call_capture_output(cmdline, error_on_nonzero=False)
if result != 0:
from pycuda.driver import CompileError
raise CompileError("nvcc preprocessing of %s failed" % source_path,
cmdline, stderr=stderr)
# sanity check
if len(stdout) < 0.5*len(source):
from pycuda.driver import CompileError
raise CompileError("nvcc preprocessing of %s failed with ridiculously "
"small code output - likely unsupported compiler." % source_path,
cmdline, stderr=stderr.decode("utf-8", "replace"))
unlink(source_path)
return stdout.decode("utf-8", "replace")
def preprocess_source(source, options, nvcc):
handle, source_path = mkstemp(suffix='.cu')
outf = open(source_path, 'w')
outf.write(source)
outf.close()
os.close(handle)
cmdline = [nvcc, '--preprocess'] + options + [source_path]
if 'win32' in sys.platform:
cmdline.extend(['--compiler-options', '-EP'])
else:
cmdline.extend(['--compiler-options', '-P'])
result, stdout, stderr = call_capture_output(cmdline,
error_on_nonzero=False)
if result != 0:
from pycuda.driver import CompileError
raise CompileError("nvcc preprocessing of %s failed" % source_path,
cmdline,
stderr=stderr)
# sanity check
if len(stdout) < 0.5 * len(source):
from pycuda.driver import CompileError
raise CompileError("nvcc preprocessing of %s failed with ridiculously "
"small code output - likely unsupported compiler." %
source_path,
cmdline,
stderr=stderr.decode("utf-8", "replace"))
unlink(source_path)
return stdout.decode("utf-8", "replace")
def build_object(self, ext_file, source_files, debug=False):
cc_cmdline = (
self._cmdline(source_files, True)
+ ["-o", ext_file]
)
from pytools.prefork import call_capture_output
if debug:
print " ".join(cc_cmdline)
result, stdout, stderr = call_capture_output(cc_cmdline)
print stderr
print stdout
if "error" in stderr:
# work around a bug in nvcc, which doesn't provide a non-zero
# return code even if it failed.
result = 1
if result != 0:
import sys
print >> sys.stderr, "FAILED compiler invocation:", \
" ".join(cc_cmdline)
raise CompileError, "module compilation failed"
def guess_toolchain():
"""Guess and return a :class:`Toolchain` instance.
Raise :exc:`ToolchainGuessError` if no toolchain could be found.
"""
kwargs = _guess_toolchain_kwargs_from_python_config()
from pytools.prefork import call_capture_output
result, version, stderr = call_capture_output([kwargs["cc"], "--version"])
if result != 0:
raise ToolchainGuessError("compiler version query failed: "+stderr)
if "Free Software Foundation" in version:
if "-Wstrict-prototypes" in kwargs["cflags"]:
kwargs["cflags"].remove("-Wstrict-prototypes")
if "darwin" in version:
# Are we running in 32-bit mode?
# The python interpreter may have been compiled as a Fat binary
# So we need to check explicitly how we're running
# And update the cflags accordingly
import sys
if sys.maxint == 0x7fffffff:
kwargs["cflags"].extend(['-arch', 'i386'])
return GCCToolchain(**kwargs)
elif "Intel Corporation" in version:
return IntelToolchain(**kwargs)
else:
raise ToolchainGuessError("unknown compiler")
def call_capture_output(*args):
from pytools.prefork import call_capture_output
import sys
encoding = sys.getdefaultencoding()
result, stdout, stderr = call_capture_output(*args)
return result, stdout.decode(encoding), stderr.decode(encoding)
def guess_toolchain():
"""Guess and return a :class:`Toolchain` instance.
Raise :exc:`ToolchainGuessError` if no toolchain could be found.
"""
kwargs = _guess_toolchain_kwargs_from_python_config()
result, version, stderr = call_capture_output([kwargs["cc"], "--version"])
if result != 0:
raise ToolchainGuessError(f"compiler version query failed: {stderr}")
if "Free Software Foundation" in version:
if "-Wstrict-prototypes" in kwargs["cflags"]:
kwargs["cflags"].remove("-Wstrict-prototypes")
if "darwin" in version:
# Are we running in 32-bit mode?
# The python interpreter may have been compiled as a Fat binary
# So we need to check explicitly how we're running
# And update the cflags accordingly
import sys
if sys.maxsize == 0x7fffffff:
kwargs["cflags"].extend(["-arch", "i386"])
return GCCToolchain(**kwargs)
else:
raise ToolchainGuessError(
"Unable to determine compiler. Tried running "
f"'{kwargs['cc']} --version' and failed.")
def detect(self, compiler):
from pytools.prefork import call_capture_output
try:
retcode, stdout, stderr = call_capture_output([compiler, "--version"])
except:
return False
return (retcode == 0)
def __enter__(self):
self.temp_dir_mgr = None
temp_dir_mgr = _TempDirManager()
try:
working_dir = temp_dir_mgr.path
from os.path import join
source_file_name = join(working_dir, "temp." + self.extension)
source_file = open(source_file_name, "w")
try:
source_file.write(self.source)
finally:
source_file.close()
output_file_name = join(working_dir, "output.msh")
cmdline = [
self.gmsh_executable,
"-%d" % self.dimensions, "-o", output_file_name, "-nopopup"
]
if self.order is not None:
cmdline.extend(["-order", str(self.order)])
if self.incomplete_elements is not None:
cmdline.extend([
"-string",
"Mesh.SecondOrderIncomplete = %d;" %
int(self.incomplete_elements)
])
cmdline.extend(self.other_options)
cmdline.append(source_file_name)
from pytools.prefork import call_capture_output
retcode, stdout, stderr = call_capture_output(cmdline, working_dir)
if stderr and "error" in stderr.lower():
msg = "gmsh execution failed with message:\n\n"
if stdout:
msg += stdout + "\n"
msg += stderr + "\n"
raise GmshError(msg)
if stderr:
from warnings import warn
msg = "gmsh issued the following messages:\n\n"
if stdout:
msg += stdout + "\n"
msg += stderr + "\n"
warn(msg)
self.output_file = open(output_file_name, "r")
self.temp_dir_mgr = temp_dir_mgr
return self.output_file
except:
temp_dir_mgr.clean_up()
raise
def version(self):
from distutils.version import LooseVersion
cmdline = [self.gmsh_executable, '-version']
from pytools.prefork import call_capture_output
retcode, stdout, stderr = call_capture_output(cmdline)
version = stderr.decode().strip()
return LooseVersion(version)
def detect(self, compiler):
from pytools.prefork import call_capture_output
try:
retcode, stdout, stderr = call_capture_output(
[compiler, "--version"])
except:
return False
return (retcode == 0)
def get_nvcc_version(nvcc):
cmdline = [nvcc, "--version"]
result, stdout, stderr = call_capture_output(cmdline)
if result != 0 or not stdout:
from warnings import warn
warn("NVCC version could not be determined.")
stdout = "nvcc unknown version"
return stdout.decode("utf-8", "replace")
def get_nvcc_version(nvcc):
cmdline = [nvcc, "--version"]
result, stdout, stderr = call_capture_output(cmdline)
if result != 0 or not stdout:
from warnings import warn
warn("NVCC version could not be determined.")
stdout = "nvcc unknown version"
return stdout.decode("utf-8", "replace")
def version(self):
from distutils.version import LooseVersion
cmdline = [self.gmsh_executable, '-version']
from pytools.prefork import call_capture_output
retcode, stdout, stderr = call_capture_output(cmdline)
# stderr can contain irregular info
import re
version = re.search(r'[0-9]+.[0-9]+.[0-9]+',
stderr.decode().strip()).group()
return LooseVersion(version)
def __enter__(self):
self.temp_dir_mgr = None
temp_dir_mgr = _TempDirManager()
try:
working_dir = temp_dir_mgr.path
from os.path import join
source_file_name = join(working_dir, "temp." + self.extension)
source_file = open(source_file_name, "w")
try:
source_file.write(self.source)
finally:
source_file.close()
output_file_name = join(working_dir, "output.msh")
cmdline = [self.gmsh_executable, "-%d" % self.dimensions, "-o", output_file_name, "-nopopup"]
if self.order is not None:
cmdline.extend(["-order", str(self.order)])
if self.incomplete_elements is not None:
cmdline.extend(["-string", "Mesh.SecondOrderIncomplete = %d;" % int(self.incomplete_elements)])
cmdline.extend(self.other_options)
cmdline.append(source_file_name)
from pytools.prefork import call_capture_output
retcode, stdout, stderr = call_capture_output(cmdline, working_dir)
if stderr and "error" in stderr.lower():
msg = "gmsh execution failed with message:\n\n"
if stdout:
msg += stdout + "\n"
msg += stderr + "\n"
raise GmshError(msg)
if stderr:
from warnings import warn
msg = "gmsh issued the following messages:\n\n"
if stdout:
msg += stdout + "\n"
msg += stderr + "\n"
warn(msg)
self.output_file = open(output_file_name, "r")
self.temp_dir_mgr = temp_dir_mgr
return self.output_file
except:
temp_dir_mgr.clean_up()
raise
def build_lib(lib, extensions, source_dir, CC, dst_dir, inc_dirs):
_build_timer.start()
_lib_filename = os.path.join(dst_dir, lib + '.so')
_lib_src_filename = os.path.join(source_dir, lib + extensions[1])
_c_cmd = CC.binary + [_lib_src_filename] + ['-o'] + \
[_lib_filename] + CC.c_flags + CC.l_flags + \
['-I' + str(d) for d in inc_dirs] + \
['-I' + str(source_dir)]
if ppmd.runtime.DEBUG > 0:
_c_cmd += CC.dbg_flags
if ppmd.runtime.OPT > 0:
_c_cmd += CC.opt_flags
_c_cmd += CC.shared_lib_flag
stdout_filename = os.path.join(dst_dir, lib + '.log')
stderr_filename = os.path.join(dst_dir, lib + '.err')
with open(stdout_filename, 'w') as stdout_fh:
with open(stderr_filename, 'w') as stderr_fh:
stdout_fh.write('# Compilation command:\n')
stdout_fh.write(' '.join(_c_cmd))
stdout_fh.write('\n\n')
stdout_fh.flush()
result, stdout, stderr = call_capture_output(
_c_cmd, error_on_nonzero=False)
stdout = stdout.decode(sys.stdout.encoding)
stderr = stderr.decode(sys.stdout.encoding)
stderr_fh.write(str(result) + '\n')
stderr_fh.write(stdout)
stderr_fh.write(stderr)
stderr_fh.flush()
if result != 0:
print("\n---- COMPILER OUTPUT START ----")
print(stdout)
print(stderr)
print("----- COMPILER OUTPUT END -----")
raise RuntimeError('PPMD build error: library not built.')
# Check library exists in the file system
if not os.path.exists(_lib_filename):
print("Critical build Error: Library not found,\n" + \
_lib_filename + "\n rank:", _MPIRANK)
raise RuntimeError('compiler call did not error, but no binary found')
_build_timer.pause()
opt.PROFILE['Build:' + CC.binary[0] + ':'] = (_build_timer.time())
return _lib_filename
def get_dependencies(self, source_files):
from codepy.tools import join_continued_lines
result, stdout, stderr = call_capture_output(
[self.cc] + ["-M"] + ["-D%s" % define for define in self.defines] +
["-U%s" % undefine for undefine in self.undefines] +
["-I%s" % idir
for idir in self.include_dirs] + self.cflags + source_files)
if result != 0:
raise CompileError("getting dependencies failed: " + stderr)
lines = join_continued_lines(stdout.split("\n"))
from pytools import flatten
return set(flatten(line.split()[2:] for line in lines))
def get_nvcc_version(nvcc):
cmdline = [nvcc, "--version"]
try:
try:
from pytools.prefork import call_capture_output
except ImportError:
from pytools.prefork import call_capture_stdout
return call_capture_stdout(cmdline)
else:
retcode, stdout, stderr = call_capture_output(cmdline)
return stdout
except OSError, e:
raise OSError("%s was not found (is it on the PATH?) [%s]"
% (nvcc, str(e)))
def llvmir_to_obj(source, llc="llc"):
"""
Returns the compiled object code for the LLVM code *source*.
"""
with tempfile.NamedTemporaryFile(mode="w", suffix=".ll") as llfp:
llfp.write(source)
llfp.file.flush()
with tempfile.NamedTemporaryFile(suffix=".o", mode="rb") as objfp:
cmdline = [llc, llfp.name, "-o", objfp.name, "-filetype=obj"]
result, stdout, stderr = call_capture_output(cmdline)
obj_code = objfp.read()
return obj_code
def get_nvcc_version(nvcc):
cmdline = [nvcc, "--version"]
try:
try:
from pytools.prefork import call_capture_output
except ImportError:
from pytools.prefork import call_capture_stdout
return call_capture_stdout(cmdline)
else:
retcode, stdout, stderr = call_capture_output(cmdline)
return stdout
except OSError, e:
raise OSError("%s was not found (is it on the PATH?) [%s]" %
(nvcc, str(e)))
def __init__(self, src, dev, cfg):
# Create a scratch directory
tmpidx = next(self._dir_seq)
tmpdir = tempfile.mkdtemp(prefix='pyfr-{0}-'.format(tmpidx))
# Find MKL
mklroot = cfg.get('backend-mic', 'mkl-root', '$MKLROOT')
try:
# File names
cn, ln = 'tmp.c', 'libtmp.so'
# Write the source code out
with open(os.path.join(tmpdir, cn), 'w') as f:
f.write(src)
# Compile and link
cmd = [
'icc',
'-shared', # Create a shared library
'-std=c99', # Enable C99 support
'-Ofast', # Optimise
'-mmic', # Compile for the MIC
'-fopenmp', # Enable OpenMP support
'-L{0}/lib/mic'.format(mklroot), # MKL stuff
'-lmkl_intel_lp64', # ...
'-lmkl_core', # ...
'-lmkl_intel_thread', # ...
'-fPIC', # Position-independent code
'-o', ln, cn
]
call_capture_output(cmd, cwd=tmpdir)
# Load
self._mod = dev.load_library(os.path.join(tmpdir, ln))
finally:
rm(tmpdir)
def __init__(self, cfg):
# Find GCC (or a compatible alternative)
self.cc = cfg.getpath('backend-openmp', 'cc', 'cc')
# User specified compiler flags
self.cflags = shlex.split(cfg.get('backend-openmp', 'cflags', ''))
# Get the processor string
self.proc = platform.processor()
# Get the compiler version string
self.version = call_capture_output([self.cc, '-v'])
# Get the base compiler command strig
self.cmd = self.cc_cmd(None, None)
def mlir_translate(source, options, mlir_translate="mlir-translate"):
"""
Calls ``mlir-translate`` on *source* with *options* as additional arguments.
:arg source: The code to be passed to mlir-translate.
:arg options: An instance of :class:`list`.
:return: Transformed *source* as emitted by ``mlir-translate``.
"""
with tempfile.NamedTemporaryFile(mode="w", suffix=".mlir", delete=False) as fp:
fp.write(source)
fp.file.flush()
cmdline = [mlir_translate, fp.name] + options
result, stdout, stderr = call_capture_output(cmdline)
return stdout.decode()
def mlir_opt(source: str, options: List[str], mlir_opt="mlir-opt"):
"""
Calls ``mlir-opt`` on *source* with *options* as additional arguments.
:arg source: The code to be passed to mlir-opt.
:arg options: An instance of :class:`list`.
:return: Transformed *source* as emitted by ``mlir-opt``.
"""
assert "-o" not in options
with tempfile.NamedTemporaryFile(mode="w", suffix=".mlir") as fp:
fp.write(source)
fp.file.flush()
cmdline = [mlir_opt, fp.name] + options
result, stdout, stderr = call_capture_output(cmdline)
return stdout.decode()
def guess_toolchain():
"""Guess and return a :class:`Toolchain` instance.
Raise :exc:`ToolchainGuessError` if no toolchain could be found.
"""
kwargs = _guess_toolchain_kwargs_from_python_config()
try:
result, version, stderr = call_capture_output(
[kwargs["cc"], "--version"])
except ExecError:
raise ToolchainGuessError("System compiler {} not found".format(
kwargs['cc']))
if result != 0:
raise ToolchainGuessError("compiler version query failed: " + stderr)
if "Free Software Foundation" in version:
if "-Wstrict-prototypes" in kwargs["cflags"]:
kwargs["cflags"].remove("-Wstrict-prototypes")
if "darwin" in version:
# Are we running in 32-bit mode?
# The python interpreter may have been compiled as a Fat binary
# So we need to check explicitly how we're running
# And update the cflags accordingly
import sys
if sys.maxsize == 0x7fffffff:
kwargs["cflags"].extend(['-arch', 'i386'])
return GCCToolchain(**kwargs)
elif "Apple LLVM" in version and "clang" in version:
if "-Wstrict-prototypes" in kwargs["cflags"]:
kwargs["cflags"].remove("-Wstrict-prototypes")
if "darwin" in version:
# Are we running in 32-bit mode?
# The python interpreter may have been compiled as a Fat binary
# So we need to check explicitly how we're running
# And update the cflags accordingly
import sys
if sys.maxsize == 0x7fffffff:
kwargs["cflags"].extend(['-arch', 'i386'])
return GCCToolchain(**kwargs)
else:
raise ToolchainGuessError("unknown compiler")
def build_object(self, ext_file, source_files, debug=False):
cc_cmdline = (self._cmdline(source_files, True) + ["-o", ext_file])
if debug:
print(" ".join(cc_cmdline))
result, stdout, stderr = call_capture_output(cc_cmdline)
print(stderr)
print(stdout)
if "error" in stderr:
# work around a bug in nvcc, which doesn't provide a non-zero
# return code even if it failed.
result = 1
if result != 0:
import sys
print("FAILED compiler invocation:" + " ".join(cc_cmdline),
file=sys.stderr)
raise CompileError("module compilation failed")
def get_dependencies(self, source_files):
from codepy.tools import join_continued_lines
from pytools.prefork import call_capture_output
result, stdout, stderr = call_capture_output(
[self.cc]
+ ["-M"]
+ ["-D%s" % define for define in self.defines]
+ ["-U%s" % undefine for undefine in self.defines]
+ ["-I%s" % idir for idir in self.include_dirs]
+ source_files
)
if result != 0:
raise CompileError("getting dependencies failed: "+stderr)
lines = join_continued_lines(stdout.split("\n"))
from pytools import flatten
return set(flatten(
line.split()[2:] for line in lines))
def preprocess_source(source, options, nvcc):
handle, source_path = mkstemp(suffix=".cu")
outf = open(source_path, "w")
outf.write(source)
outf.close()
os.close(handle)
cmdline = [nvcc, "--preprocess"] + options + [source_path]
if "win32" in sys.platform:
cmdline.extend(["--compiler-options", "-EP"])
else:
cmdline.extend(["--compiler-options", "-P"])
result, stdout, stderr = call_capture_output(cmdline,
error_on_nonzero=False)
if result != 0:
from pycuda.driver import CompileError
raise CompileError("nvcc preprocessing of %s failed" % source_path,
cmdline,
stderr=stderr)
# sanity check
if len(stdout) < 0.5 * len(source):
from pycuda.driver import CompileError
raise CompileError(
"nvcc preprocessing of %s failed with ridiculously "
"small code output - likely unsupported compiler." % source_path,
cmdline,
stderr=stderr.decode("utf-8", "replace"),
)
unlink(source_path)
preprocessed_str = stdout.decode("utf-8", "replace")
# remove the temporary filename from the preprocessed source code to get reproducible hashes
return preprocessed_str.replace(os.path.basename(source_path), "")
def get_dependencies(self, source_files):
from codepy.tools import join_continued_lines
result, stdout, stderr = call_capture_output(
[self.cc]
+ ["-M"]
+ [f"-D{define}" for define in self.defines]
+ [f"-U{undefine}" for undefine in self.undefines]
+ [f"-I{idir}" for idir in self.include_dirs]
+ self.cflags
+ source_files
)
if result != 0:
raise CompileError(f"getting dependencies failed: {stderr}")
lines = join_continued_lines(stdout.split("\n"))
lines = [line for line in lines
if not (line.strip() and line.strip()[0] == "#")]
from pytools import flatten
return set(flatten(
line.split()[2:] for line in lines))
def get_nvcc_version(nvcc):
cmdline = [nvcc, "--version"]
try:
try:
from pytools.prefork import call_capture_output
except ImportError:
from pytools.prefork import call_capture_stdout
result = call_capture_stdout(cmdline)
else:
retcode, stdout, stderr = call_capture_output(cmdline)
result = stdout
if result is None:
from warnings import warn
warn("NVCC version could not be determined.")
result = "nvcc unknown version"
return result
except OSError, e:
raise OSError("%s was not found (is it on the PATH?) [%s]"
% (nvcc, str(e)))
def compile_plain(source, options, keep, nvcc, cache_dir, target="cubin"):
from os.path import join
assert target in ["cubin", "ptx", "fatbin"]
if cache_dir:
checksum = _new_md5()
if '#include' in source:
checksum.update(preprocess_source(source, options, nvcc).encode("utf-8"))
else:
checksum.update(source.encode("utf-8"))
for option in options:
checksum.update(option.encode("utf-8"))
checksum.update(get_nvcc_version(nvcc).encode("utf-8"))
from pycuda.characterize import platform_bits
checksum.update(str(platform_bits()).encode("utf-8"))
cache_file = checksum.hexdigest()
cache_path = join(cache_dir, cache_file + "." + target)
try:
cache_file = open(cache_path, "rb")
try:
return cache_file.read()
finally:
cache_file.close()
except:
pass
from tempfile import mkdtemp
file_dir = mkdtemp()
file_root = "kernel"
cu_file_name = file_root + ".cu"
cu_file_path = join(file_dir, cu_file_name)
outf = open(cu_file_path, "w")
outf.write(str(source))
outf.close()
if keep:
options = options[:]
options.append("--keep")
print("*** compiler output in %s" % file_dir)
cmdline = [nvcc, "--" + target] + options + [cu_file_name]
result, stdout, stderr = call_capture_output(cmdline,
cwd=file_dir, error_on_nonzero=False)
try:
result_f = open(join(file_dir, file_root + "." + target), "rb")
except IOError:
no_output = True
else:
no_output = False
if result != 0 or (no_output and (stdout or stderr)):
if result == 0:
from warnings import warn
warn("PyCUDA: nvcc exited with status 0, but appears to have "
"encountered an error")
from pycuda.driver import CompileError
raise CompileError("nvcc compilation of %s failed" % cu_file_path,
cmdline, stdout=stdout.decode("utf-8", "replace"),
stderr=stderr.decode("utf-8", "replace"))
if stdout or stderr:
lcase_err_text = (stdout+stderr).decode("utf-8", "replace").lower()
from warnings import warn
if "demoted" in lcase_err_text or "demoting" in lcase_err_text:
warn("nvcc said it demoted types in source code it "
"compiled--this is likely not what you want.",
stacklevel=4)
warn("The CUDA compiler succeeded, but said the following:\n"
+ (stdout+stderr).decode("utf-8", "replace"), stacklevel=4)
result_data = result_f.read()
result_f.close()
if cache_dir:
outf = open(cache_path, "wb")
outf.write(result_data)
outf.close()
if not keep:
from os import listdir, unlink, rmdir
for name in listdir(file_dir):
unlink(join(file_dir, name))
rmdir(file_dir)
return result_data
def get_version(self):
result, stdout, stderr = call_capture_output([self.cc, "--version"])
if result != 0:
raise RuntimeError("version query failed: " + stderr)
return stdout
def __enter__(self):
self.temp_dir_mgr = None
temp_dir_mgr = _TempDirManager()
try:
working_dir = temp_dir_mgr.path
from os.path import join, abspath, exists
if isinstance(self.source, ScriptSource):
source_file_name = join(working_dir,
"temp." + self.source.extension)
with open(source_file_name, "w") as source_file:
source_file.write(self.source.source)
elif isinstance(self.source, FileSource):
source_file_name = abspath(self.source.filename)
if not exists(source_file_name):
raise IOError("'%s' does not exist" % source_file_name)
elif isinstance(self.source, ScriptWithFilesSource):
source_file_name = join(working_dir, self.source.source_name)
with open(source_file_name, "w") as source_file:
source_file.write(self.source.source)
from os.path import basename
from shutil import copyfile
for f in self.source.filenames:
copyfile(f, join(working_dir, basename(f)))
else:
raise RuntimeError("'source' type unrecognized")
output_file_name = join(working_dir, self.output_file_name)
cmdline = [
self.gmsh_executable, "-o", self.output_file_name, "-nopopup",
"-format", "msh2"
]
# NOTE: handle unit incompatibility introduced in GMSH4
# https://gitlab.onelab.info/gmsh/gmsh/issues/397
if self.version < '4.0.0':
if self.target_unit == 'M':
cmdline.extend(["-string", "Geometry.OCCScaling=1000;"])
else:
cmdline.extend([
"-string",
"Geometry.OCCTargetUnit='{}';".format(self.target_unit)
])
if self.dimensions is not None:
cmdline.append("-%d" % self.dimensions)
if self.order is not None:
cmdline.extend(["-order", str(self.order)])
if self.incomplete_elements is not None:
cmdline.extend([
"-string",
"Mesh.SecondOrderIncomplete = %d;" %
int(self.incomplete_elements)
])
cmdline.extend(self.other_options)
cmdline.append(source_file_name)
if self.dimensions is None:
cmdline.append("-")
logger.info("invoking gmsh: '%s'" % " ".join(cmdline))
from pytools.prefork import call_capture_output
retcode, stdout, stderr = call_capture_output(cmdline, working_dir)
logger.info("return from gmsh")
stdout = stdout.decode("utf-8")
stderr = stderr.decode("utf-8")
import re
error_match = re.match(r"([0-9]+)\s+error", stdout)
warning_match = re.match(r"([0-9]+)\s+warning", stdout)
if error_match is not None or warning_match is not None:
# if we have one, we expect to see both
assert error_match is not None or warning_match is not None
num_warnings = int(warning_match.group(1))
num_errors = int(error_match.group(1))
else:
num_warnings = 0
num_errors = 0
if num_errors:
msg = "gmsh execution failed with message:\n\n"
if stdout:
msg += stdout + "\n"
msg += stderr + "\n"
raise GmshError(msg)
if num_warnings:
from warnings import warn
msg = "gmsh issued the following warning messages:\n\n"
if stdout:
msg += stdout + "\n"
msg += stderr + "\n"
warn(msg)
self.output_file = open(output_file_name, "r")
if self.save_tmp_files_in:
import shutil
import errno
try:
shutil.copytree(working_dir, self.save_tmp_files_in)
except FileExistsError:
import select
import sys
print(
"%s exists! Overwrite? (Y/N, will default to Y in 10sec)."
% self.save_tmp_files_in)
decision = None
while not decision:
i, o, e = select.select([sys.stdin], [], [], 10)
if i == "N" or i == "n":
decision = 0
elif i == "Y" or i == "y" or not i:
decision = 1
else:
print("Illegal input %s, please retry." % i)
if decision == 0:
pass
else:
assert decision == 1
shutil.rmtree(self.save_tmp_files_in)
shutil.copytree(working_dir, self.save_tmp_files_in)
except OSError as exc:
if exc.errno == errno.ENOTDIR:
shutil.copy(
output_file_name, '/'.join([
self.save_tmp_files_in, self.output_file_name
]))
else:
raise
self.temp_dir_mgr = temp_dir_mgr
return self
except Exception:
temp_dir_mgr.clean_up()
raise
try:
from pytools.prefork import call_capture_output
except ImportError:
from pytools.prefork import call
try:
result = call(cmdline, cwd=file_dir)
except OSError, e:
raise OSError("%s was not found (is it on the PATH?) [%s]"
% (nvcc, str(e)))
stdout = None
stderr = None
else:
result, stdout, stderr = call_capture_output(
cmdline, cwd=file_dir,
error_on_nonzero=False)
try:
cubin_f = open(join(file_dir, file_root + ".cubin"), "rb")
except IOError:
no_output = True
else:
no_output = False
if result != 0 or (no_output and (stdout or stderr)):
if result == 0:
from warnings import warn
warn("PyCUDA: nvcc exited with status 0, but appears to have "
"encountered an error")
from pycuda.driver import CompileError
def get_version(self):
from pytools.prefork import call_capture_output
result, stdout, stderr = call_capture_output([self.cc, "--version"])
if result != 0:
raise RuntimeError("version query failed: "+stderr)
return stdout
def __enter__(self):
self.temp_dir_mgr = None
temp_dir_mgr = _TempDirManager()
try:
working_dir = temp_dir_mgr.path
from os.path import join, abspath, exists
if isinstance(self.source, ScriptSource):
source_file_name = join(
working_dir, "temp."+self.source.extension)
with open(source_file_name, "w") as source_file:
source_file.write(self.source.source)
elif isinstance(self.source, FileSource):
source_file_name = abspath(self.source.filename)
if not exists(source_file_name):
raise IOError("'%s' does not exist" % source_file_name)
elif isinstance(self.source, ScriptWithFilesSource):
source_file_name = join(
working_dir, self.source.source_name)
with open(source_file_name, "w") as source_file:
source_file.write(self.source.source)
from os.path import basename
from shutil import copyfile
for f in self.source.filenames:
copyfile(f, join(working_dir, basename(f)))
else:
raise RuntimeError("'source' type unrecognized")
output_file_name = join(working_dir, self.output_file_name)
cmdline = [
self.gmsh_executable,
"-o", self.output_file_name,
"-nopopup"]
if self.dimensions is not None:
cmdline.append("-%d" % self.dimensions)
if self.order is not None:
cmdline.extend(["-order", str(self.order)])
if self.incomplete_elements is not None:
cmdline.extend(["-string",
"Mesh.SecondOrderIncomplete = %d;"
% int(self.incomplete_elements)])
cmdline.extend(self.other_options)
cmdline.append(source_file_name)
if self.dimensions is None:
cmdline.append("-")
logger.info("invoking gmsh: '%s'" % " ".join(cmdline))
from pytools.prefork import call_capture_output
retcode, stdout, stderr = call_capture_output(
cmdline, working_dir)
logger.info("return from gmsh")
stdout = stdout.decode("utf-8")
stderr = stderr.decode("utf-8")
if stderr and "error" in stderr.lower():
msg = "gmsh execution failed with message:\n\n"
if stdout:
msg += stdout+"\n"
msg += stderr+"\n"
raise GmshError(msg)
if stderr:
from warnings import warn
msg = "gmsh issued the following messages:\n\n"
if stdout:
msg += stdout+"\n"
msg += stderr+"\n"
warn(msg)
self.output_file = open(output_file_name, "r")
self.temp_dir_mgr = temp_dir_mgr
return self
except:
temp_dir_mgr.clean_up()
raise
def compile_plain(source, options, keep, nvcc, cache_dir, target="cubin"):
from os.path import join
assert target in ["cubin", "ptx", "fatbin"]
if cache_dir:
checksum = _new_md5()
if '#include' in source:
checksum.update(
preprocess_source(source, options, nvcc).encode("utf-8"))
else:
checksum.update(source.encode("utf-8"))
for option in options:
checksum.update(option.encode("utf-8"))
checksum.update(get_nvcc_version(nvcc).encode("utf-8"))
from pycuda.characterize import platform_bits
checksum.update(str(platform_bits()).encode("utf-8"))
cache_file = checksum.hexdigest()
cache_path = join(cache_dir, cache_file + "." + target)
try:
cache_file = open(cache_path, "rb")
try:
return cache_file.read()
finally:
cache_file.close()
except:
pass
from tempfile import mkdtemp
file_dir = mkdtemp()
file_root = "kernel"
cu_file_name = file_root + ".cu"
cu_file_path = join(file_dir, cu_file_name)
outf = open(cu_file_path, "w")
outf.write(str(source))
outf.close()
if keep:
options = options[:]
options.append("--keep")
print("*** compiler output in %s" % file_dir)
cmdline = [nvcc, "--" + target] + options + [cu_file_name]
result, stdout, stderr = call_capture_output(cmdline,
cwd=file_dir,
error_on_nonzero=False)
try:
result_f = open(join(file_dir, file_root + "." + target), "rb")
except IOError:
no_output = True
else:
no_output = False
if result != 0 or (no_output and (stdout or stderr)):
if result == 0:
from warnings import warn
warn("PyCUDA: nvcc exited with status 0, but appears to have "
"encountered an error")
from pycuda.driver import CompileError
raise CompileError("nvcc compilation of %s failed" % cu_file_path,
cmdline,
stdout=stdout.decode("utf-8", "replace"),
stderr=stderr.decode("utf-8", "replace"))
if stdout or stderr:
lcase_err_text = (stdout + stderr).decode("utf-8", "replace").lower()
from warnings import warn
if "demoted" in lcase_err_text or "demoting" in lcase_err_text:
warn(
"nvcc said it demoted types in source code it "
"compiled--this is likely not what you want.",
stacklevel=4)
warn("The CUDA compiler succeeded, but said the following:\n" +
(stdout + stderr).decode("utf-8", "replace"),
stacklevel=4)
result_data = result_f.read()
result_f.close()
if cache_dir:
outf = open(cache_path, "wb")
outf.write(result_data)
outf.close()
if not keep:
from os import listdir, unlink, rmdir
for name in listdir(file_dir):
unlink(join(file_dir, name))
rmdir(file_dir)
return result_data
