def build_extension(self, ext):
# Make sure that if pythonXX-sys is used, it builds against the current
# executing python interpreter.
bindir = os.path.dirname(sys.executable)
env = os.environ.copy()
env.update({
# disables rust's pkg-config seeking for specified packages,
# which causes pythonXX-sys to fall back to detecting the
# interpreter from the path.
"PATH": bindir + os.pathsep + os.environ.get("PATH", "")
})
if not os.path.exists(ext.path):
raise DistutilsFileError(
"Can not file rust extension project file: %s" % ext.path)
features = set(ext.features)
features.update(
cpython_feature(pyo3=ext.pyo3, no_binding=ext.no_binding))
if ext.debug is None:
debug_build = self.inplace
else:
debug_build = ext.debug
debug_build = self.debug if self.debug is not None else debug_build
if self.release:
debug_build = False
quiet = self.qbuild or ext.quiet
# build cargo command
feature_args = ["--features" + " ".join(features)] if features else []
args = (["cargo", "rustc", "--lib", "--manifest-path", ext.path] +
feature_args + list(ext.args or []))
if not debug_build:
args.append("--release")
if quiet:
args.append("-q")
args.extend(["--", '--crate-type', 'cdylib'])
# OSX requires special linker argument
if sys.platform == "darwin":
args.extend(
["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"])
if not quiet:
print(" ".join(args), file=sys.stderr)
# Execute cargo
try:
output = subprocess.check_output(args, env=env)
except subprocess.CalledProcessError as e:
raise CompileError("cargo failed with code: %d\n%s" %
(e.returncode, e.output))
except OSError:
raise DistutilsExecError(
"Unable to execute 'cargo' - this package "
"requires rust to be installed and cargo to be on the PATH")
if not quiet:
if isinstance(output, bytes):
output = output.decode('latin-1')
if output:
print(output, file=sys.stderr)
# Find the shared library that cargo hopefully produced and copy
# it into the build directory as if it were produced by build_ext.
if debug_build:
suffix = "debug"
else:
suffix = "release"
target_dir = os.path.join(os.path.dirname(ext.path), "target/", suffix)
if sys.platform == "win32":
wildcard_so = "*.dll"
elif sys.platform == "darwin":
wildcard_so = "*.dylib"
else:
wildcard_so = "*.so"
try:
dylib_path = glob.glob(os.path.join(target_dir, wildcard_so))[0]
except IndexError:
raise DistutilsExecError(
"rust build failed; unable to find any %s in %s" %
(wildcard_so, target_dir))
# Ask build_ext where the shared library would go if it had built it,
# then copy it there.
build_ext = self.get_finalized_command('build_ext')
build_ext.inplace = self.inplace
target_fname = ext.name
if target_fname is None:
target_fname = os.path.basename(
os.path.splitext(os.path.basename(dylib_path)[3:])[0])
ext_path = build_ext.get_ext_fullpath(target_fname)
try:
os.makedirs(os.path.dirname(ext_path))
except OSError:
pass
shutil.copyfile(dylib_path, ext_path)
def pyx2obj(pyxpath,
objpath=None,
destdir=None,
cwd=None,
include_dirs=None,
cy_kwargs=None,
cplus=None,
**kwargs):
"""
Convenience function
If cwd is specified, pyxpath and dst are taken to be relative
If only_update is set to `True` the modification time is checked
and compilation is only run if the source is newer than the
destination
Parameters
==========
pyxpath: str
Path to Cython source file.
objpath: str (optional)
Path to object file to generate.
destdir: str (optional)
Directory to put generated C file. When ``None``: directory of ``objpath``.
cwd: str (optional)
Working directory and root of relative paths.
include_dirs: iterable of path strings (optional)
Passed onto src2obj and via cy_kwargs['include_path']
to simple_cythonize.
cy_kwargs: dict (optional)
Keyword arguments passed onto `simple_cythonize`
cplus: bool (optional)
Indicate whether C++ is used. default: auto-detect using ``.util.pyx_is_cplus``.
compile_kwargs: dict
keyword arguments passed onto src2obj
Returns
=======
Absolute path of generated object file.
"""
assert pyxpath.endswith('.pyx')
cwd = cwd or '.'
objpath = objpath or '.'
destdir = destdir or os.path.dirname(objpath)
abs_objpath = get_abspath(objpath, cwd=cwd)
if os.path.isdir(abs_objpath):
pyx_fname = os.path.basename(pyxpath)
name, ext = os.path.splitext(pyx_fname)
objpath = os.path.join(objpath, name + objext)
cy_kwargs = cy_kwargs or {}
cy_kwargs['output_dir'] = cwd
if cplus is None:
cplus = pyx_is_cplus(pyxpath)
cy_kwargs['cplus'] = cplus
interm_c_file = simple_cythonize(pyxpath,
destdir=destdir,
cwd=cwd,
**cy_kwargs)
include_dirs = include_dirs or []
flags = kwargs.pop('flags', [])
needed_flags = ('-fwrapv', '-pthread', '-fPIC')
for flag in needed_flags:
if flag not in flags:
flags.append(flag)
options = kwargs.pop('options', [])
if kwargs.pop('strict_aliasing', False):
raise CompileError("Cython requires strict aliasing to be disabled.")
# Let's be explicit about standard
if cplus:
std = kwargs.pop('std', 'c++98')
else:
std = kwargs.pop('std', 'c99')
return src2obj(interm_c_file,
objpath=objpath,
cwd=cwd,
include_dirs=include_dirs,
flags=flags,
std=std,
options=options,
inc_py=True,
strict_aliasing=False,
**kwargs)
def compile(self,
sources,
output_dir=None,
macros=None,
include_dirs=None,
debug=0,
extra_preargs=None,
extra_postargs=None,
depends=None):
if not self.initialized:
self.initialize()
compile_info = self._setup_compile(output_dir, macros, include_dirs,
sources, depends, extra_postargs)
macros, objects, extra_postargs, pp_opts, build = compile_info
compile_opts = extra_preargs or []
compile_opts.append('/c')
if debug:
compile_opts.extend(self.compile_options_debug)
else:
compile_opts.extend(self.compile_options)
for obj in objects:
try:
src, ext = build[obj]
except KeyError:
continue
if debug:
# pass the full pathname to MSVC in debug mode,
# this allows the debugger to find the source file
# without asking the user to browse for it
src = os.path.abspath(src)
if ext in self._c_extensions:
input_opt = "/Tc" + src
elif ext in self._cpp_extensions:
input_opt = "/Tp" + src
elif ext in self._rc_extensions:
# compile .RC to .RES file
input_opt = src
output_opt = "/fo" + obj
try:
self.spawn([self.rc] + pp_opts + [output_opt] +
[input_opt])
except DistutilsExecError as msg:
raise CompileError(msg)
continue
elif ext in self._mc_extensions:
# Compile .MC to .RC file to .RES file.
# * '-h dir' specifies the directory for the
# generated include file
# * '-r dir' specifies the target directory of the
# generated RC file and the binary message resource
# it includes
#
# For now (since there are no options to change this),
# we use the source-directory for the include file and
# the build directory for the RC file and message
# resources. This works at least for win32all.
h_dir = os.path.dirname(src)
rc_dir = os.path.dirname(obj)
try:
# first compile .MC to .RC and .H file
self.spawn([self.mc] + ['-h', h_dir, '-r', rc_dir] + [src])
base, _ = os.path.splitext(os.path.basename(src))
rc_file = os.path.join(rc_dir, base + '.rc')
# then compile .RC to .RES file
self.spawn([self.rc] + ["/fo" + obj] + [rc_file])
except DistutilsExecError as msg:
raise CompileError(msg)
continue
else:
# how to handle this file?
raise CompileError("Don't know how to compile %s to %s" %
(src, obj))
output_opt = "/Fo" + obj
try:
self.spawn([self.cc] + compile_opts + pp_opts +
[input_opt, output_opt] + extra_postargs)
except DistutilsExecError as msg:
raise CompileError(msg)
return objects
def check_openmp_support():
"""Check whether OpenMP test code can be compiled and run"""
ccompiler = new_compiler()
customize_compiler(ccompiler)
if os.getenv('SKTIME_NO_OPENMP'):
# Build explicitly without OpenMP support
return False
start_dir = os.path.abspath('.')
with tempfile.TemporaryDirectory() as tmp_dir:
try:
os.chdir(tmp_dir)
# Write test program
with open('test_openmp.c', 'w') as f:
f.write(CCODE)
os.mkdir('objects')
# Compile, test program
openmp_flags = get_openmp_flag(ccompiler)
ccompiler.compile(['test_openmp.c'],
output_dir='objects',
extra_postargs=openmp_flags)
# Link test program
extra_preargs = os.getenv('LDFLAGS', None)
extra_preargs = extra_preargs.split(
" ") if extra_preargs is not None else []
objects = glob.glob(
os.path.join('objects', '*' + ccompiler.obj_extension))
ccompiler.link_executable(objects,
'test_openmp',
extra_preargs=extra_preargs,
extra_postargs=openmp_flags)
# Run test program
output = subprocess.check_output('./test_openmp')
output = output.decode(sys.stdout.encoding or 'utf-8').splitlines()
# Check test program output
if 'nthreads=' in output[0]:
nthreads = int(output[0].strip().split('=')[1])
openmp_supported = (len(output) == nthreads)
else:
openmp_supported = False
except (CompileError, LinkError, subprocess.CalledProcessError):
openmp_supported = False
finally:
os.chdir(start_dir)
err_message = textwrap.dedent("""
***
It seems that sktime cannot be built with OpenMP support.
- If your compiler supports OpenMP but the build still fails, please
submit a bug report at:
'https://github.com/alan-turing-institute/sktime/issues'
- If you want to build sktime without OpenMP support, you can set
the environment variable SKTIME_NO_OPENMP and rerun the build
command. Note however that some estimators will run in sequential
mode and their `n_jobs` parameter will have no effect anymore.
- See sktime advanced installation instructions for more info:
https://alan-turing-institute.github.io/sktime/installation
.html
***
""")
if not openmp_supported:
raise CompileError(err_message)
return True
def raise_compile_error(*args, **kwargs): # type: ignore
raise CompileError('Chalice blocked C extension compiling.')
def build_extension(self, ext):
executable = ext.binding == Binding.Exec
# Make sure that if pythonXX-sys is used, it builds against the current
# executing python interpreter.
bindir = os.path.dirname(sys.executable)
env = os.environ.copy()
env.update(
{
# disables rust's pkg-config seeking for specified packages,
# which causes pythonXX-sys to fall back to detecting the
# interpreter from the path.
"PATH": os.path.join(bindir, os.environ.get("PATH", ""))
}
)
# Find where to put the temporary build files created by `cargo`
metadata_command = [
"cargo",
"metadata",
"--manifest-path",
ext.path,
"--format-version",
"1",
]
# The decoding is needed for python 3.5 compatibility
metadata = json.loads(check_output(metadata_command).decode("utf-8"))
target_dir = metadata["target_directory"]
if not os.path.exists(ext.path):
raise DistutilsFileError(
"Can not find rust extension project file: %s" % ext.path
)
features = set(ext.features)
features.update(cpython_feature(binding=ext.binding))
debug_build = ext.debug if ext.debug is not None else self.inplace
debug_build = self.debug if self.debug is not None else debug_build
if self.release:
debug_build = False
quiet = self.qbuild or ext.quiet
# build cargo command
feature_args = ["--features", " ".join(features)] if features else []
if executable:
args = (
["cargo", "build", "--manifest-path", ext.path]
+ feature_args
+ list(ext.args or [])
)
if not debug_build:
args.append("--release")
if quiet:
args.append("-q")
elif self.verbose:
args.append("--verbose")
else:
args = (
["cargo", "rustc", "--lib", "--manifest-path", ext.path]
+ feature_args
+ list(ext.args or [])
)
if not debug_build:
args.append("--release")
if quiet:
args.append("-q")
elif self.verbose:
args.append("--verbose")
args.extend(["--", "--crate-type", "cdylib"])
args.extend(ext.rustc_flags or [])
# OSX requires special linker argument
if sys.platform == "darwin":
args.extend(
["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
)
if not quiet:
print(" ".join(args), file=sys.stderr)
if ext.native:
env["RUSTFLAGS"] = "-C target-cpu=native"
# Execute cargo
try:
output = subprocess.check_output(args, env=env)
except subprocess.CalledProcessError as e:
output = e.output
if isinstance(output, bytes):
output = e.output.decode("latin-1").strip()
raise CompileError(
"cargo failed with code: %d\n%s" % (e.returncode, output)
)
except OSError:
raise DistutilsExecError(
"Unable to execute 'cargo' - this package "
"requires rust to be installed and cargo to be on the PATH"
)
if not quiet:
if isinstance(output, bytes):
output = output.decode("latin-1")
if output:
print(output, file=sys.stderr)
# Find the shared library that cargo hopefully produced and copy
# it into the build directory as if it were produced by build_ext.
if debug_build:
suffix = "debug"
else:
suffix = "release"
# location of cargo compiled files
artifactsdir = os.path.join(target_dir, suffix)
dylib_paths = []
if executable:
for name, dest in ext.target.items():
if name:
path = os.path.join(artifactsdir, name)
if os.access(path, os.X_OK):
dylib_paths.append((dest, path))
continue
else:
raise DistutilsExecError(
"rust build failed; "
'unable to find executable "%s" in %s' % (name, target_dir)
)
else:
# search executable
for name in os.listdir(artifactsdir):
path = os.path.join(artifactsdir, name)
if name.startswith(".") or not os.path.isfile(path):
continue
if os.access(path, os.X_OK):
dylib_paths.append((ext.name, path))
break
if not dylib_paths:
raise DistutilsExecError(
"rust build failed; unable to find executable in %s" % target_dir
)
else:
if sys.platform == "win32":
dylib_ext = "dll"
elif sys.platform == "darwin":
dylib_ext = "dylib"
else:
dylib_ext = "so"
wildcard_so = "*{}.{}".format(ext.get_lib_name(), dylib_ext)
try:
dylib_paths.append(
(
ext.name,
next(glob.iglob(os.path.join(artifactsdir, wildcard_so))),
)
)
except StopIteration:
raise DistutilsExecError(
"rust build failed; unable to find any %s in %s"
% (wildcard_so, artifactsdir)
)
# Ask build_ext where the shared library would go if it had built it,
# then copy it there.
build_ext = self.get_finalized_command("build_ext")
build_ext.inplace = self.inplace
for target_fname, dylib_path in dylib_paths:
if not target_fname:
target_fname = os.path.basename(
os.path.splitext(os.path.basename(dylib_path)[3:])[0]
)
if executable:
ext_path = build_ext.get_ext_fullpath(target_fname)
# remove .so extension
ext_path, _ = os.path.splitext(ext_path)
# remove python3 extension (i.e. cpython-36m)
ext_path, _ = os.path.splitext(ext_path)
ext.install_script(ext_path)
else:
ext_path = build_ext.get_ext_fullpath(target_fname)
try:
os.makedirs(os.path.dirname(ext_path))
except OSError:
pass
shutil.copyfile(dylib_path, ext_path)
if sys.platform != "win32" and not debug_build:
args = []
if ext.strip == Strip.All:
args.append("-x")
elif ext.strip == Strip.Debug:
args.append("-S")
if args:
args.insert(0, "strip")
args.append(ext_path)
try:
output = subprocess.check_output(args, env=env)
except subprocess.CalledProcessError as e:
pass
if executable:
mode = os.stat(ext_path).st_mode
mode |= (mode & 0o444) >> 2 # copy R bits to X
os.chmod(ext_path, mode)
def compile(self, sources,
output_dir=None, macros=None, include_dirs=None, debug=0,
extra_preargs=None, extra_postargs=None, depends=None):
macros, objects, extra_postargs, pp_opts, build = \
self._setup_compile(output_dir, macros, include_dirs, sources,
depends, extra_postargs)
compile_opts = extra_preargs or []
compile_opts.append ('-c')
if debug:
compile_opts.extend (self.compile_options_debug)
else:
compile_opts.extend (self.compile_options)
for obj in objects:
try:
src, ext = build[obj]
except KeyError:
continue
# XXX why do the normpath here?
src = os.path.normpath(src)
obj = os.path.normpath(obj)
# XXX _setup_compile() did a mkpath() too but before the normpath.
# Is it possible to skip the normpath?
self.mkpath(os.path.dirname(obj))
if ext == '.res':
# This is already a binary file -- skip it.
continue # the 'for' loop
if ext == '.rc':
# This needs to be compiled to a .res file -- do it now.
try:
self.spawn (["brcc32", "-fo", obj, src])
except DistutilsExecError as msg:
raise CompileError(msg)
continue # the 'for' loop
# The next two are both for the real compiler.
if ext in self._c_extensions:
input_opt = ""
elif ext in self._cpp_extensions:
input_opt = "-P"
else:
# Unknown file type -- no extra options. The compiler
# will probably fail, but let it just in case this is a
# file the compiler recognizes even if we don't.
input_opt = ""
output_opt = "-o" + obj
# Compiler command line syntax is: "bcc32 [options] file(s)".
# Note that the source file names must appear at the end of
# the command line.
try:
self.spawn ([self.cc] + compile_opts + pp_opts +
[input_opt, output_opt] +
extra_postargs + [src])
except DistutilsExecError as msg:
raise CompileError(msg)
return objects
def run(self):
CleanCommand.run(self)
try:
subprocess.check_call(['scons', '--remove'])
except subprocess.CalledProcessError:
raise CompileError("Error while cleaning Python Extensions")
def build_extension(
self, ext: RustExtension, forced_target_triple: Optional[str]
) -> List["_BuiltModule"]:
target_info = self._detect_rust_target(forced_target_triple)
if target_info is not None:
target_triple = target_info.triple
cross_lib = target_info.cross_lib
linker = target_info.linker
# We're ignoring target_info.linker_args for now because we're not
# sure if they will always do the right thing. Might help with some
# of the OS-specific logic if it does.
else:
target_triple = None
cross_lib = None
linker = None
rustc_cfgs = _get_rustc_cfgs(target_triple)
env = _prepare_build_environment(cross_lib)
if not os.path.exists(ext.path):
raise DistutilsFileError(
f"can't find Rust extension project file: {ext.path}"
)
# Find where to put the temporary build files created by `cargo`
target_dir = _base_cargo_target_dir(ext)
if target_triple is not None:
target_dir = os.path.join(target_dir, target_triple)
quiet = self.qbuild or ext.quiet
debug = self._is_debug_build(ext)
cargo_args = self._cargo_args(
ext=ext, target_triple=target_triple, release=not debug, quiet=quiet
)
if ext._uses_exec_binding():
command = [self.cargo, "build", "--manifest-path", ext.path, *cargo_args]
else:
rustc_args = [
"--crate-type",
"cdylib",
]
if ext.rustc_flags is not None:
rustc_args.extend(ext.rustc_flags)
if linker is not None:
rustc_args.extend(["-C", "linker=" + linker])
# OSX requires special linker arguments
if sys.platform == "darwin":
ext_basename = os.path.basename(self.get_dylib_ext_path(ext, ext.name))
rustc_args.extend(
[
"-C",
f"link-args=-undefined dynamic_lookup -Wl,-install_name,@rpath/{ext_basename}",
]
)
if ext.native:
rustc_args.extend(["-C", "target-cpu=native"])
# Tell musl targets not to statically link libc. See
# https://github.com/rust-lang/rust/issues/59302 for details.
if rustc_cfgs.get("target_env") == "musl":
# This must go in the env otherwise rustc will refuse to build
# the cdylib, see https://github.com/rust-lang/cargo/issues/10143
MUSL_FLAGS = "-C target-feature=-crt-static"
rustflags = env.get("RUSTFLAGS")
if rustflags is not None:
env["RUSTFLAGS"] = f"{rustflags} {MUSL_FLAGS}"
else:
env["RUSTFLAGS"] = MUSL_FLAGS
# Include this in the command-line anyway, so that when verbose
# logging enabled the user will see that this flag is in use.
rustc_args.extend(MUSL_FLAGS.split())
command = [
self.cargo,
"rustc",
"--lib",
"--manifest-path",
ext.path,
*cargo_args,
"--",
*rustc_args,
]
if not quiet:
print(" ".join(command), file=sys.stderr)
# Execute cargo
try:
output = subprocess.check_output(command, env=env, encoding="latin-1")
except subprocess.CalledProcessError as e:
raise CompileError(f"cargo failed with code: {e.returncode}\n{e.output}")
except OSError:
raise DistutilsExecError(
"Unable to execute 'cargo' - this package "
"requires Rust to be installed and cargo to be on the PATH"
)
if not quiet:
if output:
print(output, file=sys.stderr)
# Find the shared library that cargo hopefully produced and copy
# it into the build directory as if it were produced by build_ext.
artifacts_dir = os.path.join(target_dir, "debug" if debug else "release")
dylib_paths = []
if ext._uses_exec_binding():
for name, dest in ext.target.items():
if not name:
name = dest.split(".")[-1]
exe = sysconfig.get_config_var("EXE")
if exe is not None:
name += exe
path = os.path.join(artifacts_dir, name)
if os.access(path, os.X_OK):
dylib_paths.append(_BuiltModule(dest, path))
else:
raise DistutilsExecError(
"Rust build failed; "
f"unable to find executable '{name}' in '{artifacts_dir}'"
)
else:
if sys.platform == "win32" or sys.platform == "cygwin":
dylib_ext = "dll"
elif sys.platform == "darwin":
dylib_ext = "dylib"
else:
dylib_ext = "so"
wildcard_so = "*{}.{}".format(ext.get_lib_name(), dylib_ext)
try:
dylib_paths.append(
_BuiltModule(
ext.name,
next(glob.iglob(os.path.join(artifacts_dir, wildcard_so))),
)
)
except StopIteration:
raise DistutilsExecError(
f"Rust build failed; unable to find any {wildcard_so} in {artifacts_dir}"
)
return dylib_paths
def compile(sources,
output_dir=None,
macros=None,
include_dirs=None,
debug=0,
extra_preargs=None,
extra_postargs=None,
depends=None):
if not self.initialized:
self.initialize()
compile_info = self._setup_compile(output_dir, macros,
include_dirs, sources, depends,
extra_postargs)
macros, objects, extra_postargs, pp_opts, build = compile_info
compile_opts = extra_preargs or []
compile_opts.append('/c')
if debug:
compile_opts.extend(self.compile_options_debug)
else:
compile_opts.extend(self.compile_options)
add_cpp_opts = False
for obj in objects:
try:
src, ext = build[obj]
except KeyError:
continue
if debug:
# pass the full pathname to MSVC in debug mode,
# this allows the debugger to find the source file
# without asking the user to browse for it
src = os.path.abspath(src)
if ext in self._c_extensions:
input_opt = "/Tc" + src
elif ext in self._cpp_extensions:
input_opt = "/Tp" + src
add_cpp_opts = True
elif ext in self._rc_extensions:
# compile .RC to .RES file
input_opt = src
output_opt = "/fo" + obj
try:
self.spawn([self.rc] + pp_opts +
[output_opt, input_opt])
except DistutilsExecError as msg:
raise CompileError(msg)
continue
elif ext in self._mc_extensions:
# Compile .MC to .RC file to .RES file.
# * '-h dir' specifies the directory for the
# generated include file
# * '-r dir' specifies the target directory of the
# generated RC file and the binary message resource
# it includes
#
# For now (since there are no options to change this),
# we use the source-directory for the include file and
# the build directory for the RC file and message
# resources. This works at least for win32all.
h_dir = os.path.dirname(src)
rc_dir = os.path.dirname(obj)
try:
# first compile .MC to .RC and .H file
self.spawn([self.mc, '-h', h_dir, '-r', rc_dir, src])
base, _ = os.path.splitext(os.path.basename(src))
rc_file = os.path.join(rc_dir, base + '.rc')
# then compile .RC to .RES file
self.spawn([self.rc, "/fo" + obj, rc_file])
except DistutilsExecError as msg:
raise CompileError(msg)
continue
elif ext in self._cuda_extensions:
input_opt = src
else:
# how to handle this file?
raise CompileError(
"Don't know how to compile {} to {}, ext {}".format(
src, obj, ext))
# release MT
if "/MD" in compile_opts:
pass #compile_opts[compile_opts.index("/MD")] = "/MT"
# for cuda compiler
if ext in self._cuda_extensions:
args = [CUDA['nvcc']]
args.append(input_opt)
args.append("-o=" + obj)
# suppress annoying unicode warnings
args.extend(["-Xcompiler", "/wd 4819"])
args.extend(
[_arg for _arg in pp_opts if _arg.startswith('-I')])
if isinstance(extra_postargs, dict):
args.extend(extra_postargs["nvcc"])
else:
args.extend(extra_postargs)
else:
args = [self.cc] + compile_opts + pp_opts
if add_cpp_opts:
args.append('/EHsc')
args.append(input_opt)
args.append("/Fo" + obj)
if isinstance(extra_postargs, dict):
args.extend(extra_postargs["cc"])
else:
args.extend(extra_postargs)
try:
# print('-----', args)
self.spawn(args)
except DistutilsExecError as msg:
print("-----", args)
raise CompileError(msg)
return objects
def run(self):
try:
subprocess.check_call(['scons'], shell=True)
except subprocess.CalledProcessError:
raise CompileError("Error while building Python Extensions")
self.extensions = []
def run(self):
print("[gymfc] starting Gazebo plugin build.")
if self.build_plugin() != 0:
raise CompileError("Failed to compile Gazebo plugin.")
print("[gymfc] plugins built successfully.")
DistutilsBuild.run(self)
def createMakefile():
say("==== Checking supported compiler options and available libraries ====\n"
)
LINK_FLAGS = LDFLAGS.split(
) # accumulate the linker flags that will be put to Makefile.local
COMPILE_FLAGS = CFLAGS.split(
) # same for the compilation of the shared library only
# default compilation flags for both the shared library and all example programs that use it
CXXFLAGS = ['-fPIC', '-Wall', '-O2']
# [1a]: check if a compiler exists at all
if not runCompiler():
raise CompileError(
"Could not locate a compiler (set CXX=... environment variable to override)"
)
# [1b]: test if OpenMP is supported (optional)
OMP_FLAG = '-fopenmp'
OMP_CODE = "#include <omp.h>\nint main(){\n#pragma omp parallel for\nfor(int i=0; i<16; i++);\n}\n"
if runCompiler(code=OMP_CODE,
flags=OMP_FLAG + ' -Werror -Wno-unknown-pragmas'):
CXXFLAGS += [OMP_FLAG]
else:
if not ask(
"Warning, OpenMP is not supported\n" +
"If you're compiling on MacOS with clang, you'd better install another compiler such as GCC\n"
+ "Do you want to continue without OpenMP? [Y/N] "):
exit(1)
# [1c]: test if C++11 is supported (optional)
CXX11_FLAG = '-std=c++11'
if runCompiler(flags=CXX11_FLAG):
CXXFLAGS += [CXX11_FLAG]
# [1d]: test the -march flag (optional, allows architecture-dependent compiler optimizations)
ARCH_FLAG = '-march=native'
ARCH_CODE = 'int main(int c, char** v) { double x=c*3.14; return x==42; }\n'
if runCompiler(code=ARCH_CODE, flags=ARCH_FLAG):
CXXFLAGS += [ARCH_FLAG]
else:
ARCH_FLAG = '-march=core2' # try a less ambitious option
if runCompiler(code=ARCH_CODE, flags=ARCH_FLAG):
CXXFLAGS += [ARCH_FLAG]
# [1e]: special treatment for Intel compiler to restore determinism in OpenMP-parallelized loops
INTEL_FLAG = '-qno-opt-dynamic-align'
if runCompiler(flags=INTEL_FLAG):
CXXFLAGS += [INTEL_FLAG]
# [2a]: check that NumPy is present (required by the python interface)
try:
import numpy
NUMPY_INC = '-I' + numpy.get_include()
except ImportError:
raise CompileError(
"NumPy is not present - python extension cannot be compiled")
# [2b]: find out the paths to Python.h and libpythonXX.{a,so} (this is rather tricky) and
# all other relevant compilation/linking flags needed to build a shared library that uses Python
PYTHON_INC = '-I' + sysconfig.get_python_inc()
# various other system libraries that are needed at link time
PYTHON_LIB_EXTRA = \
sysconfig.get_config_var('LIBS').split() + \
sysconfig.get_config_var('SYSLIBS').split() #+ ['-lz']
# try compiling a test code with the provided link flags (in particular, the name of Python library):
# check that a sample C++ program with embedded python compiles, links and runs properly
numAttempts = 0
def tryPythonCode(PYTHON_LIB):
# test code for a shared library
PYTEST_LIB_CODE = """
#include "Python.h"
#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
#include "numpy/arrayobject.h"
void bla() {PyRun_SimpleString("from distutils import sysconfig;print(sysconfig.PREFIX);");}
void run() {Py_Initialize();bla();Py_Finalize();}
PyMODINIT_FUNC
"""
if sys.version_info[0] == 2: # Python 2.6-2.7
PYTEST_LIB_CODE += """
initpytest42(void) {
Py_InitModule3("pytest42", NULL, "doc");
import_array();
bla();
}
"""
else: # Python 3.x
PYTEST_LIB_CODE += """
PyInit_pytest42(void) {
static PyModuleDef moduledef = {PyModuleDef_HEAD_INIT, "pytest42", "doc", -1, NULL};
PyObject* mod = PyModule_Create(&moduledef);
import_array1(mod);
bla();
return mod;
}
"""
# test code for a program that loads this shared library
PYTEST_EXE_CODE = 'extern void run();int main(){run();}\n'
PYTEST_LIB_NAME = './pytest42.so'
PYTEST_EXE_NAME = './pytest42.exe'
# try compiling the test shared library
if not runCompiler(code=PYTEST_LIB_CODE, \
flags=' '.join([PYTHON_INC, NUMPY_INC, '-shared', '-fPIC'] + PYTHON_LIB), \
dest=PYTEST_LIB_NAME):
return False # the program couldn't be compiled at all (try the next variant)
# if succeeded, compile the test program that uses this library
if not runCompiler(code=PYTEST_EXE_CODE, flags=PYTEST_LIB_NAME, dest=PYTEST_EXE_NAME) \
or not os.path.isfile(PYTEST_LIB_NAME) or not os.path.isfile(PYTEST_EXE_NAME):
return False # can't find compiled test program
resultexe = runProgram(PYTEST_EXE_NAME).rstrip()
# also try loading this shared library as an extension module
proc = subprocess.Popen(sys.executable+" -c 'import pytest42'", \
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
resultpy = proc.communicate()[0].decode().rstrip()
# clean up
os.remove(PYTEST_EXE_NAME)
os.remove(PYTEST_LIB_NAME)
# check if the results (reported library path prefix) are the same as we have in this script
if resultexe != sysconfig.PREFIX or resultpy != sysconfig.PREFIX:
# return a warning, the user may still wish to continue
return "Test program doesn't seem to use the same version of Python, "+\
"or the library path is reported incorrectly: \n"+\
"Expected: "+sysconfig.PREFIX+"\n"+\
"Received: "+resultexe+"\n"+\
"From py: "+resultpy+"\n"+\
"Should we continue the build (things may go wrong at a later stage)? [Y/N] "
return True # this combination of options seems reasonable...
# explore various possible combinations of file name and path to the python library...
def findPythonLib():
PLANB_LIB = None
# try linking against the static python library libpython**.a, if this does not succeed,
# try the shared library libpython**.so**
LIBNAMES = ['LIBRARY', 'LDLIBRARY', 'INSTSONAME']
for PYTHON_LIB_FILE in [sysconfig.get_config_var(x) for x in LIBNAMES]:
for PYTHON_LIB_PATH in [
sysconfig.get_config_var(x) for x in ['LIBPL', 'LIBDIR']
]:
PYTHON_LIB_FILEPATH = os.path.join(PYTHON_LIB_PATH,
PYTHON_LIB_FILE)
if os.path.isfile(PYTHON_LIB_FILEPATH):
# other libraries depend on whether this is a static or a shared python library
PYTHON_LIB = [PYTHON_LIB_FILEPATH] + PYTHON_LIB_EXTRA
if PYTHON_LIB_FILE.endswith(
'.a'
) and not sysconfig.get_config_var('PYTHONFRAMEWORK'):
PYTHON_LIB += sysconfig.get_config_var(
'LINKFORSHARED').split()
# the stack_size flag is problematic and needs to be removed
PYTHON_LIB = [
x for x in PYTHON_LIB
if not x.startswith('-Wl,-stack_size,')
]
result = tryPythonCode(PYTHON_LIB)
if result is True:
return PYTHON_LIB # successful compilation
if result: # not True, but a warning string
# test compiled, but with a version mismatch warning, store it as a backup option
PLANB_LIB = PYTHON_LIB
PLANB_ASK = result
if not PLANB_LIB is None and ask(
PLANB_ASK
): # the user wants to continue with the backup option
return PLANB_LIB
# if none of the above combinations worked, give up...
raise CompileError(
"Could not compile test program which uses libpython" +
sysconfig.get_config_var('VERSION'))
# [2c]: find the python library and other relevant linking flags
PYTHON_LIB = findPythonLib()
COMPILE_FLAGS += ['-DHAVE_PYTHON', PYTHON_INC, NUMPY_INC]
LINK_FLAGS += PYTHON_LIB
# [3]: check that GSL is present, and find out its version (required)
# try compiling a snippet of code into a shared library (tests if GSL has been compiled with -fPIC)
GSL_CODE = """#include <gsl/gsl_version.h>
#if not defined(GSL_MAJOR_VERSION) || (GSL_MAJOR_VERSION == 1) && (GSL_MINOR_VERSION < 15)
#error "GSL version is too old (need at least 1.15)"
#endif
void dummy(){}
"""
if runCompiler(code=GSL_CODE, flags='-fPIC -lgsl -lgslcblas -shared'):
# apparently the headers and libraries can be found in some standard location,
LINK_FLAGS += ['-lgsl', '-lgslcblas'] # so we only list their names
else:
if not ask("GSL library (required) is not found\n" +
"Should we try to download and compile it now? [Y/N] "):
exit(1)
distutils.dir_util.mkpath(EXTRAS_DIR)
os.chdir(EXTRAS_DIR)
say('Downloading GSL\n')
filename = 'gsl.tar.gz'
dirname = 'gsl-2.4'
try:
urlretrieve('ftp://ftp.gnu.org/gnu/gsl/gsl-2.4.tar.gz', filename)
if os.path.isfile(filename):
subprocess.call(['tar', '-zxf',
filename]) # unpack the archive
os.remove(filename) # remove the downloaded archive
if not os.path.isdir(dirname):
raise Exception("Error unpacking GSL")
except Exception as e:
raise CompileError(
str(e) + "\nError downloading GSL library, aborting...\n" +
"You may try to manually compile GSL and install it to " +
ROOT_DIR + "/" + EXTRAS_DIR + ", so that " +
"the header files are in " + EXTRAS_DIR +
"/include and library files - in " + EXTRAS_DIR + "/lib")
say('Compiling GSL (may take a few minutes)\n')
result = subprocess.call(
'(cd ' + dirname + '; ./configure --prefix=' + os.getcwd() +
' CFLAGS="-fPIC -O2" --enable-shared=no; make; make install) > gsl-install.log',
shell=True)
if result != 0 or not os.path.isfile('lib/libgsl.a'):
raise CompileError("GSL compilation failed (check " + EXTRAS_DIR +
"/gsl-install.log)")
distutils.dir_util.remove_tree(
dirname) # clean up source and build directories
COMPILE_FLAGS += ['-I' + EXTRAS_DIR + '/include']
LINK_FLAGS += [
EXTRAS_DIR + '/lib/libgsl.a', EXTRAS_DIR + '/lib/libgslcblas.a'
]
os.chdir(ROOT_DIR)
# [4]: test if Eigen library is present (optional)
if runCompiler(code='#include <Eigen/Core>\nint main(){}\n'):
COMPILE_FLAGS += ['-DHAVE_EIGEN']
else:
if ask("Eigen library (recommended) is not found\n" +
"Should we try to download it now (no compilation needed)? [Y/N] "
):
distutils.dir_util.mkpath(EXTRAS_DIR + '/include/unsupported')
os.chdir(EXTRAS_DIR)
say('Downloading Eigen\n')
filename = 'Eigen.zip'
dirname = 'eigen-git-mirror-3.3.4'
try:
urlretrieve(
'https://github.com/eigenteam/eigen-git-mirror/archive/3.3.4.zip',
filename)
if os.path.isfile(filename):
subprocess.call('unzip ' + filename + ' >/dev/null',
shell=True) # unpack the archive
if os.path.isdir(dirname):
distutils.dir_util.copy_tree(dirname + '/Eigen',
'include/Eigen',
verbose=False)
distutils.dir_util.copy_tree(
dirname + '/unsupported/Eigen',
'include/unsupported/Eigen',
verbose=False) # copy the headers
distutils.dir_util.remove_tree(
dirname) # and delete the rest
COMPILE_FLAGS += [
'-DHAVE_EIGEN', '-I' + EXTRAS_DIR + '/include'
]
os.remove(filename) # remove the downloaded archive
except:
pass # didn't succeed with Eigen
os.chdir(ROOT_DIR)
# [5a]: test if CVXOPT is present (optional)
try:
import cvxopt # import the python module
except: # import error or some other problem, might be corrected
if ask("CVXOPT library (needed only for Schwarzschild modelling) is not found\n"
"Should we try to install it now? [Y/N] "):
try:
import pip
pip.main(['install', '--user', 'cvxopt'])
except Exception as e:
say("Failed to install CVXOPT: " + str(e) + "\n")
# [5b]: if the cvxopt module is available in Python, make sure that we also have C header files
try:
import cvxopt # if this fails, skip cvxopt altogether
if runCompiler(code='#include <cvxopt.h>\nint main(){}\n',
flags=' '.join(['-c', PYTHON_INC, NUMPY_INC])):
COMPILE_FLAGS += ['-DHAVE_CVXOPT']
else:
# download the C header file if it does not appear to be present in a default location
distutils.dir_util.mkpath(EXTRAS_DIR + '/include')
say('Downloading CVXOPT header files\n')
try:
urlretrieve(
'https://raw.githubusercontent.com/cvxopt/cvxopt/master/src/C/cvxopt.h',
EXTRAS_DIR + '/include/cvxopt.h')
urlretrieve(
'https://raw.githubusercontent.com/cvxopt/cvxopt/master/src/C/blas_redefines.h',
EXTRAS_DIR + '/include/blas_redefines.h')
except:
pass # problems in downloading, skip it
if os.path.isfile(EXTRAS_DIR+'/include/cvxopt.h') and \
os.path.isfile(EXTRAS_DIR+'/include/blas_redefines.h'):
COMPILE_FLAGS += [
'-DHAVE_CVXOPT', '-I' + EXTRAS_DIR + '/include'
]
else:
say("Failed to download CVXOPT header files, this feature will not be available\n"
)
except:
pass # cvxopt wasn't available
# [6]: test if GLPK is present (optional)
if runCompiler(code='#include <glpk.h>\nint main(){}\n', flags='-lglpk'):
COMPILE_FLAGS += ['-DHAVE_GLPK']
LINK_FLAGS += ['-lglpk']
else:
say("GLPK library (optional) is not found\n")
# [7]: test if UNSIO is present (optional)
if runCompiler(code='#include <uns.h>\nint main(){}\n',
flags='-lunsio -lnemo'):
COMPILE_FLAGS += ['-DHAVE_UNSIO']
LINK_FLAGS += ['-lunsio', '-lnemo']
else:
say("UNSIO library (optional) is not found\n")
# [99]: put everything togeter and create Makefile.local
with open('Makefile.local', 'w') as f:
f.write(
"# set the default compiler if no value is found in the environment variables or among command-line arguments\n"
+ "ifeq ($(origin CXX),default)\nCXX = g++\nendif\n" +
"ifeq ($(origin FC), default)\nFC = gfortran\nendif\nLINK = $(CXX)\n"
+
"# compilation/linking flags for both the shared library and any programs that use it\n"
+ "CXXFLAGS += " + " ".join(compressList(CXXFLAGS)) + "\n" +
"# compilation flags for the shared library only (files in src/)\n"
+ "COMPILE_FLAGS += " + " ".join(compressList(COMPILE_FLAGS)) +
"\n" + "# linking flags for the shared library only\n" +
"LINK_FLAGS += " + " ".join(compressList(LINK_FLAGS)) + "\n")
from distutils.errors import CompileError
from distutils.command.build_py import build_py as BuildPyCommand
from distutils.command.build_ext import build_ext as BuildExtCommand
from distutils.command.clean import clean as CleanCommand
from distutils.cmd import Command
import distutils.extension
import subprocess
import os.path
import os
import fnmatch
try:
subprocess.check_call(['scons'], shell=True)
except subprocess.CalledProcessError:
raise CompileError("Error while building Python Extensions")
def remove_head_directories(path, heads=1):
def explode_path(path):
head, tail = os.path.split(path)
return explode_path(head) + [tail] \
if head and head != path else [head or tail]
exploded_path = explode_path(path)
if len(exploded_path) < (heads + 1):
return ''
else:
return os.path.join(*exploded_path[heads:])
def check_openmp_support():
"""Check whether OpenMP test code can be compiled and run"""
if "PYODIDE_PACKAGE_ABI" in os.environ:
# Pyodide doesn't support OpenMP
return False
code = textwrap.dedent("""\
#include <omp.h>
#include <stdio.h>
int main(void) {
#pragma omp parallel
printf("nthreads=%d\\n", omp_get_num_threads());
return 0;
}
""")
extra_preargs = os.getenv('LDFLAGS', None)
if extra_preargs is not None:
extra_preargs = extra_preargs.strip().split(" ")
extra_preargs = [
flag for flag in extra_preargs
if flag.startswith(('-L', '-Wl,-rpath', '-l'))
]
extra_postargs = get_openmp_flag
try:
output = compile_test_program(code,
extra_preargs=extra_preargs,
extra_postargs=extra_postargs)
if output and 'nthreads=' in output[0]:
nthreads = int(output[0].strip().split('=')[1])
openmp_supported = len(output) == nthreads
elif "PYTHON_CROSSENV" in os.environ:
# Since we can't run the test program when cross-compiling
# assume that openmp is supported if the program can be
# compiled.
openmp_supported = True
else:
openmp_supported = False
except (CompileError, LinkError, subprocess.CalledProcessError):
openmp_supported = False
if not openmp_supported:
if os.getenv("SKLEARN_FAIL_NO_OPENMP"):
raise CompileError("Failed to build with OpenMP")
else:
message = textwrap.dedent("""
***********
* WARNING *
***********
It seems that scikit-learn cannot be built with OpenMP.
- Make sure you have followed the installation instructions:
https://scikit-learn.org/dev/developers/advanced_installation.html
- If your compiler supports OpenMP but you still see this
message, please submit a bug report at:
https://github.com/scikit-learn/scikit-learn/issues
- The build will continue with OpenMP-based parallelism
disabled. Note however that some estimators will run in
sequential mode instead of leveraging thread-based
parallelism.
***
""")
warnings.warn(message)
return openmp_supported
def run(self):
if 'sdist' in self.distribution.commands:
return
if not self.extensions:
return
all_optional = all(ext.optional for ext in self.extensions)
try:
version = get_rust_version()
except DistutilsPlatformError as e:
if not all_optional:
raise
else:
print(str(e))
return
if version not in MIN_VERSION:
print('Rust version mismatch: required rust%s got rust%s' %
(MIN_VERSION, version))
return
# Make sure that if pythonXX-sys is used, it builds against the current
# executing python interpreter.
bindir = os.path.dirname(sys.executable)
env = os.environ.copy()
env.update({
# disables rust's pkg-config seeking for specified packages,
# which causes pythonXX-sys to fall back to detecting the
# interpreter from the path.
"PYTHON_2.7_NO_PKG_CONFIG": "1",
"PATH": bindir + os.pathsep + os.environ.get("PATH", "")
})
for ext in self.extensions:
try:
if not os.path.exists(ext.path):
raise DistutilsFileError(
"Can not file rust extension project file: %s" %
ext.path)
features = set(ext.features)
features.update(cpython_feature(binding=ext.binding))
# check cargo command
feature_args = ["--features", " ".join(features)
] if features else []
args = (["cargo", "check", "--manifest-path", ext.path] +
feature_args + list(ext.args or []))
# Execute cargo command
try:
subprocess.check_output(args)
except subprocess.CalledProcessError as e:
raise CompileError(
"cargo failed with code: %d\n%s" %
(e.returncode, e.output.decode("utf-8")))
except OSError:
raise DistutilsExecError(
"Unable to execute 'cargo' - this package "
"requires rust to be installed and "
"cargo to be on the PATH")
else:
print("Extension '%s' checked" % ext.name)
except (DistutilsFileError, DistutilsExecError, CompileError) as e:
if not ext.optional:
raise
else:
print('Check optional Rust extension %s failed.' %
ext.name)
print(str(e))
def build_extension(self, ext):
executable = ext.binding == Binding.Exec
rust_target_info = get_rust_target_info()
# Make sure that if pythonXX-sys is used, it builds against the current
# executing python interpreter.
bindir = os.path.dirname(sys.executable)
env = os.environ.copy()
env.update({
# disables rust's pkg-config seeking for specified packages,
# which causes pythonXX-sys to fall back to detecting the
# interpreter from the path.
"PATH": os.path.join(bindir, os.environ.get("PATH", "")),
"PYTHON_SYS_EXECUTABLE": sys.executable,
"PYO3_PYTHON": sys.executable,
})
rustflags = ""
# If we are on a 64-bit machine, but running a 32-bit Python, then
# we'll target a 32-bit Rust build.
# TODO: include --target for all platforms so env vars can't break the build
target_triple = None
target_args = []
if platform.machine() == "AMD64" and self.plat_name == "win32":
target_triple = "i686-pc-windows-msvc"
target_args = ["--target", target_triple]
# Find where to put the temporary build files created by `cargo`
metadata_command = [
"cargo",
"metadata",
"--manifest-path",
ext.path,
"--format-version",
"1",
]
# The decoding is needed for python 3.5 compatibility
metadata = json.loads(check_output(metadata_command).decode("utf-8"))
target_dir = metadata["target_directory"]
if not os.path.exists(ext.path):
raise DistutilsFileError(
"Can not find rust extension project file: %s" % ext.path)
features = set(ext.features)
features.update(cpython_feature(binding=ext.binding))
debug_build = ext.debug if ext.debug is not None else self.inplace
debug_build = self.debug if self.debug is not None else debug_build
if self.release:
debug_build = False
quiet = self.qbuild or ext.quiet
# build cargo command
feature_args = ["--features", " ".join(features)] if features else []
if executable:
args = (["cargo", "build", "--manifest-path", ext.path] +
feature_args + target_args + list(ext.args or []))
if not debug_build:
args.append("--release")
if quiet:
args.append("-q")
elif self.verbose:
args.append("--verbose")
else:
args = (["cargo", "rustc", "--lib", "--manifest-path", ext.path] +
feature_args + target_args + list(ext.args or []))
if not debug_build:
args.append("--release")
if quiet:
args.append("-q")
elif self.verbose:
args.append("--verbose")
args.extend(["--", "--crate-type", "cdylib"])
args.extend(ext.rustc_flags or [])
# OSX requires special linker argument
if sys.platform == "darwin":
args.extend([
"-C", "link-arg=-undefined", "-C",
"link-arg=dynamic_lookup"
])
# Tell musl targets not to statically link libc. See
# https://github.com/rust-lang/rust/issues/59302 for details.
if b'target_env="musl"' in rust_target_info:
rustflags += " -C target-feature=-crt-static"
if not quiet:
print(" ".join(args), file=sys.stderr)
if ext.native:
rustflags += " -C target-cpu=native"
if rustflags:
env["RUSTFLAGS"] = rustflags
# Execute cargo
try:
output = subprocess.check_output(args, env=env)
except subprocess.CalledProcessError as e:
output = e.output
if isinstance(output, bytes):
output = e.output.decode("latin-1").strip()
raise CompileError("cargo failed with code: %d\n%s" %
(e.returncode, output))
except OSError:
raise DistutilsExecError(
"Unable to execute 'cargo' - this package "
"requires rust to be installed and cargo to be on the PATH")
if not quiet:
if isinstance(output, bytes):
output = output.decode("latin-1")
if output:
print(output, file=sys.stderr)
# Find the shared library that cargo hopefully produced and copy
# it into the build directory as if it were produced by build_ext.
if debug_build:
suffix = "debug"
else:
suffix = "release"
# location of cargo compiled files
artifactsdir = os.path.join(target_dir, target_triple or "", suffix)
dylib_paths = []
if executable:
for name, dest in ext.target.items():
if name:
path = os.path.join(artifactsdir, name)
if os.access(path, os.X_OK):
dylib_paths.append((dest, path))
continue
else:
raise DistutilsExecError(
"rust build failed; "
'unable to find executable "%s" in %s' %
(name, target_dir))
else:
# search executable
for name in os.listdir(artifactsdir):
path = os.path.join(artifactsdir, name)
if name.startswith(".") or not os.path.isfile(path):
continue
if os.access(path, os.X_OK):
dylib_paths.append((ext.name, path))
break
if not dylib_paths:
raise DistutilsExecError(
"rust build failed; unable to find executable in %s" %
target_dir)
else:
if sys.platform == "win32" or sys.platform == "cygwin":
dylib_ext = "dll"
elif sys.platform == "darwin":
dylib_ext = "dylib"
else:
dylib_ext = "so"
wildcard_so = "*{}.{}".format(ext.get_lib_name(), dylib_ext)
try:
dylib_paths.append((
ext.name,
next(glob.iglob(os.path.join(artifactsdir, wildcard_so))),
))
except StopIteration:
raise DistutilsExecError(
"rust build failed; unable to find any %s in %s" %
(wildcard_so, artifactsdir))
# Ask build_ext where the shared library would go if it had built it,
# then copy it there.
build_ext = self.get_finalized_command("build_ext")
build_ext.inplace = self.inplace
for target_fname, dylib_path in dylib_paths:
if not target_fname:
target_fname = os.path.basename(
os.path.splitext(os.path.basename(dylib_path)[3:])[0])
if executable:
ext_path = build_ext.get_ext_fullpath(target_fname)
# remove .so extension
ext_path, _ = os.path.splitext(ext_path)
# remove python3 extension (i.e. cpython-36m)
ext_path, _ = os.path.splitext(ext_path)
ext.install_script(ext_path)
else:
# Technically it's supposed to contain a
# `setuptools.Extension`, but in practice the only attribute it
# checks is `ext.py_limited_api`.
modpath = target_fname.split('.')[-1]
assert modpath not in build_ext.ext_map
build_ext.ext_map[modpath] = ext
try:
ext_path = build_ext.get_ext_fullpath(target_fname)
finally:
del build_ext.ext_map[modpath]
try:
os.makedirs(os.path.dirname(ext_path))
except OSError:
pass
shutil.copyfile(dylib_path, ext_path)
if sys.platform != "win32" and not debug_build:
args = []
if ext.strip == Strip.All:
args.append("-x")
elif ext.strip == Strip.Debug:
args.append("-S")
if args:
args.insert(0, "strip")
args.append(ext_path)
try:
output = subprocess.check_output(args, env=env)
except subprocess.CalledProcessError as e:
pass
# executables and win32(cygwin)-dll's need X bits
if executable or sys.platform == "win32" or sys.platform == "cygwin":
mode = os.stat(ext_path).st_mode
mode |= (mode & 0o444) >> 2 # copy R bits to X
os.chmod(ext_path, mode)
def compile(self, sources,
output_dir=None, macros=None, include_dirs=None, debug=0,
extra_preargs=None, extra_postargs=None, depends=None):
if not self.initialized:
self.initialize()
compile_info = self._setup_compile(output_dir, macros, include_dirs,
sources, depends, extra_postargs)
macros, objects, extra_postargs, pp_opts, build = compile_info
compile_opts = extra_preargs or []
compile_opts.append('/c')
if debug:
compile_opts.extend(self.compile_options_debug)
else:
compile_opts.extend(self.compile_options)
add_cpp_opts = False
for obj in objects:
try:
src, ext = build[obj]
except KeyError:
continue
if debug:
# pass the full pathname to MSVC in debug mode,
# this allows the debugger to find the source file
# without asking the user to browse for it
src = os.path.abspath(src)
# Anaconda/conda-forge customisation, we want our pdbs to be
# relocatable:
# https://developercommunity.visualstudio.com/comments/623156/view.html
d1trimfile_opts = []
if 'SRC_DIR' in os.environ:
d1trimfile_opts.append("/d1trimfile:" + os.environ['SRC_DIR'])
if ext in self._c_extensions:
input_opt = "/Tc" + src
elif ext in self._cpp_extensions:
input_opt = "/Tp" + src
add_cpp_opts = True
elif ext in self._rc_extensions:
# compile .RC to .RES file
input_opt = src
output_opt = "/fo" + obj
try:
self.spawn([self.rc] + pp_opts + [output_opt, input_opt])
except DistutilsExecError as msg:
raise CompileError(msg)
continue
elif ext in self._mc_extensions:
# Compile .MC to .RC file to .RES file.
# * '-h dir' specifies the directory for the
# generated include file
# * '-r dir' specifies the target directory of the
# generated RC file and the binary message resource
# it includes
#
# For now (since there are no options to change this),
# we use the source-directory for the include file and
# the build directory for the RC file and message
# resources. This works at least for win32all.
h_dir = os.path.dirname(src)
rc_dir = os.path.dirname(obj)
try:
# first compile .MC to .RC and .H file
self.spawn([self.mc, '-h', h_dir, '-r', rc_dir, src])
base, _ = os.path.splitext(os.path.basename (src))
rc_file = os.path.join(rc_dir, base + '.rc')
# then compile .RC to .RES file
self.spawn([self.rc, "/fo" + obj, rc_file])
except DistutilsExecError as msg:
raise CompileError(msg)
continue
else:
# how to handle this file?
raise CompileError("Don't know how to compile {} to {}"
.format(src, obj))
args = [self.cc] + compile_opts + pp_opts + d1trimfile_opts
if add_cpp_opts:
args.append('/EHsc')
args.append(input_opt)
args.append("/Fo" + obj)
args.extend(extra_postargs)
try:
self.spawn(args)
except DistutilsExecError as msg:
raise CompileError(msg)
return objects
def __init__(self,
sources,
out,
flags=None,
run_linker=True,
compiler=None,
cwd='.',
include_dirs=None,
libraries=None,
library_dirs=None,
std=None,
define=None,
undef=None,
strict_aliasing=None,
preferred_vendor=None,
**kwargs):
if isinstance(sources, str):
raise ValueError(
"Expected argument sources to be a list of strings.")
self.sources = list(sources)
self.out = out
self.flags = flags or []
self.cwd = cwd
if compiler:
self.compiler_name, self.compiler_binary = compiler
else:
# Find a compiler
if preferred_vendor is None:
preferred_vendor = os.environ.get('SYMPY_COMPILER_VENDOR',
None)
self.compiler_name, self.compiler_binary, self.compiler_vendor = self.find_compiler(
preferred_vendor)
if self.compiler_binary is None:
raise ValueError("No compiler found (searched: {})".format(
', '.join(self.compiler_dict.values())))
self.define = define or []
self.undef = undef or []
self.include_dirs = include_dirs or []
self.libraries = libraries or []
self.library_dirs = library_dirs or []
self.std = std or self.standards[0]
self.run_linker = run_linker
if self.run_linker:
# both gnu and intel compilers use '-c' for disabling linker
self.flags = list(filter(lambda x: x != '-c', self.flags))
else:
if '-c' not in self.flags:
self.flags.append('-c')
if self.std:
self.flags.append(self.std_formater[self.compiler_name](self.std))
self.linkline = []
if strict_aliasing is not None:
nsa_re = re.compile("no-strict-aliasing$")
sa_re = re.compile("strict-aliasing$")
if strict_aliasing is True:
if any(map(nsa_re.match, flags)):
raise CompileError(
"Strict aliasing cannot be both enforced and disabled")
elif any(map(sa_re.match, flags)):
pass # already enforced
else:
flags.append('-fstrict-aliasing')
elif strict_aliasing is False:
if any(map(nsa_re.match, flags)):
pass # already disabled
else:
if any(map(sa_re.match, flags)):
raise CompileError(
"Strict aliasing cannot be both enforced and disabled"
)
else:
flags.append('-fno-strict-aliasing')
else:
msg = "Expected argument strict_aliasing to be True/False, got {}"
raise ValueError(msg.format(strict_aliasing))
def check_openmp_support():
"""Check whether OpenMP test code can be compiled and run"""
code = textwrap.dedent("""\
#include <omp.h>
#include <stdio.h>
int main(void) {
#pragma omp parallel
printf("nthreads=%d\\n", omp_get_num_threads());
return 0;
}
""")
extra_preargs = os.getenv("LDFLAGS", None)
if extra_preargs is not None:
extra_preargs = extra_preargs.strip().split(" ")
extra_preargs = [
flag for flag in extra_preargs
if flag.startswith(("-L", "-Wl,-rpath", "-l"))
]
extra_postargs = get_openmp_flag
try:
output = compile_test_program(code,
extra_preargs=extra_preargs,
extra_postargs=extra_postargs)
if "nthreads=" in output[0]:
nthreads = int(output[0].strip().split("=")[1])
openmp_supported = len(output) == nthreads
else:
openmp_supported = False
except (CompileError, LinkError, subprocess.CalledProcessError):
openmp_supported = False
if not openmp_supported:
if os.getenv("NNETSAUCE_FAIL_NO_OPENMP"):
raise CompileError("Failed to build with OpenMP")
else:
message = textwrap.dedent("""
***********
* WARNING *
***********
It seems that nnetsauce cannot be built with OpenMP.
- Make sure you have followed the installing instructions:
https://github.com/thierrymoudiki/nnetsauce
- If your compiler supports OpenMP but you still see this
message, please submit a bug report at:
https://github.com/thierrymoudiki/nnetsauce/issues
- The build will continue with OpenMP-based parallelism
disabled. Note however that some estimators will run in
sequential mode instead of leveraging thread-based
parallelism.
***
""")
warnings.warn(message)
return openmp_supported
def __init__(self, compiler):
log.debug("Compiler include_dirs: %s" % compiler.include_dirs)
if hasattr(compiler, "initialize"):
compiler.initialize() # to set all variables
log.debug("Compiler include_dirs after initialize: %s" %
compiler.include_dirs)
self.compiler = compiler
log.debug(
sys.version) # contains the compiler used to build this python
# members with the info for the outside world
self.include_dirs = get_include_dirs()
self.objects = []
self.libraries = []
self.library_dirs = get_library_dirs()
self.linker_flags = []
self.compile_time_env = {}
if self.compiler.compiler_type == 'msvc':
if (sys.version_info.major, sys.version_info.minor) < (3, 3):
# The check above is a nasty hack. We're using the python
# version as a proxy for the MSVC version. 2008 doesn't
# have stdint.h, so is needed. 2010 does.
#
# We need to add the path to msvc includes
msvc_2008_path = (os.path.join(os.getcwd(), 'include',
'msvc_2008'))
self.include_dirs.append(msvc_2008_path)
elif (sys.version_info.major, sys.version_info.minor) < (3, 5):
# Actually, it seems that appveyor doesn't have a stdint that
# works, so even for 2010 we use our own (hacked) version
# of stdint.
# This should be pretty safe in whatever case.
msvc_2010_path = (os.path.join(os.getcwd(), 'include',
'msvc_2010'))
self.include_dirs.append(msvc_2010_path)
# To avoid http://bugs.python.org/issue4431
#
# C:\Program Files\Microsoft
# SDKs\Windows\v7.1\Bin\x64\mt.exe -nologo -manifest
# C:\Users\appveyor\AppData\Local\Temp\1\pyfftw-9in6l66u\a.out.exe.manifest
# -outputresource:C:\Users\appveyor\AppData\Local\Temp\1\pyfftw-9in6l66u\a.out.exe;1
# C:\Users\appveyor\AppData\Local\Temp\1\pyfftw-9in6l66u\a.out.exe.manifest
# : general error c1010070: Failed to load and parse
# the manifest. The system cannot find the file
# specified.
self.compiler.ldflags_shared.append('/MANIFEST')
if get_platform().startswith('linux'):
# needed at least libm for linker checks to succeed
self.libraries.append('m')
# main fftw3 header is required
if not self.has_header(['fftw3.h'], include_dirs=self.include_dirs):
raise CompileError("Could not find the FFTW header 'fftw3.h'")
# mpi is optional
# self.support_mpi = self.has_header(['mpi.h', 'fftw3-mpi.h'])
# TODO enable check when wrappers are included in Pyfftw
self.support_mpi = False
if self.support_mpi:
try:
import mpi4py
self.include_dirs.append(mpi4py.get_include())
except ImportError:
log.error(
"Could not import mpi4py. Skipping support for FFTW MPI.")
self.support_mpi = False
self.search_dependencies()
class CygwinCCompiler (UnixCCompiler):
compiler_type = 'cygwin'
obj_extension = ".o"
static_lib_extension = ".a"
shared_lib_extension = ".dll"
dylib_lib_extension = ".dll.a"
static_lib_format = "lib%s%s"
shared_lib_format = "%s%s"
exe_extension = ".exe"
def __init__ (self, verbose=0, dry_run=0, force=0):
UnixCCompiler.__init__ (self, verbose, dry_run, force)
(status, details) = check_config_h()
self.debug_print("Python's GCC status: %s (details: %s)" %
(status, details))
if status is not CONFIG_H_OK:
self.warn(
"Python's pyconfig.h doesn't seem to support your compiler. "
"Reason: %s. "
"Compiling may fail because of undefined preprocessor macros."
% details)
self.gcc_version, self.ld_version, self.dllwrap_version = \
get_versions()
self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" %
(self.gcc_version,
self.ld_version,
self.dllwrap_version) )
# ld_version >= "2.10.90" and < "2.13" should also be able to use
# gcc -mdll instead of dllwrap
# Older dllwraps had own version numbers, newer ones use the
# same as the rest of binutils ( also ld )
# dllwrap 2.10.90 is buggy
if self.ld_version >= "2.10.90":
self.linker_dll = "gcc"
else:
self.linker_dll = "dllwrap"
# ld_version >= "2.13" support -shared so use it instead of
# -mdll -static
if self.ld_version >= "2.13":
shared_option = "-shared"
else:
shared_option = "-mdll -static"
# Hard-code GCC because that's what this is all about.
# XXX optimization, warnings etc. should be customizable.
self.set_executables(compiler='gcc -mcygwin -O -Wall',
compiler_so='gcc -mcygwin -mdll -O -Wall',
compiler_cxx='g++ -mcygwin -O -Wall',
linker_exe='gcc -mcygwin',
linker_so=('%s -mcygwin %s' %
(self.linker_dll, shared_option)))
# cygwin and mingw32 need different sets of libraries
if self.gcc_version == "2.91.57":
# cygwin shouldn't need msvcrt, but without the dlls will crash
# (gcc version 2.91.57) -- perhaps something about initialization
self.dll_libraries=["msvcrt"]
self.warn(
"Consider upgrading to a newer version of gcc")
else:
# Include the appropriate MSVC runtime library if Python was built
# with MSVC 7.0 or later.
self.dll_libraries = get_msvcr()
# __init__ ()
def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts):
if ext == '.rc' or ext == '.res':
# gcc needs '.res' and '.rc' compiled to object files !!!
try:
self.spawn(["windres", "-i", src, "-o", obj])
except DistutilsExecError, msg:
raise CompileError, msg
elif ext == '.mc':
# Adapted from msvc9compiler:
#
# Compile .MC to .RC file to .RES file.
# * '-h dir' specifies the directory for the generated include file
# * '-r dir' specifies the target directory of the generated RC file and the binary message resource it includes
#
# For now (since there are no options to change this),
# we use the source-directory for the include file and
# the build directory for the RC file and message
# resources. This works at least for win32all.
h_dir = os.path.dirname(src)
rc_dir = os.path.dirname(obj)
try:
# first compile .MC to .RC and .H file
self.spawn(['windmc'] + ['-h', h_dir, '-r', rc_dir] + [src])
base, _ = os.path.splitext (os.path.basename (src))
rc_file = os.path.join (rc_dir, base + '.rc')
# then compile .RC to .RES file
self.spawn(['windres', '-i', rc_file, '-o', obj])
except DistutilsExecError, msg:
raise CompileError(msg)
def compile(sources,
output_dir=None, macros=None, include_dirs=None, debug=0,
extra_preargs=None, extra_postargs=None, depends=None):
if not self.initialized:
self.initialize()
compile_info = self._setup_compile(output_dir, macros, include_dirs,
sources, depends, extra_postargs)
macros, objects, extra_postargs, pp_opts, build = compile_info
compile_opts = extra_preargs or []
compile_opts.append('/c')
if debug:
compile_opts.extend(self.compile_options_debug)
else:
compile_opts.extend(self.compile_options)
add_cpp_opts = False
for obj in objects:
try:
src, ext = build[obj]
except KeyError:
continue
if debug:
# pass the full pathname to MSVC in debug mode,
# this allows the debugger to find the source file
# without asking the user to browse for it
src = os.path.abspath(src)
if ext in self._c_extensions:
input_opt = "/Tc" + src
elif ext in self._cpp_extensions:
input_opt = "/Tp" + src
add_cpp_opts = True
elif ext in self._rc_extensions:
# compile .RC to .RES file
input_opt = src
output_opt = "/fo" + obj
try:
self.spawn([self.rc] + pp_opts + [output_opt, input_opt])
except DistutilsExecError as msg:
raise CompileError(msg)
continue
elif ext in self._mc_extensions:
# Compile .MC to .RC file to .RES file.
# * '-h dir' specifies the directory for the
# generated include file
# * '-r dir' specifies the target directory of the
# generated RC file and the binary message resource
# it includes
#
# For now (since there are no options to change this),
# we use the source-directory for the include file and
# the build directory for the RC file and message
# resources. This works at least for win32all.
h_dir = os.path.dirname(src)
rc_dir = os.path.dirname(obj)
try:
# first compile .MC to .RC and .H file
self.spawn([self.mc, '-h', h_dir, '-r', rc_dir, src])
base, _ = os.path.splitext(os.path.basename(src))
rc_file = os.path.join(rc_dir, base + '.rc')
# then compile .RC to .RES file
self.spawn([self.rc, "/fo" + obj, rc_file])
except DistutilsExecError as msg:
raise CompileError(msg)
continue
elif ext == '.cu':
# a trigger for cu compile
try:
# use the cuda for .cu files
# self.set_executable('compiler_so', CUDA['nvcc'])
# use only a subset of the extra_postargs, which are 1-1 translated
# from the extra_compile_args in the Extension class
postargs = extra_postargs['nvcc']
arg = [CUDA['nvcc']] + sources + ['-odir', pjoin(output_dir,'nms')]
#arg = [CUDA['nvcc']] + sources
for include_dir in include_dirs:
arg.append('-I')
arg.append(include_dir)
arg += ['-I', py_include]
# arg += ['-lib', CUDA['lib64']]
arg += ['-Xcompiler', '/EHsc,/W3,/nologo,/Ox,/MD']
arg += postargs
self.spawn(arg)
continue
except DistutilsExecError as msg:
# raise CompileError(msg)
continue
else:
# how to handle this file?
raise CompileError("Don't know how to compile {} to {}"
.format(src, obj))
args = [self.cc] + compile_opts + pp_opts
if add_cpp_opts:
args.append('/EHsc')
args.append(input_opt)
args.append("/Fo" + obj)
args.extend(extra_postargs)
try:
self.spawn(args)
except DistutilsExecError as msg:
raise CompileError(msg)
return objects
def raise_compile_error(*args, **kwargs):
raise CompileError("Prevented C extension compiling.")
def src2obj(srcpath,
Runner=None,
objpath=None,
cwd=None,
inc_py=False,
**kwargs):
""" Compiles a source code file to an object file.
Files ending with '.pyx' assumed to be cython files and
are dispatched to pyx2obj.
Parameters
==========
srcpath: str
Path to source file.
Runner: CompilerRunner subclass (optional)
If ``None``: deduced from extension of srcpath.
objpath : str (optional)
Path to generated object. If ``None``: deduced from ``srcpath``.
cwd: str (optional)
Working directory and root of relative paths. If ``None``: current dir.
inc_py: bool
Add Python include path to kwarg "include_dirs". Default: False
\\*\\*kwargs: dict
keyword arguments passed to Runner or pyx2obj
"""
name, ext = os.path.splitext(os.path.basename(srcpath))
if objpath is None:
if os.path.isabs(srcpath):
objpath = '.'
else:
objpath = os.path.dirname(srcpath)
objpath = objpath or '.' # avoid objpath == ''
if os.path.isdir(objpath):
objpath = os.path.join(objpath, name + objext)
include_dirs = kwargs.pop('include_dirs', [])
if inc_py:
from distutils.sysconfig import get_python_inc
py_inc_dir = get_python_inc()
if py_inc_dir not in include_dirs:
include_dirs.append(py_inc_dir)
if ext.lower() == '.pyx':
return pyx2obj(srcpath,
objpath=objpath,
include_dirs=include_dirs,
cwd=cwd,
**kwargs)
if Runner is None:
Runner, std = extension_mapping[ext.lower()]
if 'std' not in kwargs:
kwargs['std'] = std
flags = kwargs.pop('flags', [])
needed_flags = ('-fPIC', )
for flag in needed_flags:
if flag not in flags:
flags.append(flag)
# src2obj implies not running the linker...
run_linker = kwargs.pop('run_linker', False)
if run_linker:
raise CompileError("src2obj called with run_linker=True")
runner = Runner([srcpath],
objpath,
include_dirs=include_dirs,
run_linker=run_linker,
cwd=cwd,
flags=flags,
**kwargs)
runner.run()
return objpath
def _single_compile(obj):
try:
src, ext = build[obj]
except KeyError:
return
add_cpp_opts = False
if debug:
# pass the full pathname to MSVC in debug mode,
# this allows the debugger to find the source file
# without asking the user to browse for it
src = os.path.abspath(src)
if ext in self._c_extensions:
input_opt = "/Tc" + src
elif ext in self._cpp_extensions:
input_opt = "/Tp" + src
add_cpp_opts = True
elif ext in self._rc_extensions:
# compile .RC to .RES file
input_opt = src
output_opt = "/fo" + obj
try:
self.spawn([self.rc] + pp_opts +
[output_opt] + [input_opt])
except DistutilsExecError as msg:
raise CompileError(msg)
return
elif ext in self._mc_extensions:
# Compile .MC to .RC file to .RES file.
h_dir = os.path.dirname(src)
rc_dir = os.path.dirname(obj)
try:
# first compile .MC to .RC and .H file
self.spawn([self.mc] +
['-h', h_dir, '-r', rc_dir] + [src])
base, _ = os.path.splitext (os.path.basename (src))
rc_file = os.path.join (rc_dir, base + '.rc')
# then compile .RC to .RES file
self.spawn([self.rc] +
["/fo" + obj] + [rc_file])
except DistutilsExecError as msg:
raise CompileError(msg)
return
else:
# how to handle this file?
raise CompileError("Don't know how to compile %s to %s"
% (src, obj))
args = [self.cc] + compile_opts + pp_opts
if add_cpp_opts:
args.append('/EHsc')
args.append(input_opt)
args.append("/Fo" + obj)
args.extend(extra_postargs)
try:
msvc_spawn_and_write_d_file(obj, src, args, env, self.dry_run)
except DistutilsExecError as msg:
raise CompileError(msg)
def compile(self,
sources,
output_dir=None,
macros=None,
include_dirs=None,
debug=0,
extra_preargs=None,
extra_postargs=None,
depends=None):
if not self.initialized:
self.initialize()
compile_info = self._setup_compile(output_dir, macros, include_dirs,
sources, depends, extra_postargs)
macros, objects, extra_postargs, pp_opts, build = compile_info
compile_opts = extra_preargs or []
compile_opts.append('/c')
if debug:
compile_opts.extend(self.compile_options_debug)
else:
compile_opts.extend(self.compile_options)
for obj in objects:
try:
src, ext = build[obj]
except KeyError:
continue
if debug:
src = os.path.abspath(src)
if ext in self._c_extensions:
input_opt = '/Tc' + src
elif ext in self._cpp_extensions:
input_opt = '/Tp' + src
elif ext in self._rc_extensions:
input_opt = src
output_opt = '/fo' + obj
try:
self.spawn([self.rc] + pp_opts + [output_opt] +
[input_opt])
except DistutilsExecError as msg:
raise CompileError(msg)
continue
elif ext in self._mc_extensions:
h_dir = os.path.dirname(src)
rc_dir = os.path.dirname(obj)
try:
self.spawn([self.mc] + ['-h', h_dir, '-r', rc_dir] + [src])
base, _ = os.path.splitext(os.path.basename(src))
rc_file = os.path.join(rc_dir, base + '.rc')
self.spawn([self.rc] + ['/fo' + obj] + [rc_file])
except DistutilsExecError as msg:
raise CompileError(msg)
continue
else:
raise CompileError("Don't know how to compile %s to %s" %
(src, obj))
output_opt = '/Fo' + obj
try:
self.spawn([self.cc] + compile_opts + pp_opts +
[input_opt, output_opt] + extra_postargs)
except DistutilsExecError as msg:
raise CompileError(msg)
return objects
def compile_single(self, obj, build, debug, pp_opts, compile_opts,
add_cpp_opts, extra_postargs):
try:
src, ext = build[obj]
except KeyError:
return
if debug:
# pass the full pathname to MSVC in debug mode,
# this allows the debugger to find the source file
# without asking the user to browse for it
src = os.path.abspath(src)
if ext in self._c_extensions:
input_opt = "/Tc" + src
elif ext in self._cpp_extensions:
input_opt = "/Tp" + src
add_cpp_opts = True
elif ext in self._rc_extensions:
# compile .RC to .RES file
input_opt = src
output_opt = "/fo" + obj
try:
self.spawn([self.rc] + pp_opts + [output_opt, input_opt])
except DistutilsExecError as msg:
raise CompileError(msg)
return
elif ext in self._mc_extensions:
# Compile .MC to .RC file to .RES file.
# * '-h dir' specifies the directory for the
# generated include file
# * '-r dir' specifies the target directory of the
# generated RC file and the binary message resource
# it includes
#
# For now (since there are no options to change this),
# we use the source-directory for the include file and
# the build directory for the RC file and message
# resources. This works at least for win32all.
h_dir = os.path.dirname(src)
rc_dir = os.path.dirname(obj)
try:
# first compile .MC to .RC and .H file
self.spawn([self.mc, "-h", h_dir, "-r", rc_dir, src])
base, _ = os.path.splitext(os.path.basename(src))
rc_file = join(rc_dir, base + ".rc")
# then compile .RC to .RES file
self.spawn([self.rc, "/fo" + obj, rc_file])
except DistutilsExecError as msg:
raise CompileError(msg)
return
else:
# how to handle this file?
raise CompileError("Don't know how to compile {} to {}".format(
src, obj))
args = [self.cc] + compile_opts + pp_opts
if add_cpp_opts:
args.append("/EHsc")
args.append(input_opt)
args.append("/Fo" + obj)
args.extend(extra_postargs)
try:
self.spawn(args)
except DistutilsExecError as msg:
raise CompileError(msg)
# first compile .MC to .RC and .H file
self.spawn ([self.mc] +
['-h', h_dir, '-r', rc_dir] + [src])
base, _ = os.path.splitext (os.path.basename (src))
rc_file = os.path.join (rc_dir, base + '.rc')
# then compile .RC to .RES file
self.spawn ([self.rc] +
["/fo" + obj] + [rc_file])
except DistutilsExecError, msg:
raise CompileError, msg
continue
else:
# how to handle this file?
raise CompileError (
"Don't know how to compile %s to %s" % \
(src, obj))
output_opt = "/Fo" + obj
try:
self.spawn ([self.cc] + compile_opts + pp_opts +
[input_opt, output_opt] +
extra_postargs)
except DistutilsExecError, msg:
raise CompileError, msg
return objects
# compile ()
def createMakefile():
say("\n ==== Checking supported compiler options and available libraries ====\n\n")
LINK_FLAGS = LDFLAGS.split() # accumulate the linker flags that will be put to Makefile.local
COMPILE_FLAGS = CFLAGS. split() # same for the compilation of the shared library only
# default compilation flags for both the shared library and all example programs that use it
CXXFLAGS = ['-fPIC', '-Wall', '-O2']
# additional compilation/linking flags for example programs
EXE_FLAGS = []
# [1a]: check if a compiler exists at all
if not runCompiler():
raise CompileError("Could not locate a compiler (set CXX=... environment variable to override)")
# [1b]: test if OpenMP is supported (optional but highly recommended)
OMP_FLAG = '-fopenmp'
OMP_FLAGS= OMP_FLAG+' -Werror -Wno-unknown-pragmas'
OMP_CODE = '#include <omp.h>\n#include <iostream>\nint main(){\nstd::cout << "Number of threads: ";\n'+\
'#pragma omp parallel\n{std::cout<<\'*\';}\nstd::cout << "\\n";\n}\n'
if runCompiler(code=OMP_CODE, flags=OMP_FLAGS):
CXXFLAGS += [OMP_FLAG]
else:
# on MacOS the clang compiler pretends not to support OpenMP, but in fact it does so
# if we insist (libomp.so/dylib or libgomp.so must be present in the system for this to work);
# in some Anaconda installations, though, linking to the system-default libomp.dylib
# leads to conflicts with libiomp5.dylib, so we first try to link to the latter explicitly.
CONDA_EXE = os.environ.get('CONDA_EXE')
if CONDA_EXE is not None and os.path.isfile(CONDA_EXE.replace('bin/conda', 'lib/libiomp5.dylib')) \
and runCompiler(code=OMP_CODE, flags=CONDA_EXE.replace('bin/conda', 'lib/libiomp5.dylib') +
' -Xpreprocessor ' + OMP_FLAGS):
CXXFLAGS += ['-Xpreprocessor', OMP_FLAG]
LINK_FLAGS += [CONDA_EXE.replace('bin/conda', 'lib/libiomp5.dylib')]
EXE_FLAGS += [CONDA_EXE.replace('bin/conda', 'lib/libiomp5.dylib')]
elif runCompiler(code=OMP_CODE, flags='-lgomp -Xpreprocessor '+OMP_FLAGS):
CXXFLAGS += ['-Xpreprocessor', OMP_FLAG]
LINK_FLAGS += ['-lgomp']
EXE_FLAGS += ['-lgomp']
elif runCompiler(code=OMP_CODE, flags='-lomp -Xpreprocessor '+OMP_FLAGS):
CXXFLAGS += ['-Xpreprocessor', OMP_FLAG]
LINK_FLAGS += ['-lomp']
EXE_FLAGS += ['-lomp']
elif not ask("Warning, OpenMP is not supported\n"+
"If you're compiling on MacOS with clang, you'd better install another compiler such as GCC\n"+
"Do you want to continue without OpenMP? [Y/N] "): exit(1)
# [1c]: test if C++11 is supported (optional)
CXX11_FLAG = '-std=c++11'
if runCompiler(flags=CXX11_FLAG):
CXXFLAGS += [CXX11_FLAG]
# [1d]: test the -march flag (optional, allows architecture-dependent compiler optimizations)
ARCH_FLAG = '-march=native'
ARCH_CODE = 'int main(int c, char** v) { double x=c*3.14; return x==42; }\n'
if runCompiler(code=ARCH_CODE, flags=ARCH_FLAG):
CXXFLAGS += [ARCH_FLAG]
else:
ARCH_FLAG = '-march=core2' # try a less ambitious option
if runCompiler(code=ARCH_CODE, flags=ARCH_FLAG):
CXXFLAGS += [ARCH_FLAG]
# [1e]: special treatment for Intel compiler to restore determinism in OpenMP-parallelized loops
INTEL_FLAG = '-qno-opt-dynamic-align'
if runCompiler(code='#ifndef __INTEL_COMPILER\n#error\n#endif\nint main(){}\n', flags=INTEL_FLAG):
CXXFLAGS += [INTEL_FLAG]
# [2a]: check that NumPy is present (required by the python interface)
try:
import numpy
NUMPY_INC = '-I'+numpy.get_include()
except ImportError:
raise CompileError("NumPy is not present - python extension cannot be compiled")
# [2b]: find out the paths to Python.h and libpythonXX.{a,so,dylib,...} (this is rather tricky)
# and all other relevant compilation/linking flags needed to build a shared library that uses Python
PYTHON_INC = '-I'+sysconfig.get_python_inc()
# various other system libraries that are needed at link time
PYTHON_LIB_EXTRA = compressList(
get_config_var('LIBS').split() +
get_config_var('SYSLIBS').split())
# try compiling a test code with the provided link flags (in particular, the name of Python library):
# check that a sample C++ program with embedded python compiles, links and runs properly
def tryPythonCode(PYTHON_SO_FLAGS, PYTHON_EXE_FLAGS=[]):
print(" **** Trying the following options for linking against Python library ****")
# test code for a shared library
PYTEST_LIB_CODE = """
#include "Python.h"
#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
#include "numpy/arrayobject.h"
void bla() {PyRun_SimpleString("import sys;print(sys.prefix);");}
void run() {Py_Initialize();bla();Py_Finalize();}
PyMODINIT_FUNC
"""
if sys.version_info[0]==2: # Python 2.6-2.7
PYTEST_LIB_CODE += """
initagamatest(void) {
Py_InitModule3("agamatest", NULL, "doc");
import_array();
bla();
}
"""
else: # Python 3.x
PYTEST_LIB_CODE += """
PyInit_agamatest(void) {
static PyModuleDef moduledef = {PyModuleDef_HEAD_INIT, "agamatest", "doc", -1, NULL};
PyObject* mod = PyModule_Create(&moduledef);
import_array1(mod);
bla();
return mod;
}
"""
# test code for a program that loads this shared library
PYTEST_EXE_CODE = 'extern void run();int main(){run();}\n'
PYTEST_LIB_NAME = './agamatest.so'
PYTEST_EXE_NAME = './agamatest.exe'
# try compiling the test shared library
if not runCompiler(code=PYTEST_LIB_CODE,
flags=' '.join([PYTHON_INC, NUMPY_INC, '-shared', '-fPIC'] + PYTHON_SO_FLAGS),
dest=PYTEST_LIB_NAME):
return False # the program couldn't be compiled at all (try the next variant)
# if succeeded, compile the test program that uses this library
if not runCompiler(code=PYTEST_EXE_CODE,
flags=' '.join([PYTEST_LIB_NAME] + PYTHON_EXE_FLAGS),
dest=PYTEST_EXE_NAME) \
or not os.path.isfile(PYTEST_LIB_NAME) \
or not os.path.isfile(PYTEST_EXE_NAME):
return False # can't find compiled test program
resultexe = subprocess.Popen(PYTEST_EXE_NAME,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()[0].decode().rstrip()
# the test program might not be able to find the python home, in which case manually provide it
if 'Could not find platform independent libraries <prefix>' in resultexe:
resultexe = subprocess.Popen(PYTEST_EXE_NAME,
env=dict(os.environ, PYTHONHOME=sys.prefix),
stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()[0].decode().rstrip()
# also try loading this shared library as an extension module
procpy = subprocess.Popen(sys.executable+" -c 'import agamatest'", shell=True, \
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
resultpy = procpy.communicate()[0].decode().rstrip()
returnpy = procpy.returncode
# clean up
os.remove(PYTEST_EXE_NAME)
os.remove(PYTEST_LIB_NAME)
# check if the results (reported library path prefix) are the same as we have in this script
sysprefix = os.path.realpath(sys.prefix)
if os.path.realpath(resultexe) != sysprefix or os.path.realpath(resultpy) != sysprefix:
print("Test program doesn't seem to use the same version of Python, "+\
"or the library path is reported incorrectly: \n"+\
"Expected: "+sysprefix+"\n"+\
"Received: "+resultexe+"\n"+\
"From py: "+resultpy+('' if returnpy==0 else ' (crashed with error '+str(returnpy)+')'))
return False
print(" **** Successfully linked using these options ****")
return True # this combination of options seems reasonable...
# explore various possible combinations of file name and path to the python library...
def findPythonLib():
# try linking against the static python library libpython**.a, if this does not succeed,
# try the shared library libpython**.so** or libpython**.dylib
for PYTHON_LIB_FILENAME in compressList([sysconfig.get_config_var(x) for x in ['LIBRARY', 'LDLIBRARY', 'INSTSONAME']]):
for PYTHON_LIB_PATH in compressList([sysconfig.get_config_var(x) for x in ['LIBPL', 'LIBDIR']]):
# obtain full path to the python library
PYTHON_LIB_FILEPATH = os.path.join(PYTHON_LIB_PATH, PYTHON_LIB_FILENAME)
# check if the file exists at all at the given location
if os.path.isfile(PYTHON_LIB_FILEPATH):
# flags for compiling the shared library which serves as a Python extension module
PYTHON_SO_FLAGS = [PYTHON_LIB_FILEPATH] + PYTHON_LIB_EXTRA
# other libraries depend on whether this is a static or a shared python library
if PYTHON_LIB_FILENAME.endswith('.a') and not sysconfig.get_config_var('PYTHONFRAMEWORK'):
PYTHON_SO_FLAGS += get_config_var('LINKFORSHARED').split()
# the stack_size flag is problematic and needs to be removed
PYTHON_SO_FLAGS = [x for x in PYTHON_SO_FLAGS if not x.startswith('-Wl,-stack_size,')]
if tryPythonCode(PYTHON_SO_FLAGS):
return PYTHON_SO_FLAGS, [] # successful compilation
elif not PYTHON_LIB_FILENAME.endswith('.a'):
# sometimes the python installation is so wrecked that the linker can find and use
# the shared library libpython***.so, but this library is not in LD_LIBRARY_PATH and
# cannot be found when loading the python extension module outside python itself.
# the (inelegant) fix is to hardcode the path to this libpython***.so as -rpath.
print("Trying rpath")
RPATH = ['-Wl,-rpath,'+PYTHON_LIB_PATH] # extend the linker options and try again
if tryPythonCode(PYTHON_SO_FLAGS + RPATH):
return PYTHON_SO_FLAGS + RPATH, []
if "-undefined dynamic_lookup" in sysconfig.get_config_var('LDSHARED'):
print("Trying the last resort solution")
PYTHON_SO_FLAGS = ['-undefined dynamic_lookup'] + PYTHON_LIB_EXTRA
PYTHON_EXE_FLAGS = RPATH + [PYTHON_LIB_FILEPATH]
if tryPythonCode(PYTHON_SO_FLAGS, PYTHON_EXE_FLAGS):
return PYTHON_SO_FLAGS, PYTHON_EXE_FLAGS
# if none of the above combinations worked, give up...
raise CompileError("Could not compile test program which uses libpython" +
sysconfig.get_config_var('VERSION'))
# [2c]: find the python library and other relevant linking flags
PYTHON_SO_FLAGS, PYTHON_EXE_FLAGS = findPythonLib()
COMPILE_FLAGS += ['-DHAVE_PYTHON', PYTHON_INC, NUMPY_INC]
LINK_FLAGS += PYTHON_SO_FLAGS
EXE_FLAGS += PYTHON_EXE_FLAGS
# [3]: check that GSL is present, and find out its version (required)
# try compiling a snippet of code into a shared library (tests if GSL has been compiled with -fPIC),
# then compiling a test program that loads this library (tests if the correct version of GSL is loaded at link time)
GSL_TEST_CODE = """#include <gsl/gsl_version.h>
#if not defined(GSL_MAJOR_VERSION) || (GSL_MAJOR_VERSION == 1) && (GSL_MINOR_VERSION < 15)
#error "GSL version is too old (need at least 1.15)"
#endif
#include <gsl/gsl_integration.h>
void run() { gsl_integration_cquad_workspace_alloc(10); }
"""
if runCompileShared(GSL_TEST_CODE, '-lgsl -lgslcblas'):
# apparently the headers and libraries can be found in some standard location,
LINK_FLAGS += ['-lgsl', '-lgslcblas'] # so we only list their names
else:
if not ask("GSL library (required) is not found\n"+
"Should we try to download and compile it now? [Y/N] "): exit(1)
distutils.dir_util.mkpath(EXTRAS_DIR)
os.chdir(EXTRAS_DIR)
say('Downloading GSL\n')
filename = 'gsl.tar.gz'
dirname = 'gsl-2.6'
try:
urlretrieve('ftp://ftp.gnu.org/gnu/gsl/gsl-2.6.tar.gz', filename)
if os.path.isfile(filename):
subprocess.call(['tar', '-zxf', filename]) # unpack the archive
os.remove(filename) # remove the downloaded archive
if not os.path.isdir(dirname): raise Exception("Error unpacking GSL")
except Exception as e:
raise CompileError(str(e) + "\nError downloading GSL library, aborting...\n"+
"You may try to manually compile GSL and install it to "+ROOT_DIR+"/"+EXTRAS_DIR+", so that "+
"the header files are in "+EXTRAS_DIR+"/include and library files - in "+EXTRAS_DIR+"/lib")
say('Compiling GSL (may take a few minutes)\n')
result = subprocess.call('(cd '+dirname+'; ./configure --prefix='+os.getcwd()+
' CFLAGS="-fPIC -O2" --enable-shared=no; make; make install) > gsl-install.log', shell=True)
if result != 0 or not os.path.isfile('lib/libgsl.a'):
raise CompileError("GSL compilation failed (check "+EXTRAS_DIR+"/gsl-install.log)")
distutils.dir_util.remove_tree(dirname) # clean up source and build directories
COMPILE_FLAGS += ['-I'+EXTRAS_DIR+'/include']
LINK_FLAGS += [EXTRAS_DIR+'/lib/libgsl.a', EXTRAS_DIR+'/lib/libgslcblas.a']
os.chdir(ROOT_DIR)
# [4]: test if Eigen library is present (optional)
if runCompiler(code='#include <Eigen/Core>\nint main(){}\n'):
COMPILE_FLAGS += ['-DHAVE_EIGEN']
else:
if ask("Eigen library (recommended) is not found\n"+
"Should we try to download it now (no compilation needed)? [Y/N] "):
distutils.dir_util.mkpath(EXTRAS_DIR+'/include/unsupported')
os.chdir(EXTRAS_DIR)
say('Downloading Eigen\n')
filename = 'Eigen.zip'
dirname = 'eigen-git-mirror-3.3.7'
try:
urlretrieve('https://github.com/eigenteam/eigen-git-mirror/archive/3.3.7.zip', filename)
if os.path.isfile(filename):
subprocess.call('unzip '+filename+' >/dev/null', shell=True) # unpack the archive
if os.path.isdir(dirname):
distutils.dir_util.copy_tree(dirname+'/Eigen', 'include/Eigen', verbose=False) # copy the headers
distutils.dir_util.copy_tree(dirname+'/unsupported/Eigen', 'include/unsupported/Eigen', verbose=False)
distutils.dir_util.remove_tree(dirname) # and delete the rest
COMPILE_FLAGS += ['-DHAVE_EIGEN', '-I'+EXTRAS_DIR+'/include']
os.remove(filename) # remove the downloaded archive
except Exception as e:
say("Failed to install Eigen: "+str(e)+"\n") # didn't succeed with Eigen
os.chdir(ROOT_DIR)
# [5a]: test if CVXOPT is present (optional); install if needed
try:
import cvxopt # import the python module
except: # import error or some other problem, might be corrected
if ask("CVXOPT library (needed only for Schwarzschild modelling) is not found\n"
"Should we try to install it now? [Y/N] "):
try:
subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--user', 'cvxopt'])
except Exception as e:
say("Failed to install CVXOPT: "+str(e)+"\n")
# [5b]: if the cvxopt module is available in Python, make sure that we also have C header files
try:
import cvxopt # if this fails, skip cvxopt altogether
if runCompiler(code='#include <cvxopt.h>\nint main(){import_cvxopt();}\n', flags=' '.join(['-c', PYTHON_INC, NUMPY_INC])):
COMPILE_FLAGS += ['-DHAVE_CVXOPT']
else:
# download the C header file if it does not appear to be present in a default location
distutils.dir_util.mkpath(EXTRAS_DIR+'/include')
say('Downloading CVXOPT header files\n')
try:
urlretrieve('https://raw.githubusercontent.com/cvxopt/cvxopt/master/src/C/cvxopt.h',
EXTRAS_DIR+'/include/cvxopt.h')
urlretrieve('https://raw.githubusercontent.com/cvxopt/cvxopt/master/src/C/blas_redefines.h',
EXTRAS_DIR+'/include/blas_redefines.h')
except: pass # problems in downloading, skip it
if os.path.isfile(EXTRAS_DIR+'/include/cvxopt.h') and \
os.path.isfile(EXTRAS_DIR+'/include/blas_redefines.h'):
COMPILE_FLAGS += ['-DHAVE_CVXOPT', '-I'+EXTRAS_DIR+'/include']
else:
say("Failed to download CVXOPT header files, this feature will not be available\n")
except: pass # cvxopt wasn't available
# [6]: test if GLPK is present (optional - ignored if not found)
if runCompileShared('#include <glpk.h>\nvoid run() { glp_create_prob(); }\n', '-lglpk'):
COMPILE_FLAGS += ['-DHAVE_GLPK']
LINK_FLAGS += ['-lglpk']
else:
say("GLPK library (optional) is not found\n")
# [7]: test if UNSIO is present (optional), download and compile if needed
if runCompileShared('#include <uns.h>\nvoid run() { }\n', '-lunsio -lnemo'):
COMPILE_FLAGS += ['-DHAVE_UNSIO']
LINK_FLAGS += ['-lunsio', '-lnemo']
else:
if ask("UNSIO library (optional; used for input/output of N-body snapshots) is not found\n"+
"Should we try to download and compile it now? [Y/N] "):
distutils.dir_util.mkpath(EXTRAS_DIR)
distutils.dir_util.mkpath(EXTRAS_DIR+'/include')
distutils.dir_util.mkpath(EXTRAS_DIR+'/lib')
say('Downloading UNSIO\n')
filename = EXTRAS_DIR+'/unsio-master.zip'
dirname = EXTRAS_DIR+'/unsio-master'
try:
urlretrieve('https://github.com/GalacticDynamics-Oxford/unsio/archive/master.zip', filename)
if os.path.isfile(filename):
subprocess.call('(cd '+EXTRAS_DIR+'; unzip ../'+filename+') >/dev/null', shell=True) # unpack
os.remove(filename) # remove the downloaded archive
say("Compiling UNSIO\n")
result = subprocess.call('(cd '+dirname+'; make) > '+EXTRAS_DIR+'/unsio-install.log', shell=True)
if result == 0 and os.path.isfile(dirname+'/libnemo.a') and os.path.isfile(dirname+'/libunsio.a'):
# successfully compiled: copy the header files to extras/include
for hfile in ['componentrange.h', 'ctools.h', 'snapshotinterface.h', 'uns.h', 'userselection.h']:
distutils.file_util.copy_file(dirname+'/unsio/'+hfile, EXTRAS_DIR+'/include')
# copy the static libraries to extras/lib
distutils.file_util.copy_file(dirname+'/libnemo.a', EXTRAS_DIR+'/lib')
distutils.file_util.copy_file(dirname+'/libunsio.a', EXTRAS_DIR+'/lib')
# delete the compiled directory
distutils.dir_util.remove_tree(dirname)
UNSIO_COMPILE_FLAGS = ['-I'+EXTRAS_DIR+'/include']
UNSIO_LINK_FLAGS = [EXTRAS_DIR+'/lib/libunsio.a', EXTRAS_DIR+'/lib/libnemo.a']
if runCompiler(code='#include <uns.h>\nint main(){}\n', flags=' '.join(UNSIO_COMPILE_FLAGS+UNSIO_LINK_FLAGS)):
COMPILE_FLAGS += ['-DHAVE_UNSIO'] + UNSIO_COMPILE_FLAGS
LINK_FLAGS += UNSIO_LINK_FLAGS
else:
raise CompileError('Failed to link against the just compiled UNSIO library')
else: raise CompileError(
"Failed compiling UNSIO (check "+EXTRAS_DIR+"/unsio-install.log)")
except Exception as e: # didn't succeed with UNSIO
say(str(e)+'\n')
# [99]: put everything together and create Makefile.local
with open('Makefile.local','w') as f: f.write(
"# set the default compiler if no value is found in the environment variables or among command-line arguments\n" +
"ifeq ($(origin CXX),default)\nCXX = " + CC + "\nendif\n" +
"ifeq ($(origin FC), default)\nFC = gfortran\nendif\nLINK = $(CXX)\n" +
"# compilation/linking flags for both the shared library and any programs that use it\n" +
"CXXFLAGS += " + " ".join(compressList(CXXFLAGS)) + "\n" +
"# compilation flags for the shared library only (files in src/)\n" +
"COMPILE_FLAGS += " + " ".join(compressList(COMPILE_FLAGS)) + "\n" +
"# linking flags for the shared library only\n" +
"LINK_FLAGS += " + " ".join(compressList(LINK_FLAGS)) + "\n" +
("# linking flags for the example/test programs\n" +
"EXE_FLAGS += " + " ".join(compressList(EXE_FLAGS)) + "\n" if EXE_FLAGS else "") )
