def build_java_ext(self, ext):
"""Run command."""
java = self.distribution.enable_build_jar
javac = "javac"
try:
if os.path.exists(
os.path.join(os.environ['JAVA_HOME'], 'bin', 'javac')):
javac = '"%s"' % os.path.join(os.environ['JAVA_HOME'], 'bin',
'javac')
except KeyError:
pass
jar = "jar"
try:
if os.path.exists(
os.path.join(os.environ['JAVA_HOME'], 'bin', 'jar')):
jar = '"%s"' % os.path.join(os.environ['JAVA_HOME'], 'bin',
'jar')
except KeyError:
pass
# Try to use the cache if we are not requested build
if not java:
src = os.path.join('native', 'jars')
dest = os.path.join('build', 'lib')
if os.path.exists(src):
distutils.log.info("Using Jar cache")
copy_tree(src, dest)
return
classpath = "."
if ext.libraries:
classpath = os.path.pathsep.join(ext.libraries)
distutils.log.info(
"Jar cache is missing, using --enable-build-jar to recreate it.")
coverage = self.distribution.enable_coverage
target_version = "1.8"
# build the jar
try:
dirname = os.path.dirname(self.get_ext_fullpath("JAVA"))
jarFile = os.path.join(dirname, ext.name + ".jar")
build_dir = os.path.join(self.build_temp, ext.name, "classes")
os.makedirs(build_dir, exist_ok=True)
os.makedirs(dirname, exist_ok=True)
cmd1 = shlex.split(
'%s -cp "%s" -d "%s" -g:none -source %s -target %s' %
(javac, classpath, build_dir, target_version, target_version))
cmd1.extend(ext.sources)
debug = "-g:none"
if coverage:
debug = "-g:lines,vars,source"
os.makedirs("build/classes", exist_ok=True)
self.announce(" %s" % " ".join(cmd1), level=distutils.log.INFO)
subprocess.check_call(cmd1)
try:
for file in glob.iglob("native/java/**/*.*", recursive=True):
if file.endswith(".java") or os.path.isdir(file):
continue
p = os.path.join(build_dir,
os.path.relpath(file, "native/java"))
print("Copy file", file, p)
shutil.copyfile(file, p)
except Exception as ex:
print("FAIL", ex)
pass
cmd3 = shlex.split('%s cvf "%s" -C "%s" .' %
(jar, jarFile, build_dir))
self.announce(" %s" % " ".join(cmd3), level=distutils.log.INFO)
subprocess.check_call(cmd3)
except subprocess.CalledProcessError as exc:
distutils.log.error(exc.output)
raise DistutilsPlatformError("Error executing {}".format(exc.cmd))
def runtime_library_dir_option(self, dir):
raise DistutilsPlatformError(
"don't know how to set runtime library search path for MSVC++")
result[key] = removeDuplicates(value)
finally:
popen.stdout.close()
popen.stderr.close()
if len(result) != len(interesting):
raise ValueError(str(list(result.keys())))
return result
# More globals
VERSION = get_build_version()
if VERSION < 8.0:
raise DistutilsPlatformError("VC %0.1f is not supported by this module" % VERSION)
# MACROS = MacroExpander(VERSION)
class MSVCCompiler(CCompiler):
"""Concrete class that implements an interface to Microsoft Visual C++,
as defined by the CCompiler abstract class."""
compiler_type = "msvc"
# Just set this so CCompiler's constructor doesn't barf. We currently
# don't use the 'set_executables()' bureaucracy provided by CCompiler,
# as it really isn't necessary for this sort of single-compiler class.
# Would be nice to have a consistent interface with UnixCCompiler,
# though, so it's worth thinking about.
executables = {}
def cython_sources(self, sources, extension):
"""
Walk the list of source files in 'sources', looking for Cython
source files (.pyx and .py). Run Cython on all that are
found, and return a modified 'sources' list with Cython source
files replaced by the generated C (or C++) files.
"""
try:
from Cython.Compiler.Main \
import CompilationOptions, \
default_options as cython_default_options, \
compile as cython_compile
from Cython.Compiler.Errors import PyrexError
except ImportError:
e = sys.exc_info()[1]
print("failed to import Cython: %s" % e)
raise DistutilsPlatformError("Cython does not appear to be installed")
new_sources = []
cython_sources = []
cython_targets = {}
# Setup create_list and cplus from the extension options if
# Cython.Distutils.extension.Extension is used, otherwise just
# use what was parsed from the command-line or the configuration file.
# cplus will also be set to true is extension.language is equal to
# 'C++' or 'c++'.
#try:
# create_listing = self.cython_create_listing or \
# extension.cython_create_listing
# cplus = self.cython_cplus or \
# extension.cython_cplus or \
# (extension.language != None and \
# extension.language.lower() == 'c++')
#except AttributeError:
# create_listing = self.cython_create_listing
# cplus = self.cython_cplus or \
# (extension.language != None and \
# extension.language.lower() == 'c++')
create_listing = self.cython_create_listing or \
getattr(extension, 'cython_create_listing', 0)
line_directives = self.cython_line_directives or \
getattr(extension, 'cython_line_directives', 0)
no_c_in_traceback = self.no_c_in_traceback or \
getattr(extension, 'no_c_in_traceback', 0)
cplus = self.cython_cplus or getattr(extension, 'cython_cplus', 0) or \
(extension.language and extension.language.lower() == 'c++')
cython_gen_pxi = self.cython_gen_pxi or getattr(extension, 'cython_gen_pxi', 0)
cython_gdb = self.cython_gdb or getattr(extension, 'cython_gdb', False)
cython_compile_time_env = self.cython_compile_time_env or \
getattr(extension, 'cython_compile_time_env', None)
# Set up the include_path for the Cython compiler:
# 1. Start with the command line option.
# 2. Add in any (unique) paths from the extension
# cython_include_dirs (if Cython.Distutils.extension is used).
# 3. Add in any (unique) paths from the extension include_dirs
includes = self.cython_include_dirs
try:
for i in extension.cython_include_dirs:
if not i in includes:
includes.append(i)
except AttributeError:
pass
for i in extension.include_dirs:
if not i in includes:
includes.append(i)
# Set up Cython compiler directives:
# 1. Start with the command line option.
# 2. Add in any (unique) entries from the extension
# cython_directives (if Cython.Distutils.extension is used).
directives = self.cython_directives
if hasattr(extension, "cython_directives"):
directives.update(extension.cython_directives)
# Set the target_ext to '.c'. Cython will change this to '.cpp' if
# needed.
if cplus:
target_ext = '.cpp'
else:
target_ext = '.c'
# Decide whether to drop the generated C files into the temp dir
# or the source tree.
if not self.inplace and (self.cython_c_in_temp
or getattr(extension, 'cython_c_in_temp', 0)):
target_dir = os.path.join(self.build_temp, "pyrex")
for package_name in extension.name.split('.')[:-1]:
target_dir = os.path.join(target_dir, package_name)
else:
target_dir = None
newest_dependency = None
for source in sources:
(base, ext) = os.path.splitext(os.path.basename(source))
if ext == ".py":
# FIXME: we might want to special case this some more
ext = '.pyx'
if ext == ".pyx": # Cython source file
output_dir = target_dir or os.path.dirname(source)
new_sources.append(os.path.join(output_dir, base + target_ext))
cython_sources.append(source)
cython_targets[source] = new_sources[-1]
elif ext == '.pxi' or ext == '.pxd':
if newest_dependency is None \
or newer(source, newest_dependency):
newest_dependency = source
else:
new_sources.append(source)
if not cython_sources:
return new_sources
module_name = extension.name
for source in cython_sources:
target = cython_targets[source]
depends = [source] + list(extension.depends or ())
if(source[-4:].lower()==".pyx" and os.path.isfile(source[:-3]+"pxd")):
depends += [source[:-3]+"pxd"]
rebuild = self.force or newer_group(depends, target, 'newer')
if not rebuild and newest_dependency is not None:
rebuild = newer(newest_dependency, target)
if rebuild:
log.info("cythoning %s to %s", source, target)
self.mkpath(os.path.dirname(target))
if self.inplace:
output_dir = os.curdir
else:
output_dir = self.build_lib
options = CompilationOptions(cython_default_options,
use_listing_file = create_listing,
include_path = includes,
compiler_directives = directives,
output_file = target,
cplus = cplus,
emit_linenums = line_directives,
c_line_in_traceback = not no_c_in_traceback,
generate_pxi = cython_gen_pxi,
output_dir = output_dir,
gdb_debug = cython_gdb,
compile_time_env = cython_compile_time_env)
result = cython_compile(source, options=options,
full_module_name=module_name)
else:
log.info("skipping '%s' Cython extension (up-to-date)", target)
return new_sources
my_msg = "invalid Python installation: unable to open %s" % filename
if hasattr(msg, "strerror"):
my_msg = my_msg + " (%s)" % msg.strerror
raise DistutilsPlatformError(my_msg)
# load the installed pyconfig.h:
try:
filename = get_config_h_filename()
parse_config_h(file(filename), g)
except IOError, msg:
my_msg = "invalid Python installation: unable to open %s" % filename
if hasattr(msg, "strerror"):
my_msg = my_msg + " (%s)" % msg.strerror
raise DistutilsPlatformError(my_msg)
# On MacOSX we need to check the setting of the environment variable
# MACOSX_DEPLOYMENT_TARGET: configure bases some choices on it so
# it needs to be compatible.
# If it isn't set we set it to the configure-time value
if sys.platform == 'darwin' and 'MACOSX_DEPLOYMENT_TARGET' in g:
cfg_target = g['MACOSX_DEPLOYMENT_TARGET']
cur_target = os.getenv('MACOSX_DEPLOYMENT_TARGET', '')
if cur_target == '':
cur_target = cfg_target
os.putenv('MACOSX_DEPLOYMENT_TARGET', cfg_target)
elif map(int, cfg_target.split('.')) > map(int, cur_target.split('.')):
my_msg = (
'$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" during configure'
% (cur_target, cfg_target))
def run(self, msg=msg):
raise DistutilsPlatformError(msg)
def initialize(self, plat_name=None):
if not not self.initialized:
raise AssertionError("don't init multiple times")
if plat_name is None:
plat_name = get_platform()
ok_plats = ('win32', 'win-amd64', 'win-ia64')
if plat_name not in ok_plats:
raise DistutilsPlatformError('--plat-name must be one of %s' %
(ok_plats, ))
if 'DISTUTILS_USE_SDK' in os.environ and 'MSSdk' in os.environ and self.find_exe(
'cl.exe'):
self.cc = 'cl.exe'
self.linker = 'link.exe'
self.lib = 'lib.exe'
self.rc = 'rc.exe'
self.mc = 'mc.exe'
else:
if plat_name == get_platform() or plat_name == 'win32':
plat_spec = PLAT_TO_VCVARS[plat_name]
else:
plat_spec = PLAT_TO_VCVARS[
get_platform()] + '_' + PLAT_TO_VCVARS[plat_name]
vc_env = query_vcvarsall(VERSION, plat_spec)
self.__paths = vc_env['path'].encode('mbcs').split(os.pathsep)
os.environ['lib'] = vc_env['lib'].encode('mbcs')
os.environ['include'] = vc_env['include'].encode('mbcs')
if len(self.__paths) == 0:
raise DistutilsPlatformError(
"Python was built with %s, and extensions need to be built with the same version of the compiler, but it isn't installed."
% self.__product)
self.cc = self.find_exe('cl.exe')
self.linker = self.find_exe('link.exe')
self.lib = self.find_exe('lib.exe')
self.rc = self.find_exe('rc.exe')
self.mc = self.find_exe('mc.exe')
try:
for p in os.environ['path'].split(';'):
self.__paths.append(p)
except KeyError:
pass
self.__paths = normalize_and_reduce_paths(self.__paths)
os.environ['path'] = ';'.join(self.__paths)
self.preprocess_options = None
if self.__arch == 'x86':
self.compile_options = [
'/nologo', '/Ox', '/MD', '/W3', '/DNDEBUG'
]
self.compile_options_debug = [
'/nologo', '/Od', '/MDd', '/W3', '/Z7', '/D_DEBUG'
]
else:
self.compile_options = [
'/nologo', '/Ox', '/MD', '/W3', '/GS-', '/DNDEBUG'
]
self.compile_options_debug = [
'/nologo', '/Od', '/MDd', '/W3', '/GS-', '/Z7', '/D_DEBUG'
]
self.ldflags_shared = ['/DLL', '/nologo', '/INCREMENTAL:NO']
self.ldflags_shared_debug = self.__version >= 7 and [
'/DLL', '/nologo', '/INCREMENTAL:no', '/DEBUG', '/pdb:None'
]
self.ldflags_static = ['/nologo']
self.initialized = True
return
from distutils.util import get_platform
from distutils.errors import DistutilsPlatformError
import sys
import ctypes
import os.path
# Determine the target platform
platform = '%s-%s.%s' % (get_platform(), sys.version_info[0],
sys.version_info[1])
# Determine how to copy the precompiled binary.
dlls = os.path.join(sys.prefix, 'DLLs')
pyd = os.path.join(platform, 'pydasm.pyd')
if not os.path.exists(pyd):
raise DistutilsPlatformError()
if os.path.exists(dlls):
data_files = [(dlls, [pyd])]
else:
data_files = [pyd]
# Set the parameters for the setup script
params = {
# Setup instructions
'provides': ['pydasm'],
'data_files':
data_files,
# Metadata
'name':
def _fixup_compiler(use_ccache):
if "CC" in os.environ:
# CC is in the environment, always use explicit
# overrides.
return
try:
# Newer version of python have support for dealing with
# the compiler mess w.r.t. various versions of Apple's SDKs
import _osx_support
_osx_support.customize_compiler(get_config_vars())
except (ImportError, AttributeError, KeyError):
pass
cc = oldcc = get_config_var("CC").split()[0]
cc = _find_executable(cc)
if cc is not None and os.path.basename(cc).startswith("gcc"):
# Check if compiler is LLVM-GCC, that's known to
# generate bad code.
with os.popen("'%s' --version 2>/dev/null" %
(cc.replace("'", "'\"'\"'"), )) as fp:
data = fp.read()
if "llvm-gcc" in data:
cc = None
if cc is not None and not _working_compiler(cc):
cc = None
if cc is None:
# Default compiler is not useable, try finding 'clang'
cc = _find_executable("clang")
if cc is None:
cc = os.popen("/usr/bin/xcrun -find clang").read()
if not cc:
raise DistutilsPlatformError("Cannot locate compiler candidate")
if not _working_compiler(cc):
raise DistutilsPlatformError("Cannot locate a working compiler")
if use_ccache:
p = _find_executable("ccache")
if p is not None:
log.info("Detected and using 'ccache'")
cc = "%s %s" % (p, cc)
if cc != oldcc:
log.info("Use '%s' instead of '%s' as the compiler" % (cc, oldcc))
config_vars = get_config_vars()
for env in ("BLDSHARED", "LDSHARED", "CC", "CXX"):
if env in config_vars and env not in os.environ:
split = config_vars[env].split()
split[0] = cc if env != "CXX" else cc + "++"
config_vars[env] = " ".join(split)
cflags = get_config_var("CFLAGS")
if re.search(r"-arch\s+i386", cflags) is not None:
raise DistutilsPlatformError(
"i386 (32-bit) is not supported by PyObjC")
if re.search(r"-arch\s+ppc", cflags) is not None:
raise DistutilsPlatformError("PowerPC is not supported by PyObjC")
def get_nvcc_cmd() -> str:
from shutil import which
nvcc_cmd = which('nvcc')
if nvcc_cmd is None:
raise DistutilsPlatformError('Unable to find NVCC compiler')
return nvcc_cmd
stderr=subprocess.STDOUT)
except OSError, ex:
if ex.errno == errno.ENOENT:
print(
"Could not find protoc command. Make sure protobuf is "
"installed and your PATH environment is set.")
raise DistutilsPlatformError("Failed to generate protbuf "
"CPP files with protoc.")
else:
raise
out = proc.communicate()[0]
result = proc.wait()
if result != 0:
print "Error during protbuf files generation with protoc:"
print out
raise DistutilsPlatformError("Failed to generate protbuf "
"CPP files with protoc.")
def run(self):
self.generate_protoc()
try:
self.generate_c_file()
except DistutilsPlatformError:
if os.path.exists('imposm/cache/tc.c'):
print 'Found existing C file. Ignoring previous error.'
else:
raise
build_ext.run(self)
import imposm.version
version = imposm.version.__version__
def link(self,
target_desc,
objects,
output_filename,
output_dir=None,
libraries=None,
library_dirs=None,
runtime_library_dirs=None,
export_symbols=None,
debug=0,
extra_preargs=None,
extra_postargs=None,
build_temp=None,
target_lang=None):
objects, output_dir = self._fix_object_args(objects, output_dir)
fixed_args = self._fix_lib_args(libraries, library_dirs,
runtime_library_dirs)
libraries, library_dirs, runtime_library_dirs = fixed_args
lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs,
libraries)
if not isinstance(output_dir, (str, type(None))):
raise TypeError("'output_dir' must be a string or None")
if output_dir is not None:
output_filename = os.path.join(output_dir, output_filename)
if self._need_link(objects, output_filename):
ld_args = (objects + self.objects + lib_opts +
['-o', output_filename])
if debug:
ld_args[:0] = ['-g']
if extra_preargs:
ld_args[:0] = extra_preargs
if extra_postargs:
ld_args.extend(extra_postargs)
self.mkpath(os.path.dirname(output_filename))
try:
if target_desc == CCompiler.EXECUTABLE:
linker = self.linker_exe[:]
else:
linker = self.linker_so[:]
if target_lang == "c++" and self.compiler_cxx:
# skip over environment variable settings if /usr/bin/env
# is used to set up the linker's environment.
# This is needed on OSX. Note: this assumes that the
# normal and C++ compiler have the same environment
# settings.
i = 0
if os.path.basename(linker[0]) == "env":
i = 1
while '=' in linker[i]:
i += 1
linker[i] = self.compiler_cxx[i]
if sys.platform == 'darwin':
linker = _osx_support.compiler_fixup(linker, ld_args)
# self.spawn(linker + ld_args)
raise DistutilsPlatformError("clang not available on iOS for ",
linker + ld_args)
except DistutilsExecError as msg:
raise LinkError(msg)
else:
log.debug("skipping %s (up-to-date)", output_filename)
def finalize_unix(self):
"""Finalizes options for posix platforms."""
if self.install_base is not None or self.install_platbase is not None:
if ((self.install_lib is None and
self.install_purelib is None and
self.install_platlib is None) or
self.install_headers is None or
self.install_scripts is None or
self.install_data is None):
raise DistutilsOptionError(
"install-base or install-platbase supplied, but "
"installation scheme is incomplete")
return
if self.user:
if self.install_userbase is None:
raise DistutilsPlatformError(
"User base directory is not specified")
self.install_base = self.install_platbase = self.install_userbase
self.select_scheme("unix_user")
elif self.home is not None:
self.install_base = self.install_platbase = self.home
self.select_scheme("unix_home")
else:
self.prefix_option = self.prefix
if self.prefix is None:
if self.exec_prefix is not None:
raise DistutilsOptionError(
"must not supply exec-prefix without prefix")
self.prefix = os.path.normpath(sys.prefix)
self.exec_prefix = os.path.normpath(sys.exec_prefix)
else:
if self.exec_prefix is None:
self.exec_prefix = self.prefix
self.install_base = self.prefix
self.install_platbase = self.exec_prefix
if self.install_layout:
if self.install_layout.lower() in ['deb']:
import sysconfig
self.multiarch = sysconfig.get_config_var('MULTIARCH')
self.select_scheme("deb_system")
elif self.install_layout.lower() in ['unix']:
self.select_scheme("unix_prefix")
else:
raise DistutilsOptionError(
"unknown value for --install-layout")
elif ((self.prefix_option and
os.path.normpath(self.prefix) != '/usr/local')
or sys.base_prefix != sys.prefix
or 'PYTHONUSERBASE' in os.environ
or 'VIRTUAL_ENV' in os.environ
or 'real_prefix' in sys.__dict__):
self.select_scheme("unix_prefix")
else:
if os.path.normpath(self.prefix) == '/usr/local':
self.prefix = self.exec_prefix = '/usr'
self.install_base = self.install_platbase = '/usr'
self.select_scheme("unix_local")
def get_rdma_dirs(build_ext):
rdma_include_dirs = []
rdma_lib_dirs = []
rdma_link_flags = []
rdma_lib = []
rdma_home = os.environ.get('T_RING_RDMA_HOME')
if rdma_home:
rdma_include_dirs += ['%s/include' % rdma_home]
rdma_lib_dirs += ['%s/lib' % rdma_home, '%s/lib64' % rdma_home]
rdma_include = os.environ.get('T_RING_RDMA_INCLUDE')
if rdma_include:
rdma_include_dirs += [rdma_include]
rdma_lib = os.environ.get('T_RING_CUDA_LIB')
if rdma_lib:
rdma_lib_dirs += [rdma_lib]
else:
rdma_lib = []
if not rdma_include_dirs and not rdma_lib_dirs:
# default to /usr/local/cuda
rdma_include_dirs += ['/usr/src/mlnx-ofed-kernel-4.1/include']
rdma_lib_dirs += [
'/usr/src/mlnx-ofed-kernel-4.1/lib',
'/usr/src/mlnx-ofed-kernel-4.1/lib64'
]
rdma_lib += ['rdmacm', 'ibverbs']
try:
test_compile(build_ext,
'test_rdma',
include_dirs=rdma_include_dirs,
library_dirs=rdma_lib_dirs,
libraries=rdma_lib,
code=textwrap.dedent('''\
#include <rdma/rdma_cma.h>
void test()
{
char *buffer =NULL;
struct ibv_pd * pd = NULL;
rdma_create_event_channel();
ibv_reg_mr(
pd,
buffer,
1024,
IBV_ACCESS_REMOTE_WRITE);
}
'''))
except (CompileError, LinkError):
raise DistutilsPlatformError(
'RDMA library was not found (see error above).\n'
'Please specify correct RDMA location with the BCUBE_RDMA_HOME '
'environment variable or combination of BCUBE_RDMA_INCLUDE and '
'BCUBE_RDMA_LIB environment variables.\n\n'
'BCUBE_RDMA_HOME - path where RDMA include and lib directories can be found\n'
'BCUBE_RDMA_INCLUDE - path to RDMA include directory\n'
'BCUBE_RDMA_LIB - path to RDMA lib directory')
for lib in rdma_lib:
rdma_link_flags.append('-l%s' % lib)
return rdma_include_dirs, rdma_lib_dirs, rdma_link_flags
def initialize(self, plat_name=None):
# multi-init means we would need to check platform same each time...
assert not self.initialized, "don't init multiple times"
if plat_name is None:
plat_name = get_platform()
# sanity check for platforms to prevent obscure errors later.
if plat_name not in PLAT_TO_VCVARS:
raise DistutilsPlatformError(
"--plat-name must be one of {}".format(tuple(PLAT_TO_VCVARS)))
# On x86, 'vcvarsall.bat amd64' creates an env that doesn't work;
# to cross compile, you use 'x86_amd64'.
# On AMD64, 'vcvarsall.bat amd64' is a native build env; to cross
# compile use 'x86' (ie, it runs the x86 compiler directly)
if plat_name == get_platform() or plat_name == 'win32':
# native build or cross-compile to win32
plat_spec = PLAT_TO_VCVARS[plat_name]
else:
# cross compile from win32 -> some 64bit
plat_spec = '{}_{}'.format(PLAT_TO_VCVARS[get_platform()],
PLAT_TO_VCVARS[plat_name])
vc_env = _get_vc_env(plat_spec)
if not vc_env:
raise DistutilsPlatformError("Unable to find a compatible "
"Visual Studio installation.")
self._paths = vc_env.get('path', '')
paths = self._paths.split(os.pathsep)
self.cc = _find_exe("cl.exe", paths)
self.linker = _find_exe("link.exe", paths)
self.lib = _find_exe("lib.exe", paths)
self.rc = _find_exe("rc.exe", paths) # resource compiler
self.mc = _find_exe("mc.exe", paths) # message compiler
self.mt = _find_exe("mt.exe", paths) # message compiler
for dir in vc_env.get('include', '').split(os.pathsep):
if dir:
self.add_include_dir(dir)
for dir in vc_env.get('lib', '').split(os.pathsep):
if dir:
self.add_library_dir(dir)
self.preprocess_options = None
# Use /MT[d] to build statically, then switch from libucrt[d].lib to ucrt[d].lib
# later to dynamically link to ucrtbase but not vcruntime.
self.compile_options = [
'/nologo', '/Ox', '/MT', '/W3', '/GL', '/DNDEBUG'
]
self.compile_options_debug = [
'/nologo', '/Od', '/MTd', '/Zi', '/W3', '/D_DEBUG'
]
ldflags = [
'/nologo',
'/INCREMENTAL:NO',
'/LTCG',
'/nodefaultlib:libucrt.lib',
'ucrt.lib',
]
ldflags_debug = [
'/nologo',
'/INCREMENTAL:NO',
'/LTCG',
'/DEBUG:FULL',
'/nodefaultlib:libucrtd.lib',
'ucrtd.lib',
]
self.ldflags_exe = [*ldflags, '/MANIFEST:EMBED,ID=1']
self.ldflags_exe_debug = [*ldflags_debug, '/MANIFEST:EMBED,ID=1']
self.ldflags_shared = [
*ldflags, '/DLL', '/MANIFEST:EMBED,ID=2', '/MANIFESTUAC:NO'
]
self.ldflags_shared_debug = [
*ldflags_debug, '/DLL', '/MANIFEST:EMBED,ID=2', '/MANIFESTUAC:NO'
]
self.ldflags_static = [*ldflags]
self.ldflags_static_debug = [*ldflags_debug]
self._ldflags = {
(CCompiler.EXECUTABLE, None): self.ldflags_exe,
(CCompiler.EXECUTABLE, False): self.ldflags_exe,
(CCompiler.EXECUTABLE, True): self.ldflags_exe_debug,
(CCompiler.SHARED_OBJECT, None): self.ldflags_shared,
(CCompiler.SHARED_OBJECT, False): self.ldflags_shared,
(CCompiler.SHARED_OBJECT, True): self.ldflags_shared_debug,
(CCompiler.SHARED_LIBRARY, None): self.ldflags_static,
(CCompiler.SHARED_LIBRARY, False): self.ldflags_static,
(CCompiler.SHARED_LIBRARY, True): self.ldflags_static_debug,
}
self.initialized = True
def _spawn_posix(cmd, search_path=1, verbose=0, dry_run=0):
global _cfg_target
global _cfg_target_split
log.info(' '.join(cmd))
if dry_run:
return
else:
executable = cmd[0]
exec_fn = search_path and os.execvp or os.execv
env = None
if sys.platform == 'darwin':
if _cfg_target is None:
_cfg_target = sysconfig.get_config_var(
'MACOSX_DEPLOYMENT_TARGET') or ''
if _cfg_target:
_cfg_target_split = [
int(x) for x in _cfg_target.split('.')
]
if _cfg_target:
cur_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET',
_cfg_target)
if _cfg_target_split > [int(x) for x in cur_target.split('.')]:
my_msg = '$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" during configure' % (
cur_target, _cfg_target)
raise DistutilsPlatformError(my_msg)
env = dict(os.environ, MACOSX_DEPLOYMENT_TARGET=cur_target)
exec_fn = search_path and os.execvpe or os.execve
pid = os.fork()
if pid == 0:
try:
if env is None:
exec_fn(executable, cmd)
else:
exec_fn(executable, cmd, env)
except OSError as e:
if not DEBUG:
cmd = executable
sys.stderr.write('unable to execute %r: %s\n' %
(cmd, e.strerror))
os._exit(1)
if not DEBUG:
cmd = executable
sys.stderr.write('unable to execute %r for unknown reasons' % cmd)
os._exit(1)
else:
while 1:
try:
pid, status = os.waitpid(pid, 0)
except OSError as exc:
import errno
if exc.errno == errno.EINTR:
continue
if not DEBUG:
cmd = executable
raise DistutilsExecError, 'command %r failed: %s' % (
cmd, exc[-1])
if os.WIFSIGNALED(status):
if not DEBUG:
cmd = executable
raise DistutilsExecError, 'command %r terminated by signal %d' % (
cmd, os.WTERMSIG(status))
if os.WIFEXITED(status):
exit_status = os.WEXITSTATUS(status)
if exit_status == 0:
return
if not DEBUG:
cmd = executable
raise DistutilsExecError, 'command %r failed with exit status %d' % (
cmd, exit_status)
if os.WIFSTOPPED(status):
continue
if not DEBUG:
cmd = executable
raise DistutilsExecError, 'unknown error executing %r: termination status %d' % (
cmd, status)
return
def initialize(self):
self.__paths = []
if "DISTUTILS_USE_SDK" in os.environ and "MSSdk" in os.environ and self.find_exe(
"cl.exe"):
# Assume that the SDK set up everything alright; don't try to be
# smarter
self.cc = "cl.exe"
self.linker = "link.exe"
self.lib = "lib.exe"
self.rc = "rc.exe"
self.mc = "mc.exe"
else:
self.__paths = self.get_msvc_paths("path")
if len(self.__paths) == 0:
raise DistutilsPlatformError(
"Python was built with %s, "
"and extensions need to be built with the same "
"version of the compiler, but it isn't installed." %
self.__product)
self.cc = self.find_exe("cl.exe")
self.linker = self.find_exe("link.exe")
self.lib = self.find_exe("lib.exe")
self.rc = self.find_exe("rc.exe") # resource compiler
self.mc = self.find_exe("mc.exe") # message compiler
self.set_path_env_var('lib')
self.set_path_env_var('include')
# extend the MSVC path with the current path
try:
for p in os.environ['path'].split(';'):
self.__paths.append(p)
except KeyError:
pass
self.__paths = normalize_and_reduce_paths(self.__paths)
os.environ['path'] = ";".join(self.__paths)
self.preprocess_options = None
if self.__arch == "Intel":
self.compile_options = [
'/nologo', '/Ox', '/MD', '/W3', '/GX', '/DNDEBUG'
]
self.compile_options_debug = [
'/nologo', '/Od', '/MDd', '/W3', '/GX', '/Z7', '/D_DEBUG'
]
else:
# Win64
self.compile_options = [
'/nologo', '/Ox', '/MD', '/W3', '/GS-', '/DNDEBUG'
]
self.compile_options_debug = [
'/nologo', '/Od', '/MDd', '/W3', '/GS-', '/Z7', '/D_DEBUG'
]
self.ldflags_shared = ['/DLL', '/nologo', '/INCREMENTAL:NO']
if self.__version >= 7:
self.ldflags_shared_debug = [
'/DLL', '/nologo', '/INCREMENTAL:no', '/DEBUG'
]
else:
self.ldflags_shared_debug = [
'/DLL', '/nologo', '/INCREMENTAL:no', '/pdb:None', '/DEBUG'
]
self.ldflags_static = ['/nologo']
self.initialized = True
def compile(self,
sources,
output_dir=None,
macros=None,
include_dirs=None,
debug=0,
extra_preargs=None,
extra_postargs=None,
depends=None):
macros = macros or []
include_dirs = include_dirs or []
extra_preargs = extra_preargs or []
extra_postargs = extra_postargs or []
pythonVersionOpts = self.versionOpts()
if not os.path.exists(output_dir):
os.makedirs(output_dir)
binpath = _qp(self._binpath)
if self.build_exe:
compileOpts = self._exeCompileOpts
else:
compileOpts = self._compileOpts
outputOpts = self._outputOpts
includePathOpts = []
# All object files will be placed in one of three directories:
# infra - All of the infrastructure's object files.
# project - The project's own object files.
# outside - Any source files specified by the project which are not
# contained in the project's own directory.
orig_sources = sources
sources = []
for source in orig_sources:
if os.path.abspath(source).startswith(os.getcwd()):
sources.append((winpath(source, self.winonly), 'project'))
else:
sources.append((winpath(source, self.winonly), 'outside'))
if self.with_pyd:
for file in _pydFiles:
filePath = os.path.join(_infraDir, 'pyd', file)
if not os.path.isfile(filePath):
raise DistutilsPlatformError(
"Required Pyd source file '%s' is"
" missing." % filePath)
sources.append((winpath(filePath, self.winonly), 'infra'))
for file in _utilFiles:
filePath = os.path.join(_infraDir, 'util', file)
if not os.path.isfile(filePath):
raise DistutilsPlatformError(
"Required util source file '%s' is"
" missing." % filePath)
sources.append((winpath(filePath, self.winonly), 'infra'))
if self.build_deimos:
for file in _deimosFiles:
filePath = os.path.join(_infraDir, 'deimos', 'python', file)
if not os.path.isfile(filePath):
raise DistutilsPlatformError("Required deimos header "
"file '%s' is missing." %
filePath)
sources.append((winpath(filePath, self.winonly), 'infra'))
# If using PydMain, parse the template file
if self.build_exe:
pass
elif self.with_main:
name = self.proj_name
# Store the finished pydmain.d file alongside the object files
infra_output_dir = winpath(os.path.join(output_dir, 'infra'),
self.winonly)
if not os.path.exists(infra_output_dir):
os.makedirs(infra_output_dir)
mainFilename = os.path.join(infra_output_dir, 'pydmain.d')
make_pydmain(mainFilename, name)
sources.append((winpath(mainFilename, self.winonly), 'infra'))
# Add the infraDir to the include path for pyd and utils.
includePathOpts += self._includeOpts
includePathOpts[-1] = includePathOpts[-1] % winpath(
os.path.join(_infraDir), self.winonly)
for include_dir in include_dirs:
includePathOpts += self._includeOpts
includePathOpts[-1] %= winpath(include_dir, self.winonly)
if self.build_exe:
pass
else:
# Add DLL/SO boilerplate code file.
if _isPlatWin:
boilerplatePath = os.path.join(
_infraDir, 'd', 'python_dll_windows_boilerplate.d')
else:
boilerplatePath = os.path.join(
_infraDir, 'd', 'python_so_linux_boilerplate.d')
if not os.path.isfile(boilerplatePath):
raise DistutilsFileError('Required supporting code file "%s"'
' is missing.' % boilerplatePath)
sources.append((winpath(boilerplatePath, self.winonly), 'infra'))
for imp in self.string_imports_from_ext:
if not (os.path.isfile(imp) or os.path.isdir(imp)):
raise DistutilsFileError(
'String import file "%s" does not exist' % imp)
userVersionAndDebugOpts = (
[self._versionOpt % v for v in self.version_flags_from_ext] +
[self._debugOpt % v for v in self.debug_flags_from_ext])
# Optimization opts
if debug:
optimizationOpts = self._debugOptimizeOpts
elif self.optimize:
optimizationOpts = self._releaseOptimizeOpts
else:
optimizationOpts = self._defaultOptimizeOpts
unittestOpt = []
if self.unittest_from_ext:
unittestOpt.append(self._unittestOpt)
if self.property_from_ext:
unittestOpt.append(self._propertyOpt)
if self.string_imports_from_ext:
imps = set()
for imp in self.string_imports_from_ext:
if os.path.isfile(imp):
imps.add(os.path.dirname(os.path.abspath(imp)))
else:
imps.add(os.path.abspath(imp))
unittestOpt.extend(
[self._stringImportOpt % (imp, ) for imp in imps])
objFiles = []
if self.lump:
objName = self._make_object_name(output_dir, 'infra', 'temp')
outOpts = outputOpts[:]
outOpts[-1] = outOpts[-1] % objName
cmdElements = ([binpath] + extra_preargs + unittestOpt +
compileOpts + pythonVersionOpts + optimizationOpts +
includePathOpts + outOpts +
userVersionAndDebugOpts +
[_qp(source[0])
for source in sources] + extra_postargs)
cmdElements = [el for el in cmdElements if el]
wrapped_spawn(self, cmdElements, 'pyd_compile')
return [objName]
else:
for source, source_type in sources:
outOpts = outputOpts[:]
objFilename = os.path.splitext(source)[0]
if source_type == 'project':
objName = self._make_object_name(output_dir, 'project',
objFilename)
elif source_type == 'outside':
objName = self._make_object_name(
output_dir, 'outside', os.path.basename(objFilename))
else: # infra
objName = self._make_object_name(
output_dir, 'infra', os.path.basename(objFilename))
objFiles.append(objName)
outOpts[-1] = outOpts[-1] % objName
cmdElements = ([binpath] + extra_preargs + unittestOpt +
compileOpts + pythonVersionOpts +
optimizationOpts + includePathOpts + outOpts +
userVersionAndDebugOpts + [_qp(source)] +
extra_postargs)
cmdElements = [el for el in cmdElements if el]
spawn0(self, cmdElements)
return objFiles
def initialize(self, plat_name=None):
# multi-init means we would need to check platform same each time...
assert not self.initialized, "don't init multiple times"
if plat_name is None:
plat_name = get_platform()
# sanity check for platforms to prevent obscure errors later.
if plat_name not in PLAT_TO_VCVARS:
raise DistutilsPlatformError(
"--plat-name must be one of {}".format(tuple(PLAT_TO_VCVARS)))
# Get the vcvarsall.bat spec for the requested platform.
plat_spec = PLAT_TO_VCVARS[plat_name]
vc_env = _get_vc_env(plat_spec)
if not vc_env:
raise DistutilsPlatformError("Unable to find a compatible "
"Visual Studio installation.")
self._paths = vc_env.get('path', '')
paths = self._paths.split(os.pathsep)
self.cc = _find_exe("cl.exe", paths)
self.linker = _find_exe("link.exe", paths)
self.lib = _find_exe("lib.exe", paths)
self.rc = _find_exe("rc.exe", paths) # resource compiler
self.mc = _find_exe("mc.exe", paths) # message compiler
self.mt = _find_exe("mt.exe", paths) # message compiler
self._vcruntime_redist = vc_env.get('py_vcruntime_redist', '')
for dir in vc_env.get('include', '').split(os.pathsep):
if dir:
self.add_include_dir(dir)
for dir in vc_env.get('lib', '').split(os.pathsep):
if dir:
self.add_library_dir(dir)
self.preprocess_options = None
# If vcruntime_redist is available, link against it dynamically. Otherwise,
# use /MT[d] to build statically, then switch from libucrt[d].lib to ucrt[d].lib
# later to dynamically link to ucrtbase but not vcruntime.
self.compile_options = ['/nologo', '/Ox', '/W3', '/GL', '/DNDEBUG']
self.compile_options.append('/MD' if self._vcruntime_redist else '/MT')
self.compile_options_debug = [
'/nologo', '/Od', '/MDd', '/Zi', '/W3', '/D_DEBUG'
]
ldflags = ['/nologo', '/INCREMENTAL:NO', '/LTCG']
if not self._vcruntime_redist:
ldflags.extend(('/nodefaultlib:libucrt.lib', 'ucrt.lib'))
ldflags_debug = ['/nologo', '/INCREMENTAL:NO', '/LTCG', '/DEBUG:FULL']
self.ldflags_exe = [*ldflags, '/MANIFEST:EMBED,ID=1']
self.ldflags_exe_debug = [*ldflags_debug, '/MANIFEST:EMBED,ID=1']
self.ldflags_shared = [
*ldflags, '/DLL', '/MANIFEST:EMBED,ID=2', '/MANIFESTUAC:NO'
]
self.ldflags_shared_debug = [
*ldflags_debug, '/DLL', '/MANIFEST:EMBED,ID=2', '/MANIFESTUAC:NO'
]
self.ldflags_static = [*ldflags]
self.ldflags_static_debug = [*ldflags_debug]
self._ldflags = {
(CCompiler.EXECUTABLE, None): self.ldflags_exe,
(CCompiler.EXECUTABLE, False): self.ldflags_exe,
(CCompiler.EXECUTABLE, True): self.ldflags_exe_debug,
(CCompiler.SHARED_OBJECT, None): self.ldflags_shared,
(CCompiler.SHARED_OBJECT, False): self.ldflags_shared,
(CCompiler.SHARED_OBJECT, True): self.ldflags_shared_debug,
(CCompiler.SHARED_LIBRARY, None): self.ldflags_static,
(CCompiler.SHARED_LIBRARY, False): self.ldflags_static,
(CCompiler.SHARED_LIBRARY, True): self.ldflags_static_debug,
}
self.initialized = True
def spawn(cmd, search_path=1, verbose=0, dry_run=0, env=None):
"""Run another program, specified as a command list 'cmd', in a new process.
'cmd' is just the argument list for the new process, ie.
cmd[0] is the program to run and cmd[1:] are the rest of its arguments.
There is no way to run a program with a name different from that of its
executable.
If 'search_path' is true (the default), the system's executable
search path will be used to find the program; otherwise, cmd[0]
must be the exact path to the executable. If 'dry_run' is true,
the command will not actually be run.
Raise DistutilsExecError if running the program fails in any way; just
return on success.
"""
# cmd is documented as a list, but just in case some code passes a tuple
# in, protect our %-formatting code against horrible death
cmd = list(cmd)
log.info(' '.join(cmd))
if dry_run:
return
if search_path:
executable = find_executable(cmd[0])
if executable is not None:
cmd[0] = executable
env = env if env is not None else dict(os.environ)
if sys.platform == 'darwin':
global _cfg_target, _cfg_target_split
if _cfg_target is None:
from distutils import sysconfig
_cfg_target = sysconfig.get_config_var(
'MACOSX_DEPLOYMENT_TARGET') or ''
if _cfg_target:
_cfg_target_split = [int(x) for x in _cfg_target.split('.')]
if _cfg_target:
# ensure that the deployment target of build process is not less
# than that used when the interpreter was built. This ensures
# extension modules are built with correct compatibility values
cur_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', _cfg_target)
if _cfg_target_split > [int(x) for x in cur_target.split('.')]:
my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: '
'now "%s" but "%s" during configure'
% (cur_target, _cfg_target))
raise DistutilsPlatformError(my_msg)
env.update(MACOSX_DEPLOYMENT_TARGET=cur_target)
try:
proc = subprocess.Popen(cmd, env=env)
proc.wait()
exitcode = proc.returncode
except OSError as exc:
if not DEBUG:
cmd = cmd[0]
raise DistutilsExecError(
"command %r failed: %s" % (cmd, exc.args[-1])) from exc
if exitcode:
if not DEBUG:
cmd = cmd[0]
raise DistutilsExecError(
"command %r failed with exit code %s" % (cmd, exitcode))
def run(self):
raise DistutilsPlatformError("bdist_wininst is not supported "
"in this Python distribution")
def finalize_options(self):
if sys.version_info[:2] != (2, 4):
raise DistutilsPlatformError(
'check_py24 command require Python 2.4')
def initialize(self, plat_name=None):
# multi-init means we would need to check platform same each time...
assert not self.initialized, "don't init multiple times"
if plat_name is None:
plat_name = get_platform()
# sanity check for platforms to prevent obscure errors later.
if plat_name not in PLAT_TO_VCVARS:
raise DistutilsPlatformError(
"--plat-name must be one of {}".format(tuple(PLAT_TO_VCVARS)))
# Get the vcvarsall.bat spec for the requested platform.
plat_spec = PLAT_TO_VCVARS[plat_name]
vc_env = _get_vc_env(plat_spec)
if not vc_env:
raise DistutilsPlatformError("Unable to find a compatible "
"Visual Studio installation.")
self._paths = vc_env.get("path", "")
paths = self._paths.split(os.pathsep)
self.cc = _find_exe("cl.exe", paths)
self.linker = _find_exe("link.exe", paths)
self.lib = _find_exe("lib.exe", paths)
self.rc = _find_exe("rc.exe", paths) # resource compiler
self.mc = _find_exe("mc.exe", paths) # message compiler
self.mt = _find_exe("mt.exe", paths) # message compiler
for dir in vc_env.get("include", "").split(os.pathsep):
if dir:
self.add_include_dir(dir.rstrip(os.sep))
for dir in vc_env.get("lib", "").split(os.pathsep):
if dir:
self.add_library_dir(dir.rstrip(os.sep))
self.preprocess_options = None
# bpo-38597: Always compile with dynamic linking
# Future releases of Python 3.x will include all past
# versions of vcruntime*.dll for compatibility.
self.compile_options = [
"/nologo", "/Ox", "/W3", "/GL", "/DNDEBUG", "/MD"
]
self.compile_options_debug = [
"/nologo",
"/Od",
"/MDd",
"/Zi",
"/W3",
"/D_DEBUG",
]
ldflags = ["/nologo", "/INCREMENTAL:NO", "/LTCG"]
ldflags_debug = ["/nologo", "/INCREMENTAL:NO", "/LTCG", "/DEBUG:FULL"]
self.ldflags_exe = [*ldflags, "/MANIFEST:EMBED,ID=1"]
self.ldflags_exe_debug = [*ldflags_debug, "/MANIFEST:EMBED,ID=1"]
self.ldflags_shared = [
*ldflags,
"/DLL",
"/MANIFEST:EMBED,ID=2",
"/MANIFESTUAC:NO",
]
self.ldflags_shared_debug = [
*ldflags_debug,
"/DLL",
"/MANIFEST:EMBED,ID=2",
"/MANIFESTUAC:NO",
]
self.ldflags_static = [*ldflags]
self.ldflags_static_debug = [*ldflags_debug]
self._ldflags = {
(CCompiler.EXECUTABLE, None): self.ldflags_exe,
(CCompiler.EXECUTABLE, False): self.ldflags_exe,
(CCompiler.EXECUTABLE, True): self.ldflags_exe_debug,
(CCompiler.SHARED_OBJECT, None): self.ldflags_shared,
(CCompiler.SHARED_OBJECT, False): self.ldflags_shared,
(CCompiler.SHARED_OBJECT, True): self.ldflags_shared_debug,
(CCompiler.SHARED_LIBRARY, None): self.ldflags_static,
(CCompiler.SHARED_LIBRARY, False): self.ldflags_static,
(CCompiler.SHARED_LIBRARY, True): self.ldflags_static_debug,
}
self.initialized = True
def configure_mpi(ext, config_cmd):
from textwrap import dedent
from distutils import log
from distutils.errors import DistutilsPlatformError
headers = ['stdlib.h', 'mpi.h']
#
log.info("checking for MPI compile and link ...")
ConfigTest = dedent("""\
int main(int argc, char **argv)
{
(void)MPI_Init(&argc, &argv);
(void)MPI_Finalize();
return 0;
}
""")
errmsg = "Cannot %s MPI programs. Check your configuration!!!"
ok = config_cmd.try_compile(ConfigTest, headers=headers)
if not ok: raise DistutilsPlatformError(errmsg % "compile")
ok = config_cmd.try_link(ConfigTest, headers=headers)
if not ok: raise DistutilsPlatformError(errmsg % "link")
#
log.info("checking for missing MPI functions/symbols ...")
tests = ["defined(%s)" % macro for macro in
("OPEN_MPI", "MSMPI_VER",)]
tests += ["(defined(MPICH_NAME)&&(MPICH_NAME==3))"]
tests += ["(defined(MPICH_NAME)&&(MPICH_NAME==2))"]
ConfigTest = dedent("""\
#if !(%s)
#error "Unknown MPI implementation"
#endif
""") % "||".join(tests)
ok = config_cmd.try_compile(ConfigTest, headers=headers)
if not ok:
from mpidistutils import ConfigureMPI
configure = ConfigureMPI(config_cmd)
results = configure.run()
configure.dump(results)
ext.define_macros += [('HAVE_CONFIG_H', 1)]
else:
for function, arglist in (
('MPI_Type_create_f90_integer', '0,(MPI_Datatype*)0'),
('MPI_Type_create_f90_real', '0,0,(MPI_Datatype*)0'),
('MPI_Type_create_f90_complex', '0,0,(MPI_Datatype*)0'),
('MPI_Status_c2f', '(MPI_Status*)0,(MPI_Fint*)0'),
('MPI_Status_f2c', '(MPI_Fint*)0,(MPI_Status*)0'),
):
ok = config_cmd.check_function_call(
function, arglist, headers=headers)
if not ok:
macro = 'PyMPI_MISSING_' + function
ext.define_macros += [(macro, 1)]
for symbol, stype in (
('MPI_LB', 'MPI_Datatype'),
('MPI_UB', 'MPI_Datatype'),
):
ok = config_cmd.check_symbol(
symbol, type=stype, headers=headers)
if not ok:
macro = 'PyMPI_MISSING_' + symbol
ext.define_macros += [(macro, 1)]
#
if os.name == 'posix':
configure_dl(ext, config_cmd)
def initialize(self, plat_name=None):
# multi-init means we would need to check platform same each time...
assert not self.initialized, "don't init multiple times"
if plat_name is None:
plat_name = get_platform()
# sanity check for platforms to prevent obscure errors later.
ok_plats = 'win32', 'win-amd64', 'win-ia64'
if plat_name not in ok_plats:
raise DistutilsPlatformError("--plat-name must be one of %s" %
(ok_plats, ))
if "DISTUTILS_USE_SDK" in os.environ and "MSSdk" in os.environ and self.find_exe(
"cl.exe"):
# Assume that the SDK set up everything alright; don't try to be
# smarter
self.cc = "cl.exe"
self.linker = "link.exe"
self.lib = "lib.exe"
self.rc = "rc.exe"
self.mc = "mc.exe"
else:
# On x86, 'vcvars32.bat amd64' creates an env that doesn't work;
# to cross compile, you use 'x86_amd64'.
# On AMD64, 'vcvars32.bat amd64' is a native build env; to cross
# compile use 'x86' (ie, it runs the x86 compiler directly)
# No idea how itanium handles this, if at all.
if plat_name == get_platform() or plat_name == 'win32':
# native build or cross-compile to win32
plat_spec = PLAT_TO_VCVARS[plat_name]
else:
# cross compile from win32 -> some 64bit
plat_spec = PLAT_TO_VCVARS[get_platform()] + '_' + \
PLAT_TO_VCVARS[plat_name]
vc_env = query_vcvarsall(VERSION, plat_spec)
# take care to only use strings in the environment.
self.__paths = vc_env['path'].encode('mbcs').split(os.pathsep)
os.environ['lib'] = vc_env['lib'].encode('mbcs')
os.environ['include'] = vc_env['include'].encode('mbcs')
if len(self.__paths) == 0:
raise DistutilsPlatformError(
"Python was built with %s, "
"and extensions need to be built with the same "
"version of the compiler, but it isn't installed." %
self.__product)
self.cc = self.find_exe("cl.exe")
self.linker = self.find_exe("link.exe")
self.lib = self.find_exe("lib.exe")
self.rc = self.find_exe("rc.exe") # resource compiler
self.mc = self.find_exe("mc.exe") # message compiler
#self.set_path_env_var('lib')
#self.set_path_env_var('include')
# extend the MSVC path with the current path
try:
for p in os.environ['path'].split(';'):
self.__paths.append(p)
except KeyError:
pass
self.__paths = normalize_and_reduce_paths(self.__paths)
os.environ['path'] = ";".join(self.__paths)
self.preprocess_options = None
if self.__arch == "x86":
self.compile_options = ['/nologo', '/Ox', '/MD', '/W3', '/DNDEBUG']
self.compile_options_debug = [
'/nologo', '/Od', '/MDd', '/W3', '/Z7', '/D_DEBUG'
]
else:
# Win64
self.compile_options = [
'/nologo', '/Ox', '/MD', '/W3', '/GS-', '/DNDEBUG'
]
self.compile_options_debug = [
'/nologo', '/Od', '/MDd', '/W3', '/GS-', '/Z7', '/D_DEBUG'
]
self.ldflags_shared = ['/DLL', '/nologo', '/INCREMENTAL:NO']
if self.__version >= 7:
self.ldflags_shared_debug = [
'/DLL', '/nologo', '/INCREMENTAL:no', '/DEBUG', '/pdb:None'
]
self.ldflags_static = ['/nologo']
self.initialized = True
def build_tf_extension(build_ext, tf_lib, options):
check_tf_version()
tf_compile_flags, tf_link_flags = get_tf_flags(build_ext, options['COMPILE_FLAGS'])
tf_lib.define_macros = options['MACROS'] + tf_lib.define_macros
tf_lib.include_dirs = options['INCLUDES'] + tf_lib.include_dirs
tf_lib.sources = options['SOURCES'] + tf_lib.sources
tf_lib.extra_compile_args = options['COMPILE_FLAGS'] + tf_compile_flags + tf_lib.extra_compile_args
tf_lib.extra_link_args = options['LINK_FLAGS'] + tf_link_flags + tf_lib.extra_link_args
tf_lib.library_dirs = options['LIBRARY_DIRS'] + tf_lib.library_dirs
tf_lib.libraries = options['LIBRARIES'] + tf_lib.libraries
cc_compiler = cxx_compiler = None
if not sys.platform.startswith('linux'):
raise EnvironmentError("Only Linux Systems are supported")
if not os.getenv('CC') and not os.getenv('CXX'):
# Determine g++ version compatible with this TensorFlow installation
import tensorflow as tf
if hasattr(tf, 'version'):
# Since TensorFlow 1.13.0
tf_compiler_version = LooseVersion(tf.version.COMPILER_VERSION)
else:
tf_compiler_version = LooseVersion(tf.COMPILER_VERSION)
if tf_compiler_version.version[0] == 4:
# g++ 4.x is ABI-incompatible with g++ 5.x+ due to std::function change
# See: https://github.com/tensorflow/tensorflow/issues/27067
maximum_compiler_version = LooseVersion('5')
else:
maximum_compiler_version = LooseVersion('999')
# Find the compatible compiler of the highest version
compiler_version = LooseVersion('0')
for candidate_cxx_compiler, candidate_compiler_version in find_gxx_compiler_in_path():
if tf_compiler_version <= candidate_compiler_version < maximum_compiler_version:
candidate_cc_compiler = find_matching_gcc_compiler_path(candidate_compiler_version)
if candidate_cc_compiler and candidate_compiler_version > compiler_version:
cc_compiler = candidate_cc_compiler
cxx_compiler = candidate_cxx_compiler
compiler_version = candidate_compiler_version
else:
print("===========================================================================================")
print(
'INFO: Compiler %s (version %s) is not usable for this TensorFlow '
'installation. Require g++ (version >=%s, <%s).' %
(candidate_cxx_compiler, candidate_compiler_version, tf_compiler_version, maximum_compiler_version)
)
print("===========================================================================================")
if cc_compiler:
print("===========================================================================================")
print('INFO: Compilers %s and %s (version %s) selected for TensorFlow plugin build.' % (
cc_compiler, cxx_compiler, compiler_version
))
print("===========================================================================================")
else:
raise DistutilsPlatformError(
'Could not find compiler compatible with this TensorFlow installation.\n'
'Please check the NVTX-Plugins Github Repository for recommended compiler versions.\n'
'To force a specific compiler version, set CC and CXX environment variables.')
cflags, cppflags, ldshared = remove_offensive_gcc_compiler_options(compiler_version)
try:
with env(CC=cc_compiler, CXX=cxx_compiler, CFLAGS=cflags, CPPFLAGS=cppflags, LDSHARED=ldshared):
customize_compiler(build_ext.compiler)
try:
build_ext.compiler.compiler.remove("-DNDEBUG")
except (AttributeError, ValueError):
pass
try:
build_ext.compiler.compiler_so.remove("-DNDEBUG")
except (AttributeError, ValueError):
pass
try:
build_ext.compiler.compiler_so.remove("-Wstrict-prototypes")
except (AttributeError, ValueError):
pass
try:
build_ext.compiler.linker_so.remove("-Wl,-O1")
except (AttributeError, ValueError):
pass
build_ext.build_extension(tf_lib)
finally:
# Revert to the default compiler settings
customize_compiler(build_ext.compiler)
def _spawn_posix(cmd, search_path=1, verbose=0, dry_run=0):
log.info(' '.join(cmd))
if dry_run:
return
executable = cmd[0]
exec_fn = search_path and os.execvp or os.execv
env = None
if sys.platform == 'darwin':
global _cfg_target, _cfg_target_split
if _cfg_target is None:
_cfg_target = sysconfig.get_config_var(
'MACOSX_DEPLOYMENT_TARGET') or ''
if _cfg_target:
_cfg_target_split = [int(x) for x in _cfg_target.split('.')]
if _cfg_target:
# ensure that the deployment target of build process is not less
# than that used when the interpreter was built. This ensures
# extension modules are built with correct compatibility values
cur_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET',
_cfg_target)
if _cfg_target_split > [int(x) for x in cur_target.split('.')]:
my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: '
'now "%s" but "%s" during configure' %
(cur_target, _cfg_target))
raise DistutilsPlatformError(my_msg)
env = dict(os.environ, MACOSX_DEPLOYMENT_TARGET=cur_target)
exec_fn = search_path and os.execvpe or os.execve
pid = os.fork()
if pid == 0: # in the child
try:
if env is None:
exec_fn(executable, cmd)
else:
exec_fn(executable, cmd, env)
except OSError as e:
if not DEBUG:
cmd = executable
sys.stderr.write("unable to execute %r: %s\n" % (cmd, e.strerror))
os._exit(1)
if not DEBUG:
cmd = executable
sys.stderr.write("unable to execute %r for unknown reasons" % cmd)
os._exit(1)
else: # in the parent
# Loop until the child either exits or is terminated by a signal
# (ie. keep waiting if it's merely stopped)
while True:
try:
pid, status = os.waitpid(pid, 0)
except OSError as exc:
if not DEBUG:
cmd = executable
raise DistutilsExecError("command %r failed: %s" %
(cmd, exc.args[-1]))
if os.WIFSIGNALED(status):
if not DEBUG:
cmd = executable
raise DistutilsExecError("command %r terminated by signal %d" %
(cmd, os.WTERMSIG(status)))
elif os.WIFEXITED(status):
exit_status = os.WEXITSTATUS(status)
if exit_status == 0:
return # hey, it succeeded!
else:
if not DEBUG:
cmd = executable
raise DistutilsExecError(
"command %r failed with exit status %d" %
(cmd, exit_status))
elif os.WIFSTOPPED(status):
continue
else:
if not DEBUG:
cmd = executable
raise DistutilsExecError(
"unknown error executing %r: termination status %d" %
(cmd, status))
def check_torch_import():
try:
import torch
except ImportError:
raise DistutilsPlatformError(
'import torch failed, is it installed?\n\n%s' % traceback.format_exc())
def initialize(self, plat_name=None):
# multi-init means we would need to check platform same each time...
assert not self.initialized, "don't init multiple times"
if plat_name is None:
plat_name = get_platform()
# sanity check for platforms to prevent obscure errors later.
ok_plats = "win32", "win-amd64"
if plat_name not in ok_plats:
raise DistutilsPlatformError("--plat-name must be one of %s" % (ok_plats,))
if (
"DISTUTILS_USE_SDK" in os.environ
and "MSSdk" in os.environ
and self.find_exe("cl.exe")
):
# Assume that the SDK set up everything alright; don't try to be
# smarter
self.cc = "cl.exe"
self.linker = "link.exe"
self.lib = "lib.exe"
self.rc = "rc.exe"
self.mc = "mc.exe"
else:
# On x86, 'vcvars32.bat amd64' creates an env that doesn't work;
# to cross compile, you use 'x86_amd64'.
# On AMD64, 'vcvars32.bat amd64' is a native build env; to cross
# compile use 'x86' (ie, it runs the x86 compiler directly)
if plat_name == get_platform() or plat_name == "win32":
# native build or cross-compile to win32
plat_spec = PLAT_TO_VCVARS[plat_name]
else:
# cross compile from win32 -> some 64bit
plat_spec = (
PLAT_TO_VCVARS[get_platform()] + "_" + PLAT_TO_VCVARS[plat_name]
)
vc_env = query_vcvarsall(VERSION, plat_spec)
self.__paths = vc_env["path"].split(os.pathsep)
os.environ["lib"] = vc_env["lib"]
os.environ["include"] = vc_env["include"]
if len(self.__paths) == 0:
raise DistutilsPlatformError(
"Python was built with %s, "
"and extensions need to be built with the same "
"version of the compiler, but it isn't installed." % self.__product
)
self.cc = self.find_exe("cl.exe")
self.linker = self.find_exe("link.exe")
self.lib = self.find_exe("lib.exe")
self.rc = self.find_exe("rc.exe") # resource compiler
self.mc = self.find_exe("mc.exe") # message compiler
# self.set_path_env_var('lib')
# self.set_path_env_var('include')
# extend the MSVC path with the current path
try:
for p in os.environ["path"].split(";"):
self.__paths.append(p)
except KeyError:
pass
self.__paths = normalize_and_reduce_paths(self.__paths)
os.environ["path"] = ";".join(self.__paths)
self.preprocess_options = None
if self.__arch == "x86":
self.compile_options = ["/nologo", "/Ox", "/MD", "/W3", "/DNDEBUG"]
self.compile_options_debug = [
"/nologo",
"/Od",
"/MDd",
"/W3",
"/Z7",
"/D_DEBUG",
]
else:
# Win64
self.compile_options = ["/nologo", "/Ox", "/MD", "/W3", "/GS-", "/DNDEBUG"]
self.compile_options_debug = [
"/nologo",
"/Od",
"/MDd",
"/W3",
"/GS-",
"/Z7",
"/D_DEBUG",
]
self.ldflags_shared = ["/DLL", "/nologo", "/INCREMENTAL:NO"]
if self.__version >= 7:
self.ldflags_shared_debug = ["/DLL", "/nologo", "/INCREMENTAL:no", "/DEBUG"]
self.ldflags_static = ["/nologo"]
self.initialized = True
def link(self,
target_desc,
objects,
output_filename,
output_dir=None,
libraries=None,
library_dirs=None,
runtime_library_dirs=None,
export_symbols=None,
debug=0,
extra_preargs=None,
extra_postargs=None,
build_temp=None,
target_lang=None):
# First fixup.
(objects, output_dir) = self._fix_object_args(objects, output_dir)
(libraries, library_dirs, runtime_library_dirs) = \
self._fix_lib_args (libraries, library_dirs, runtime_library_dirs)
# First examine a couple of options for things that aren't implemented yet
if not target_desc in (self.SHARED_LIBRARY, self.SHARED_OBJECT):
raise DistutilsPlatformError(
'Can only make SHARED_LIBRARY or SHARED_OBJECT targets on the Mac'
)
if runtime_library_dirs:
raise DistutilsPlatformError(
'Runtime library dirs not implemented yet')
if extra_preargs or extra_postargs:
raise DistutilsPlatformError(
'Runtime library dirs not implemented yet')
if len(export_symbols) != 1:
raise DistutilsPlatformError('Need exactly one export symbol')
# Next there are various things for which we need absolute pathnames.
# This is because we (usually) create the project in a subdirectory of
# where we are now, and keeping the paths relative is too much work right
# now.
sources = [self._filename_to_abs(s) for s in self.__sources]
include_dirs = [self._filename_to_abs(d) for d in self.__include_dirs]
if objects:
objects = [self._filename_to_abs(o) for o in objects]
else:
objects = []
if build_temp:
build_temp = self._filename_to_abs(build_temp)
else:
build_temp = os.curdir()
if output_dir:
output_filename = os.path.join(output_dir, output_filename)
# The output filename needs special handling: splitting it into dir and
# filename part. Actually I'm not sure this is really needed, but it
# can't hurt.
output_filename = self._filename_to_abs(output_filename)
output_dir, output_filename = os.path.split(output_filename)
# Now we need the short names of a couple of things for putting them
# into the project.
if output_filename[-8:] == '.ppc.slb':
basename = output_filename[:-8]
elif output_filename[-11:] == '.carbon.slb':
basename = output_filename[:-11]
else:
basename = os.path.strip(output_filename)[0]
projectname = basename + '.mcp'
targetname = basename
xmlname = basename + '.xml'
exportname = basename + '.mcp.exp'
prefixname = 'mwerks_%s_config.h' % basename
# Create the directories we need
distutils.dir_util.mkpath(build_temp, dry_run=self.dry_run)
distutils.dir_util.mkpath(output_dir, dry_run=self.dry_run)
# And on to filling in the parameters for the project builder
settings = {}
settings['mac_exportname'] = exportname
settings['mac_outputdir'] = output_dir
settings['mac_dllname'] = output_filename
settings['mac_targetname'] = targetname
settings['sysprefix'] = sys.prefix
settings['mac_sysprefixtype'] = 'Absolute'
sourcefilenames = []
sourcefiledirs = []
for filename in sources + objects:
dirname, filename = os.path.split(filename)
sourcefilenames.append(filename)
if not dirname in sourcefiledirs:
sourcefiledirs.append(dirname)
settings['sources'] = sourcefilenames
settings['libraries'] = libraries
settings[
'extrasearchdirs'] = sourcefiledirs + include_dirs + library_dirs
if self.dry_run:
print('CALLING LINKER IN', os.getcwd())
for key, value in settings.items():
print('%20.20s %s' % (key, value))
return
# Build the export file
exportfilename = os.path.join(build_temp, exportname)
log.debug("\tCreate export file %s", exportfilename)
fp = open(exportfilename, 'w')
fp.write('%s\n' % export_symbols[0])
fp.close()
# Generate the prefix file, if needed, and put it in the settings
if self.__macros:
prefixfilename = os.path.join(os.getcwd(),
os.path.join(build_temp, prefixname))
fp = open(prefixfilename, 'w')
fp.write('#include "mwerks_shcarbon_config.h"\n')
for name, value in self.__macros:
if value is None:
fp.write('#define %s\n' % name)
else:
fp.write('#define %s %s\n' % (name, value))
fp.close()
settings['prefixname'] = prefixname
# Build the XML file. We need the full pathname (only lateron, really)
# because we pass this pathname to CodeWarrior in an AppleEvent, and CW
# doesn't have a clue about our working directory.
xmlfilename = os.path.join(os.getcwd(),
os.path.join(build_temp, xmlname))
log.debug("\tCreate XML file %s", xmlfilename)
import mkcwproject
xmlbuilder = mkcwproject.cwxmlgen.ProjectBuilder(settings)
xmlbuilder.generate()
xmldata = settings['tmp_projectxmldata']
fp = open(xmlfilename, 'w')
fp.write(xmldata)
fp.close()
# Generate the project. Again a full pathname.
projectfilename = os.path.join(os.getcwd(),
os.path.join(build_temp, projectname))
log.debug('\tCreate project file %s', projectfilename)
mkcwproject.makeproject(xmlfilename, projectfilename)
# And build it
log.debug('\tBuild project')
mkcwproject.buildproject(projectfilename)
