def _setup_extensions(self):
ext = {e.name: e for e in self.extensions}
compiler = new_compiler(compiler=self.compiler)
customize_compiler(compiler)
def add_dependency(ext, name):
add_ext_pkg_config_dep(ext, compiler.compiler_type, name)
def add_pycairo(ext):
ext.include_dirs += [get_pycairo_include_dir()]
gi_ext = ext["gi._gi"]
add_dependency(gi_ext, "glib-2.0")
add_dependency(gi_ext, "gio-2.0")
add_dependency(gi_ext, "gobject-introspection-1.0")
add_dependency(gi_ext, "libffi")
add_ext_compiler_flags(gi_ext, compiler)
if WITH_CAIRO:
gi_cairo_ext = ext["gi._gi_cairo"]
add_dependency(gi_cairo_ext, "glib-2.0")
add_dependency(gi_cairo_ext, "gio-2.0")
add_dependency(gi_cairo_ext, "gobject-introspection-1.0")
add_dependency(gi_cairo_ext, "libffi")
add_dependency(gi_cairo_ext, "cairo")
add_dependency(gi_cairo_ext, "cairo-gobject")
add_pycairo(gi_cairo_ext)
add_ext_compiler_flags(gi_cairo_ext, compiler)
def __init__(self, debug=False):
self._compiler = new_compiler()
log.set_threshold(log.DEBUG if debug else log.INFO)
customize_compiler(self._compiler)
self._build_ext = build_ext(Distribution())
self._build_ext.finalize_options()
self._py_lib_dirs = self._build_ext.library_dirs
def _have_sqlite_extension_support():
import shutil
import tempfile
from distutils.ccompiler import new_compiler
from distutils.sysconfig import customize_compiler
libraries = ['sqlite3']
c_code = ('#include <sqlite3.h>\n\n'
'int main(int argc, char **argv) { return 0; }')
tmp_dir = tempfile.mkdtemp(prefix='tmp_pw_sqlite3_')
bin_file = os.path.join(tmp_dir, 'test_pw_sqlite3')
src_file = bin_file + '.c'
with open(src_file, 'w') as fh:
fh.write(c_code)
compiler = new_compiler()
customize_compiler(compiler)
success = False
try:
compiler.link_executable(
compiler.compile([src_file], output_dir=tmp_dir),
bin_file,
libraries=['sqlite3'])
except CCompilerError:
print('unable to compile sqlite3 C extensions - missing headers?')
except DistutilsExecError:
print('unable to compile sqlite3 C extensions - no c compiler?')
except DistutilsPlatformError:
print('unable to compile sqlite3 C extensions - platform error')
else:
success = True
shutil.rmtree(tmp_dir)
return success
def make_compiler(compiler_cmd='g++'):
executables = {
'preprocessor': None,
'compiler': [compiler_cmd],
'compiler_so': [compiler_cmd],
'compiler_cxx': [compiler_cmd],
'linker_so': [compiler_cmd, "-shared"],
'linker_exe': [compiler_cmd],
'archiver': ["ar", "-cr"],
'ranlib': None,
}
compiler = new_compiler(verbose=1)
compiler.set_executables(**executables)
compiler.add_include_dir('/usr/local/include')
compiler.add_include_dir('/usr/include')
compiler.add_include_dir('/sw/include')
compiler.add_include_dir(distutils.sysconfig.get_python_inc())
compiler.add_include_dir('/sw/lib')
compiler.add_library_dir('/usr/local/lib')
compiler.add_library_dir('/usr/lib')
compiler.add_library_dir('.')
for i in numpy.distutils.misc_util.get_numpy_include_dirs():
compiler.add_include_dir(i)
output_dir = '.'
compiler.add_library('m')
compiler.add_library('blas')
return compiler
def check_sanity():
"""
Test if development headers and library for rbd is available by compiling a dummy C program.
"""
CEPH_SRC_DIR = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'..',
'..'
)
tmp_dir = tempfile.mkdtemp(dir=os.environ.get('TMPDIR', os.path.dirname(__file__)))
tmp_file = os.path.join(tmp_dir, 'rbd_dummy.c')
with open(tmp_file, 'w') as fp:
dummy_prog = textwrap.dedent("""
#include <stddef.h>
#include <rbd/librbd.h>
int main(void) {
rados_t cluster;
rados_create(&cluster, NULL);
return 0;
}
""")
fp.write(dummy_prog)
compiler = new_compiler()
distutils.sysconfig.customize_compiler(compiler)
if {'MAKEFLAGS', 'MFLAGS', 'MAKELEVEL'}.issubset(set(os.environ.keys())):
# The setup.py has been invoked by a top-level Ceph make.
# Set the appropriate CFLAGS and LDFLAGS
compiler.set_include_dirs([os.path.join(CEPH_SRC_DIR, 'include')])
compiler.set_library_dirs([os.environ.get('CEPH_LIBDIR')])
try:
compiler.define_macro('_FILE_OFFSET_BITS', '64')
link_objects = compiler.compile(
sources=[tmp_file],
output_dir=tmp_dir
)
compiler.link_executable(
objects=link_objects,
output_progname=os.path.join(tmp_dir, 'rbd_dummy'),
libraries=['rbd', 'rados'],
output_dir=tmp_dir,
)
except CompileError:
print('\nCompile Error: RBD development headers not found', file=sys.stderr)
return False
except LinkError:
print('\nLink Error: RBD library not found', file=sys.stderr)
return False
else:
return True
finally:
shutil.rmtree(tmp_dir)
def run(self):
# This should always work when building a Debian package.
if os.path.exists("/usr/bin/qmake-qt4"):
retval = subprocess.call(
"(cd qt/frontend; qmake-qt4; make)", shell=True)
if retval:
raise SystemExit(retval)
DATA_FILES.append(("lib/checkbox/qt/",
["qt/frontend/checkbox-qt-service"]))
super(checkbox_build, self).run()
cc = new_compiler()
for source in self.sources:
executable = os.path.splitext(source)[0]
cc.link_executable(
[source], executable,
libraries=["rt", "pthread", "nl-3", "nl-genl-3"],
# Enforce security with CFLAGS + LDFLAGS (see dpkg-buildflags)
extra_preargs=[
"-O2", "-fstack-protector",
"--param=ssp-buffer-size=4", "-Wformat",
"-Werror=format-security",
"-Wl,-Bsymbolic-functions",
"-Wl,-z,relro",
"-I/usr/include/libnl3"])
def run(self):
cc = new_compiler()
customize_compiler(cc)
o_files = cc.compile(['githome/gh_client.c'])
cc.link_executable(o_files, 'githome/gh_client')
install.run(self) # run normal build command
def run(self):
for exe in self.distribution.executables:
exe.include_dirs.append(get_python_inc())
compiler = new_compiler( # compiler=self.compiler,
verbose=self.verbose,
dry_run=self.dry_run,
force=self.force)
customize_compiler(compiler)
compiler.set_include_dirs(exe.include_dirs)
for (name, value) in exe.define_macros:
compiler.define_macro(name, value)
objects = compiler.compile(exe.sources, output_dir=self.build_temp)
# This is a hack copied from distutils.commands.build_exe (where it is also called
# a hack).
self._build_objects = objects[:]
library_dirs = [os.path.join(sys.exec_prefix, 'lib')]
exe_path = join(self.build_lib, exe.name.split('.')[-1])
compiler.link(CCompiler.EXECUTABLE,
objects=objects,
output_filename=exe_path,
library_dirs=library_dirs,
libraries=exe.libraries
)
def __init__(self, disable_openmp):
self.msvc = new_compiler().compiler_type == 'msvc'
if disable_openmp:
self.openmp_enabled = False
else:
self.openmp_enabled, openmp_needs_gomp = self._detect_openmp()
self.sse3_enabled = self._detect_sse3() if not self.msvc else True
self.sse41_enabled = self._detect_sse41() if not self.msvc else True
self.compiler_args_sse2 = ['-msse2'] if not self.msvc else ['/arch:SSE2']
self.compiler_args_sse3 = ['-mssse3'] if (self.sse3_enabled and not self.msvc) else []
self.compiler_args_sse41, self.define_macros_sse41 = [], []
if self.sse41_enabled:
self.define_macros_sse41 = [('__SSE4__', 1), ('__SSE4_1__', 1)]
if not self.msvc:
self.compiler_args_sse41 = ['-msse4']
if self.openmp_enabled:
self.compiler_libraries_openmp = ['gomp'] if openmp_needs_gomp else []
if self.msvc:
self.compiler_args_openmp = ['/openmp']
else:
self.compiler_args_openmp = ['-fopenmp']
else:
self.compiler_libraries_openmp = []
self.compiler_args_openmp = []
if self.msvc:
self.compiler_args_opt = ['/O2']
else:
self.compiler_args_opt = ['-O3', '-funroll-loops']
def test_run(self):
# can't test on windows
if sys.platform == "win32":
return
pkg_dir, dist = self.create_dist()
cmd = build_clib(dist)
foo_c = os.path.join(pkg_dir, "foo.c")
self.write_file(foo_c, "int main(void) { return 1;}\n")
cmd.libraries = [("foo", {"sources": [foo_c]})]
build_temp = os.path.join(pkg_dir, "build")
os.mkdir(build_temp)
cmd.build_temp = build_temp
cmd.build_clib = build_temp
# before we run the command, we want to make sure
# all commands are present on the system
# by creating a compiler and checking its executables
from distutils.ccompiler import new_compiler, customize_compiler
compiler = new_compiler()
customize_compiler(compiler)
for ccmd in compiler.executables.values():
if ccmd is None:
continue
if find_executable(ccmd[0]) is None:
return # can't test
# this should work
cmd.run()
# let's check the result
self.assertTrue("libfoo.a" in os.listdir(build_temp))
def find_python_library():
"Return the full path to the Python library (empty string if not found)"
pyver = sysconfig.get_python_version()
libpython_names = [
"python%s" % pyver.replace(".", ""),
"python%smu" % pyver,
"python%sm" % pyver,
"python%su" % pyver,
"python%s" % pyver,
]
dirs = [
"%s/lib" % os.environ.get("PYTHON_DIR", ""),
"%s" % sysconfig.get_config_vars().get("LIBDIR", ""),
"/usr/lib/%s" % sysconfig.get_config_vars().get("MULTIARCH", ""),
"/usr/local/lib",
"/opt/local/lib",
"/usr/lib",
"/usr/lib64",
]
libpython = None
cc = new_compiler()
for name in libpython_names:
libpython = cc.find_library_file(dirs, name)
if libpython is not None:
break
return libpython or ""
def cc(filenames, dest,
library_dirs=None, include_dirs=None,
libraries=None, compile_args=None,
runtime_lib_dirs=None):
compiler = ccompiler.new_compiler()
distutils.sysconfig.customize_compiler(compiler)
if library_dirs:
compiler.set_library_dirs(library_dirs)
if include_dirs:
compiler.set_include_dirs(include_dirs)
if libraries:
compiler.set_libraries(libraries)
if runtime_lib_dirs:
compiler.set_runtime_library_dirs(runtime_lib_dirs)
try:
objnames = compiler.compile(filenames, output_dir=dest,
extra_postargs=compile_args)
for obj in objnames:
execname, ext = os.path.splitext(obj)
compiler.link_executable(
[obj], execname, output_dir=dest,
target_lang="c++",
)
except (CompileError, LinkError):
return False
return True
def rmsd_extension():
openmp_enabled, needs_gomp = detect_openmp()
compiler_args = ["-msse2" if not detect_sse3() else "-mssse3", "-O3", "-funroll-loops"]
if new_compiler().compiler_type == "msvc":
compiler_args.append("/arch:SSE2")
if openmp_enabled:
compiler_args.append("-fopenmp")
compiler_libraries = ["gomp"] if needs_gomp else []
# compiler_defs = [('USE_OPENMP', None)] if openmp_enabled else []
rmsd = Extension(
"mdtraj._rmsd",
sources=[
"MDTraj/rmsd/src/theobald_rmsd.c",
"MDTraj/rmsd/src/rotation.c",
"MDTraj/rmsd/src/center.c",
"MDTraj/rmsd/_rmsd." + cython_extension,
],
include_dirs=["MDTraj/rmsd/include", numpy.get_include()],
extra_compile_args=compiler_args,
# define_macros=compiler_defs,
libraries=compiler_libraries,
)
return rmsd
def run(self):
"""Runs the build extension."""
compiler = new_compiler(compiler=self.compiler)
if compiler.compiler_type == "msvc":
self.define = [
("UNICODE", ""),
]
else:
command = "sh configure --disable-shared-libs"
output = self._RunCommand(command)
print_line = False
for line in output.split("\n"):
line = line.rstrip()
if line == "configure:":
print_line = True
if print_line:
print(line)
self.define = [
("HAVE_CONFIG_H", ""),
("LOCALEDIR", "\"/usr/share/locale\""),
]
build_ext.run(self)
def __init__(self):
self.debug = False
self._compiler = new_compiler()
customize_compiler(self._compiler)
self._build_ext = build_ext(Distribution())
self._build_ext.finalize_options()
self._py_lib_dirs = self._build_ext.library_dirs
def run(self):
from distutils.ccompiler import new_compiler
if not self.extensions:
return
else:
if self.distribution.has_c_libraries():
build_clib = self.get_finalized_command('build_clib')
self.libraries.extend(build_clib.get_library_names() or [])
self.library_dirs.append(build_clib.build_clib)
self.compiler = new_compiler(compiler=self.compiler, verbose=self.verbose, dry_run=self.dry_run, force=self.force)
customize_compiler(self.compiler)
if os.name == 'nt' and self.plat_name != get_platform():
self.compiler.initialize(self.plat_name)
if self.include_dirs is not None:
self.compiler.set_include_dirs(self.include_dirs)
if self.define is not None:
for name, value in self.define:
self.compiler.define_macro(name, value)
if self.undef is not None:
for macro in self.undef:
self.compiler.undefine_macro(macro)
if self.libraries is not None:
self.compiler.set_libraries(self.libraries)
if self.library_dirs is not None:
self.compiler.set_library_dirs(self.library_dirs)
if self.rpath is not None:
self.compiler.set_runtime_library_dirs(self.rpath)
if self.link_objects is not None:
self.compiler.set_link_objects(self.link_objects)
self.build_extensions()
return
def setup_shlib_compiler(self):
compiler = self.shlib_compiler = new_compiler(
compiler=self.compiler, dry_run=self.dry_run, force=self.force
)
_customize_compiler_for_shlib(compiler)
if self.include_dirs is not None:
compiler.set_include_dirs(self.include_dirs)
if self.define is not None:
# 'define' option is a list of (name,value) tuples
for (name, value) in self.define:
compiler.define_macro(name, value)
if self.undef is not None:
for macro in self.undef:
compiler.undefine_macro(macro)
if self.libraries is not None:
compiler.set_libraries(self.libraries)
if self.library_dirs is not None:
compiler.set_library_dirs(self.library_dirs)
if self.rpath is not None:
compiler.set_runtime_library_dirs(self.rpath)
if self.link_objects is not None:
compiler.set_link_objects(self.link_objects)
# hack so distutils' build_extension() builds a library instead
compiler.link_shared_object = link_shared_object.__get__(compiler)
def setUp(self):
"""
Run gdb and have cygdb import the debug information from the code
defined in TestParseTreeTransforms's setUp method
"""
self.tempdir = tempfile.mkdtemp()
self.destfile = os.path.join(self.tempdir, 'codefile.pyx')
self.debug_dest = os.path.join(self.tempdir,
'cython_debug',
'cython_debug_info_codefile')
self.cfuncs_destfile = os.path.join(self.tempdir, 'cfuncs')
self.cwd = os.getcwd()
os.chdir(self.tempdir)
shutil.copy(codefile, self.destfile)
shutil.copy(cfuncs_file, self.cfuncs_destfile + '.c')
compiler = ccompiler.new_compiler()
compiler.compile(['cfuncs.c'], debug=True)
ext = Cython.Distutils.extension.Extension(
'codefile',
['codefile.pyx'],
pyrex_debug=True,
extra_objects=['cfuncs.o'])
distutils.core.setup(
script_args=['build_ext', '--inplace'],
ext_modules=[ext],
cmdclass=dict(build_ext=Cython.Distutils.build_ext)
)
def setup_shlib_compiler(self):
compiler = self.shlib_compiler = new_compiler(compiler=self.compiler, dry_run=self.dry_run, force=self.force)
if sys.platform == 'darwin':
tmp = _config_vars.copy()
try:
_config_vars['LDSHARED'] = 'gcc -Wl,-x -dynamiclib -undefined dynamic_lookup'
_config_vars['CCSHARED'] = ' -dynamiclib'
_config_vars['SO'] = '.dylib'
customize_compiler(compiler)
finally:
_config_vars.clear()
_config_vars.update(tmp)
else:
customize_compiler(compiler)
if self.include_dirs is not None:
compiler.set_include_dirs(self.include_dirs)
if self.define is not None:
for name, value in self.define:
compiler.define_macro(name, value)
if self.undef is not None:
for macro in self.undef:
compiler.undefine_macro(macro)
if self.libraries is not None:
compiler.set_libraries(self.libraries)
if self.library_dirs is not None:
compiler.set_library_dirs(self.library_dirs)
if self.rpath is not None:
compiler.set_runtime_library_dirs(self.rpath)
if self.link_objects is not None:
compiler.set_link_objects(self.link_objects)
compiler.link_shared_object = link_shared_object.__get__(compiler)
return
def test_run(self):
pkg_dir, dist = self.create_dist()
cmd = build_clib(dist)
foo_c = os.path.join(pkg_dir, 'foo.c')
self.write_file(foo_c, 'int main(void) { return 1;}\n')
cmd.libraries = [('foo', {'sources': [foo_c]})]
build_temp = os.path.join(pkg_dir, 'build')
os.mkdir(build_temp)
cmd.build_temp = build_temp
cmd.build_clib = build_temp
# before we run the command, we want to make sure
# all commands are present on the system
# by creating a compiler and checking its executables
from distutils.ccompiler import new_compiler
from distutils.sysconfig import customize_compiler
compiler = new_compiler()
customize_compiler(compiler)
for ccmd in compiler.executables.values():
if ccmd is None:
continue
if find_executable(ccmd[0]) is None:
self.skipTest('The %r command is not found' % ccmd[0])
# this should work
cmd.run()
# let's check the result
self.assertIn('libfoo.a', os.listdir(build_temp))
def test_compilation(cfile, compiler=None, **compiler_attrs):
"""Test simple compilation with given settings"""
if compiler is None or isinstance(compiler, str):
cc = ccompiler.new_compiler(compiler=compiler)
customize_compiler(cc)
if cc.compiler_type == 'mingw32':
customize_mingw(cc)
else:
cc = compiler
for name, val in compiler_attrs.items():
setattr(cc, name, val)
efile, ext = os.path.splitext(cfile)
cpreargs = lpreargs = None
if sys.platform == 'darwin':
# use appropriate arch for compiler
if platform.architecture()[0]=='32bit':
if platform.processor() == 'powerpc':
cpu = 'ppc'
else:
cpu = 'i386'
cpreargs = ['-arch', cpu]
lpreargs = ['-arch', cpu, '-undefined', 'dynamic_lookup']
else:
# allow for missing UB arch, since it will still work:
lpreargs = ['-undefined', 'dynamic_lookup']
extra = compiler_attrs.get('extra_compile_args', None)
objs = cc.compile([cfile],extra_preargs=cpreargs, extra_postargs=extra)
cc.link_executable(objs, efile, extra_preargs=lpreargs)
return efile
def get_cxxflags():
from distutils.ccompiler import new_compiler
from distutils.sysconfig import customize_compiler
from distutils import sysconfig
from platform import system
if system() == DARWIN_KEY:
CXX_FLAGS["gcc"] = CXX_FLAGS["gcc-mac"]
CXX_FLAGS["cc"] = CXX_FLAGS["clang"]
CXX_FLAGS["c++"] = CXX_FLAGS["clang"]
elif system() == LINUX_KEY:
CXX_FLAGS["gcc"] = CXX_FLAGS["gcc-linux"]
CXX_FLAGS["cc"] = CXX_FLAGS["gcc"]
CXX_FLAGS["c++"] = CXX_FLAGS["gcc"]
else:
raise UnsupportedCompilerException("System: %s is not supported by HOPE" % system())
sysconfig.get_config_vars() #init vars
compiler = new_compiler()
customize_compiler(compiler)
compiler_name = compiler.compiler[0].split("/")[-1]
_check_version(compiler_name, compiler.compiler[0])
for name, flags in CXX_FLAGS.items():
if compiler_name.startswith(name):
return flags
raise UnsupportedCompilerException("Unknown compiler: {0}".format(compiler_name))
def find_ssl():
# Detect SSL support for the socket module (via _ssl)
from distutils.ccompiler import new_compiler
compiler = new_compiler()
inc_dirs = compiler.include_dirs + ['/usr/include']
search_for_ssl_incs_in = [
'/usr/local/ssl/include',
'/usr/contrib/ssl/include/'
]
ssl_incs = find_file('openssl/ssl.h', inc_dirs,
search_for_ssl_incs_in
)
if ssl_incs is not None:
krb5_h = find_file('krb5.h', inc_dirs,
['/usr/kerberos/include'])
if krb5_h:
ssl_incs += krb5_h
ssl_libs = find_library_file(compiler, 'ssl',
['/usr/lib'],
['/usr/local/lib',
'/usr/local/ssl/lib',
'/usr/contrib/ssl/lib/'
] )
if (ssl_incs is not None and ssl_libs is not None):
return ssl_incs, ssl_libs, ['ssl', 'crypto']
raise Exception("No SSL support found")
def gecode_version():
global GECODE_VERSION
if GECODE_VERSION is not None:
return GECODE_VERSION
from distutils.ccompiler import new_compiler
try:
from distutils.ccompiler import customize_compiler
except:
from distutils.sysconfig import customize_compiler
import os
cxx = new_compiler()
customize_compiler(cxx)
pid = os.getpid()
file_hh = "_gecode_version_%d.hh" % pid
file_txt = "_gecode_version_%d.txt" % pid
f = file(file_hh,"w")
f.write("""#include "gecode/support/config.hpp"
@@GECODE_VERSION""")
f.close()
cxx.preprocess(file_hh,output_file=file_txt)
if True:
f = open(file_txt)
version = ""
for line in f:
if line.startswith("@@"):
version = line[3:-2]
break
f.close()
os.remove(file_hh)
os.remove(file_txt)
else:
version = "4.4.0"
GECODE_VERSION = version
return version
def _build(self):
from distutils.ccompiler import new_compiler
compiler = new_compiler(force=1)
if self.compiler_exe is not None:
for c in """compiler compiler_so compiler_cxx
linker_exe linker_so""".split():
compiler.executables[c][0] = self.compiler_exe
compiler.spawn = log_spawned_cmd(compiler.spawn)
objects = []
for cfile in self.cfilenames:
cfile = py.path.local(cfile)
old = cfile.dirpath().chdir()
try:
res = compiler.compile(
[cfile.basename], include_dirs=self.eci.include_dirs, extra_preargs=self.compile_extra
)
assert len(res) == 1
cobjfile = py.path.local(res[0])
assert cobjfile.check()
objects.append(str(cobjfile))
finally:
old.chdir()
compiler.link_executable(
objects,
str(self.outputfilename),
libraries=self.eci.libraries,
extra_preargs=self.link_extra,
library_dirs=self.eci.library_dirs,
)
def _build(self):
from distutils.ccompiler import new_compiler
compiler = new_compiler(force=1)
if self.compiler_exe is not None:
for c in '''compiler compiler_so compiler_cxx
linker_exe linker_so'''.split():
compiler.executables[c][0] = self.compiler_exe
compiler.spawn = log_spawned_cmd(compiler.spawn)
objects = []
for cfile in self.cfilenames:
old = os.getcwd()
os.chdir(os.path.dirname(cfile))
try:
res = compiler.compile([os.path.basename(cfile)],
include_dirs=self.eci.include_dirs,
extra_preargs=self.compile_extra)
assert len(res) == 1
cobjfile = res[0]
assert os.path.exists(cobjfile)
objects.append(cobjfile)
compiler.link_executable(objects, str(self.outputfilename),
libraries=self.eci.libraries,
extra_preargs=self.link_extra,
library_dirs=self.eci.library_dirs)
finally:
os.chdir(old)
def run(self):
# mostly copied from distutils.command.build_ext
from distutils.ccompiler import new_compiler
if not self.extensions:
return
self.compiler = new_compiler(
compiler=self.compiler or patch_distutils.get_default_compiler(),
verbose=self.verbose,
dry_run=self.dry_run,
force=self.force)
from pyd import dcompiler
assert isinstance(self.compiler, dcompiler.DCompiler)
self.compiler.build_exe = True
self.compiler.optimize = self.optimize
# irrelevant for D compilers?
#customize_compiler(self.compiler)
if self.include_dirs is not None:
self.compiler.set_include_dirs(self.include_dirs)
if self.libraries is not None:
self.compiler.set_libraries(self.libraries)
if self.library_dirs is not None:
self.compiler.set_library_dirs(self.library_dirs)
if self.link_objects is not None:
self.compiler.set_link_objects(self.link_objects)
if self.print_flags:
print( ' '.join(self.compiler.versionOpts()) )
else:
for ext in self.extensions:
self.per_ext(ext)
def run(self):
# Build our own POTFILES.in as DistUtilsExtra does not include rfc822
# files automatically
with open('po/POTFILES.in', 'w') as potfiles_in:
potfiles_in.write('[encoding: UTF-8]\n')
for f in glob("provider_jobs/*"):
potfiles_in.write('[type: gettext/rfc822deb] ' + f + '\n')
for f in glob("provider_bin/*"):
potfiles_in.write(f + '\n')
build_extra.build_extra.run(self)
cc = new_compiler()
for source in glob('provider_bin/*.c'):
executable = os.path.splitext(source)[0]
try:
cc.link_executable(
[source], executable,
libraries=["rt", "pthread", "nl-3", "nl-genl-3"],
# Enforce security with CFLAGS + LDFLAGS
# See dpkg-buildflags
extra_preargs=[
"-O2", "-fstack-protector",
"--param=ssp-buffer-size=4", "-Wformat",
"-Werror=format-security",
"-Wl,-Bsymbolic-functions",
"-Wl,-z,relro",
"-I/usr/include/libnl3"])
except LinkError as e:
print('Please install libnl-genl-3-dev on Debian systems')
raise
os.unlink('po/POTFILES.in')
def detect_gmp(self):
# It's far from perfect, but we try to compile a code that uses
# Python long. If it fails, _whatever the reason_ we just disable gmp
print('Trying to compile GMP dependencies.')
cc = ccompiler.new_compiler(verbose=False)
# try to compile a code that requires gmp support
with NamedTemporaryFile(suffix='.cpp', delete=False) as temp:
temp.write('''
#include <gmpxx.h>
int main() {
mpz_class a(1);
return a == 0;
};
'''.encode('ascii'))
srcs = [temp.name]
exe = "a.out"
try:
objs = cc.compile(srcs)
cc.link(ccompiler.CCompiler.EXECUTABLE,
objs, exe,
libraries=['gmp', 'gmpxx'])
except Exception:
# failure: remove the gmp dependency
print('Failed to compile GMP source, disabling long support.')
for cfg in glob.glob(os.path.join(pythrandir, "pythran-*.cfg")):
with open(cfg, "r+") as cfg:
content = cfg.read()
content = content.replace('USE_GMP', '')
content = content.replace('gmp gmpxx', '')
cfg.seek(0)
cfg.write(content)
map(os.remove, objs + srcs + [exe])
def run(self):
from distutils.ccompiler import new_compiler
if not self.extensions:
return
# Setup the CCompiler object that we'll use to do all the
# compiling and linking
self.compiler = new_compiler(compiler=self.compiler,
verbose=self.verbose,
dry_run=self.dry_run,
force=self.force)
customize_compiler(self.compiler)
# And make sure that any compile/link-related options (which might
# come from the command-line or from the setup script) are set in
# that CCompiler object -- that way, they automatically apply to
# all compiling and linking done here.
if self.include_dirs is not None:
self.compiler.set_include_dirs(self.include_dirs)
if self.define is not None:
# 'define' option is a list of (name,value) tuples
for (name, value) in self.define:
self.compiler.define_macro(name, value)
if self.undef is not None:
for macro in self.undef:
self.compiler.undefine_macro(macro)
if self.libraries is not None:
self.compiler.set_libraries(self.libraries)
if self.library_dirs is not None:
self.compiler.set_library_dirs(self.library_dirs)
if self.rpath is not None:
self.compiler.set_runtime_library_dirs(self.rpath)
if self.link_objects is not None:
self.compiler.set_link_objects(self.link_objects)
# Now actually compile and link everything.
self.build_extensions()
def mypycify(
paths: List[str],
*,
only_compile_paths: Optional[Iterable[str]] = None,
verbose: bool = False,
opt_level: str = '3',
strip_asserts: bool = False,
multi_file: bool = False,
separate: Union[bool, List[Tuple[List[str], Optional[str]]]] = False,
skip_cgen_input: Optional[Any] = None,
target_dir: Optional[str] = None,
include_runtime_files: Optional[bool] = None) -> List['Extension']:
"""Main entry point to building using mypyc.
This produces a list of Extension objects that should be passed as the
ext_modules parameter to setup.
Arguments:
paths: A list of file paths to build. It may also contain mypy options.
only_compile_paths: If not None, an iterable of paths that are to be
the only modules compiled, even if other modules
appear in the mypy command line given to paths.
(These modules must still be passed to paths.)
verbose: Should mypyc be more verbose. Defaults to false.
opt_level: The optimization level, as a string. Defaults to '3' (meaning '-O3').
strip_asserts: Should asserts be stripped from the generated code.
multi_file: Should each Python module be compiled into its own C source file.
This can reduce compile time and memory requirements at the likely
cost of runtime performance of compiled code. Defaults to false.
separate: Should compiled modules be placed in separate extension modules.
If False, all modules are placed in a single shared library.
If True, every module is placed in its own library.
Otherwise separate should be a list of
(file name list, optional shared library name) pairs specifying
groups of files that should be placed in the same shared library
(while all other modules will be placed in its own library).
Each group can be compiled independently, which can
speed up compilation, but calls between groups can
be slower than calls within a group and can't be
inlined.
target_dir: The directory to write C output files. Defaults to 'build'.
include_runtime_files: If not None, whether the mypyc runtime library
should be directly #include'd instead of linked
separately in order to reduce compiler invocations.
Defaults to False in multi_file mode, True otherwise.
"""
# Figure out our configuration
compiler_options = CompilerOptions(
strip_asserts=strip_asserts,
multi_file=multi_file,
verbose=verbose,
separate=separate is not False,
target_dir=target_dir,
include_runtime_files=include_runtime_files,
)
# Generate all the actual important C code
groups, group_cfilenames = mypyc_build(
paths,
only_compile_paths=only_compile_paths,
compiler_options=compiler_options,
separate=separate,
skip_cgen_input=skip_cgen_input,
)
# Mess around with setuptools and actually get the thing built
setup_mypycify_vars()
# Create a compiler object so we can make decisions based on what
# compiler is being used. typeshed is missing some attribues on the
# compiler object so we give it type Any
compiler = ccompiler.new_compiler() # type: Any
sysconfig.customize_compiler(compiler)
build_dir = compiler_options.target_dir
cflags = [] # type: List[str]
if compiler.compiler_type == 'unix':
cflags += [
'-O{}'.format(opt_level),
'-Werror',
'-Wno-unused-function',
'-Wno-unused-label',
'-Wno-unreachable-code',
'-Wno-unused-variable',
'-Wno-unused-command-line-argument',
'-Wno-unknown-warning-option',
]
if 'gcc' in compiler.compiler[0]:
# This flag is needed for gcc but does not exist on clang.
cflags += ['-Wno-unused-but-set-variable']
elif compiler.compiler_type == 'msvc':
if opt_level == '3':
opt_level = '2'
cflags += [
'/O{}'.format(opt_level),
'/wd4102', # unreferenced label
'/wd4101', # unreferenced local variable
'/wd4146', # negating unsigned int
]
if multi_file:
# Disable whole program optimization in multi-file mode so
# that we actually get the compilation speed and memory
# use wins that multi-file mode is intended for.
cflags += [
'/GL-',
'/wd9025', # warning about overriding /GL
]
# If configured to (defaults to yes in multi-file mode), copy the
# runtime library in. Otherwise it just gets #included to save on
# compiler invocations.
shared_cfilenames = []
if not compiler_options.include_runtime_files:
for name in RUNTIME_C_FILES:
rt_file = os.path.join(build_dir, name)
with open(os.path.join(include_dir(), name),
encoding='utf-8') as f:
write_file(rt_file, f.read())
shared_cfilenames.append(rt_file)
extensions = []
for (group_sources, lib_name), (cfilenames,
deps) in zip(groups, group_cfilenames):
if lib_name:
extensions.extend(
build_using_shared_lib(group_sources, lib_name,
cfilenames + shared_cfilenames, deps,
build_dir, cflags))
else:
extensions.extend(
build_single_module(group_sources,
cfilenames + shared_cfilenames, cflags))
return extensions
def setup_package():
root = os.path.abspath(os.path.dirname(__file__))
if len(sys.argv) > 1 and sys.argv[1] == 'clean':
return clean(root)
with chdir(root):
with io.open(os.path.join(root, 'spacy', 'about.py'),
encoding='utf8') as f:
about = {}
exec(f.read(), about)
with io.open(os.path.join(root, 'README.rst'), encoding='utf8') as f:
readme = f.read()
include_dirs = [
get_python_inc(plat_specific=True),
os.path.join(root, 'include')
]
if (ccompiler.new_compiler().compiler_type == 'msvc'
and msvccompiler.get_build_version() == 9):
include_dirs.append(os.path.join(root, 'include', 'msvc9'))
ext_modules = []
for mod_name in MOD_NAMES:
mod_path = mod_name.replace('.', '/') + '.cpp'
extra_link_args = []
# ???
# Imported from patch from @mikepb
# See Issue #267. Running blind here...
if sys.platform == 'darwin':
dylib_path = ['..' for _ in range(mod_name.count('.'))]
dylib_path = '/'.join(dylib_path)
dylib_path = '@loader_path/%s/spacy/platform/darwin/lib' % dylib_path
extra_link_args.append('-Wl,-rpath,%s' % dylib_path)
ext_modules.append(
Extension(mod_name, [mod_path],
language='c++',
include_dirs=include_dirs,
extra_link_args=extra_link_args))
if not is_source_release(root):
generate_cython(root, 'spacy')
setup(
name=about['__title__'],
zip_safe=False,
packages=PACKAGES,
package_data=PACKAGE_DATA,
description=about['__summary__'],
long_description=readme,
author=about['__author__'],
author_email=about['__email__'],
version=about['__version__'],
url=about['__uri__'],
license=about['__license__'],
ext_modules=ext_modules,
scripts=['bin/spacy'],
install_requires=[
'numpy>=1.7', 'murmurhash>=0.28,<0.29', 'cymem>=1.30,<1.32',
'preshed>=1.0.0,<2.0.0', 'thinc>=6.10.1,<6.11.0',
'plac<1.0.0,>=0.9.6', 'pathlib', 'ujson>=1.35',
'dill>=0.2,<0.3', 'regex==2017.4.5', 'requests>=2.13.0,<3.0.0'
],
classifiers=[
'Development Status :: 5 - Production/Stable',
'Environment :: Console', 'Intended Audience :: Developers',
'Intended Audience :: Science/Research',
'License :: OSI Approved :: MIT License',
'Operating System :: POSIX :: Linux',
'Operating System :: MacOS :: MacOS X',
'Operating System :: Microsoft :: Windows',
'Programming Language :: Cython',
'Programming Language :: Python :: 2',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Topic :: Scientific/Engineering'
],
cmdclass={'build_ext': build_ext_subclass},
)
yield
finally:
if oldstdchannel is not None:
os.dup2(oldstdchannel, stdchannel.fileno())
if dest_file is not None:
dest_file.close()
def add_dir_if_exists(filtered_dirs, *dirs):
for d in dirs:
if osp.exists(d):
filtered_dirs.append(d)
compiler = ccompiler.new_compiler()
_extra_compile_args = ['-DMSDBLIB']
WINDOWS = False
SYSTEM = platform.system()
print("setup.py: platform.system() => %r" % SYSTEM)
print("setup.py: platform.architecture() => %r" % (platform.architecture(), ))
print("setup.py: platform.linux_distribution() => %r" %
(platform.linux_distribution(), ))
print("setup.py: platform.libc_ver() => %r" % (platform.libc_ver(), ))
# 32 bit or 64 bit system?
BITNESS = struct.calcsize("P") * 8
def run(self):
from distutils.ccompiler import new_compiler
# 'self.extensions', as supplied by setup.py, is a list of
# Extension instances. See the documentation for Extension (in
# distutils.extension) for details.
#
# For backwards compatibility with Distutils 0.8.2 and earlier, we
# also allow the 'extensions' list to be a list of tuples:
# (ext_name, build_info)
# where build_info is a dictionary containing everything that
# Extension instances do except the name, with a few things being
# differently named. We convert these 2-tuples to Extension
# instances as needed.
if not self.extensions:
return
# If we were asked to build any C/C++ libraries, make sure that the
# directory where we put them is in the library search path for
# linking extensions.
if self.distribution.has_c_libraries():
build_clib = self.get_finalized_command('build_clib')
self.libraries.extend(build_clib.get_library_names() or [])
self.library_dirs.append(build_clib.build_clib)
# Setup the CCompiler object that we'll use to do all the
# compiling and linking
self.compiler = new_compiler(compiler=self.compiler,
verbose=self.verbose,
dry_run=self.dry_run,
force=self.force)
customize_compiler(self.compiler)
# If we are cross-compiling, init the compiler now (if we are not
# cross-compiling, init would not hurt, but people may rely on
# late initialization of compiler even if they shouldn't...)
if os.name == 'nt' and self.plat_name != get_platform():
self.compiler.initialize(self.plat_name)
# And make sure that any compile/link-related options (which might
# come from the command-line or from the setup script) are set in
# that CCompiler object -- that way, they automatically apply to
# all compiling and linking done here.
if self.include_dirs is not None:
self.compiler.set_include_dirs(self.include_dirs)
if self.define is not None:
# 'define' option is a list of (name,value) tuples
for (name, value) in self.define:
self.compiler.define_macro(name, value)
if self.undef is not None:
for macro in self.undef:
self.compiler.undefine_macro(macro)
if self.libraries is not None:
self.compiler.set_libraries(self.libraries)
if self.library_dirs is not None:
self.compiler.set_library_dirs(self.library_dirs)
if self.rpath is not None:
self.compiler.set_runtime_library_dirs(self.rpath)
if self.link_objects is not None:
self.compiler.set_link_objects(self.link_objects)
# Now actually compile and link everything.
self.build_extensions()
def has_configure(self):
compiler = new_compiler(compiler=self.compiler)
return compiler.compiler_type != 'msvc'
class CommandDependency(Dependency):
exe_extension = _distutils_ccompiler.new_compiler().exe_extension
def __init__(self,
command,
paths=None,
version_options=('--version', ),
stdin=None,
version_regexp=None,
version_stream='stdout',
**kwargs):
if 'name' not in kwargs:
kwargs['name'] = command
super(CommandDependency, self).__init__(**kwargs)
self.command = command
self.paths = paths
self.version_options = version_options
self.stdin = None
if not version_regexp:
regexp = r'([\d][\d{0}]*[\d])'.format(self.version_delimiter)
version_regexp = _re.compile(regexp)
self.version_regexp = version_regexp
self.version_stream = version_stream
def _get_command_version_stream(self,
command=None,
stdin=None,
expect=(0, )):
if command is None:
command = self.command + (self.exe_extension or '')
if not stdin:
stdin = self.stdin
if stdin:
popen_stdin = _subprocess.PIPE
else:
popen_stdin = None
try:
p = _subprocess.Popen([command] + list(self.version_options),
stdin=popen_stdin,
stdout=_subprocess.PIPE,
stderr=_subprocess.PIPE,
universal_newlines=True)
except OSError as e:
raise DependencyError(
checker=self,
message="could not find '{0}' executable".format(command),
) # from e
stdout, stderr = p.communicate(stdin)
status = p.wait()
if status not in expect:
lines = [
"failed to execute: {0} {1}".format(
command, ' '.join(
_shlex.quote(arg) for arg in self.version_options)),
'status: {0}'.format(status),
]
for name, string in [('stdout', stdout), ('stderr', stderr)]:
if string:
lines.extend([name + ':', string])
raise DependencyError(checker=self, message='\n'.join(lines))
for name, string in [('stdout', stdout), ('stderr', stderr)]:
if name == self.version_stream:
if not string:
raise DependencyError(
checker=self,
message='empty version stream on {0} for {1}'.format(
self.version_stream, command))
return string
raise NotImplementedError(self.version_stream)
def _get_version_stream(self, **kwargs):
paths = [self.command + (self.exe_extension or '')]
if self.exe_extension:
paths.append(self.command) # also look at the extension-less path
if self.paths:
paths.extend(self.paths)
or_errors = []
for path in paths:
try:
return self._get_command_version_stream(command=path, **kwargs)
except DependencyError as e:
or_errors.append(e)
raise DependencyError(checker=self,
message='errors finding {0} version'.format(
self.full_name()),
causes=or_errors)
def _get_version(self):
version_stream = self._get_version_stream()
match = self.version_regexp.search(version_stream)
if not match:
raise DependencyError(
checker=self,
message='no version string in output:\n{0}'.format(
version_stream))
return match.group(1)
def setUp(self):
self.compiler = ccompiler.new_compiler()
self.settings = build.get_compiler_setting()
def get_compilation_flags():
import os
import tempfile
from distutils.ccompiler import new_compiler, CompileError, LinkError
import shutil
print("INFO: Checking for OpenMP support...")
cc = new_compiler()
if cc.compiler_type == "msvc":
compile_flags = ["/openmp"]
link_flags = []
else:
compile_flags = ["-fopenmp"]
link_flags = ["-fopenmp"]
# see http://openmp.org/wp/openmp-compilers/
omp_test = \
b"""
#include <omp.h>\n
#include <stdio.h>\n
int main() {\n
#pragma omp parallel\n
printf("Hello from thread %d, nthreads %d\\n", omp_get_thread_num(), omp_get_num_threads());\n
}\n
"""
tmpdir = tempfile.mkdtemp()
curdir = os.getcwd()
os.chdir(tmpdir)
# Write test.c
filename = 'test.c'
with open(filename, 'wb') as fp:
fp.write(omp_test)
try:
buf = StringIO()
with std_redirector(buf):
objects = cc.compile([filename],
output_dir=tmpdir,
extra_postargs=compile_flags)
cc.link_executable(objects,
os.path.join(tmpdir, "a.out"),
extra_postargs=link_flags)
except (CompileError, LinkError):
print("WARNING: Could not find OpenMP, parallelism will "
"not be available!")
compile_flags = []
link_flags = []
else:
print("INFO: OpenMP is present and will be used!")
os.chdir(curdir)
# Clean up
shutil.rmtree(tmpdir)
# Add generic flags
if cc.compiler_type == "msvc":
compile_flags += ["/wd4244", "/wd4267", "/wd4018", "/wd4996"]
else:
compile_flags += [
"-Wno-maybe-uninitialized", "-Wno-unused-function", "-Wno-cpp",
"-Wno-shorten-64-to-32", "-Wno-unneeded-internal-declaration"
]
return compile_flags, link_flags
def mypycify(paths: List[str],
mypy_options: Optional[List[str]] = None,
opt_level: str = '3',
multi_file: bool = False,
skip_cgen: bool = False,
verbose: bool = False) -> List[MypycifyExtension]:
"""Main entry point to building using mypyc.
This produces a list of Extension objects that should be passed as the
ext_modules parameter to setup.
Arguments:
* paths: A list of file paths to build. It may contain globs.
* mypy_options: Optionally, a list of command line flags to pass to mypy.
(This can also contain additional files, for compatibility reasons.)
* opt_level: The optimization level, as a string. Defaults to '3' (meaning '-O3').
"""
setup_mypycify_vars()
# Create a compiler object so we can make decisions based on what
# compiler is being used. typeshed is missing some attribues on the
# compiler object so we give it type Any
compiler = ccompiler.new_compiler() # type: Any
sysconfig.customize_compiler(compiler)
expanded_paths = []
for path in paths:
expanded_paths.extend(glob.glob(path))
build_dir = 'build' # TODO: can this be overridden??
try:
os.mkdir(build_dir)
except FileExistsError:
pass
sources, options = get_mypy_config(expanded_paths, mypy_options)
# We generate a shared lib if there are multiple modules or if any
# of the modules are in package. (Because I didn't want to fuss
# around with making the single module code handle packages.)
use_shared_lib = len(sources) > 1 or any('.' in x.module for x in sources)
lib_name = shared_lib_name([source.module for source in sources
]) if use_shared_lib else None
# We let the test harness make us skip doing the full compilation
# so that it can do a corner-cutting version without full stubs.
# TODO: Be able to do this based on file mtimes?
if not skip_cgen:
cfiles, ops_text = generate_c(sources, options, multi_file, lib_name,
verbose)
# TODO: unique names?
with open(os.path.join(build_dir, 'ops.txt'), 'w') as f:
f.write(ops_text)
cfilenames = []
for cfile, ctext in cfiles:
cfile = os.path.join(build_dir, cfile)
with open(cfile, 'w', encoding='utf-8') as f:
f.write(ctext)
if os.path.splitext(cfile)[1] == '.c':
cfilenames.append(cfile)
else:
cfilenames = glob.glob(os.path.join(build_dir, '*.c'))
cflags = [] # type: List[str]
if compiler.compiler_type == 'unix':
cflags += [
'-O{}'.format(opt_level), '-Werror', '-Wno-unused-function',
'-Wno-unused-label', '-Wno-unreachable-code',
'-Wno-unused-variable', '-Wno-trigraphs',
'-Wno-unused-command-line-argument'
]
if 'gcc' in compiler.compiler[0]:
# This flag is needed for gcc but does not exist on clang.
cflags += ['-Wno-unused-but-set-variable']
elif compiler.compiler_type == 'msvc':
if opt_level == '3':
opt_level = '2'
cflags += [
'/O{}'.format(opt_level),
'/wd4102', # unreferenced label
'/wd4101', # unreferenced local variable
'/wd4146', # negating unsigned int
]
if multi_file:
# Disable whole program optimization in multi-file mode so
# that we actually get the compilation speed and memory
# use wins that multi-file mode is intended for.
cflags += [
'/GL-',
'/wd9025', # warning about overriding /GL
]
# Copy the runtime library in
rt_file = os.path.join(build_dir, 'CPy.c')
shutil.copyfile(os.path.join(include_dir(), 'CPy.c'), rt_file)
cfilenames.append(rt_file)
if use_shared_lib:
assert lib_name
extensions = build_using_shared_lib(sources, lib_name, cfilenames,
build_dir, cflags)
else:
extensions = build_single_module(sources, cfilenames, cflags)
return extensions
if use_cython and not has_cython:
if is_dev:
raise RuntimeError('Cython required to build dev version of cyksuid.')
print('WARNING: Cython not installed. Building without Cython.')
use_cython = False
if use_cython:
suffix = '.pyx'
else:
suffix = '.c'
include_dirs = []
if ccompiler.new_compiler().compiler_type == 'msvc' and msvccompiler.get_build_version() == 9:
root = os.path.abspath(os.path.dirname(__file__))
include_dirs.append(os.path.join(root, 'include', 'msvc9'))
ext_modules = []
for modname in ['fast_base62', 'ksuid']:
ext_modules.append(Extension('cyksuid.' + modname.replace('/', '.'),
['cyksuid/' + modname + suffix],
include_dirs=include_dirs))
if use_cython:
try:
from Cython.Compiler.Options import get_directive_defaults
directive_defaults = get_directive_defaults()
def check_openmp_support(openmp_flags=None):
"""
Check whether OpenMP test code can be compiled and run.
Parameters
----------
openmp_flags : dict, optional
This should be a dictionary with keys ``compiler_flags`` and
``linker_flags`` giving the compiliation and linking flags respectively.
These are passed as `extra_postargs` to `compile()` and
`link_executable()` respectively. If this is not set, the flags will
be automatically determined using environment variables.
Returns
-------
result : bool
`True` if the test passed, `False` otherwise.
"""
ccompiler = new_compiler()
customize_compiler(ccompiler)
if not openmp_flags:
# customize_compiler() extracts info from os.environ. If certain keys
# exist it uses these plus those from sysconfig.get_config_vars().
# If the key is missing in os.environ it is not extracted from
# sysconfig.get_config_var(). E.g. 'LDFLAGS' get left out, preventing
# clang from finding libomp.dylib because -L<path> is not passed to
# linker. Call get_openmp_flags() to get flags missed by
# customize_compiler().
openmp_flags = get_openmp_flags()
compile_flags = openmp_flags.get('compiler_flags')
link_flags = openmp_flags.get('linker_flags')
tmp_dir = tempfile.mkdtemp()
start_dir = os.path.abspath('.')
try:
os.chdir(tmp_dir)
# Write test program
with open('test_openmp.c', 'w') as f:
f.write(CCODE)
os.mkdir('objects')
# Compile, test program
ccompiler.compile(['test_openmp.c'],
output_dir='objects',
extra_postargs=compile_flags)
# Link test program
objects = glob.glob(
os.path.join('objects', '*' + ccompiler.obj_extension))
ccompiler.link_executable(objects,
'test_openmp',
extra_postargs=link_flags)
# Run test program
output = subprocess.check_output('./test_openmp')
output = output.decode(sys.stdout.encoding or 'utf-8').splitlines()
if 'nthreads=' in output[0]:
nthreads = int(output[0].strip().split('=')[1])
if len(output) == nthreads:
is_openmp_supported = True
else:
log.warn(
"Unexpected number of lines from output of test OpenMP "
"program (output was {0})".format(output))
is_openmp_supported = False
else:
log.warn("Unexpected output from test OpenMP "
"program (output was {0})".format(output))
is_openmp_supported = False
except (CompileError, LinkError, subprocess.CalledProcessError):
is_openmp_supported = False
finally:
os.chdir(start_dir)
return is_openmp_supported
def setup_package():
root = os.path.abspath(os.path.dirname(__file__))
if len(sys.argv) > 1 and sys.argv[1] == "clean":
return clean(root)
with chdir(root):
with io.open(os.path.join(root, "spacy", "about.py"), encoding="utf8") as f:
about = {}
exec(f.read(), about)
with io.open(os.path.join(root, "README.md"), encoding="utf8") as f:
readme = f.read()
include_dirs = [
get_python_inc(plat_specific=True),
os.path.join(root, "include"),
]
if (
ccompiler.new_compiler().compiler_type == "msvc"
and msvccompiler.get_build_version() == 9
):
include_dirs.append(os.path.join(root, "include", "msvc9"))
ext_modules = []
for mod_name in MOD_NAMES:
mod_path = mod_name.replace(".", "/") + ".cpp"
extra_link_args = []
extra_compile_args = []
# ???
# Imported from patch from @mikepb
# See Issue #267. Running blind here...
if sys.platform == "darwin":
dylib_path = [".." for _ in range(mod_name.count("."))]
dylib_path = "/".join(dylib_path)
dylib_path = "@loader_path/%s/spacy/platform/darwin/lib" % dylib_path
extra_link_args.append("-Wl,-rpath,%s" % dylib_path)
ext_modules.append(
Extension(
mod_name,
[mod_path],
language="c++",
include_dirs=include_dirs,
extra_link_args=extra_link_args,
)
)
if not is_source_release(root):
generate_cython(root, "spacy")
setup(
name="spacy",
zip_safe=False,
packages=PACKAGES,
package_data=PACKAGE_DATA,
description=about["__summary__"],
long_description=readme,
long_description_content_type="text/markdown",
author=about["__author__"],
author_email=about["__email__"],
version=about["__version__"],
url=about["__uri__"],
license=about["__license__"],
ext_modules=ext_modules,
scripts=["bin/spacy"],
install_requires=[
"numpy>=1.15.0",
"murmurhash>=0.28.0,<1.1.0",
"cymem>=2.0.2,<2.1.0",
"preshed>=2.0.1,<2.1.0",
"thinc>=7.0.8,<7.1.0",
"blis>=0.2.2,<0.3.0",
"plac<1.0.0,>=0.9.6",
"requests>=2.13.0,<3.0.0",
"wasabi>=0.2.0,<1.1.0",
"srsly>=0.0.6,<1.1.0",
'pathlib==1.0.1; python_version < "3.4"',
],
setup_requires=["wheel"],
extras_require={
"cuda": ["thinc_gpu_ops>=0.0.1,<0.1.0", "cupy>=5.0.0b4"],
"cuda80": ["thinc_gpu_ops>=0.0.1,<0.1.0", "cupy-cuda80>=5.0.0b4"],
"cuda90": ["thinc_gpu_ops>=0.0.1,<0.1.0", "cupy-cuda90>=5.0.0b4"],
"cuda91": ["thinc_gpu_ops>=0.0.1,<0.1.0", "cupy-cuda91>=5.0.0b4"],
"cuda92": ["thinc_gpu_ops>=0.0.1,<0.1.0", "cupy-cuda92>=5.0.0b4"],
"cuda100": ["thinc_gpu_ops>=0.0.1,<0.1.0", "cupy-cuda100>=5.0.0b4"],
# Language tokenizers with external dependencies
"ja": ["mecab-python3==0.7"],
"ko": ["natto-py==0.9.0"],
},
python_requires=">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*",
classifiers=[
"Development Status :: 5 - Production/Stable",
"Environment :: Console",
"Intended Audience :: Developers",
"Intended Audience :: Science/Research",
"License :: OSI Approved :: MIT License",
"Operating System :: POSIX :: Linux",
"Operating System :: MacOS :: MacOS X",
"Operating System :: Microsoft :: Windows",
"Programming Language :: Cython",
"Programming Language :: Python :: 2",
"Programming Language :: Python :: 2.7",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.4",
"Programming Language :: Python :: 3.5",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Topic :: Scientific/Engineering",
],
cmdclass={"build_ext": build_ext_subclass},
)
"-DH5Tarray_create_vers=2",
# "-DH5Tcommit_vers=2",
"-DH5Tget_array_dims_vers=2",
# "-DH5Topen_vers=2",
"-DH5Z_class_t_vers=2",
])
# H5Oget_info_by_name seems to have performance issues (see gh-402), so we
# need to use teh deprecated H5Gget_objinfo function
# CFLAGS.append("-DH5_NO_DEPRECATED_SYMBOLS")
# Do not use numpy deprecated API
# CFLAGS.append("-DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION")
# Try to locate the compulsory and optional libraries.
lzo2_enabled = False
compiler = new_compiler()
for (package, location) in [(hdf5_package, HDF5_DIR),
(lzo2_package, LZO_DIR),
(lzo1_package, LZO_DIR),
(bzip2_package, BZIP2_DIR),
(blosc_package, BLOSC_DIR)]:
if package.tag == 'LZO' and lzo2_enabled:
print(
"* Skipping detection of %s since %s has already been found." %
(lzo1_package.name, lzo2_package.name))
continue # do not use LZO 1 if LZO 2 is available
# if a package location is not specified, try to find it in conda env
if not location and CONDA_PREFIX:
location = CONDA_PREFIX
def get_device_extensions_and_extras(library_dirs=None):
library_dirs = [] if library_dirs is None else library_dirs
cur_dir = os.path.dirname(os.path.realpath(__file__))
include_dirs = []
device_extras = dict()
if os.path.isdir(os.path.join(cur_dir, "lib/shared")):
# Device libs are packaged, so we are in release mode
include_dirs.insert(
0, os.path.realpath(os.path.join(cur_dir, "lib/shared/include")))
library_dirs.insert(
0, os.path.realpath(os.path.join(cur_dir, "lib/shared")))
if sys.platform == "darwin":
# On Mac OS X clang is by default not smart enough to search in the lib dir
# see: https://github.com/jopohl/urh/issues/173
if os.path.isdir("/opt/local/lib"):
library_dirs.append("/opt/local/lib")
library_dirs.append("/usr/local/lib")
if os.path.isdir("/opt/local/include"):
include_dirs.append("/opt/local/include")
result = []
# None = automatic (depending on lib is installed)
# 1 = install extension always
# 0 = Do not install extension
build_device_extensions = defaultdict(lambda: None)
for dev_name in DEVICES:
with_option = "--with-" + dev_name
without_option = "--without-" + dev_name
if with_option in sys.argv and without_option in sys.argv:
print("ambiguous options for " + dev_name)
sys.exit(1)
elif without_option in sys.argv:
build_device_extensions[dev_name] = 0
sys.argv.remove(without_option)
elif with_option in sys.argv:
build_device_extensions[dev_name] = 1
sys.argv.remove(with_option)
sys.path.append(os.path.realpath(os.path.join(cur_dir, "lib")))
compiler = ccompiler.new_compiler()
for dev_name, params in DEVICES.items():
if build_device_extensions[dev_name] == 0:
print("Skipping native {0} support".format(dev_name))
continue
if build_device_extensions[dev_name] == 1:
print("Enforcing native {0} support".format(dev_name))
elif compiler_has_function(compiler, params["test_function"],
(params["lib"], ), library_dirs,
include_dirs):
print("Found {0} lib. Will compile with native {1} support".format(
params["lib"], dev_name))
else:
print("Skipping native support for {0}".format(dev_name))
continue
device_extras.update(
get_device_extras(compiler, dev_name, [params["lib"]],
library_dirs, include_dirs))
if "api_version_check_code" in params:
env_name = dev_name.upper() + "_API_VERSION"
ver = os.getenv(env_name)
if ver is not None:
try:
ver = float(ver)
except Exception as e:
print(" Could not convert content of {} to float: {}".
format(env_name, e))
print(
" Will now try to automatically detect API version."
)
ver = None
else:
print(
" Environment variable {} is unset, try to automatically detect API version"
.format(env_name))
if ver is None:
ver = check_api_version(compiler,
params["api_version_check_code"],
(params["lib"], ), library_dirs,
include_dirs)
device_extras[env_name] = ver
print(" Using {}={}".format(env_name, ver))
extension = get_device_extension(dev_name, [params["lib"]],
library_dirs, include_dirs)
result.append(extension)
return result, device_extras
def pre_build_check():
"""
Try to verify build tools
"""
if os.environ.get('CASS_DRIVER_NO_PRE_BUILD_CHECK'):
return True
try:
from distutils.ccompiler import new_compiler
from distutils.sysconfig import customize_compiler
from distutils.dist import Distribution
# base build_ext just to emulate compiler option setup
be = build_ext(Distribution())
be.initialize_options()
be.finalize_options()
# First, make sure we have a Python include directory
have_python_include = any(
os.path.isfile(os.path.join(p, 'Python.h'))
for p in be.include_dirs)
if not have_python_include:
sys.stderr.write("Did not find 'Python.h' in %s.\n" %
(be.include_dirs, ))
return False
compiler = new_compiler(compiler=be.compiler)
customize_compiler(compiler)
try:
# We must be able to initialize the compiler if it has that method
if hasattr(compiler, "initialize"):
compiler.initialize()
except:
return False
executables = []
if compiler.compiler_type in ('unix', 'cygwin'):
executables = [
compiler.executables[exe][0]
for exe in ('compiler_so', 'linker_so')
]
elif compiler.compiler_type == 'nt':
executables = [getattr(compiler, exe) for exe in ('cc', 'linker')]
if executables:
from distutils.spawn import find_executable
for exe in executables:
if not find_executable(exe):
sys.stderr.write(
"Failed to find %s for compiler type %s.\n" %
(exe, compiler.compiler_type))
return False
except Exception as exc:
sys.stderr.write('%s\n' % str(exc))
sys.stderr.write("Failed pre-build check. Attempting anyway.\n")
# if we are unable to positively id the compiler type, or one of these assumptions fails,
# just proceed as we would have without the check
return True
def run(self):
if not self.libraries:
return
# Make sure that library sources are complete.
languages = []
# Make sure that extension sources are complete.
self.run_command('build_src')
for (lib_name, build_info) in self.libraries:
l = build_info.get('language', None)
if l and l not in languages:
languages.append(l)
from distutils.ccompiler import new_compiler
self.compiler = new_compiler(compiler=self.compiler,
dry_run=self.dry_run,
force=self.force)
self.compiler.customize(self.distribution,
need_cxx=self.have_cxx_sources())
if self.warn_error:
self.compiler.compiler.append('-Werror')
self.compiler.compiler_so.append('-Werror')
libraries = self.libraries
self.libraries = None
self.compiler.customize_cmd(self)
self.libraries = libraries
self.compiler.show_customization()
if not self.disable_optimization:
dispatch_hpath = os.path.join("numpy", "distutils", "include",
"npy_cpu_dispatch_config.h")
dispatch_hpath = os.path.join(
self.get_finalized_command("build_src").build_src,
dispatch_hpath)
opt_cache_path = os.path.abspath(
os.path.join(self.build_temp, 'ccompiler_opt_cache_clib.py'))
if hasattr(self, "compiler_opt"):
# By default `CCompilerOpt` update the cache at the exit of
# the process, which may lead to duplicate building
# (see build_extension()/force_rebuild) if run() called
# multiple times within the same os process/thread without
# giving the chance the previous instances of `CCompilerOpt`
# to update the cache.
self.compiler_opt.cache_flush()
self.compiler_opt = new_ccompiler_opt(
compiler=self.compiler,
dispatch_hpath=dispatch_hpath,
cpu_baseline=self.cpu_baseline,
cpu_dispatch=self.cpu_dispatch,
cache_path=opt_cache_path)
def report(copt):
log.info(
"\n########### CLIB COMPILER OPTIMIZATION ###########")
log.info(copt.report(full=True))
import atexit
atexit.register(report, self.compiler_opt)
if self.have_f_sources():
from numpy.distutils.fcompiler import new_fcompiler
self._f_compiler = new_fcompiler(compiler=self.fcompiler,
verbose=self.verbose,
dry_run=self.dry_run,
force=self.force,
requiref90='f90' in languages,
c_compiler=self.compiler)
if self._f_compiler is not None:
self._f_compiler.customize(self.distribution)
libraries = self.libraries
self.libraries = None
self._f_compiler.customize_cmd(self)
self.libraries = libraries
self._f_compiler.show_customization()
else:
self._f_compiler = None
self.build_libraries(self.libraries)
if self.inplace:
for l in self.distribution.installed_libraries:
libname = self.compiler.library_filename(l.name)
source = os.path.join(self.build_clib, libname)
target = os.path.join(l.target_dir, libname)
self.mkpath(l.target_dir)
shutil.copy(source, target)
def bundle_libzmq_extension(self):
bundledir = "bundled"
ext_modules = self.distribution.ext_modules
if ext_modules and any(m.name == 'zmq.libzmq' for m in ext_modules):
# I've already been run
return
line()
info("Using bundled libzmq")
# fetch sources for libzmq extension:
if not os.path.exists(bundledir):
os.makedirs(bundledir)
fetch_libzmq(bundledir)
stage_platform_hpp(pjoin(bundledir, 'zeromq'))
# construct the Extensions:
libzmq = Extension(
'zmq.libzmq',
sources=[pjoin('buildutils', 'initlibzmq.c')] +
glob(pjoin(bundledir, 'zeromq', 'src', '*.cpp')),
include_dirs=[
pjoin(bundledir, 'zeromq', 'include'),
],
)
# register the extension:
self.distribution.ext_modules.insert(0, libzmq)
# select polling subsystem based on platform
if sys.platform == 'darwin' or 'bsd' in sys.platform:
libzmq.define_macros.append(('ZMQ_USE_KQUEUE', 1))
elif 'linux' in sys.platform:
libzmq.define_macros.append(('ZMQ_USE_EPOLL', 1))
elif sys.platform.startswith('win'):
libzmq.define_macros.append(('ZMQ_USE_SELECT', 1))
else:
# this may not be sufficiently precise
libzmq.define_macros.append(('ZMQ_USE_POLL', 1))
if sys.platform.startswith('win'):
# include defines from zeromq msvc project:
libzmq.define_macros.append(('FD_SETSIZE', 1024))
libzmq.define_macros.append(('DLL_EXPORT', 1))
libzmq.define_macros.append(('_CRT_SECURE_NO_WARNINGS', 1))
# When compiling the C++ code inside of libzmq itself, we want to
# avoid "warning C4530: C++ exception handler used, but unwind
# semantics are not enabled. Specify /EHsc".
if self.compiler_type == 'msvc':
libzmq.extra_compile_args.append('/EHsc')
elif self.compiler_type == 'mingw32':
libzmq.define_macros.append(('ZMQ_HAVE_MINGW32', 1))
# And things like sockets come from libraries that must be named.
libzmq.libraries.extend(['rpcrt4', 'ws2_32', 'advapi32'])
# link against libsodium in build dir:
suffix = ''
if sys.version_info >= (3, 5):
# Python 3.5 adds EXT_SUFFIX to libs
ext_suffix = distutils.sysconfig.get_config_var('EXT_SUFFIX')
suffix = os.path.splitext(ext_suffix)[0]
if self.debug:
suffix = '_d' + suffix
libzmq.libraries.append('libsodium' + suffix)
libzmq.library_dirs.append(pjoin(self.build_temp, 'buildutils'))
else:
libzmq.include_dirs.append(bundledir)
# check if we need to link against Realtime Extensions library
cc = new_compiler(compiler=self.compiler_type)
cc.output_dir = self.build_temp
if not sys.platform.startswith(('darwin', 'freebsd')):
line()
info("checking for timer_create")
if not cc.has_function('timer_create'):
info("no timer_create, linking librt")
libzmq.libraries.append('rt')
else:
info("ok")
if pypy:
# seem to need explicit libstdc++ on linux + pypy
# not sure why
libzmq.libraries.append("stdc++")
# Also bundle libsodium, even on Windows.
self.bundle_libsodium_extension(libzmq)
# update other extensions, with bundled settings
self.config['libzmq_extension'] = True
self.init_settings_from_config()
self.save_config('config', self.config)
def get_device_extensions(use_cython: bool, library_dirs=None):
library_dirs = [] if library_dirs is None else library_dirs
cur_dir = os.path.dirname(os.path.realpath(__file__))
include_dirs = [os.path.realpath(os.path.join(cur_dir, "includes"))]
if sys.platform == "win32":
if platform.architecture()[0] != "64bit":
return [
] # only 64 bit python supported for native device backends
result = []
lib_dir = os.path.realpath(os.path.join(cur_dir, "lib/win"))
for dev_name, params in DEVICES.items():
result.append(
get_device_extension(dev_name, [params["lib"]], [lib_dir],
include_dirs))
return result
if sys.platform == "darwin":
# On Mac OS X clang is by default not smart enough to search in the lib dir
# see: https://github.com/jopohl/urh/issues/173
library_dirs.append("/usr/local/lib")
result = []
# None = automatic (depending on lib is installed)
# 1 = install extension always
# 0 = Do not install extension
build_device_extensions = defaultdict(lambda: None)
for dev_name in DEVICES:
with_option = "--with-" + dev_name
without_option = "--without-" + dev_name
if with_option in sys.argv and without_option in sys.argv:
print("ambiguous options for " + dev_name)
sys.exit(1)
elif without_option in sys.argv:
build_device_extensions[dev_name] = 0
sys.argv.remove(without_option)
elif with_option in sys.argv:
build_device_extensions[dev_name] = 1
sys.argv.remove(with_option)
sys.path.append(os.path.realpath(os.path.join(cur_dir, "lib")))
compiler = ccompiler.new_compiler()
for dev_name, params in DEVICES.items():
if build_device_extensions[dev_name] == 0:
print("\nSkipping native {0} support\n".format(dev_name))
continue
if build_device_extensions[dev_name] == 1:
print("\nEnforcing native {0} support\n".format(dev_name))
result.append(
get_device_extension(dev_name, [params["lib"]], library_dirs,
include_dirs, use_cython))
continue
if compiler.has_function(params["test_function"],
libraries=(params["lib"], ),
library_dirs=library_dirs,
include_dirs=include_dirs):
print("\nFound {0} lib. Will compile with native {1} support\n".
format(params["lib"], dev_name))
result.append(
get_device_extension(dev_name, [params["lib"]], library_dirs,
include_dirs, use_cython))
elif dev_name in FALLBACKS:
print("Trying fallback for {0}".format(dev_name))
params = FALLBACKS[dev_name]
dev_name += "_fallback"
if compiler.has_function(params["test_function"],
libraries=(params["lib"], ),
library_dirs=library_dirs,
include_dirs=include_dirs):
print(
"\nFound fallback. Will compile with native {0} support\n".
format(dev_name))
result.append(
get_device_extension(dev_name, [params["lib"]],
library_dirs, include_dirs,
use_cython))
else:
print("\nSkipping native support for {1}\n".format(
params["lib"], dev_name))
# remove Temp file for checking
try:
os.remove("a.out")
except OSError:
pass
for filename in os.listdir(tempfile.gettempdir()):
dev_name = dev_name.replace("_fallback", "")
func_names = [DEVICES[dev_name]["test_function"]]
if dev_name in FALLBACKS:
func_names.append(FALLBACKS[dev_name]["test_function"])
if any(filename.startswith(func_name)
for func_name in func_names) and filename.endswith(".c"):
os.remove(os.path.join(tempfile.gettempdir(), filename))
return result
def do_custom_build(self):
# We're using a system freetype
if options.get('system_freetype'):
return
src_path = Path('build', f'freetype-{LOCAL_FREETYPE_VERSION}')
# We've already built freetype
if sys.platform == 'win32':
libfreetype = 'libfreetype.lib'
else:
libfreetype = 'libfreetype.a'
# bailing because it is already built
if (src_path / 'objs' / '.libs' / libfreetype).is_file():
return
# do we need to download / load the source from cache?
if not src_path.exists():
os.makedirs('build', exist_ok=True)
tarball = f'freetype-{LOCAL_FREETYPE_VERSION}.tar.gz'
target_urls = [
(f'https://downloads.sourceforge.net/project/freetype'
f'/freetype2/{LOCAL_FREETYPE_VERSION}/{tarball}'),
(f'https://download.savannah.gnu.org/releases/freetype'
f'/{tarball}')
]
for tarball_url in target_urls:
try:
tar_contents = download_or_cache(tarball_url,
LOCAL_FREETYPE_HASH)
break
except Exception:
pass
else:
raise IOError(
f"Failed to download FreeType. Please download one of "
f"{target_urls} and extract it into {src_path} at the "
f"top-level of the source repository.")
print(f"Extracting {tarball}")
with tarfile.open(fileobj=tar_contents, mode="r:gz") as tgz:
tgz.extractall("build")
print(f"Building freetype in {src_path}")
if sys.platform != 'win32': # compilation on non-windows
env = {
**os.environ, "CFLAGS":
"{} -fPIC".format(os.environ.get("CFLAGS", ""))
}
subprocess.check_call([
"./configure", "--with-zlib=no", "--with-bzip2=no",
"--with-png=no", "--with-harfbuzz=no"
],
env=env,
cwd=src_path)
subprocess.check_call(["make"], env=env, cwd=src_path)
else: # compilation on windows
shutil.rmtree(src_path / "objs", ignore_errors=True)
msbuild_platform = ('x64' if platform.architecture()[0] == '64bit'
else 'Win32')
base_path = Path("build/freetype-2.6.1/builds/windows")
vc = 'vc2010'
sln_path = (base_path / vc / "freetype.sln")
# https://developercommunity.visualstudio.com/comments/190992/view.html
(sln_path.parent / "Directory.Build.props").write_text("""
<Project>
<PropertyGroup>
<!-- The following line *cannot* be split over multiple lines. -->
<WindowsTargetPlatformVersion>$([Microsoft.Build.Utilities.ToolLocationHelper]::GetLatestSDKTargetPlatformVersion('Windows', '10.0'))</WindowsTargetPlatformVersion>
</PropertyGroup>
</Project>
""")
# It is not a trivial task to determine PlatformToolset to plug it
# into msbuild command, and Directory.Build.props will not override
# the value in the project file.
# The DefaultPlatformToolset is from Microsoft.Cpp.Default.props
with open(base_path / vc / "freetype.vcxproj", 'r+b') as f:
toolset_repl = b'PlatformToolset>$(DefaultPlatformToolset)<'
vcxproj = f.read().replace(b'PlatformToolset>v100<',
toolset_repl)
assert toolset_repl in vcxproj, (
'Upgrading Freetype might break this')
f.seek(0)
f.truncate()
f.write(vcxproj)
cc = ccompiler.new_compiler()
cc.initialize() # Get msbuild in the %PATH% of cc.spawn.
cc.spawn([
"msbuild",
str(sln_path), "/t:Clean;Build",
f"/p:Configuration=Release;Platform={msbuild_platform}"
])
# Move to the corresponding Unix build path.
(src_path / "objs" / ".libs").mkdir()
# Be robust against change of FreeType version.
lib_path, = (src_path / "objs" / vc /
msbuild_platform).glob("freetype*.lib")
shutil.copy2(lib_path, src_path / "objs/.libs/libfreetype.lib")
def finalize_options(self):
build.finalize_options(self)
# Check that build options are available
compiler = ccompiler.new_compiler(compiler=self.compiler, force=True)
sysconfig.customize_compiler(compiler)
if self.cpp11:
if compiler.compiler_type == 'msvc':
self.cpp11 = sys.version_info[:2] >= (3, 5)
else:
self.cpp1 = check_compile_flag(compiler,
'-std=c++11',
extension='.cc')
if not self.cpp11:
logger.warning("C++11 disabled: not available")
if self.sse2:
if compiler.compiler_type == 'msvc':
self.sse2 = sys.version_info[0] >= 3
else:
self.sse2 = check_compile_flag(compiler, '-msse2')
if not self.sse2:
logger.warning("SSE2 disabled: not available")
if self.avx2:
if compiler.compiler_type == 'msvc':
self.avx2 = sys.version_info[:2] >= (3, 5)
else:
self.avx2 = check_compile_flag(compiler, '-mavx2')
if not self.avx2:
logger.warning("AVX2 disabled: not available")
if self.openmp:
prefix = '/' if compiler.compiler_type == 'msvc' else '-f'
self.openmp = check_compile_flag(compiler, prefix + 'openmp')
if not self.openmp:
logger.warning("OpenMP disabled: not available")
if self.native:
is_cpu_sse2, is_cpu_avx2 = get_cpu_sse2_avx2()
self.sse2 = self.sse2 and is_cpu_sse2
self.avx2 = self.avx2 and is_cpu_avx2
logger.info("Building with C++11: %r", bool(self.cpp11))
logger.info('Building with native option: %r', bool(self.native))
logger.info("Building with SSE2: %r", bool(self.sse2))
logger.info("Building with AVX2: %r", bool(self.avx2))
logger.info("Building with OpenMP: %r", bool(self.openmp))
# Filter out C++11 libraries if cpp11 option is False
self.distribution.libraries = [
(name, info) for name, info in self.distribution.libraries
if self.cpp11 or '-std=c++11' not in info.get('cflags', [])
]
# Filter out C++11-only extensions if cpp11 option is False
self.distribution.ext_modules = [
ext for ext in self.distribution.ext_modules if self.cpp11 or not (
isinstance(ext, HDF5PluginExtension) and ext.cpp11_required)
]
def do_custom_build(self, env):
# We're using a system freetype
if options.get('system_freetype'):
return
tarball = f'freetype-{LOCAL_FREETYPE_VERSION}.tar.gz'
src_path = get_and_extract_tarball(
urls=[(f'https://downloads.sourceforge.net/project/freetype'
f'/freetype2/{LOCAL_FREETYPE_VERSION}/{tarball}'),
(f'https://download.savannah.gnu.org/releases/freetype'
f'/{tarball}')],
sha=LOCAL_FREETYPE_HASH,
dirname=f'freetype-{LOCAL_FREETYPE_VERSION}',
)
if sys.platform == 'win32':
libfreetype = 'libfreetype.lib'
else:
libfreetype = 'libfreetype.a'
if (src_path / 'objs' / '.libs' / libfreetype).is_file():
return # Bail out because we have already built FreeType.
print(f"Building freetype in {src_path}")
if sys.platform != 'win32': # compilation on non-windows
env = {**env, "CFLAGS": "{} -fPIC".format(env.get("CFLAGS", ""))}
subprocess.check_call([
"./configure", "--with-zlib=no", "--with-bzip2=no",
"--with-png=no", "--with-harfbuzz=no", "--enable-static",
"--disable-shared"
],
env=env,
cwd=src_path)
if 'GNUMAKE' in env:
make = env['GNUMAKE']
elif 'MAKE' in env:
make = env['MAKE']
else:
try:
output = subprocess.check_output(['make', '-v'],
stderr=subprocess.DEVNULL)
except subprocess.CalledProcessError:
output = b''
if b'GNU' not in output and b'makepp' not in output:
make = 'gmake'
else:
make = 'make'
subprocess.check_call([make], env=env, cwd=src_path)
else: # compilation on windows
shutil.rmtree(src_path / "objs", ignore_errors=True)
msbuild_platform = ('x64' if platform.architecture()[0] == '64bit'
else 'Win32')
base_path = Path("build/freetype-2.6.1/builds/windows")
vc = 'vc2010'
sln_path = (base_path / vc / "freetype.sln")
# https://developercommunity.visualstudio.com/comments/190992/view.html
(sln_path.parent / "Directory.Build.props").write_text("""
<Project>
<PropertyGroup>
<!-- The following line *cannot* be split over multiple lines. -->
<WindowsTargetPlatformVersion>$([Microsoft.Build.Utilities.ToolLocationHelper]::GetLatestSDKTargetPlatformVersion('Windows', '10.0'))</WindowsTargetPlatformVersion>
</PropertyGroup>
</Project>
""")
# It is not a trivial task to determine PlatformToolset to plug it
# into msbuild command, and Directory.Build.props will not override
# the value in the project file.
# The DefaultPlatformToolset is from Microsoft.Cpp.Default.props
with open(base_path / vc / "freetype.vcxproj", 'r+b') as f:
toolset_repl = b'PlatformToolset>$(DefaultPlatformToolset)<'
vcxproj = f.read().replace(b'PlatformToolset>v100<',
toolset_repl)
assert toolset_repl in vcxproj, (
'Upgrading Freetype might break this')
f.seek(0)
f.truncate()
f.write(vcxproj)
cc = ccompiler.new_compiler()
cc.initialize() # Get msbuild in the %PATH% of cc.spawn.
cc.spawn([
"msbuild",
str(sln_path), "/t:Clean;Build",
f"/p:Configuration=Release;Platform={msbuild_platform}"
])
# Move to the corresponding Unix build path.
(src_path / "objs" / ".libs").mkdir()
# Be robust against change of FreeType version.
lib_path, = (src_path / "objs" / vc /
msbuild_platform).glob("freetype*.lib")
shutil.copy2(lib_path, src_path / "objs/.libs/libfreetype.lib")
def run(self):
if not self.extensions:
return
# Make sure that extension sources are complete.
self.run_command('build_src')
if self.distribution.has_c_libraries():
if self.inplace:
if self.distribution.have_run.get('build_clib'):
log.warn('build_clib already run, it is too late to '
'ensure in-place build of build_clib')
build_clib = self.distribution.get_command_obj(
'build_clib')
else:
build_clib = self.distribution.get_command_obj(
'build_clib')
build_clib.inplace = 1
build_clib.ensure_finalized()
build_clib.run()
self.distribution.have_run['build_clib'] = 1
else:
self.run_command('build_clib')
build_clib = self.get_finalized_command('build_clib')
self.library_dirs.append(build_clib.build_clib)
else:
build_clib = None
# Not including C libraries to the list of
# extension libraries automatically to prevent
# bogus linking commands. Extensions must
# explicitly specify the C libraries that they use.
from distutils.ccompiler import new_compiler
from numpy.distutils.fcompiler import new_fcompiler
compiler_type = self.compiler
# Initialize C compiler:
self.compiler = new_compiler(compiler=compiler_type,
verbose=self.verbose,
dry_run=self.dry_run,
force=self.force)
self.compiler.customize(self.distribution)
self.compiler.customize_cmd(self)
if self.warn_error:
self.compiler.compiler.append('-Werror')
self.compiler.compiler_so.append('-Werror')
self.compiler.show_customization()
if not self.disable_optimization:
dispatch_hpath = os.path.join("numpy", "distutils", "include",
"npy_cpu_dispatch_config.h")
dispatch_hpath = os.path.join(
self.get_finalized_command("build_src").build_src,
dispatch_hpath)
opt_cache_path = os.path.abspath(
os.path.join(self.build_temp, 'ccompiler_opt_cache_ext.py'))
if hasattr(self, "compiler_opt"):
# By default `CCompilerOpt` update the cache at the exit of
# the process, which may lead to duplicate building
# (see build_extension()/force_rebuild) if run() called
# multiple times within the same os process/thread without
# giving the chance the previous instances of `CCompilerOpt`
# to update the cache.
self.compiler_opt.cache_flush()
self.compiler_opt = new_ccompiler_opt(
compiler=self.compiler,
dispatch_hpath=dispatch_hpath,
cpu_baseline=self.cpu_baseline,
cpu_dispatch=self.cpu_dispatch,
cache_path=opt_cache_path)
def report(copt):
log.info("\n########### EXT COMPILER OPTIMIZATION ###########")
log.info(copt.report(full=True))
import atexit
atexit.register(report, self.compiler_opt)
# Setup directory for storing generated extra DLL files on Windows
self.extra_dll_dir = os.path.join(self.build_temp, '.libs')
if not os.path.isdir(self.extra_dll_dir):
os.makedirs(self.extra_dll_dir)
# Create mapping of libraries built by build_clib:
clibs = {}
if build_clib is not None:
for libname, build_info in build_clib.libraries or []:
if libname in clibs and clibs[libname] != build_info:
log.warn('library %r defined more than once,'
' overwriting build_info\n%s... \nwith\n%s...' %
(libname, repr(clibs[libname])[:300],
repr(build_info)[:300]))
clibs[libname] = build_info
# .. and distribution libraries:
for libname, build_info in self.distribution.libraries or []:
if libname in clibs:
# build_clib libraries have a precedence before distribution ones
continue
clibs[libname] = build_info
# Determine if C++/Fortran 77/Fortran 90 compilers are needed.
# Update extension libraries, library_dirs, and macros.
all_languages = set()
for ext in self.extensions:
ext_languages = set()
c_libs = []
c_lib_dirs = []
macros = []
for libname in ext.libraries:
if libname in clibs:
binfo = clibs[libname]
c_libs += binfo.get('libraries', [])
c_lib_dirs += binfo.get('library_dirs', [])
for m in binfo.get('macros', []):
if m not in macros:
macros.append(m)
for l in clibs.get(libname, {}).get('source_languages', []):
ext_languages.add(l)
if c_libs:
new_c_libs = ext.libraries + c_libs
log.info('updating extension %r libraries from %r to %r' %
(ext.name, ext.libraries, new_c_libs))
ext.libraries = new_c_libs
ext.library_dirs = ext.library_dirs + c_lib_dirs
if macros:
log.info('extending extension %r defined_macros with %r' %
(ext.name, macros))
ext.define_macros = ext.define_macros + macros
# determine extension languages
if has_f_sources(ext.sources):
ext_languages.add('f77')
if has_cxx_sources(ext.sources):
ext_languages.add('c++')
l = ext.language or self.compiler.detect_language(ext.sources)
if l:
ext_languages.add(l)
# reset language attribute for choosing proper linker
if 'c++' in ext_languages:
ext_language = 'c++'
elif 'f90' in ext_languages:
ext_language = 'f90'
elif 'f77' in ext_languages:
ext_language = 'f77'
else:
ext_language = 'c' # default
if l and l != ext_language and ext.language:
log.warn('resetting extension %r language from %r to %r.' %
(ext.name, l, ext_language))
ext.language = ext_language
# global language
all_languages.update(ext_languages)
need_f90_compiler = 'f90' in all_languages
need_f77_compiler = 'f77' in all_languages
need_cxx_compiler = 'c++' in all_languages
# Initialize C++ compiler:
if need_cxx_compiler:
self._cxx_compiler = new_compiler(compiler=compiler_type,
verbose=self.verbose,
dry_run=self.dry_run,
force=self.force)
compiler = self._cxx_compiler
compiler.customize(self.distribution, need_cxx=need_cxx_compiler)
compiler.customize_cmd(self)
compiler.show_customization()
self._cxx_compiler = compiler.cxx_compiler()
else:
self._cxx_compiler = None
# Initialize Fortran 77 compiler:
if need_f77_compiler:
ctype = self.fcompiler
self._f77_compiler = new_fcompiler(compiler=self.fcompiler,
verbose=self.verbose,
dry_run=self.dry_run,
force=self.force,
requiref90=False,
c_compiler=self.compiler)
fcompiler = self._f77_compiler
if fcompiler:
ctype = fcompiler.compiler_type
fcompiler.customize(self.distribution)
if fcompiler and fcompiler.get_version():
fcompiler.customize_cmd(self)
fcompiler.show_customization()
else:
self.warn('f77_compiler=%s is not available.' % (ctype))
self._f77_compiler = None
else:
self._f77_compiler = None
# Initialize Fortran 90 compiler:
if need_f90_compiler:
ctype = self.fcompiler
self._f90_compiler = new_fcompiler(compiler=self.fcompiler,
verbose=self.verbose,
dry_run=self.dry_run,
force=self.force,
requiref90=True,
c_compiler=self.compiler)
fcompiler = self._f90_compiler
if fcompiler:
ctype = fcompiler.compiler_type
fcompiler.customize(self.distribution)
if fcompiler and fcompiler.get_version():
fcompiler.customize_cmd(self)
fcompiler.show_customization()
else:
self.warn('f90_compiler=%s is not available.' % (ctype))
self._f90_compiler = None
else:
self._f90_compiler = None
# Build extensions
self.build_extensions()
# Copy over any extra DLL files
# FIXME: In the case where there are more than two packages,
# we blindly assume that both packages need all of the libraries,
# resulting in a larger wheel than is required. This should be fixed,
# but it's so rare that I won't bother to handle it.
pkg_roots = {
self.get_ext_fullname(ext.name).split('.')[0]
for ext in self.extensions
}
for pkg_root in pkg_roots:
shared_lib_dir = os.path.join(pkg_root, '.libs')
if not self.inplace:
shared_lib_dir = os.path.join(self.build_lib, shared_lib_dir)
for fn in os.listdir(self.extra_dll_dir):
if not os.path.isdir(shared_lib_dir):
os.makedirs(shared_lib_dir)
if not fn.lower().endswith('.dll'):
continue
runtime_lib = os.path.join(self.extra_dll_dir, fn)
copy_file(runtime_lib, shared_lib_dir)
def bundle_libqpid_proton_extension(self):
"""The proper version of libqpid-proton is not present on the system,
so attempt to retrieve the proper libqpid-proton sources and
include them in the extension.
"""
setup_path = os.path.dirname(os.path.realpath(__file__))
base = self.get_finalized_command('build').build_base
build_include = os.path.join(base, 'include')
log.info("Bundling qpid-proton into the extension")
# QPID_PROTON_SRC - (optional) pathname to the Proton C sources. Can
# be used to override where this setup gets the Proton C sources from
# (see bundle.fetch_libqpid_proton())
if 'QPID_PROTON_SRC' not in os.environ:
if not os.path.exists(os.path.join(setup_path, 'tox.ini')):
bundledir = os.path.join(base, "bundled")
if not os.path.exists(bundledir):
os.makedirs(bundledir)
bundle.fetch_libqpid_proton(bundledir)
libqpid_proton_dir = os.path.abspath(os.path.join(bundledir, 'qpid-proton'))
else:
# This should happen just in **dev** environemnts since
# tox.ini is not shipped with the driver. It should only
# be triggered when calling `setup.py`. This can happen either
# manually or when calling `tox` in the **sdist** step. Tox will
# defined the `QPID_PROTON_SRC` itself.
proton_c = os.path.join(setup_path, 'qpid-proton')
libqpid_proton_dir = os.path.abspath(proton_c)
else:
libqpid_proton_dir = os.path.abspath(os.environ['QPID_PROTON_SRC'])
log.debug("Using libqpid-proton src: %s" % libqpid_proton_dir)
proton_base = os.path.join(libqpid_proton_dir, 'proton-c')
proton_src = os.path.join(proton_base, 'src')
proton_include = os.path.join(proton_base, 'include')
#
# Create any generated header files, and put them in build_include:
#
if not os.path.exists(build_include):
os.makedirs(build_include)
os.mkdir(os.path.join(build_include, 'proton'))
# Generate `protocol.h` by calling the python
# script found in the source dir.
with open(os.path.join(build_include, 'protocol.h'), 'wb') as header:
subprocess.Popen([sys.executable, os.path.join(proton_src, 'protocol.h.py')],
env={'PYTHONPATH': proton_base}, stdout=header)
# Generate `encodings.h` by calling the python
# script found in the source dir.
with open(os.path.join(build_include, 'encodings.h'), 'wb') as header:
subprocess.Popen([sys.executable,
os.path.join(proton_src, 'codec', 'encodings.h.py')],
env={'PYTHONPATH': proton_base}, stdout=header)
# Create a custom, temporary, version.h file mapping the
# major and minor versions from the downloaded tarbal. This version should
# match the ones in the bundle module
with open(os.path.join(build_include, 'proton', 'version.h'), "wb") as ver:
version_text = """
#ifndef _PROTON_VERSION_H
#define _PROTON_VERSION_H 1
#define PN_VERSION_MAJOR %i
#define PN_VERSION_MINOR %i
#endif /* version.h */
""" % bundle.min_qpid_proton
ver.write(version_text.encode('utf-8'))
# Collect all the Proton C files that need to be built.
# we could've used `glob(.., '*', '*.c')` but I preferred going
# with an explicit list of subdirs that we can control and expand
# depending on the version. Specifically, lets avoid adding things
# we don't need.
sources = []
for subdir in ['object', 'framing', 'codec', 'dispatcher',
'engine', 'events', 'transport',
'message', 'reactor', 'messenger',
'handlers', 'posix']:
sources.extend(glob.glob(os.path.join(proton_src, subdir, '*.c')))
sources.extend(filter(lambda x: not x.endswith('dump.c'),
glob.iglob(os.path.join(proton_src, '*.c'))))
# Look for any optional libraries that proton needs, and adjust the
# source list as necessary.
libraries = []
# Check whether openssl is installed by poking
# pkg-config for a minimum version 0. If it's installed, it should
# return True and we'll use it. Otherwise, we'll use the stub.
if misc.pkg_config_version(atleast='0', module='openssl'):
libraries += ['ssl', 'crypto']
sources.append(os.path.join(proton_src, 'ssl', 'openssl.c'))
else:
sources.append(os.path.join(proton_src, 'ssl', 'ssl_stub.c'))
# create a temp compiler to check for optional compile-time features
cc = new_compiler(compiler=self.compiler_type)
cc.output_dir = self.build_temp
# Some systems need to link to
# `rt`. Check whether `clock_getttime` is around
# and if not, link on rt.
if not cc.has_function('clock_getttime'):
libraries.append('rt')
# 0.10 added an implementation for cyrus. Check
# if it is available before adding the implementation to the sources
# list. Eventually, `sasl.c` will be added and one of the existing
# implementations will be used.
if cc.has_function('sasl_client_done', includes=['sasl/sasl.h'],
libraries=['sasl2']):
libraries.append('sasl2')
sources.append(os.path.join(proton_src, 'sasl', 'cyrus_sasl.c'))
else:
sources.append(os.path.join(proton_src, 'sasl', 'none_sasl.c'))
sources.append(os.path.join(proton_src, 'sasl', 'sasl.c'))
# compile all the proton sources. We'll add the resulting list of
# objects to the _cproton extension as 'extra objects'. We do this
# instead of just lumping all the sources into the extension to prevent
# any proton-specific compilation flags from affecting the compilation
# of the generated swig code
cc = new_compiler(compiler=self.compiler_type)
ds_sys.customize_compiler(cc)
objects = cc.compile(sources,
# -D flags (None means no value, just define)
macros=[('qpid_proton_EXPORTS', None),
('USE_ATOLL', None),
('USE_CLOCK_GETTIME', None),
('USE_STRERROR_R', None)],
include_dirs=[build_include,
proton_include,
proton_src],
# compiler command line options:
extra_postargs=['-std=gnu99'],
output_dir=self.build_temp)
#
# Now update the _cproton extension instance to include the objects and
# libraries
#
_cproton = self.distribution.ext_modules[-1]
_cproton.extra_objects = objects
_cproton.include_dirs.append(build_include)
_cproton.include_dirs.append(proton_include)
# swig will need to access the proton headers:
_cproton.swig_opts.append('-I%s' % build_include)
_cproton.swig_opts.append('-I%s' % proton_include)
# lastly replace the libqpid-proton dependency with libraries required
# by the Proton objects:
_cproton.libraries=libraries
Extension('mercurial.diffhelpers', ['mercurial/diffhelpers.c']),
Extension('mercurial.mpatch', ['mercurial/mpatch.c']),
Extension('mercurial.parsers', ['mercurial/parsers.c']),
]
# disable osutil.c under windows + python 2.4 (issue1364)
if sys.platform == 'win32' and sys.version_info < (2, 5, 0, 'final'):
pymodules.append('mercurial.pure.osutil')
else:
extmodules.append(Extension('mercurial.osutil', ['mercurial/osutil.c']))
if sys.platform.startswith('linux') and os.uname()[2] > '2.6':
# The inotify extension is only usable with Linux 2.6 kernels.
# You also need a reasonably recent C library.
# In any case, if it fails to build the error will be skipped ('optional').
cc = new_compiler()
if hasfunction(cc, 'inotify_add_watch'):
inotify = Extension('hgext.inotify.linux._inotify',
['hgext/inotify/linux/_inotify.c'], ['mercurial'])
inotify.optional = True
extmodules.append(inotify)
packages.extend(['hgext.inotify', 'hgext.inotify.linux'])
packagedata = {'mercurial': ['locale/*/LC_MESSAGES/hg.mo', 'help/*.txt']}
def ordinarypath(p):
return p and p[0] != '.' and p[-1] != '~'
for root in ('templates', ):
def test_compilation(program,
extra_cc_options=None,
extra_libraries=None,
msg=''):
"""Test if a certain C program can be compiled."""
# Create a temporary file with the C program
if not os.path.exists("build"):
os.makedirs("build")
fname = os.path.join("build", "test1.c")
f = open(fname, 'w')
f.write(program)
f.close()
# Name for the temporary executable
oname = os.path.join("build", "test1.out")
debug = False
# Mute the compiler and the linker
if msg:
print("Testing support for %s" % msg)
if not (debug or os.name == 'nt'):
old_stdout = os.dup(sys.stdout.fileno())
old_stderr = os.dup(sys.stderr.fileno())
dev_null = open(os.devnull, "w")
os.dup2(dev_null.fileno(), sys.stdout.fileno())
os.dup2(dev_null.fileno(), sys.stderr.fileno())
objects = []
try:
compiler = ccompiler.new_compiler()
distutils.sysconfig.customize_compiler(compiler)
if compiler.compiler_type in ['msvc']:
# Force creation of the manifest file (http://bugs.python.org/issue16296)
# as needed by VS2010
extra_linker_options = ["/MANIFEST"]
else:
extra_linker_options = []
# In Unix, force the linker step to use CFLAGS and not CC alone (see GH#180)
if compiler.compiler_type in ['unix']:
compiler.set_executables(linker_exe=compiler.compiler)
objects = compiler.compile([fname], extra_postargs=extra_cc_options)
compiler.link_executable(objects,
oname,
libraries=extra_libraries,
extra_preargs=extra_linker_options)
result = True
except CCompilerError:
result = False
for f in objects + [fname, oname]:
try:
os.remove(f)
except OSError:
pass
# Restore stdout and stderr
if not (debug or os.name == 'nt'):
if old_stdout is not None:
os.dup2(old_stdout, sys.stdout.fileno())
if old_stderr is not None:
os.dup2(old_stderr, sys.stderr.fileno())
if dev_null is not None:
dev_null.close()
if msg:
if result:
x = ""
else:
x = " not"
print("Target does%s support %s" % (x, msg))
return result
def using_clang():
"""Will we be using a clang compiler?"""
compiler = new_compiler()
customize_compiler(compiler)
compiler_ver = getoutput("{0} -v".format(compiler.compiler[0]))
return 'clang' in compiler_ver
def check_sanity():
"""
Test if development headers and library for cephfs is available by compiling a dummy C program.
"""
CEPH_SRC_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'..', '..')
tmp_dir = tempfile.mkdtemp(
dir=os.environ.get('TMPDIR', os.path.dirname(__file__)))
tmp_file = os.path.join(tmp_dir, 'cephfs_dummy.c')
with open(tmp_file, 'w') as fp:
dummy_prog = textwrap.dedent("""
#include <stddef.h>
#include "cephfs/libcephfs.h"
int main(void) {
struct ceph_mount_info *cmount = NULL;
ceph_init(cmount);
return 0;
}
""")
fp.write(dummy_prog)
compiler = new_compiler()
distutils.sysconfig.customize_compiler(compiler)
if 'CEPH_LIBDIR' in os.environ:
# The setup.py has been invoked by a top-level Ceph make.
# Set the appropriate CFLAGS and LDFLAGS
compiler.set_library_dirs([os.environ.get('CEPH_LIBDIR')])
try:
compiler.define_macro('_FILE_OFFSET_BITS', '64')
link_objects = compiler.compile(
sources=[tmp_file],
output_dir=tmp_dir,
extra_preargs=[
'-iquote{path}'.format(
path=os.path.join(CEPH_SRC_DIR, 'include'))
])
compiler.link_executable(
objects=link_objects,
output_progname=os.path.join(tmp_dir, 'cephfs_dummy'),
libraries=['cephfs'],
output_dir=tmp_dir,
)
except CompileError:
print('\nCompile Error: Ceph FS development headers not found',
file=sys.stderr)
return False
except LinkError:
print('\nLink Error: Ceph FS library not found', file=sys.stderr)
return False
else:
return True
finally:
shutil.rmtree(tmp_dir)
def run(self):
if not self.libraries:
return
# Make sure that library sources are complete.
languages = []
# Make sure that extension sources are complete.
self.run_command('build_src')
for (lib_name, build_info) in self.libraries:
l = build_info.get('language', None)
if l and l not in languages:
languages.append(l)
from distutils.ccompiler import new_compiler
self.compiler = new_compiler(compiler=self.compiler,
dry_run=self.dry_run,
force=self.force)
self.compiler.customize(self.distribution,
need_cxx=self.have_cxx_sources())
if self.warn_error:
self.compiler.compiler.append('-Werror')
self.compiler.compiler_so.append('-Werror')
libraries = self.libraries
self.libraries = None
self.compiler.customize_cmd(self)
self.libraries = libraries
self.compiler.show_customization()
if self.have_f_sources():
from numpy.distutils.fcompiler import new_fcompiler
self._f_compiler = new_fcompiler(compiler=self.fcompiler,
verbose=self.verbose,
dry_run=self.dry_run,
force=self.force,
requiref90='f90' in languages,
c_compiler=self.compiler)
if self._f_compiler is not None:
self._f_compiler.customize(self.distribution)
libraries = self.libraries
self.libraries = None
self._f_compiler.customize_cmd(self)
self.libraries = libraries
self._f_compiler.show_customization()
else:
self._f_compiler = None
self.build_libraries(self.libraries)
if self.inplace:
for l in self.distribution.installed_libraries:
libname = self.compiler.library_filename(l.name)
source = os.path.join(self.build_clib, libname)
target = os.path.join(l.target_dir, libname)
self.mkpath(l.target_dir)
shutil.copy(source, target)
def add_openmp_flags_if_available(extension):
"""
Add OpenMP compilation flags, if available (if not a warning will be
printed to the console and no flags will be added)
Returns `True` if the flags were added, `False` otherwise.
"""
ccompiler = new_compiler()
customize_compiler(ccompiler)
tmp_dir = tempfile.mkdtemp()
start_dir = os.path.abspath('.')
if get_compiler_option() == 'msvc':
compile_flag = '-openmp'
link_flag = ''
else:
compile_flag = '-fopenmp'
link_flag = '-fopenmp'
try:
os.chdir(tmp_dir)
with open('test_openmp.c', 'w') as f:
f.write(CCODE)
os.mkdir('objects')
# Compile, link, and run test program
ccompiler.compile(['test_openmp.c'],
output_dir='objects',
extra_postargs=[compile_flag])
ccompiler.link_executable(glob.glob(
os.path.join('objects', '*' + ccompiler.obj_extension)),
'test_openmp',
extra_postargs=[link_flag])
output = subprocess.check_output('./test_openmp').decode(
sys.stdout.encoding or 'utf-8').splitlines()
if 'nthreads=' in output[0]:
nthreads = int(output[0].strip().split('=')[1])
if len(output) == nthreads:
using_openmp = True
else:
log.warn(
"Unexpected number of lines from output of test OpenMP "
"program (output was {0})".format(output))
using_openmp = False
else:
log.warn("Unexpected output from test OpenMP "
"program (output was {0})".format(output))
using_openmp = False
except (CompileError, LinkError):
using_openmp = False
finally:
os.chdir(start_dir)
if using_openmp:
log.info("Compiling Cython extension with OpenMP support")
extension.extra_compile_args.append(compile_flag)
extension.extra_link_args.append(link_flag)
else:
log.warn(
"Cannot compile Cython extension with OpenMP, reverting to non-parallel code"
)
return using_openmp
def mypycify(
paths: List[str],
mypy_options: Optional[List[str]] = None,
*,
verbose: bool = False,
opt_level: str = '3',
strip_asserts: bool = False,
multi_file: bool = False,
separate: Union[bool, List[Tuple[List[str], Optional[str]]]] = False,
skip_cgen_input: Optional[Any] = None
) -> List['Extension']:
"""Main entry point to building using mypyc.
This produces a list of Extension objects that should be passed as the
ext_modules parameter to setup.
Arguments:
paths: A list of file paths to build. It may contain globs.
mypy_options: Optionally, a list of command line flags to pass to mypy.
(This can also contain additional files, for compatibility reasons.)
verbose: Should mypyc be more verbose. Defaults to false.
opt_level: The optimization level, as a string. Defaults to '3' (meaning '-O3').
strip_asserts: Should asserts be stripped from the generated code.
multi_file: Should each Python module be compiled into its own C source file.
This can reduce compile time and memory requirements at the likely
cost of runtime performance of compiled code. Defaults to false.
separate: Should compiled modules be placed in separate extension modules.
If False, all modules are placed in a single shared library.
If True, every module is placed in its own library.
Otherwise separate should be a list of
(file name list, optional shared library name) pairs specifying
groups of files that should be placed in the same shared library
(while all other modules will be placed in its own library).
Each group can be compiled independently, which can
speed up compilation, but calls between groups can
be slower than calls within a group and can't be
inlined.
"""
setup_mypycify_vars()
compiler_options = CompilerOptions(strip_asserts=strip_asserts,
multi_file=multi_file, verbose=verbose)
# Create a compiler object so we can make decisions based on what
# compiler is being used. typeshed is missing some attribues on the
# compiler object so we give it type Any
compiler = ccompiler.new_compiler() # type: Any
sysconfig.customize_compiler(compiler)
expanded_paths = []
for path in paths:
expanded_paths.extend(glob.glob(path))
build_dir = 'build' # TODO: can this be overridden??
try:
os.mkdir(build_dir)
except FileExistsError:
pass
sources, options = get_mypy_config(expanded_paths, mypy_options)
# We generate a shared lib if there are multiple modules or if any
# of the modules are in package. (Because I didn't want to fuss
# around with making the single module code handle packages.)
use_shared_lib = len(sources) > 1 or any('.' in x.module for x in sources)
groups = construct_groups(sources, separate, use_shared_lib)
# We let the test harness just pass in the c file contents instead
# so that it can do a corner-cutting version without full stubs.
if not skip_cgen_input:
group_cfiles, ops_text = generate_c(sources, options, groups,
compiler_options=compiler_options)
# TODO: unique names?
with open(os.path.join(build_dir, 'ops.txt'), 'w') as f:
f.write(ops_text)
else:
group_cfiles = skip_cgen_input
# Write out the generated C and collect the files for each group
group_cfilenames = [] # type: List[Tuple[List[str], List[str]]]
for cfiles in group_cfiles:
cfilenames = []
for cfile, ctext in cfiles:
cfile = os.path.join(build_dir, cfile)
write_file(cfile, ctext)
if os.path.splitext(cfile)[1] == '.c':
cfilenames.append(cfile)
deps = [os.path.join(build_dir, dep) for dep in get_header_deps(cfiles)]
group_cfilenames.append((cfilenames, deps))
cflags = [] # type: List[str]
if compiler.compiler_type == 'unix':
cflags += [
'-O{}'.format(opt_level), '-Werror', '-Wno-unused-function', '-Wno-unused-label',
'-Wno-unreachable-code', '-Wno-unused-variable', '-Wno-trigraphs',
'-Wno-unused-command-line-argument', '-Wno-unknown-warning-option',
]
if 'gcc' in compiler.compiler[0]:
# This flag is needed for gcc but does not exist on clang.
cflags += ['-Wno-unused-but-set-variable']
elif compiler.compiler_type == 'msvc':
if opt_level == '3':
opt_level = '2'
cflags += [
'/O{}'.format(opt_level),
'/wd4102', # unreferenced label
'/wd4101', # unreferenced local variable
'/wd4146', # negating unsigned int
]
if multi_file:
# Disable whole program optimization in multi-file mode so
# that we actually get the compilation speed and memory
# use wins that multi-file mode is intended for.
cflags += [
'/GL-',
'/wd9025', # warning about overriding /GL
]
# In multi-file mode, copy the runtime library in.
# Otherwise it just gets #included to save on compiler invocations
shared_cfilenames = []
if multi_file:
for name in ['CPy.c', 'getargs.c']:
rt_file = os.path.join(build_dir, name)
with open(os.path.join(include_dir(), name), encoding='utf-8') as f:
write_file(rt_file, f.read())
shared_cfilenames.append(rt_file)
extensions = []
for (group_sources, lib_name), (cfilenames, deps) in zip(groups, group_cfilenames):
if use_shared_lib:
assert lib_name
extensions.extend(build_using_shared_lib(
group_sources, lib_name, cfilenames + shared_cfilenames, deps, build_dir, cflags))
else:
extensions.extend(build_single_module(
group_sources, cfilenames + shared_cfilenames, cflags))
return extensions
