def cythonize_extension(self, ext):
if not HAVE_CYTHON:
if not ext.output_file.is_file():
raise DistutilsFileError(
"expected cythonized file %r not found" %
str(ext.output_file))
else:
return
is_newer = self.check_newer(ext.cython_source, ext.output_file)
if not self.force and not is_newer:
return
self.announce("cythonizing %r extension" % ext.name, 2)
try:
ext.output_dir.mkdir(parents=True)
except FileExistsError:
pass
options = CompilationOptions(
defaults=None,
output_file=str(ext.output_file),
language_level=ext.language_level,
cplus=ext.cplus,
annotate=ext.annotate,
)
source = str(ext.cython_source)
output_file = str(ext.output_file)
self.announce("cythonizing %r -> %r" % (source, output_file), 2)
result = compile_single(source, options, ext.name)
def create_dependency_tree(ctx=None, quiet=False):
global _dep_tree
if _dep_tree is None:
if ctx is None:
ctx = Context(["."], CompilationOptions(default_options))
_dep_tree = DependencyTree(ctx, quiet=quiet)
return _dep_tree
def cythonize(source,
includes=(),
destdir_c=None,
destdir_h=None,
wdir=None):
from Cython.Compiler.Main import \
CompilationOptions, default_options, \
compile, \
PyrexError
from Cython.Compiler import Options
cwd = os.getcwd()
try:
name, ext = os.path.splitext(source)
outputs_c = [name+'.c']
outputs_h = [name+'.h', name+'_api.h']
# change working directory
if wdir:
os.chdir(wdir)
# run Cython on source
options = CompilationOptions(default_options)
options.output_file = outputs_c[0]
options.include_path = list(includes)
Options.generate_cleanup_code = 3
any_failures = 0
try:
result = compile(source, options)
if result.num_errors > 0:
any_failures = 1
except (EnvironmentError, PyrexError):
e = sys.exc_info()[1]
sys.stderr.write(str(e) + '\n')
any_failures = 1
if any_failures:
for output in outputs_c + outputs_h:
try:
os.remove(output)
except OSError:
pass
return 1
# move ouputs
for destdir, outputs in (
(destdir_c, outputs_c),
(destdir_h, outputs_h)):
if destdir is None: continue
for output in outputs:
dest = os.path.join(
destdir, os.path.basename(output))
try:
os.remove(dest)
except OSError:
pass
os.rename(output, dest)
#
return 0
#
finally:
os.chdir(cwd)
def gen_mpyrandom(ext, build_dir):
from Cython.Compiler.Main import compile_single, CompilationOptions, default_options
target = join(build_dir, 'mpyrandom.c')
src = join('micpy', 'random', 'mpyrandom.pyx')
header = join('micpy', 'random', 'mpyrandom.pxd')
if newer(src, target) or newer(header, target):
# create target file
options = CompilationOptions(default_options)
options.output_file = target
result = compile_single(src, options)
return target
def run_cython(inc_dirs, extra_opts, out):
from Cython.Compiler.Main import compile, CompilationOptions
from Cython.Compiler.Options import directive_defaults
directive_defaults["boundscheck"] = False
directive_defaults["wraparound"] = False
options = dict(include_path=inc_dirs,
compiler_directives=directive_defaults,
cplus=True)
if extra_opts is not None:
options.update(extra_opts)
options = CompilationOptions(**options)
compile(out, options=options)
def cythonize_one(pyx_file,
c_file,
fingerprint,
quiet,
options=None,
raise_on_failure=True):
from Cython.Compiler.Main import compile, default_options
from Cython.Compiler.Errors import CompileError, PyrexError
if fingerprint:
if not os.path.exists(options.cache):
try:
os.mkdir(options.cache)
except:
if not os.path.exists(options.cache):
raise
# Cython-generated c files are highly compressible.
# (E.g. a compression ratio of about 10 for Sage).
fingerprint_file = join_path(
options.cache,
"%s-%s%s" % (os.path.basename(c_file), fingerprint, gzip_ext))
if os.path.exists(fingerprint_file):
if not quiet:
print("Found compiled %s in cache" % pyx_file)
os.utime(fingerprint_file, None)
g = gzip_open(fingerprint_file, 'rb')
try:
f = open(c_file, 'wb')
try:
shutil.copyfileobj(g, f)
finally:
f.close()
finally:
g.close()
return
if not quiet:
print("Cythonizing %s" % pyx_file)
if options is None:
options = CompilationOptions(default_options)
options.output_file = c_file
any_failures = 0
try:
result = compile([pyx_file], options)
if result.num_errors > 0:
any_failures = 1
except (EnvironmentError, PyrexError), e:
sys.stderr.write('%s\n' % e)
any_failures = 1
# XXX
import traceback
traceback.print_exc()
def simple_cythonize(src, destdir=None, cwd=None, **cy_kwargs):
""" Generates a C file from a Cython source file.
Parameters
==========
src: str
Path to Cython source.
destdir: str (optional)
Path to output directory (default: '.').
cwd: path string (optional)
Root of relative paths (default: '.').
**cy_kwargs:
Second argument passed to cy_compile. Generates a .cpp file if ``cplus=True`` in ``cy_kwargs``,
else a .c file.
"""
from Cython.Compiler.Main import (
default_options, CompilationOptions
)
from Cython.Compiler.Main import compile as cy_compile
assert src.lower().endswith('.pyx') or src.lower().endswith('.py')
cwd = cwd or '.'
destdir = destdir or '.'
ext = '.cpp' if cy_kwargs.get('cplus', False) else '.c'
c_name = os.path.splitext(os.path.basename(src))[0] + ext
dstfile = os.path.join(destdir, c_name)
if cwd:
ori_dir = os.getcwd()
else:
ori_dir = '.'
os.chdir(cwd)
try:
cy_options = CompilationOptions(default_options)
cy_options.__dict__.update(cy_kwargs)
# Set language_level if not set by cy_kwargs
# as not setting it is deprecated
if 'language_level' not in cy_kwargs:
cy_options.__dict__['language_level'] = 3
cy_result = cy_compile([src], cy_options)
if cy_result.num_errors > 0:
raise ValueError("Cython compilation failed.")
if os.path.realpath(os.path.dirname(src)) != os.path.realpath(destdir):
if os.path.exists(dstfile):
os.unlink(dstfile)
shutil.move(os.path.join(os.path.dirname(src), c_name), destdir)
finally:
os.chdir(ori_dir)
return dstfile
def finalize_options(self):
"""Sanitize and set defaults for final values of options.
"""
SDistCommand.finalize_options(self)
self.ensure_string_list('exclude_sources')
self.ensure_string_list('source_extensions')
self.ensure_string_list('include_path')
if self.exclude_sources is None:
self.exclude_sources = ['setup.py', '**/__init__.py']
if self.source_extensions is None:
self.source_extensions = ['py', 'pyx']
self.python_sources = {}
self.compilation_options = CompilationOptions()
self.compilation_options.cplus = False
self.compilation_options.include_path = self.include_path or []
def cythonize(source, output=None, includes=(), workdir=None):
import cyautodoc
from Cython.Compiler import Options
from Cython.Compiler.Main import (
CompilationOptions,
default_options,
compile,
PyrexError,
)
cwd = os.getcwd()
try:
# compute output filenames
if output is None:
name, _ = os.path.splitext(source)
output = name + '.c'
else:
name, _ = os.path.splitext(output)
outputs_c = [output]
outputs_h = [name + '.h', name + '_api.h']
# run Cython on source
options = CompilationOptions(default_options)
lang_level = Options.directive_types.get('language_level', int)
options.language_level = lang_level(3)
options.output_file = output
options.include_path = list(includes)
options.working_path = workdir or ""
Options.generate_cleanup_code = 3
any_failures = 0
try:
if options.working_path:
os.chdir(options.working_path)
result = compile(source, options)
if result.num_errors > 0:
any_failures = 1
except (EnvironmentError, PyrexError) as e:
sys.stderr.write(str(e) + '\n')
any_failures = 1
if any_failures:
for out in outputs_c + outputs_h:
try:
os.remove(out)
except OSError:
pass
return 1
return 0
#
finally:
os.chdir(cwd)
def cythonize_one(pyx_file, c_file, options=None):
from Cython.Compiler.Main import compile, default_options
from Cython.Compiler.Errors import CompileError, PyrexError
if options is None:
options = CompilationOptions(default_options)
options.output_file = c_file
any_failures = 0
try:
result = compile([pyx_file], options)
if result.num_errors > 0:
any_failures = 1
except (EnvironmentError, PyrexError), e:
sys.stderr.write(str(e) + '\n')
any_failures = 1
def simple_cythonize(src, destdir=None, cwd=None, **cy_kwargs):
""" Generates a C file from a Cython source file.
Parameters
==========
src: str
Path to Cython source.
destdir: str (optional)
Path to output directory (default: '.').
cwd: path string (optional)
Root of relative paths (default: '.').
**cy_kwargs:
Second argument passed to cy_compile. Generates a .cpp file if ``cplus=True`` in ``cy_kwargs``,
else a .c file.
"""
from Cython.Compiler.Main import default_options, CompilationOptions
from Cython.Compiler.Main import compile as cy_compile
assert src.lower().endswith(".pyx") or src.lower().endswith(".py")
cwd = cwd or "."
destdir = destdir or "."
ext = ".cpp" if cy_kwargs.get("cplus", False) else ".c"
c_name = os.path.splitext(os.path.basename(src))[0] + ext
dstfile = os.path.join(destdir, c_name)
if cwd:
ori_dir = os.getcwd()
else:
ori_dir = "."
os.chdir(cwd)
try:
cy_options = CompilationOptions(default_options)
cy_options.__dict__.update(cy_kwargs)
cy_result = cy_compile([src], cy_options)
if cy_result.num_errors > 0:
raise ValueError("Cython compilation failed.")
if os.path.abspath(os.path.dirname(src)) != os.path.abspath(destdir):
if os.path.exists(dstfile):
os.unlink(dstfile)
shutil.move(os.path.join(os.path.dirname(src), c_name), destdir)
finally:
os.chdir(ori_dir)
return dstfile
def run(self):
"""Run - pass execution to generate_qcd"""
filepaths = generate_qcd(self.num_colours, self.precision,
self.representation)
from Cython.Build.Dependencies import cythonize_one
from Cython.Compiler.Main import CompilationOptions
options = CompilationOptions()
options.cplus = True
for src_path in filepaths:
if not src_path.endswith(".pyx"):
raise ValueError(
"Aborted attempt to cythonize file '{}'".format(src_path))
dest_path = src_path[-3:] + "cpp"
cythonize_one(src_path, dest_path, None, False, options=options)
def cythonize():
try:
from Cython.Compiler.Main import CompilationOptions, default_options, \
compile, PyrexError
from Cython.Compiler import Options
import subprocess
for code in cythoncodes:
source = code[0] + '.pyx'
options = CompilationOptions(default_options)
options.output_file = code[0] + '.c'
options.include_path = code[1]
Options.generate_cleanup_code = 3
any_failures = False
try:
result = compile(source, options)
if result.num_errors > 0:
any_failures = True
if not any_failures:
callist = [
cythcompiler, '-shared', '-fwrapv', '-Wall',
'-fno-strict-aliasing'
]
for x in CFLAGS:
callist.append(x)
for x in code[1]:
callist.append('-L' + x)
callist.append('-o')
callist.append('_' + code[0] + '.so')
callist.append(code[0] + '.c')
subprocess.call(callist)
except (EnvironmentError, PyrexError):
e = sys.exc_info()[1]
sys.stderr.write(str(e) + '\n')
any_failures = True
if any_failures:
try:
os.remove(code[0] + '.c')
except OSError:
pass
except:
raise ValueError
def run_cython(inc_dirs, extra_opts, out):
from Cython.Compiler.Main import compile, CompilationOptions
# Try to get directive_defaults (API differs from 0.25 on)
try:
from Cython.Compiler.Options import directive_defaults
except ImportError:
# Cython 0.25
import Cython.Compiler.Options
directive_defaults = Cython.Compiler.Options.get_directive_defaults()
directive_defaults["boundscheck"] = False
directive_defaults["wraparound"] = False
options = dict(include_path=inc_dirs,
compiler_directives=directive_defaults,
cplus=True)
if extra_opts is not None:
options.update(extra_opts)
options = CompilationOptions(**options)
compile(out, options=options)
def extract(path, **kwargs):
name = os.path.splitext(os.path.relpath(path))[0].replace('/', '.')
options = CompilationOptions()
options.include_path.append('include')
options.language_level = 2
options.compiler_directives = dict(
c_string_type='str',
c_string_encoding='ascii',
)
context = options.create_context()
tree = parse_from_strings(name,
open(path).read().decode('utf8'), context,
**kwargs)
extractor = Visitor({'file': path})
extractor.visit(tree)
return extractor.events
def extract(path, **kwargs):
name = os.path.splitext(os.path.relpath(path))[0].replace("/", ".")
options = CompilationOptions()
options.include_path.append("include")
options.language_level = 2
options.compiler_directives = dict(c_string_type="str",
c_string_encoding="ascii")
context = options.create_context()
tree = parse_from_strings(
name,
open(path).read(),
context,
level="module_pxd" if path.endswith(".pxd") else None,
**kwargs)
extractor = Visitor({"file": path})
extractor.visit(tree)
return extractor.events
def cythonize(source, includes=(), output_h=os.curdir):
name, ext = os.path.splitext(source)
output_c = name + '.c'
#
from Cython.Compiler.Main import \
CompilationOptions, default_options, \
compile, \
PyrexError
#
options = CompilationOptions(default_options)
options.output_file = output_c
options.include_path = includes
#
from Cython.Compiler import Options
Options.generate_cleanup_code = 3
#
any_failures = 0
try:
result = compile(source, options)
if result.num_errors > 0:
any_failures = 1
except (EnvironmentError, PyrexError), e:
sys.stderr.write(str(e) + '\n')
any_failures = 1
def cythonize(module_list,
exclude=[],
nthreads=0,
aliases=None,
quiet=False,
force=False,
**options):
if 'include_path' not in options:
options['include_path'] = ['.']
c_options = CompilationOptions(**options)
cpp_options = CompilationOptions(**options)
cpp_options.cplus = True
ctx = c_options.create_context()
module_list = create_extension_list(module_list,
exclude=exclude,
ctx=ctx,
aliases=aliases)
deps = create_dependency_tree(ctx)
to_compile = []
for m in module_list:
new_sources = []
for source in m.sources:
base, ext = os.path.splitext(source)
if ext in ('.pyx', '.py'):
if m.language == 'c++':
c_file = base + '.cpp'
options = cpp_options
else:
c_file = base + '.c'
options = c_options
if os.path.exists(c_file):
c_timestamp = os.path.getmtime(c_file)
else:
c_timestamp = -1
# Priority goes first to modified files, second to direct
# dependents, and finally to indirect dependents.
if c_timestamp < deps.timestamp(source):
dep_timestamp, dep = deps.timestamp(source), source
priority = 0
else:
dep_timestamp, dep = deps.newest_dependency(source)
priority = 2 - (dep in deps.immediate_dependencies(source))
if force or c_timestamp < dep_timestamp:
if not quiet:
if source == dep:
print("Compiling %s because it changed." % source)
else:
print("Compiling %s because it depends on %s." %
(source, dep))
to_compile.append((priority, source, c_file, options))
new_sources.append(c_file)
else:
new_sources.append(source)
m.sources = new_sources
to_compile.sort()
if nthreads:
# Requires multiprocessing (or Python >= 2.6)
try:
import multiprocessing
pool = multiprocessing.Pool(nthreads)
pool.map(cythonize_one_helper, to_compile)
except ImportError:
print("multiprocessing required for parallel cythonization")
nthreads = 0
if not nthreads:
for priority, pyx_file, c_file, options in to_compile:
cythonize_one(pyx_file, c_file, quiet, options)
return module_list
def cythonize(module_list,
exclude=[],
nthreads=0,
aliases=None,
quiet=False,
force=False,
exclude_failures=False,
**options):
"""
Compile a set of source modules into C/C++ files and return a list of distutils
Extension objects for them.
As module list, pass either a glob pattern, a list of glob patterns or a list of
Extension objects. The latter allows you to configure the extensions separately
through the normal distutils options.
When using glob patterns, you can exclude certain module names explicitly
by passing them into the 'exclude' option.
For parallel compilation, set the 'nthreads' option to the number of
concurrent builds.
For a broad 'try to compile' mode that ignores compilation failures and
simply excludes the failed extensions, pass 'exclude_failures=True'. Note
that this only really makes sense for compiling .py files which can also
be used without compilation.
Additional compilation options can be passed as keyword arguments.
"""
if 'include_path' not in options:
options['include_path'] = ['.']
if 'common_utility_include_dir' in options:
if options.get('cache'):
raise NotImplementedError, "common_utility_include_dir does not yet work with caching"
if not os.path.exists(options['common_utility_include_dir']):
os.makedirs(options['common_utility_include_dir'])
c_options = CompilationOptions(**options)
cpp_options = CompilationOptions(**options)
cpp_options.cplus = True
ctx = c_options.create_context()
options = c_options
module_list = create_extension_list(module_list,
exclude=exclude,
ctx=ctx,
quiet=quiet,
exclude_failures=exclude_failures,
aliases=aliases)
deps = create_dependency_tree(ctx, quiet=quiet)
build_dir = getattr(options, 'build_dir', None)
modules_by_cfile = {}
to_compile = []
for m in module_list:
if build_dir:
root = os.path.realpath(
os.path.abspath(find_root_package_dir(m.sources[0])))
def copy_to_build_dir(filepath, root=root):
filepath_abs = os.path.realpath(os.path.abspath(filepath))
if os.path.isabs(filepath):
filepath = filepath_abs
if filepath_abs.startswith(root):
mod_dir = os.path.join(
build_dir, os.path.dirname(_relpath(filepath, root)))
if not os.path.isdir(mod_dir):
os.makedirs(mod_dir)
shutil.copy(filepath, mod_dir)
for dep in m.depends:
copy_to_build_dir(dep)
new_sources = []
for source in m.sources:
base, ext = os.path.splitext(source)
if ext in ('.pyx', '.py'):
if m.language == 'c++':
c_file = base + '.cpp'
options = cpp_options
else:
c_file = base + '.c'
options = c_options
# setup for out of place build directory if enabled
if build_dir:
c_file = os.path.join(build_dir, c_file)
dir = os.path.dirname(c_file)
if not os.path.isdir(dir):
os.makedirs(dir)
if os.path.exists(c_file):
c_timestamp = os.path.getmtime(c_file)
else:
c_timestamp = -1
# Priority goes first to modified files, second to direct
# dependents, and finally to indirect dependents.
if c_timestamp < deps.timestamp(source):
dep_timestamp, dep = deps.timestamp(source), source
priority = 0
else:
dep_timestamp, dep = deps.newest_dependency(source)
priority = 2 - (dep in deps.immediate_dependencies(source))
if force or c_timestamp < dep_timestamp:
if not quiet:
if source == dep:
print("Compiling %s because it changed." % source)
else:
print("Compiling %s because it depends on %s." %
(source, dep))
if not force and hasattr(options, 'cache'):
extra = m.language
fingerprint = deps.transitive_fingerprint(
source, extra)
else:
fingerprint = None
to_compile.append((priority, source, c_file, fingerprint,
quiet, options, not exclude_failures))
new_sources.append(c_file)
if c_file not in modules_by_cfile:
modules_by_cfile[c_file] = [m]
else:
modules_by_cfile[c_file].append(m)
else:
new_sources.append(source)
if build_dir:
copy_to_build_dir(source)
m.sources = new_sources
if hasattr(options, 'cache'):
if not os.path.exists(options.cache):
os.makedirs(options.cache)
to_compile.sort()
if nthreads:
# Requires multiprocessing (or Python >= 2.6)
try:
import multiprocessing
pool = multiprocessing.Pool(nthreads)
except (ImportError, OSError):
print("multiprocessing required for parallel cythonization")
nthreads = 0
else:
pool.map(cythonize_one_helper, to_compile)
if not nthreads:
for args in to_compile:
cythonize_one(*args[1:])
if exclude_failures:
failed_modules = set()
for c_file, modules in modules_by_cfile.iteritems():
if not os.path.exists(c_file):
failed_modules.update(modules)
elif os.path.getsize(c_file) < 200:
f = io_open(c_file, 'r', encoding='iso8859-1')
try:
if f.read(len('#error ')) == '#error ':
# dead compilation result
failed_modules.update(modules)
finally:
f.close()
if failed_modules:
for module in failed_modules:
module_list.remove(module)
print("Failed compilations: %s" %
', '.join(sorted([module.name
for module in failed_modules])))
if hasattr(options, 'cache'):
cleanup_cache(options.cache,
getattr(options, 'cache_size', 1024 * 1024 * 100))
# cythonize() is often followed by the (non-Python-buffered)
# compiler output, flush now to avoid interleaving output.
sys.stdout.flush()
return module_list
def cython_sources(self, sources, extension):
"""
Walk the list of source files in 'sources', looking for Cython
source files (.pyx and .py). Run Cython on all that are
found, and return a modified 'sources' list with Cython source
files replaced by the generated C (or C++) files.
"""
new_sources = []
cython_sources = []
cython_targets = {}
# Setup create_list and cplus from the extension options if
# Cython.Distutils.extension.Extension is used, otherwise just
# use what was parsed from the command-line or the configuration file.
# cplus will also be set to true is extension.language is equal to
# 'C++' or 'c++'.
#try:
# create_listing = self.cython_create_listing or \
# extension.cython_create_listing
# cplus = self.cython_cplus or \
# extension.cython_cplus or \
# (extension.language != None and \
# extension.language.lower() == 'c++')
#except AttributeError:
# create_listing = self.cython_create_listing
# cplus = self.cython_cplus or \
# (extension.language != None and \
# extension.language.lower() == 'c++')
create_listing = self.cython_create_listing or \
getattr(extension, 'cython_create_listing', 0)
line_directives = self.cython_line_directives or \
getattr(extension, 'cython_line_directives', 0)
no_c_in_traceback = self.no_c_in_traceback or \
getattr(extension, 'no_c_in_traceback', 0)
cplus = self.cython_cplus or getattr(extension, 'cython_cplus', 0) or \
(extension.language and extension.language.lower() == 'c++')
cython_gen_pxi = self.cython_gen_pxi or getattr(
extension, 'cython_gen_pxi', 0)
cython_gdb = self.cython_gdb or getattr(extension, 'cython_gdb', False)
cython_compile_time_env = self.cython_compile_time_env or \
getattr(extension, 'cython_compile_time_env', None)
# Set up the include_path for the Cython compiler:
# 1. Start with the command line option.
# 2. Add in any (unique) paths from the extension
# cython_include_dirs (if Cython.Distutils.extension is used).
# 3. Add in any (unique) paths from the extension include_dirs
includes = list(self.cython_include_dirs)
try:
for i in extension.cython_include_dirs:
if i not in includes:
includes.append(i)
except AttributeError:
pass
# In case extension.include_dirs is a generator, evaluate it and keep
# result
extension.include_dirs = list(extension.include_dirs)
for i in extension.include_dirs:
if i not in includes:
includes.append(i)
# Set up Cython compiler directives:
# 1. Start with the command line option.
# 2. Add in any (unique) entries from the extension
# cython_directives (if Cython.Distutils.extension is used).
directives = dict(self.cython_directives)
if hasattr(extension, "cython_directives"):
directives.update(extension.cython_directives)
# Set the target file extension for C/C++ mode.
if cplus:
target_ext = '.cpp'
else:
target_ext = '.c'
# Decide whether to drop the generated C files into the temp dir
# or the source tree.
if not self.inplace and (self.cython_c_in_temp
or getattr(extension, 'cython_c_in_temp', 0)):
target_dir = os.path.join(self.build_temp, "pyrex")
for package_name in extension.name.split('.')[:-1]:
target_dir = os.path.join(target_dir, package_name)
else:
target_dir = None
newest_dependency = None
for source in sources:
(base, ext) = os.path.splitext(os.path.basename(source))
if ext == ".py":
# FIXME: we might want to special case this some more
ext = '.pyx'
if ext == ".pyx": # Cython source file
output_dir = target_dir or os.path.dirname(source)
new_sources.append(os.path.join(output_dir, base + target_ext))
cython_sources.append(source)
cython_targets[source] = new_sources[-1]
elif ext == '.pxi' or ext == '.pxd':
if newest_dependency is None \
or newer(source, newest_dependency):
newest_dependency = source
else:
new_sources.append(source)
if not cython_sources:
return new_sources
try:
from Cython.Compiler.Main \
import CompilationOptions, \
default_options as cython_default_options, \
compile as cython_compile
from Cython.Compiler.Errors import PyrexError
except ImportError:
e = sys.exc_info()[1]
print("failed to import Cython: %s" % e)
raise DistutilsPlatformError(
"Cython does not appear to be installed")
module_name = extension.name
for source in cython_sources:
target = cython_targets[source]
depends = [source] + list(extension.depends or ())
if (source[-4:].lower() == ".pyx"
and os.path.isfile(source[:-3] + "pxd")):
depends += [source[:-3] + "pxd"]
rebuild = self.force or newer_group(depends, target, 'newer')
if not rebuild and newest_dependency is not None:
rebuild = newer(newest_dependency, target)
if rebuild:
log.info("cythoning %s to %s", source, target)
self.mkpath(os.path.dirname(target))
if self.inplace:
output_dir = os.curdir
else:
output_dir = self.build_lib
options = CompilationOptions(
cython_default_options,
use_listing_file=create_listing,
include_path=includes,
compiler_directives=directives,
output_file=target,
cplus=cplus,
emit_linenums=line_directives,
c_line_in_traceback=not no_c_in_traceback,
generate_pxi=cython_gen_pxi,
output_dir=output_dir,
gdb_debug=cython_gdb,
compile_time_env=cython_compile_time_env)
result = cython_compile(source,
options=options,
full_module_name=module_name)
else:
log.info("skipping '%s' Cython extension (up-to-date)", target)
return new_sources
def parse_command_line(args):
from Cython.Compiler.Main import \
CompilationOptions, default_options
def pop_arg():
if args:
return args.pop(0)
else:
bad_usage()
def get_param(option):
tail = option[2:]
if tail:
return tail
else:
return pop_arg()
options = CompilationOptions(default_options)
sources = []
while args:
if args[0].startswith("-"):
option = pop_arg()
if option in ("-V", "--version"):
options.show_version = 1
elif option in ("-l", "--create-listing"):
options.use_listing_file = 1
elif option in ("-+", "--cplus"):
options.cplus = 1
elif option.startswith("--embed"):
ix = option.find('=')
if ix == -1:
Options.embed = "main"
else:
Options.embed = option[ix+1:]
elif option.startswith("-I"):
options.include_path.append(get_param(option))
elif option == "--include-dir":
options.include_path.append(pop_arg())
elif option in ("-w", "--working"):
options.working_path = pop_arg()
elif option in ("-o", "--output-file"):
options.output_file = pop_arg()
elif option in ("-r", "--recursive"):
options.recursive = 1
elif option in ("-t", "--timestamps"):
options.timestamps = 1
elif option in ("-f", "--force"):
options.timestamps = 0
elif option in ("-v", "--verbose"):
options.verbose += 1
elif option in ("-p", "--embed-positions"):
Options.embed_pos_in_docstring = 1
elif option in ("-z", "--pre-import"):
Options.pre_import = pop_arg()
elif option == "--cleanup":
Options.generate_cleanup_code = int(pop_arg())
elif option in ("-D", "--no-docstrings"):
Options.docstrings = False
elif option in ("-a", "--annotate"):
Options.annotate = True
elif option == "--convert-range":
Options.convert_range = True
elif option == "--line-directives":
options.emit_linenums = True
elif option == "--no-c-in-traceback":
options.c_line_in_traceback = False
elif option == "--gdb":
options.gdb_debug = True
options.output_dir = os.curdir
elif option == '-2':
options.language_level = 2
elif option == '-3':
options.language_level = 3
elif option == "--fast-fail":
Options.fast_fail = True
elif option in ('-Werror', '--warning-errors'):
Options.warning_errors = True
elif option == "--disable-function-redefinition":
Options.disable_function_redefinition = True
elif option == "--directive" or option.startswith('-X'):
if option.startswith('-X') and option[2:].strip():
x_args = option[2:]
else:
x_args = pop_arg()
try:
options.compiler_directives = Options.parse_directive_list(
x_args, relaxed_bool=True,
current_settings=options.compiler_directives)
except ValueError, e:
sys.stderr.write("Error in compiler directive: %s\n" % e.args[0])
sys.exit(1)
elif option.startswith('--debug'):
option = option[2:].replace('-', '_')
import DebugFlags
if option in dir(DebugFlags):
setattr(DebugFlags, option, True)
else:
sys.stderr.write("Unknown debug flag: %s\n" % option)
bad_usage()
elif option in ('-h', '--help'):
sys.stdout.write(usage)
sys.exit(0)
else:
sys.stderr.write("Unknown compiler flag: %s\n" % option)
sys.exit(1)
else:
arg = pop_arg()
if arg.endswith(".pyx"):
sources.append(arg)
elif arg.endswith(".py"):
# maybe do some other stuff, but this should work for now
sources.append(arg)
else:
sys.stderr.write(
"cython: %s: Unknown filename suffix\n" % arg)
def parse_command_line(args):
from Cython.Compiler.Main import \
CompilationOptions, default_options
def pop_arg():
if args:
return args.pop(0)
else:
bad_usage()
def get_param(option):
tail = option[2:]
if tail:
return tail
else:
return pop_arg()
options = CompilationOptions(default_options)
sources = []
while args:
if args[0].startswith("-"):
option = pop_arg()
if option in ("-V", "--version"):
options.show_version = 1
elif option in ("-l", "--create-listing"):
options.use_listing_file = 1
elif option in ("-C", "--compile"):
options.c_only = 0
elif option in ("-X", "--link"):
options.c_only = 0
options.obj_only = 0
elif option in ("-+", "--cplus"):
options.cplus = 1
elif option.startswith("-I"):
options.include_path.append(get_param(option))
elif option == "--include-dir":
options.include_path.append(pop_arg())
elif option in ("-w", "--working"):
options.working_path = pop_arg()
elif option in ("-o", "--output-file"):
options.output_file = pop_arg()
elif option in ("-r", "--recursive"):
options.recursive = 1
elif option in ("-t", "--timestamps"):
options.timestamps = 1
elif option in ("-f", "--force"):
options.timestamps = 0
elif option in ("-v", "--verbose"):
options.verbose += 1
elif option in ("-p", "--embed-positions"):
Options.embed_pos_in_docstring = 1
elif option in ("-z", "--pre-import"):
Options.pre_import = pop_arg()
elif option == "--incref-local-binop":
Options.incref_local_binop = 1
elif option == "--cleanup":
Options.generate_cleanup_code = int(pop_arg())
elif option in ("-D", "--no-docstrings"):
Options.docstrings = False
elif option in ("-a", "--annotate"):
Options.annotate = True
elif option == "--convert-range":
Options.convert_range = True
elif option == "--line-directives":
options.emit_linenums = True
elif option in ("-X", "--directive"):
try:
options.pragma_overrides = Options.parse_option_list(pop_arg())
except ValueError, e:
sys.stderr.write("Error in compiler directive: %s\n" % e.message)
sys.exit(1)
else:
bad_usage()
else:
arg = pop_arg()
if arg.endswith(".pyx"):
sources.append(arg)
elif arg.endswith(".py"):
# maybe do some other stuff, but this should work for now
sources.append(arg)
elif arg.endswith(".o"):
options.objects.append(arg)
else:
sys.stderr.write(
"cython: %s: Unknown filename suffix\n" % arg)
from Cython.Compiler.Main import compile_single, CompilationOptions
from Cython.Compiler.TreeFragment import parse_from_strings
from Cython.Compiler.Visitor import TreeVisitor
from Cython.Compiler import Nodes
from Cython.Compiler.AutoDocTransforms import EmbedSignature
options = CompilationOptions()
options.include_path.append('include')
options.language_level = 2
options.compiler_directives = dict(
c_string_type='str',
c_string_encoding='ascii',
)
ctx = options.create_context()
tree = parse_from_strings('include.libavutil.avutil',
open('scratchpad/test.pxd').read().decode('utf8'),
ctx)
class Visitor(TreeVisitor):
def __init__(self, state=None):
super(Visitor, self).__init__()
self.state = dict(state or {})
self.events = []
def record_event(self, node, **kw):
state = self.state.copy()
state.update(**kw)
state['pos'] = node.pos
def cython_sources(self, sources, extension):
"""
Walk the list of source files in 'sources', looking for Cython
source files (.pyx and .py). Run Cython on all that are
found, and return a modified 'sources' list with Cython source
files replaced by the generated C (or C++) files.
"""
if PyrexError == None:
raise DistutilsPlatformError, \
("Cython does not appear to be installed "
"on platform '%s'") % os.name
new_sources = []
pyrex_sources = []
pyrex_targets = {}
# Setup create_list and cplus from the extension options if
# Cython.Distutils.extension.Extension is used, otherwise just
# use what was parsed from the command-line or the configuration file.
# cplus will also be set to true is extension.language is equal to
# 'C++' or 'c++'.
#try:
# create_listing = self.pyrex_create_listing or \
# extension.pyrex_create_listing
# cplus = self.pyrex_cplus or \
# extension.pyrex_cplus or \
# (extension.language != None and \
# extension.language.lower() == 'c++')
#except AttributeError:
# create_listing = self.pyrex_create_listing
# cplus = self.pyrex_cplus or \
# (extension.language != None and \
# extension.language.lower() == 'c++')
create_listing = self.pyrex_create_listing or \
getattr(extension, 'pyrex_create_listing', 0)
cplus = self.pyrex_cplus or getattr(extension, 'pyrex_cplus', 0) or \
(extension.language and extension.language.lower() == 'c++')
pyrex_gen_pxi = self.pyrex_gen_pxi or getattr(extension,
'pyrex_gen_pxi', 0)
# Set up the include_path for the Cython compiler:
# 1. Start with the command line option.
# 2. Add in any (unique) paths from the extension
# pyrex_include_dirs (if Cython.Distutils.extension is used).
# 3. Add in any (unique) paths from the extension include_dirs
includes = self.pyrex_include_dirs
try:
for i in extension.pyrex_include_dirs:
if not i in includes:
includes.append(i)
except AttributeError:
pass
for i in extension.include_dirs:
if not i in includes:
includes.append(i)
# Set the target_ext to '.c'. Cython will change this to '.cpp' if
# needed.
if cplus:
target_ext = '.cpp'
else:
target_ext = '.c'
# Decide whether to drop the generated C files into the temp dir
# or the source tree.
if not self.inplace and (self.pyrex_c_in_temp
or getattr(extension, 'pyrex_c_in_temp', 0)):
target_dir = os.path.join(self.build_temp, "pyrex")
else:
target_dir = None
newest_dependency = None
for source in sources:
(base, ext) = os.path.splitext(os.path.basename(source))
if ext == ".py":
# FIXME: we might want to special case this some more
ext = '.pyx'
if ext == ".pyx": # Cython source file
output_dir = target_dir or os.path.dirname(source)
new_sources.append(os.path.join(output_dir, base + target_ext))
pyrex_sources.append(source)
pyrex_targets[source] = new_sources[-1]
elif ext == '.pxi' or ext == '.pxd':
if newest_dependency is None \
or newer(source, newest_dependency):
newest_dependency = source
else:
new_sources.append(source)
if not pyrex_sources:
return new_sources
module_name = extension.name
for source in pyrex_sources:
target = pyrex_targets[source]
rebuild = self.force or newer(source, target)
if not rebuild and newest_dependency is not None:
rebuild = newer(newest_dependency, target)
if rebuild:
log.info("cythoning %s to %s", source, target)
self.mkpath(os.path.dirname(target))
options = CompilationOptions(pyrex_default_options,
use_listing_file=create_listing,
include_path=includes,
output_file=target,
cplus=cplus,
generate_pxi=pyrex_gen_pxi)
result = cython_compile(source,
options=options,
full_module_name=module_name)
return new_sources
def godopy_cython():
source, outfile_path = sys.argv[1:]
pyinit_src_symbol = 'PyInit_' + os.path.basename(outfile_path[:-4])
pyinit_dst_symbol = 'PyInit_' + strip_internal_path(outfile_path).replace(
os.sep, '__')[:-4]
tempfile = outfile_path.replace('.cpp', '.temp.cpp')
tempheaderfile = outfile_path.replace('.cpp', '.temp.h')
header_path = outfile_path.replace('.cpp', '.hpp')
from Cython.Compiler import Options
from Cython.Compiler.Main import compile, default_options, CompilationOptions
directives = {'c_string_encoding': 'utf-8'}
options = CompilationOptions(default_options,
compiler_directives=directives)
Options.fast_fail = True
options.output_file = tempfile
options.cplus = 1
options.language_level = 3
result = compile(source, options)
if result.num_errors > 0:
raise SystemExit('Cython compilation finished with errors')
def clean_line(line):
if pyinit_src_symbol in line:
line = line.replace(pyinit_src_symbol, pyinit_dst_symbol)
# Undefined by #define NO_IMPORT_ARRAY
if '_import_array()' in line:
line = line.replace('_import_array()', '0')
# Fix variable declarations with GDCALLINGCONV, GDCALLINGCONV is valid only for functions
if line.lstrip().startswith('GDCALLINGCONV_'):
line = re.sub(r'^(\s+)(GDCALLINGCONV_VOID_PTR)(\s\w+;)$',
r'\1void *\3', line)
line = re.sub(r'^(\s+)(GDCALLINGCONV_VOID)(\s\w+;)$', r'\1void\3',
line)
line = re.sub(r'^(\s+)(GDCALLINGCONV_BOOL)(\s\w+;)$', r'\1bool\3',
line)
# Remove "tp_print" deprecation warnings
if line.strip(
) == '#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000':
line = line.replace(
'PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000',
'PY_VERSION_HEX < 0x030800b1')
# FIXME: undeclared identifier, looks like a bug in Cython
if line.strip() == '__Pyx_CppExn2PyErr();':
line = ''
return line
with open(outfile_path, 'w', encoding='utf-8') as outfile:
with open(tempfile, 'r', encoding='utf-8') as fp:
for line in fp:
outfile.write(clean_line(line))
os.unlink(tempfile)
if os.path.exists(tempheaderfile):
with open(header_path, 'w', encoding='utf-8') as outheaderfile:
with open(tempheaderfile, 'r', encoding='utf-8') as fp:
for line in fp:
outheaderfile.write(clean_line(line))
os.unlink(tempheaderfile)
def cythonize(module_list,
exclude=[],
nthreads=0,
aliases=None,
quiet=False,
force=False,
exclude_failures=False,
**options):
"""
Compile a set of source modules into C/C++ files and return a list of distutils
Extension objects for them.
As module list, pass either a glob pattern, a list of glob patterns or a list of
Extension objects. The latter allows you to configure the extensions separately
through the normal distutils options.
When using glob patterns, you can exclude certain module names explicitly
by passing them into the 'exclude' option.
For parallel compilation, set the 'nthreads' option to the number of
concurrent builds.
For a broad 'try to compile' mode that ignores compilation failures and
simply excludes the failed extensions, pass 'exclude_failures=True'. Note
that this only really makes sense for compiling .py files which can also
be used without compilation.
Additional compilation options can be passed as keyword arguments.
"""
if 'include_path' not in options:
options['include_path'] = ['.']
c_options = CompilationOptions(**options)
cpp_options = CompilationOptions(**options)
cpp_options.cplus = True
ctx = c_options.create_context()
module_list = create_extension_list(module_list,
exclude=exclude,
ctx=ctx,
quiet=quiet,
exclude_failures=exclude_failures,
aliases=aliases)
deps = create_dependency_tree(ctx, quiet=quiet)
modules_by_cfile = {}
to_compile = []
for m in module_list:
new_sources = []
for source in m.sources:
base, ext = os.path.splitext(source)
if ext in ('.pyx', '.py'):
if m.language == 'c++':
c_file = base + '.cpp'
options = cpp_options
else:
c_file = base + '.c'
options = c_options
if os.path.exists(c_file):
c_timestamp = os.path.getmtime(c_file)
else:
c_timestamp = -1
# Priority goes first to modified files, second to direct
# dependents, and finally to indirect dependents.
if c_timestamp < deps.timestamp(source):
dep_timestamp, dep = deps.timestamp(source), source
priority = 0
else:
dep_timestamp, dep = deps.newest_dependency(source)
priority = 2 - (dep in deps.immediate_dependencies(source))
if force or c_timestamp < dep_timestamp:
if not quiet:
if source == dep:
print("Compiling %s because it changed." % source)
else:
print("Compiling %s because it depends on %s." %
(source, dep))
if not force and hasattr(options, 'cache'):
extra = m.language
fingerprint = deps.transitive_fingerprint(
source, extra)
else:
fingerprint = None
to_compile.append((priority, source, c_file, fingerprint,
quiet, options, not exclude_failures))
new_sources.append(c_file)
if c_file not in modules_by_cfile:
modules_by_cfile[c_file] = [m]
else:
modules_by_cfile[c_file].append(m)
else:
new_sources.append(source)
m.sources = new_sources
if hasattr(options, 'cache'):
if not os.path.exists(options.cache):
os.mkdir(options.cache)
to_compile.sort()
if nthreads:
# Requires multiprocessing (or Python >= 2.6)
try:
import multiprocessing
pool = multiprocessing.Pool(nthreads)
pool.map(cythonize_one_helper, to_compile)
except ImportError:
print("multiprocessing required for parallel cythonization")
nthreads = 0
if not nthreads:
for args in to_compile:
cythonize_one(*args[1:])
if exclude_failures:
failed_modules = set()
for c_file, modules in modules_by_cfile.iteritems():
if not os.path.exists(c_file):
failed_modules.update(modules)
for module in failed_modules:
module_list.remove(module)
if hasattr(options, 'cache'):
cleanup_cache(options.cache,
getattr(options, 'cache_size', 1024 * 1024 * 100))
# cythonize() is often followed by the (non-Python-buffered)
# compiler output, flush now to avoid interleaving output.
sys.stdout.flush()
return module_list
def parse_command_line(args):
def parse_add_transform(transforms, param):
def import_symbol(fqn):
modsplitpt = fqn.rfind(".")
if modsplitpt == -1: bad_usage()
modulename = fqn[:modsplitpt]
symbolname = fqn[modsplitpt+1:]
module = __import__(modulename, globals(), locals(), [symbolname])
return getattr(module, symbolname)
stagename, factoryname = param.split(":")
if not stagename in Transform.PHASES:
bad_usage()
factory = import_symbol(factoryname)
transform = factory()
transforms[stagename].append(transform)
from Cython.Compiler.Main import \
CompilationOptions, default_options
def pop_arg():
if args:
return args.pop(0)
else:
bad_usage()
def get_param(option):
tail = option[2:]
if tail:
return tail
else:
return pop_arg()
options = CompilationOptions(default_options)
sources = []
while args:
if args[0].startswith("-"):
option = pop_arg()
if option in ("-v", "--version"):
options.show_version = 1
elif option in ("-l", "--create-listing"):
options.use_listing_file = 1
elif option in ("-C", "--compile"):
options.c_only = 0
elif option in ("-X", "--link"):
options.c_only = 0
options.obj_only = 0
elif option in ("-+", "--cplus"):
options.cplus = 1
elif option.startswith("-I"):
options.include_path.append(get_param(option))
elif option == "--include-dir":
options.include_path.append(pop_arg())
elif option in ("-o", "--output-file"):
options.output_file = pop_arg()
elif option in ("-p", "--embed-positions"):
Options.embed_pos_in_docstring = 1
elif option in ("-z", "--pre-import"):
Options.pre_import = pop_arg()
elif option == "--incref-local-binop":
Options.incref_local_binop = 1
elif option == "--cleanup":
Options.generate_cleanup_code = int(pop_arg())
elif option in ("-D", "--no-docstrings"):
Options.docstrings = False
elif option in ("-a", "--annotate"):
Options.annotate = True
elif option == "--convert-range":
Options.convert_range = True
elif option.startswith("-T"):
parse_add_transform(options.transforms, get_param(option))
# Note: this can occur multiple times, each time appends
else:
bad_usage()
else:
arg = pop_arg()
if arg.endswith(".pyx"):
sources.append(arg)
elif arg.endswith(".o"):
options.objects.append(arg)
else:
print >>sys.stderr, \
"cython: %s: Unknown filename suffix" % arg
if options.objects and len(sources) > 1:
print >>sys.stderr, \
"cython: Only one source file allowed together with .o files"
if options.use_listing_file and len(sources) > 1:
print >>sys.stderr, \
"cython: Only one source file allowed when using -o"
sys.exit(1)
if len(sources) == 0 and not options.show_version:
bad_usage()
return options, sources
try:
import cPickle
except ImportError:
import _pickle as cPickle
persisted_data_path = "include_dir.bin"
try:
autowrap_include_dirs = cPickle.load(open(persisted_data_path, "rb"))
except IOError:
print(
"The file include_dir.bin does not yet exist, please run setup.py first to create it."
)
from Cython.Compiler.Main import compile, CompilationOptions
import Cython
print("Will try to compile with Cython version", Cython.__version__)
# Prepare options
print("include:", autowrap_include_dirs)
options = dict(
include_path=autowrap_include_dirs,
#output_dir=".",
#gdb_debug=True,
cplus=True)
# Do Cython compile
import sys
out = sys.argv[1]
options = CompilationOptions(**options)
compile(out, options=options)
