def run(self):
"""Run - pass execution to generate_qcd"""
filepaths = generate_qcd(self.num_colours, self.precision,
self.representation)
from Cython.Build.Dependencies import cythonize_one
from Cython.Compiler.Main import CompilationOptions
options = CompilationOptions()
options.cplus = True
for src_path in filepaths:
if not src_path.endswith(".pyx"):
raise ValueError(
"Aborted attempt to cythonize file '{}'".format(src_path))
dest_path = src_path[-3:] + "cpp"
cythonize_one(src_path, dest_path, None, False, options=options)
def run(self):
"""Run - pass execution to generate_qcd"""
filepaths = generate_qcd(self.num_colours, self.precision,
self.representation)
from Cython.Build.Dependencies import cythonize_one
from Cython.Compiler.Main import CompilationOptions
options = CompilationOptions()
options.cplus = True
for src_path in filepaths:
if not src_path.endswith(".pyx"):
raise ValueError("Aborted attempt to cythonize file '{}'"
.format(src_path))
dest_path = src_path[-3:] + "cpp"
cythonize_one(src_path, dest_path, None, False, options=options)
def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, force=False,
exclude_failures=False, **options):
"""
Compile a set of source modules into C/C++ files and return a list of distutils
Extension objects for them.
As module list, pass either a glob pattern, a list of glob patterns or a list of
Extension objects. The latter allows you to configure the extensions separately
through the normal distutils options.
When using glob patterns, you can exclude certain module names explicitly
by passing them into the 'exclude' option.
For parallel compilation, set the 'nthreads' option to the number of
concurrent builds.
For a broad 'try to compile' mode that ignores compilation failures and
simply excludes the failed extensions, pass 'exclude_failures=True'. Note
that this only really makes sense for compiling .py files which can also
be used without compilation.
Additional compilation options can be passed as keyword arguments.
"""
if 'include_path' not in options:
options['include_path'] = ['.']
if 'common_utility_include_dir' in options:
if options.get('cache'):
raise NotImplementedError("common_utility_include_dir does not yet work with caching")
if not os.path.exists(options['common_utility_include_dir']):
os.makedirs(options['common_utility_include_dir'])
c_options = CompilationOptions(**options)
cpp_options = CompilationOptions(**options); cpp_options.cplus = True
ctx = c_options.create_context()
options = c_options
module_list = create_extension_list(
module_list,
exclude=exclude,
ctx=ctx,
quiet=quiet,
exclude_failures=exclude_failures,
aliases=aliases)
deps = create_dependency_tree(ctx, quiet=quiet)
build_dir = getattr(options, 'build_dir', None)
modules_by_cfile = {}
to_compile = []
for m in module_list:
if build_dir:
root = os.path.realpath(os.path.abspath(find_root_package_dir(m.sources[0])))
def copy_to_build_dir(filepath, root=root):
filepath_abs = os.path.realpath(os.path.abspath(filepath))
if os.path.isabs(filepath):
filepath = filepath_abs
if filepath_abs.startswith(root):
mod_dir = os.path.join(build_dir,
os.path.dirname(_relpath(filepath, root)))
if not os.path.isdir(mod_dir):
os.makedirs(mod_dir)
shutil.copy(filepath, mod_dir)
for dep in m.depends:
copy_to_build_dir(dep)
new_sources = []
for source in m.sources:
base, ext = os.path.splitext(source)
if ext in ('.pyx', '.py'):
if m.language == 'c++':
c_file = base + '.cpp'
options = cpp_options
else:
c_file = base + '.c'
options = c_options
# setup for out of place build directory if enabled
if build_dir:
c_file = os.path.join(build_dir, c_file)
dir = os.path.dirname(c_file)
if not os.path.isdir(dir):
os.makedirs(dir)
if os.path.exists(c_file):
c_timestamp = os.path.getmtime(c_file)
else:
c_timestamp = -1
# Priority goes first to modified files, second to direct
# dependents, and finally to indirect dependents.
if c_timestamp < deps.timestamp(source):
dep_timestamp, dep = deps.timestamp(source), source
priority = 0
else:
dep_timestamp, dep = deps.newest_dependency(source)
priority = 2 - (dep in deps.immediate_dependencies(source))
if force or c_timestamp < dep_timestamp:
if not quiet:
if source == dep:
print("Compiling %s because it changed." % source)
else:
print("Compiling %s because it depends on %s." % (source, dep))
if not force and hasattr(options, 'cache'):
extra = m.language
fingerprint = deps.transitive_fingerprint(source, extra)
else:
fingerprint = None
to_compile.append((priority, source, c_file, fingerprint, quiet,
options, not exclude_failures))
new_sources.append(c_file)
if c_file not in modules_by_cfile:
modules_by_cfile[c_file] = [m]
else:
modules_by_cfile[c_file].append(m)
else:
new_sources.append(source)
if build_dir:
copy_to_build_dir(source)
m.sources = new_sources
if hasattr(options, 'cache'):
if not os.path.exists(options.cache):
os.makedirs(options.cache)
to_compile.sort()
if nthreads:
# Requires multiprocessing (or Python >= 2.6)
try:
import multiprocessing
pool = multiprocessing.Pool(nthreads)
except (ImportError, OSError):
print("multiprocessing required for parallel cythonization")
nthreads = 0
else:
try:
pool.map(cythonize_one_helper, to_compile)
finally:
pool.close()
if not nthreads:
for args in to_compile:
cythonize_one(*args[1:])
if exclude_failures:
failed_modules = set()
for c_file, modules in modules_by_cfile.iteritems():
if not os.path.exists(c_file):
failed_modules.update(modules)
elif os.path.getsize(c_file) < 200:
f = io_open(c_file, 'r', encoding='iso8859-1')
try:
if f.read(len('#error ')) == '#error ':
# dead compilation result
failed_modules.update(modules)
finally:
f.close()
if failed_modules:
for module in failed_modules:
module_list.remove(module)
print("Failed compilations: %s" % ', '.join(sorted([
module.name for module in failed_modules])))
if hasattr(options, 'cache'):
cleanup_cache(options.cache, getattr(options, 'cache_size', 1024 * 1024 * 100))
# cythonize() is often followed by the (non-Python-buffered)
# compiler output, flush now to avoid interleaving output.
sys.stdout.flush()
return module_list
def cythonize(module_list,
exclude=[],
nthreads=0,
aliases=None,
quiet=False,
force=False,
exclude_failures=False,
**options):
"""
Compile a set of source modules into C/C++ files and return a list of distutils
Extension objects for them.
As module list, pass either a glob pattern, a list of glob patterns or a list of
Extension objects. The latter allows you to configure the extensions separately
through the normal distutils options.
When using glob patterns, you can exclude certain module names explicitly
by passing them into the 'exclude' option.
For parallel compilation, set the 'nthreads' option to the number of
concurrent builds.
For a broad 'try to compile' mode that ignores compilation failures and
simply excludes the failed extensions, pass 'exclude_failures=True'. Note
that this only really makes sense for compiling .py files which can also
be used without compilation.
Additional compilation options can be passed as keyword arguments.
"""
if 'include_path' not in options:
options['include_path'] = ['.']
if 'common_utility_include_dir' in options:
if options.get('cache'):
raise NotImplementedError, "common_utility_include_dir does not yet work with caching"
if not os.path.exists(options['common_utility_include_dir']):
os.makedirs(options['common_utility_include_dir'])
c_options = CompilationOptions(**options)
cpp_options = CompilationOptions(**options)
cpp_options.cplus = True
ctx = c_options.create_context()
options = c_options
module_list = create_extension_list(module_list,
exclude=exclude,
ctx=ctx,
quiet=quiet,
exclude_failures=exclude_failures,
aliases=aliases)
deps = create_dependency_tree(ctx, quiet=quiet)
build_dir = getattr(options, 'build_dir', None)
modules_by_cfile = {}
to_compile = []
for m in module_list:
if build_dir:
root = os.path.realpath(
os.path.abspath(find_root_package_dir(m.sources[0])))
def copy_to_build_dir(filepath, root=root):
filepath_abs = os.path.realpath(os.path.abspath(filepath))
if os.path.isabs(filepath):
filepath = filepath_abs
if filepath_abs.startswith(root):
mod_dir = os.path.join(
build_dir, os.path.dirname(_relpath(filepath, root)))
if not os.path.isdir(mod_dir):
os.makedirs(mod_dir)
shutil.copy(filepath, mod_dir)
for dep in m.depends:
copy_to_build_dir(dep)
new_sources = []
for source in m.sources:
base, ext = os.path.splitext(source)
if ext in ('.pyx', '.py'):
if m.language == 'c++':
c_file = base + '.cpp'
options = cpp_options
else:
c_file = base + '.c'
options = c_options
# setup for out of place build directory if enabled
if build_dir:
c_file = os.path.join(build_dir, c_file)
dir = os.path.dirname(c_file)
if not os.path.isdir(dir):
os.makedirs(dir)
if os.path.exists(c_file):
c_timestamp = os.path.getmtime(c_file)
else:
c_timestamp = -1
# Priority goes first to modified files, second to direct
# dependents, and finally to indirect dependents.
if c_timestamp < deps.timestamp(source):
dep_timestamp, dep = deps.timestamp(source), source
priority = 0
else:
dep_timestamp, dep = deps.newest_dependency(source)
priority = 2 - (dep in deps.immediate_dependencies(source))
if force or c_timestamp < dep_timestamp:
if not quiet:
if source == dep:
print("Compiling %s because it changed." % source)
else:
print("Compiling %s because it depends on %s." %
(source, dep))
if not force and hasattr(options, 'cache'):
extra = m.language
fingerprint = deps.transitive_fingerprint(
source, extra)
else:
fingerprint = None
to_compile.append((priority, source, c_file, fingerprint,
quiet, options, not exclude_failures))
new_sources.append(c_file)
if c_file not in modules_by_cfile:
modules_by_cfile[c_file] = [m]
else:
modules_by_cfile[c_file].append(m)
else:
new_sources.append(source)
if build_dir:
copy_to_build_dir(source)
m.sources = new_sources
if hasattr(options, 'cache'):
if not os.path.exists(options.cache):
os.makedirs(options.cache)
to_compile.sort()
if nthreads:
# Requires multiprocessing (or Python >= 2.6)
try:
import multiprocessing
pool = multiprocessing.Pool(nthreads)
except (ImportError, OSError):
print("multiprocessing required for parallel cythonization")
nthreads = 0
else:
pool.map(cythonize_one_helper, to_compile)
if not nthreads:
for args in to_compile:
cythonize_one(*args[1:])
if exclude_failures:
failed_modules = set()
for c_file, modules in modules_by_cfile.iteritems():
if not os.path.exists(c_file):
failed_modules.update(modules)
elif os.path.getsize(c_file) < 200:
f = io_open(c_file, 'r', encoding='iso8859-1')
try:
if f.read(len('#error ')) == '#error ':
# dead compilation result
failed_modules.update(modules)
finally:
f.close()
if failed_modules:
for module in failed_modules:
module_list.remove(module)
print("Failed compilations: %s" %
', '.join(sorted([module.name
for module in failed_modules])))
if hasattr(options, 'cache'):
cleanup_cache(options.cache,
getattr(options, 'cache_size', 1024 * 1024 * 100))
# cythonize() is often followed by the (non-Python-buffered)
# compiler output, flush now to avoid interleaving output.
sys.stdout.flush()
return module_list
def parse_command_line(args):
from Cython.Compiler.Main import \
CompilationOptions, default_options
def pop_arg():
if args:
return args.pop(0)
else:
bad_usage()
def get_param(option):
tail = option[2:]
if tail:
return tail
else:
return pop_arg()
options = CompilationOptions(default_options)
sources = []
while args:
if args[0].startswith("-"):
option = pop_arg()
if option in ("-V", "--version"):
options.show_version = 1
elif option in ("-l", "--create-listing"):
options.use_listing_file = 1
elif option in ("-+", "--cplus"):
options.cplus = 1
elif option.startswith("--embed"):
ix = option.find('=')
if ix == -1:
Options.embed = "main"
else:
Options.embed = option[ix+1:]
elif option.startswith("-I"):
options.include_path.append(get_param(option))
elif option == "--include-dir":
options.include_path.append(pop_arg())
elif option in ("-w", "--working"):
options.working_path = pop_arg()
elif option in ("-o", "--output-file"):
options.output_file = pop_arg()
elif option in ("-r", "--recursive"):
options.recursive = 1
elif option in ("-t", "--timestamps"):
options.timestamps = 1
elif option in ("-f", "--force"):
options.timestamps = 0
elif option in ("-v", "--verbose"):
options.verbose += 1
elif option in ("-p", "--embed-positions"):
Options.embed_pos_in_docstring = 1
elif option in ("-z", "--pre-import"):
Options.pre_import = pop_arg()
elif option == "--cleanup":
Options.generate_cleanup_code = int(pop_arg())
elif option in ("-D", "--no-docstrings"):
Options.docstrings = False
elif option in ("-a", "--annotate"):
Options.annotate = True
elif option == "--convert-range":
Options.convert_range = True
elif option == "--line-directives":
options.emit_linenums = True
elif option == "--no-c-in-traceback":
options.c_line_in_traceback = False
elif option == "--gdb":
options.gdb_debug = True
options.output_dir = os.curdir
elif option == '-2':
options.language_level = 2
elif option == '-3':
options.language_level = 3
elif option == "--fast-fail":
Options.fast_fail = True
elif option in ('-Werror', '--warning-errors'):
Options.warning_errors = True
elif option == "--disable-function-redefinition":
Options.disable_function_redefinition = True
elif option == "--directive" or option.startswith('-X'):
if option.startswith('-X') and option[2:].strip():
x_args = option[2:]
else:
x_args = pop_arg()
try:
options.compiler_directives = Options.parse_directive_list(
x_args, relaxed_bool=True,
current_settings=options.compiler_directives)
except ValueError, e:
sys.stderr.write("Error in compiler directive: %s\n" % e.args[0])
sys.exit(1)
elif option.startswith('--debug'):
option = option[2:].replace('-', '_')
import DebugFlags
if option in dir(DebugFlags):
setattr(DebugFlags, option, True)
else:
sys.stderr.write("Unknown debug flag: %s\n" % option)
bad_usage()
elif option in ('-h', '--help'):
sys.stdout.write(usage)
sys.exit(0)
else:
sys.stderr.write("Unknown compiler flag: %s\n" % option)
sys.exit(1)
else:
arg = pop_arg()
if arg.endswith(".pyx"):
sources.append(arg)
elif arg.endswith(".py"):
# maybe do some other stuff, but this should work for now
sources.append(arg)
else:
sys.stderr.write(
"cython: %s: Unknown filename suffix\n" % arg)
def parse_command_line(args):
from Cython.Compiler.Main import \
CompilationOptions, default_options
def pop_arg():
if args:
return args.pop(0)
else:
bad_usage()
def get_param(option):
tail = option[2:]
if tail:
return tail
else:
return pop_arg()
options = CompilationOptions(default_options)
sources = []
while args:
if args[0].startswith("-"):
option = pop_arg()
if option in ("-V", "--version"):
options.show_version = 1
elif option in ("-l", "--create-listing"):
options.use_listing_file = 1
elif option in ("-C", "--compile"):
options.c_only = 0
elif option in ("-X", "--link"):
options.c_only = 0
options.obj_only = 0
elif option in ("-+", "--cplus"):
options.cplus = 1
elif option.startswith("-I"):
options.include_path.append(get_param(option))
elif option == "--include-dir":
options.include_path.append(pop_arg())
elif option in ("-w", "--working"):
options.working_path = pop_arg()
elif option in ("-o", "--output-file"):
options.output_file = pop_arg()
elif option in ("-r", "--recursive"):
options.recursive = 1
elif option in ("-t", "--timestamps"):
options.timestamps = 1
elif option in ("-f", "--force"):
options.timestamps = 0
elif option in ("-v", "--verbose"):
options.verbose += 1
elif option in ("-p", "--embed-positions"):
Options.embed_pos_in_docstring = 1
elif option in ("-z", "--pre-import"):
Options.pre_import = pop_arg()
elif option == "--incref-local-binop":
Options.incref_local_binop = 1
elif option == "--cleanup":
Options.generate_cleanup_code = int(pop_arg())
elif option in ("-D", "--no-docstrings"):
Options.docstrings = False
elif option in ("-a", "--annotate"):
Options.annotate = True
elif option == "--convert-range":
Options.convert_range = True
elif option == "--line-directives":
options.emit_linenums = True
elif option in ("-X", "--directive"):
try:
options.pragma_overrides = Options.parse_option_list(pop_arg())
except ValueError, e:
sys.stderr.write("Error in compiler directive: %s\n" % e.message)
sys.exit(1)
else:
bad_usage()
else:
arg = pop_arg()
if arg.endswith(".pyx"):
sources.append(arg)
elif arg.endswith(".py"):
# maybe do some other stuff, but this should work for now
sources.append(arg)
elif arg.endswith(".o"):
options.objects.append(arg)
else:
sys.stderr.write(
"cython: %s: Unknown filename suffix\n" % arg)
def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, **options):
if 'include_path' not in options:
options['include_path'] = ['.']
c_options = CompilationOptions(**options)
cpp_options = CompilationOptions(**options); cpp_options.cplus = True
ctx = c_options.create_context()
module_list = create_extension_list(
module_list,
exclude=exclude,
ctx=ctx,
aliases=aliases)
deps = create_dependency_tree(ctx)
to_compile = []
for m in module_list:
new_sources = []
for source in m.sources:
base, ext = os.path.splitext(source)
if ext in ('.pyx', '.py'):
if m.language == 'c++':
c_file = base + '.cpp'
options = cpp_options
else:
c_file = base + '.c'
options = c_options
if os.path.exists(c_file):
c_timestamp = os.path.getmtime(c_file)
else:
c_timestamp = -1
# Priority goes first to modified files, second to direct
# dependents, and finally to indirect dependents.
if c_timestamp < deps.timestamp(source):
dep_timestamp, dep = deps.timestamp(source), source
priority = 0
else:
dep_timestamp, dep = deps.newest_dependency(source)
priority = 2 - (dep in deps.immediate_dependencies(source))
if c_timestamp < dep_timestamp:
if not quiet:
if source == dep:
print("Compiling %s because it changed." % source)
else:
print("Compiling %s because it depends on %s." % (source, dep))
to_compile.append((priority, source, c_file, options))
new_sources.append(c_file)
else:
new_sources.append(source)
m.sources = new_sources
to_compile.sort()
if nthreads:
# Requires multiprocessing (or Python >= 2.6)
try:
import multiprocessing
pool = multiprocessing.Pool(nthreads)
pool.map(cythonize_one_helper, to_compile)
except ImportError:
print("multiprocessing required for parallel cythonization")
nthreads = 0
if not nthreads:
for priority, pyx_file, c_file, options in to_compile:
cythonize_one(pyx_file, c_file, options)
return module_list
def cythonize(module_list,
exclude=[],
nthreads=0,
aliases=None,
quiet=False,
force=False,
**options):
if 'include_path' not in options:
options['include_path'] = ['.']
c_options = CompilationOptions(**options)
cpp_options = CompilationOptions(**options)
cpp_options.cplus = True
ctx = c_options.create_context()
module_list = create_extension_list(module_list,
exclude=exclude,
ctx=ctx,
aliases=aliases)
deps = create_dependency_tree(ctx)
to_compile = []
for m in module_list:
new_sources = []
for source in m.sources:
base, ext = os.path.splitext(source)
if ext in ('.pyx', '.py'):
if m.language == 'c++':
c_file = base + '.cpp'
options = cpp_options
else:
c_file = base + '.c'
options = c_options
if os.path.exists(c_file):
c_timestamp = os.path.getmtime(c_file)
else:
c_timestamp = -1
# Priority goes first to modified files, second to direct
# dependents, and finally to indirect dependents.
if c_timestamp < deps.timestamp(source):
dep_timestamp, dep = deps.timestamp(source), source
priority = 0
else:
dep_timestamp, dep = deps.newest_dependency(source)
priority = 2 - (dep in deps.immediate_dependencies(source))
if force or c_timestamp < dep_timestamp:
if not quiet:
if source == dep:
print("Compiling %s because it changed." % source)
else:
print("Compiling %s because it depends on %s." %
(source, dep))
to_compile.append((priority, source, c_file, options))
new_sources.append(c_file)
else:
new_sources.append(source)
m.sources = new_sources
to_compile.sort()
if nthreads:
# Requires multiprocessing (or Python >= 2.6)
try:
import multiprocessing
pool = multiprocessing.Pool(nthreads)
pool.map(cythonize_one_helper, to_compile)
except ImportError:
print("multiprocessing required for parallel cythonization")
nthreads = 0
if not nthreads:
for priority, pyx_file, c_file, options in to_compile:
cythonize_one(pyx_file, c_file, quiet, options)
return module_list
def parse_command_line(args):
def parse_add_transform(transforms, param):
def import_symbol(fqn):
modsplitpt = fqn.rfind(".")
if modsplitpt == -1: bad_usage()
modulename = fqn[:modsplitpt]
symbolname = fqn[modsplitpt+1:]
module = __import__(modulename, globals(), locals(), [symbolname])
return getattr(module, symbolname)
stagename, factoryname = param.split(":")
if not stagename in Transform.PHASES:
bad_usage()
factory = import_symbol(factoryname)
transform = factory()
transforms[stagename].append(transform)
from Cython.Compiler.Main import \
CompilationOptions, default_options
def pop_arg():
if args:
return args.pop(0)
else:
bad_usage()
def get_param(option):
tail = option[2:]
if tail:
return tail
else:
return pop_arg()
options = CompilationOptions(default_options)
sources = []
while args:
if args[0].startswith("-"):
option = pop_arg()
if option in ("-v", "--version"):
options.show_version = 1
elif option in ("-l", "--create-listing"):
options.use_listing_file = 1
elif option in ("-C", "--compile"):
options.c_only = 0
elif option in ("-X", "--link"):
options.c_only = 0
options.obj_only = 0
elif option in ("-+", "--cplus"):
options.cplus = 1
elif option.startswith("-I"):
options.include_path.append(get_param(option))
elif option == "--include-dir":
options.include_path.append(pop_arg())
elif option in ("-o", "--output-file"):
options.output_file = pop_arg()
elif option in ("-p", "--embed-positions"):
Options.embed_pos_in_docstring = 1
elif option in ("-z", "--pre-import"):
Options.pre_import = pop_arg()
elif option == "--incref-local-binop":
Options.incref_local_binop = 1
elif option == "--cleanup":
Options.generate_cleanup_code = int(pop_arg())
elif option in ("-D", "--no-docstrings"):
Options.docstrings = False
elif option in ("-a", "--annotate"):
Options.annotate = True
elif option == "--convert-range":
Options.convert_range = True
elif option.startswith("-T"):
parse_add_transform(options.transforms, get_param(option))
# Note: this can occur multiple times, each time appends
else:
bad_usage()
else:
arg = pop_arg()
if arg.endswith(".pyx"):
sources.append(arg)
elif arg.endswith(".o"):
options.objects.append(arg)
else:
print >>sys.stderr, \
"cython: %s: Unknown filename suffix" % arg
if options.objects and len(sources) > 1:
print >>sys.stderr, \
"cython: Only one source file allowed together with .o files"
if options.use_listing_file and len(sources) > 1:
print >>sys.stderr, \
"cython: Only one source file allowed when using -o"
sys.exit(1)
if len(sources) == 0 and not options.show_version:
bad_usage()
return options, sources
def cythonize(module_list,
exclude=[],
nthreads=0,
aliases=None,
quiet=False,
force=False,
exclude_failures=False,
**options):
"""
Compile a set of source modules into C/C++ files and return a list of distutils
Extension objects for them.
As module list, pass either a glob pattern, a list of glob patterns or a list of
Extension objects. The latter allows you to configure the extensions separately
through the normal distutils options.
When using glob patterns, you can exclude certain module names explicitly
by passing them into the 'exclude' option.
For parallel compilation, set the 'nthreads' option to the number of
concurrent builds.
For a broad 'try to compile' mode that ignores compilation failures and
simply excludes the failed extensions, pass 'exclude_failures=True'. Note
that this only really makes sense for compiling .py files which can also
be used without compilation.
Additional compilation options can be passed as keyword arguments.
"""
if 'include_path' not in options:
options['include_path'] = ['.']
c_options = CompilationOptions(**options)
cpp_options = CompilationOptions(**options)
cpp_options.cplus = True
ctx = c_options.create_context()
module_list = create_extension_list(module_list,
exclude=exclude,
ctx=ctx,
quiet=quiet,
exclude_failures=exclude_failures,
aliases=aliases)
deps = create_dependency_tree(ctx, quiet=quiet)
modules_by_cfile = {}
to_compile = []
for m in module_list:
new_sources = []
for source in m.sources:
base, ext = os.path.splitext(source)
if ext in ('.pyx', '.py'):
if m.language == 'c++':
c_file = base + '.cpp'
options = cpp_options
else:
c_file = base + '.c'
options = c_options
if os.path.exists(c_file):
c_timestamp = os.path.getmtime(c_file)
else:
c_timestamp = -1
# Priority goes first to modified files, second to direct
# dependents, and finally to indirect dependents.
if c_timestamp < deps.timestamp(source):
dep_timestamp, dep = deps.timestamp(source), source
priority = 0
else:
dep_timestamp, dep = deps.newest_dependency(source)
priority = 2 - (dep in deps.immediate_dependencies(source))
if force or c_timestamp < dep_timestamp:
if not quiet:
if source == dep:
print("Compiling %s because it changed." % source)
else:
print("Compiling %s because it depends on %s." %
(source, dep))
if not force and hasattr(options, 'cache'):
extra = m.language
fingerprint = deps.transitive_fingerprint(
source, extra)
else:
fingerprint = None
to_compile.append((priority, source, c_file, fingerprint,
quiet, options, not exclude_failures))
new_sources.append(c_file)
if c_file not in modules_by_cfile:
modules_by_cfile[c_file] = [m]
else:
modules_by_cfile[c_file].append(m)
else:
new_sources.append(source)
m.sources = new_sources
if hasattr(options, 'cache'):
if not os.path.exists(options.cache):
os.mkdir(options.cache)
to_compile.sort()
if nthreads:
# Requires multiprocessing (or Python >= 2.6)
try:
import multiprocessing
pool = multiprocessing.Pool(nthreads)
pool.map(cythonize_one_helper, to_compile)
except ImportError:
print("multiprocessing required for parallel cythonization")
nthreads = 0
if not nthreads:
for args in to_compile:
cythonize_one(*args[1:])
if exclude_failures:
failed_modules = set()
for c_file, modules in modules_by_cfile.iteritems():
if not os.path.exists(c_file):
failed_modules.update(modules)
for module in failed_modules:
module_list.remove(module)
if hasattr(options, 'cache'):
cleanup_cache(options.cache,
getattr(options, 'cache_size', 1024 * 1024 * 100))
# cythonize() is often followed by the (non-Python-buffered)
# compiler output, flush now to avoid interleaving output.
sys.stdout.flush()
return module_list
def parse_command_line(args):
from Cython.Compiler.Main import \
CompilationOptions, default_options
def pop_arg():
if args:
return args.pop(0)
else:
bad_usage()
def get_param(option):
tail = option[2:]
if tail:
return tail
else:
return pop_arg()
options = CompilationOptions(default_options)
sources = []
while args:
if args[0].startswith("-"):
option = pop_arg()
if option in ("-V", "--version"):
options.show_version = 1
elif option in ("-l", "--create-listing"):
options.use_listing_file = 1
elif option in ("-+", "--cplus"):
options.cplus = 1
elif option == "--embed":
Options.embed = "main"
elif option.startswith("--embed="):
Options.embed = option[8:]
elif option.startswith("-I"):
options.include_path.append(get_param(option))
elif option == "--include-dir":
options.include_path.append(pop_arg())
elif option in ("-w", "--working"):
options.working_path = pop_arg()
elif option in ("-o", "--output-file"):
options.output_file = pop_arg()
elif option in ("-t", "--timestamps"):
options.timestamps = 1
elif option in ("-f", "--force"):
options.timestamps = 0
elif option in ("-v", "--verbose"):
options.verbose += 1
elif option in ("-p", "--embed-positions"):
Options.embed_pos_in_docstring = 1
elif option in ("-z", "--pre-import"):
Options.pre_import = pop_arg()
elif option == "--cleanup":
Options.generate_cleanup_code = int(pop_arg())
elif option in ("-D", "--no-docstrings"):
Options.docstrings = False
elif option in ("-a", "--annotate"):
Options.annotate = True
elif option == "--convert-range":
Options.convert_range = True
elif option == "--line-directives":
options.emit_linenums = True
elif option == "--no-c-in-traceback":
options.c_line_in_traceback = False
elif option == "--gdb":
options.gdb_debug = True
options.output_dir = os.curdir
elif option == "--gdb-outdir":
options.gdb_debug = True
options.output_dir = pop_arg()
elif option == "--lenient":
Options.error_on_unknown_names = False
Options.error_on_uninitialized = False
elif option == '-2':
options.language_level = 2
elif option == '-3':
options.language_level = 3
elif option == "--capi-reexport-cincludes":
options.capi_reexport_cincludes = True
elif option == "--fast-fail":
Options.fast_fail = True
elif option in ('-Werror', '--warning-errors'):
Options.warning_errors = True
elif option in ('-Wextra', '--warning-extra'):
options.compiler_directives.update(Options.extra_warnings)
elif option == "--old-style-globals":
Options.old_style_globals = True
elif option == "--directive" or option.startswith('-X'):
if option.startswith('-X') and option[2:].strip():
x_args = option[2:]
else:
x_args = pop_arg()
try:
options.compiler_directives = Options.parse_directive_list(
x_args, relaxed_bool=True,
current_settings=options.compiler_directives)
except ValueError as e:
sys.stderr.write("Error in compiler directive: %s\n" % e.args[0])
sys.exit(1)
elif option.startswith('--debug'):
option = option[2:].replace('-', '_')
from . import DebugFlags
if option in dir(DebugFlags):
setattr(DebugFlags, option, True)
else:
sys.stderr.write("Unknown debug flag: %s\n" % option)
bad_usage()
elif option in ('-h', '--help'):
sys.stdout.write(usage)
sys.exit(0)
else:
sys.stderr.write("Unknown compiler flag: %s\n" % option)
sys.exit(1)
else:
sources.append(pop_arg())
if options.use_listing_file and len(sources) > 1:
sys.stderr.write(
"cython: Only one source file allowed when using -o\n")
sys.exit(1)
if len(sources) == 0 and not options.show_version:
bad_usage()
if Options.embed and len(sources) > 1:
sys.stderr.write(
"cython: Only one source file allowed when using -embed\n")
sys.exit(1)
return options, sources
def godopy_cython():
source, outfile_path = sys.argv[1:]
pyinit_src_symbol = 'PyInit_' + os.path.basename(outfile_path[:-4])
pyinit_dst_symbol = 'PyInit_' + strip_internal_path(outfile_path).replace(
os.sep, '__')[:-4]
tempfile = outfile_path.replace('.cpp', '.temp.cpp')
tempheaderfile = outfile_path.replace('.cpp', '.temp.h')
header_path = outfile_path.replace('.cpp', '.hpp')
from Cython.Compiler import Options
from Cython.Compiler.Main import compile, default_options, CompilationOptions
directives = {'c_string_encoding': 'utf-8'}
options = CompilationOptions(default_options,
compiler_directives=directives)
Options.fast_fail = True
options.output_file = tempfile
options.cplus = 1
options.language_level = 3
result = compile(source, options)
if result.num_errors > 0:
raise SystemExit('Cython compilation finished with errors')
def clean_line(line):
if pyinit_src_symbol in line:
line = line.replace(pyinit_src_symbol, pyinit_dst_symbol)
# Undefined by #define NO_IMPORT_ARRAY
if '_import_array()' in line:
line = line.replace('_import_array()', '0')
# Fix variable declarations with GDCALLINGCONV, GDCALLINGCONV is valid only for functions
if line.lstrip().startswith('GDCALLINGCONV_'):
line = re.sub(r'^(\s+)(GDCALLINGCONV_VOID_PTR)(\s\w+;)$',
r'\1void *\3', line)
line = re.sub(r'^(\s+)(GDCALLINGCONV_VOID)(\s\w+;)$', r'\1void\3',
line)
line = re.sub(r'^(\s+)(GDCALLINGCONV_BOOL)(\s\w+;)$', r'\1bool\3',
line)
# Remove "tp_print" deprecation warnings
if line.strip(
) == '#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000':
line = line.replace(
'PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000',
'PY_VERSION_HEX < 0x030800b1')
# FIXME: undeclared identifier, looks like a bug in Cython
if line.strip() == '__Pyx_CppExn2PyErr();':
line = ''
return line
with open(outfile_path, 'w', encoding='utf-8') as outfile:
with open(tempfile, 'r', encoding='utf-8') as fp:
for line in fp:
outfile.write(clean_line(line))
os.unlink(tempfile)
if os.path.exists(tempheaderfile):
with open(header_path, 'w', encoding='utf-8') as outheaderfile:
with open(tempheaderfile, 'r', encoding='utf-8') as fp:
for line in fp:
outheaderfile.write(clean_line(line))
os.unlink(tempheaderfile)
def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, force=False,
exclude_failures=False, **options):
"""
Compile a set of source modules into C/C++ files and return a list of distutils
Extension objects for them.
As module list, pass either a glob pattern, a list of glob patterns or a list of
Extension objects. The latter allows you to configure the extensions separately
through the normal distutils options.
When using glob patterns, you can exclude certain module names explicitly
by passing them into the 'exclude' option.
For parallel compilation, set the 'nthreads' option to the number of
concurrent builds.
For a broad 'try to compile' mode that ignores compilation failures and
simply excludes the failed extensions, pass 'exclude_failures=True'. Note
that this only really makes sense for compiling .py files which can also
be used without compilation.
Additional compilation options can be passed as keyword arguments.
"""
if 'include_path' not in options:
options['include_path'] = ['.']
c_options = CompilationOptions(**options)
cpp_options = CompilationOptions(**options); cpp_options.cplus = True
ctx = c_options.create_context()
module_list = create_extension_list(
module_list,
exclude=exclude,
ctx=ctx,
quiet=quiet,
exclude_failures=exclude_failures,
aliases=aliases)
deps = create_dependency_tree(ctx, quiet=quiet)
modules_by_cfile = {}
to_compile = []
for m in module_list:
new_sources = []
for source in m.sources:
base, ext = os.path.splitext(source)
if ext in ('.pyx', '.py'):
if m.language == 'c++':
c_file = base + '.cpp'
options = cpp_options
else:
c_file = base + '.c'
options = c_options
if os.path.exists(c_file):
c_timestamp = os.path.getmtime(c_file)
else:
c_timestamp = -1
# Priority goes first to modified files, second to direct
# dependents, and finally to indirect dependents.
if c_timestamp < deps.timestamp(source):
dep_timestamp, dep = deps.timestamp(source), source
priority = 0
else:
dep_timestamp, dep = deps.newest_dependency(source)
priority = 2 - (dep in deps.immediate_dependencies(source))
if force or c_timestamp < dep_timestamp:
if not quiet:
if source == dep:
print("Compiling %s because it changed." % source)
else:
print("Compiling %s because it depends on %s." % (source, dep))
if not force and hasattr(options, 'cache'):
extra = m.language
fingerprint = deps.transitive_fingerprint(source, extra)
else:
fingerprint = None
to_compile.append((priority, source, c_file, fingerprint, quiet,
options, not exclude_failures))
new_sources.append(c_file)
if c_file not in modules_by_cfile:
modules_by_cfile[c_file] = [m]
else:
modules_by_cfile[c_file].append(m)
else:
new_sources.append(source)
m.sources = new_sources
if hasattr(options, 'cache'):
if not os.path.exists(options.cache):
os.mkdir(options.cache)
to_compile.sort()
if nthreads:
# Requires multiprocessing (or Python >= 2.6)
try:
import multiprocessing
pool = multiprocessing.Pool(nthreads)
pool.map(cythonize_one_helper, to_compile)
except ImportError:
print("multiprocessing required for parallel cythonization")
nthreads = 0
if not nthreads:
for args in to_compile:
cythonize_one(*args[1:])
if exclude_failures:
failed_modules = set()
for c_file, modules in modules_by_cfile.iteritems():
if not os.path.exists(c_file):
failed_modules.update(modules)
for module in failed_modules:
module_list.remove(module)
if hasattr(options, 'cache'):
cleanup_cache(options.cache, getattr(options, 'cache_size', 1024 * 1024 * 100))
return module_list
