def cython_build(name, file=None, force=False, quiet=True, cythonize_args={},
lib_dir=os.path.join(get_cython_cache_dir(), 'inline/lib'),
tmp_dir=os.path.join(get_cython_cache_dir(), 'inline/tmp'),
**extension_args):
"""Build a cython extension.
"""
if file is not None:
_append_args(extension_args, 'sources', file)
with _suppress_output(quiet=quiet):
extension = Extension(name, **extension_args)
extensions = cythonize([extension], force=force, **cythonize_args)
build_extension = _get_build_extension()
build_extension.extensions = extensions
build_extension.build_lib = lib_dir
build_extension.build_temp = tmp_dir
build_extension.run()
def cython_build(name,
file=None,
force=False,
quiet=True,
cythonize_args={},
lib_dir=os.path.join(get_cython_cache_dir(), 'inline/lib'),
tmp_dir=os.path.join(get_cython_cache_dir(), 'inline/tmp'),
**extension_args):
"""Build a cython extension.
"""
if file is not None:
_append_args(extension_args, 'sources', file)
with _suppress_output(quiet=quiet):
extension = Extension(name, **extension_args)
extensions = cythonize([extension], force=force, **cythonize_args)
build_extension = _get_build_extension()
build_extension.extensions = extensions
build_extension.build_lib = lib_dir
build_extension.build_temp = tmp_dir
build_extension.run()
def create_extension(
self,
code,
force=False,
name=None,
define_macros=None,
include_dirs=None,
library_dirs=None,
runtime_library_dirs=None,
extra_compile_args=None,
extra_link_args=None,
libraries=None,
compiler=None,
owner_name='',
):
self._simplify_paths()
if Cython is None:
raise ImportError('Cython is not available')
code = deindent(code)
lib_dir = prefs.codegen.runtime.cython.cache_dir
if lib_dir is None:
lib_dir = os.path.join(get_cython_cache_dir(), 'brian_extensions')
if '~' in lib_dir:
lib_dir = os.path.expanduser(lib_dir)
try:
os.makedirs(lib_dir)
except OSError:
if not os.path.exists(lib_dir):
raise IOError(
"Couldn't create Cython cache directory '%s', try setting the "
"cache directly with prefs.codegen.runtime.cython.cache_dir."
% lib_dir)
numpy_version = '.'.join(
numpy.__version__.split('.')[:2]) # Only use major.minor version
key = code, sys.version_info, sys.executable, Cython.__version__, numpy_version
if force:
# Force a new module name by adding the current time to the
# key which is hashed to determine the module name.
key += time.time(),
if key in self._code_cache:
return self._code_cache[key]
if name is not None:
module_name = name #py3compat.unicode_to_str(args.name)
else:
module_name = "_cython_magic_" + hashlib.md5(
str(key).encode('utf-8')).hexdigest()
if owner_name:
logger.diagnostic(
'"{owner_name}" using Cython module "{module_name}"'.format(
owner_name=owner_name, module_name=module_name))
module_path = os.path.join(lib_dir, module_name + self.so_ext)
if prefs['codegen.runtime.cython.multiprocess_safe']:
lock_file = os.path.join(lib_dir, module_name + '.lock')
with open(lock_file, 'w') as f:
if msvcrt:
msvcrt.locking(f.fileno(), msvcrt.LK_RLCK,
os.stat(lock_file).st_size)
else:
fcntl.flock(f, fcntl.LOCK_EX)
return self._load_module(
module_path,
define_macros=define_macros,
include_dirs=include_dirs,
library_dirs=library_dirs,
extra_compile_args=extra_compile_args,
extra_link_args=extra_link_args,
libraries=libraries,
code=code,
lib_dir=lib_dir,
module_name=module_name,
runtime_library_dirs=runtime_library_dirs,
compiler=compiler,
key=key)
else:
return self._load_module(module_path,
define_macros=define_macros,
include_dirs=include_dirs,
library_dirs=library_dirs,
extra_compile_args=extra_compile_args,
extra_link_args=extra_link_args,
libraries=libraries,
code=code,
lib_dir=lib_dir,
module_name=module_name,
runtime_library_dirs=runtime_library_dirs,
compiler=compiler,
key=key)
def cython_inline(code,
get_type=unsafe_type,
lib_dir=os.path.join(get_cython_cache_dir(), 'inline'),
cython_include_dirs=['.'],
force=False,
quiet=False,
locals=None,
globals=None,
**kwds):
if get_type is None:
get_type = lambda x: 'object'
code = to_unicode(code)
orig_code = code
code, literals = strip_string_literals(code)
code = strip_common_indent(code)
ctx = Context(cython_include_dirs, default_options)
if locals is None:
locals = inspect.currentframe().f_back.f_back.f_locals
if globals is None:
globals = inspect.currentframe().f_back.f_back.f_globals
try:
for symbol in unbound_symbols(code):
if symbol in kwds:
continue
elif symbol in locals:
kwds[symbol] = locals[symbol]
elif symbol in globals:
kwds[symbol] = globals[symbol]
else:
print("Couldn't find ", symbol)
except AssertionError:
if not quiet:
# Parsing from strings not fully supported (e.g. cimports).
print("Could not parse code as a string (to extract unbound symbols).")
cimports = []
for name, arg in kwds.items():
if arg is cython_module:
cimports.append('\ncimport cython as %s' % name)
del kwds[name]
arg_names = kwds.keys()
arg_names.sort()
arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names])
key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__
module_name = "_cython_inline_" + hashlib.md5(str(key).encode('utf-8')).hexdigest()
so_ext = [ ext for ext,_,mod_type in imp.get_suffixes() if mod_type == imp.C_EXTENSION ][0]
module_path = os.path.join(lib_dir, module_name+so_ext)
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
if force or not os.path.isfile(module_path):
cflags = []
c_include_dirs = []
qualified = re.compile(r'([.\w]+)[.]')
for type, _ in arg_sigs:
m = qualified.match(type)
if m:
cimports.append('\ncimport %s' % m.groups()[0])
# one special case
if m.groups()[0] == 'numpy':
import numpy
c_include_dirs.append(numpy.get_include())
# cflags.append('-Wno-unused')
module_body, func_body = extract_func_code(code)
params = ', '.join(['%s %s' % a for a in arg_sigs])
module_code = """
%(module_body)s
%(cimports)s
def __invoke(%(params)s):
%(func_body)s
""" % {'cimports': '\n'.join(cimports), 'module_body': module_body, 'params': params, 'func_body': func_body }
for key, value in literals.items():
module_code = module_code.replace(key, value)
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
fh = open(pyx_file, 'w')
try:
fh.write(module_code)
finally:
fh.close()
extension = Extension(
name = module_name,
sources = [pyx_file],
include_dirs = c_include_dirs,
extra_compile_args = cflags)
dist = Distribution()
# Ensure the build respects distutils configuration by parsing
# the configuration files
config_files = dist.find_config_files()
dist.parse_config_files(config_files)
build_extension = build_ext(dist)
build_extension.finalize_options()
build_extension.extensions = cythonize([extension], ctx=ctx, quiet=quiet)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
build_extension.run()
_code_cache[key] = module_name
module = imp.load_dynamic(module_name, module_path)
arg_list = [kwds[arg] for arg in arg_names]
return module.__invoke(*arg_list)
def cythonmagic(code, export=None, name=None, force=False,
quiet=True, smart=True, fast_indexing=False,
directives={}, cimport_dirs=[], cythonize_args={},
lib_dir=os.path.join(get_cython_cache_dir(), 'inline/lib'),
tmp_dir=os.path.join(get_cython_cache_dir(), 'inline/tmp'),
environ={}, **extension_args):
"""Compile a code snippet in string.
The contents of the code are written to a `.pyx` file in the
cython cache directory using a filename with the hash of the
code. This file is then cythonized and compiled.
Parameters
----------
code : str
The code to compile.
It can also be a file path, but must start with "./", "/", "X:", or "~",
and end with ".py" or ".pyx".
Strings like "import abc.pyx" or "a=1; b=a.pyx" will be treated as
code snippet.
export : dict
Export the variables from the compiled module to a dict.
`export=globals()` is equivalent to `from module import *`.
name : str, optional
Name of compiled module. If not given, it will be generated
automatically by hash of the code and options (recommended).
force : bool
Force the compilation of a new module, even if the source
has been previously compiled.
quiet : bool
Suppress compiler's outputs/warnings.
smart : bool
If True, numpy and openmp will be auto-detected from the code.
fast_indexing : bool
If True, `boundscheck` and `wraparound` are turned off
for better array indexing performance (at cost of safety).
This setting can be overridden by `directives`.
directives : dict
Cython compiler directives, including
binding, boundscheck, wraparound, initializedcheck, nonecheck,
overflowcheck, overflowcheck.fold, embedsignature, cdivision, cdivision_warnings,
always_allow_keywords, profile, linetrace, infer_types, language_level, etc.
Ref http://docs.cython.org/en/latest/src/userguide/source_files_and_compilation.html#compiler-directives
This setting can be overridden by `cythonize_args['compiler_directives']`.
cimport_dirs : list of str
Directories for finding cimport modules (.pxd files).
This setting can be overridden by `cythonize_args['include_path']`.
cythonize_args : dict
Arguments for `Cython.Build.cythonize`, including
aliases, quiet, force, language, annotate, build_dir, output_file,
include_path, compiler_directives, etc.
Ref http://docs.cython.org/en/latest/src/userguide/source_files_and_compilation.html#cythonize-arguments
environ : dict
Temporary environment variables for compilation.
lib_dir : str
Directory to put the compiled module.
tmp_dir : str
Directory to put the temporary files.
**extension_args :
Arguments for `distutils.core.Extension`, including
name, sources, define_macros, undef_macros,
include_dirs, library_dirs, runtime_library_dirs,
libraries, extra_compile_args, extra_link_args,
extra_objects, export_symbols, depends, language
Ref https://docs.python.org/2/distutils/apiref.html#distutils.core.Extension
Examples
--------
Basic usage:
code = r'''
def func(x):
return 2.0 * x
'''
pyx = cythonmagic(code)
pyx.func(1)
Raw string is recommended to avoid breaking escape character.
Export the names from compiled module:
cythonmagic(code, globals())
func(1)
Get better performance (with risk) with arrays:
cythonmagic(code, fast_indexing=True)
Compile OpenMP codes with gcc:
cythonmagic(openmpcode, openmp='-fopenmp')
# or equivalently
cythonmagic(openmpcode,
extra_compile_args=['-fopenmp'],
extra_link_args=['-fopenmp'],
)
# use '-openmp' or '-qopenmp' (>=15.0) for Intel
# use '/openmp' for Microsoft Visual C++ Compiler
# use '-fopenmp=libomp' for Clang
Use icc to compile:
cythonmagic(code, environ={'CC':'icc', 'LDSHARED':'icc -shared'})
Ref https://software.intel.com/en-us/articles/thread-parallelism-in-cython
Set directory for searching cimport (.pxd file):
cythonmagic(code, cimport_dirs=[custom_path]})
# or equivalently
cythonmagic(code, cythonize_args={'include_path': [custom_path]})
Try setting `cimport_dirs=sys.path` if Cython can not find installed
cimport module.
The cython `directives` and distutils `extension_args` can also be
set in a directive comment at the top of the code, e.g.:
# cython: boundscheck=False, wraparound=False, cdivision=True
# distutils: extra_compile_args = -fopenmp
# distutils: extra_link_args = -fopenmp
...code...
Example of using gsl library, assuming gsl is installed at /opt/gsl/
code = r'''
cdef extern from "gsl/gsl_math.h":
double gsl_pow_int (double x, int n)
def pow(double x, int n):
y = gsl_pow_int(x, n)
return y
'''
pyx = cythonmagic(
code,
include_dirs=['/opt/gsl/include/'],
library_dirs=['/opt/gsl/lib'],
libraries=['gsl', 'gslcblas']
)
pyx.pow(2, 6)
References
----------
https://github.com/cython/cython/blob/master/Cython/Build/IpythonMagic.py
https://github.com/cython/cython/blob/master/Cython/Build/Inline.py
"""
# get working directories
# assume all paths are relative to the directory of the caller's frame
cur_dir = get_frame_dir(depth=1) # where cythonmagic is called
lib_dir = join_path(cur_dir, lib_dir)
tmp_dir = join_path(cur_dir, tmp_dir)
if not os.path.isdir(lib_dir):
os.makedirs(lib_dir)
if not os.path.isdir(tmp_dir):
os.makedirs(tmp_dir)
# check if `code` is a code snippet or a .pyx/.py file
reg_pyx = re.compile(r"^ ( ~ | [\.]? [/\\] | [a-zA-Z]:) .* \.pyx? $ | "
r"^ [^\s=;]+ \.pyx? $", re.X | re.S)
is_file = reg_pyx.match(code)
if is_file:
file = join_path(cur_dir, code)
code = io.open(file, 'r', encoding='utf-8').read()
if name is None:
name = get_basename(file)
# it might exist related .pyd file in the same directory
cimport_dirs = cimport_dirs + [os.path.dirname(file)]
else:
cimport_dirs = cimport_dirs + [cur_dir]
code = strip_common_indent(to_unicode(code))
# update arguments
directives = directives.copy()
if fast_indexing:
directives.setdefault('boundscheck', False)
directives.setdefault('wraparound', False)
directives.setdefault('embedsignature', True) # recommended setting
cythonize_args = cythonize_args.copy()
cythonize_args.setdefault('compiler_directives', directives)
cythonize_args.setdefault('include_path', cimport_dirs)
# if any extra dependencies
extra_depends = any(extension_args.get(k, [])
for k in ['sources', 'extra_objects', 'depends'])
# module signature
key = (code, name, cythonize_args, extension_args, environ, os.environ,
sys.executable, sys.version_info, Cython.__version__)
key_bytes = u"{}".format(key).encode('utf-8') # for 2, 3 compatibility
signature = hashlib.md5(key_bytes).hexdigest()
# embed module signature?
# code = u"{}\n\n# added by cythonmagic\n{} = '{}'".format(
# code, '__cythonmagic_signature__', signature)
# module name and path
pyx_name = "__cythonmagic__{}".format(signature)
ext_name = pyx_name if name is None else name
pyx_file = os.path.join(tmp_dir, pyx_name + '.pyx') # path of source file
ext_file = os.path.join(lib_dir, ext_name + so_ext()) # path of extension
# write pyx file
if force or not os.path.isfile(pyx_file):
with io.open(pyx_file, 'w', encoding='utf-8') as f:
f.write(code)
if os.path.isfile(ext_file):
os.remove(ext_file) # dangerous?
# build
# if existing extra depends, let distutils to decide whether rebuild or not
if not os.path.isfile(ext_file) or extra_depends:
with set_env(**environ):
_update_flag(code, extension_args, smart=smart)
cython_build(ext_name, file=pyx_file, force=force,
quiet=quiet, cythonize_args=cythonize_args,
lib_dir=lib_dir, tmp_dir=tmp_dir,
**extension_args)
# import
module = load_dynamic(ext_name, ext_file)
# module.__pyx_file__ = pyx_file
if export is not None:
_export_all(module.__dict__, export)
return module
def create_extension(self, code, force=False, name=None,
include_dirs=None,
library_dirs=None,
runtime_library_dirs=None,
extra_compile_args=None,
extra_link_args=None,
libraries=None,
compiler=None,
owner_name='',
):
self._simplify_paths()
if Cython is None:
raise ImportError('Cython is not available')
code = deindent(code)
lib_dir = prefs.codegen.runtime.cython.cache_dir
if lib_dir is None:
lib_dir = os.path.join(get_cython_cache_dir(), 'brian_extensions')
if '~' in lib_dir:
lib_dir = os.path.expanduser(lib_dir)
try:
os.makedirs(lib_dir)
except OSError:
if not os.path.exists(lib_dir):
raise IOError("Couldn't create Cython cache directory '%s', try setting the "
"cache directly with prefs.codegen.runtime.cython.cache_dir." % lib_dir)
key = code, sys.version_info, sys.executable, Cython.__version__
if force:
# Force a new module name by adding the current time to the
# key which is hashed to determine the module name.
key += time.time(),
if key in self._code_cache:
return self._code_cache[key]
if name is not None:
module_name = name#py3compat.unicode_to_str(args.name)
else:
module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest()
if owner_name:
logger.diagnostic('"{owner_name}" using Cython module "{module_name}"'.format(owner_name=owner_name,
module_name=module_name))
module_path = os.path.join(lib_dir, module_name + self.so_ext)
if prefs['codegen.runtime.cython.multiprocess_safe']:
lock_file = os.path.join(lib_dir, module_name + '.lock')
with open(lock_file, 'w') as f:
if msvcrt:
msvcrt.locking(f.fileno(), msvcrt.LK_RLCK,
os.stat(lock_file).st_size)
else:
fcntl.flock(f, fcntl.LOCK_EX)
return self._load_module(module_path, include_dirs,
library_dirs,
extra_compile_args, extra_link_args,
libraries, code, lib_dir, module_name,
runtime_library_dirs, compiler, key)
else:
return self._load_module(module_path, include_dirs, library_dirs,
extra_compile_args, extra_link_args,
libraries, code, lib_dir, module_name,
runtime_library_dirs, compiler, key)
def cythonize_new(include_dirs, libraries_dirs, libraries, c_flags, l_flags,
o_dir, py_version, pyx_files):
# inline directory.
lib_dir = o_dir
if not lib_dir:
lib_dir = os.path.join(get_cython_cache_dir(), 'inline')
if pyx_files is None or len(pyx_files) != 1:
raise ValueError('expected single file but got "%s"' % (pyx_files, ))
pyx_file = pyx_files[0]
if not os.path.isfile(pyx_file):
raise ValueError('"%s" not a valid file' % (pyx_file, ))
f_name = os.path.split(pyx_file)[1]
module_name = os.path.splitext(f_name)[0]
# todo: check if module_name is valid.
build_extension = get_build_extension()
if build_extension is None:
raise ValueError('fail')
mod_ext = build_extension.get_ext_filename('')
full_module_name = '%s%s' % (module_name, mod_ext)
module_path = os.path.join(lib_dir, full_module_name)
quiet = True
print('cythonize {')
print(' include_dirs: %s' % (include_dirs, ))
print(' lib_dirs: %s' % (libraries_dirs, ))
print(' libs: %s' % (libraries, ))
print(' cxx_flags: %s' % (c_flags, ))
print(' l_flags: %s' % (l_flags, ))
print(' file: %s' % (pyx_file, ))
print(' out_file: %s' % (module_path, ))
print(' py_version: %s' % (py_version, ))
print('}')
#pyx_file = "C:/Users/user/source/repos/openfst/OpenFst/src/python/openpyfst.pyx"
#module_path = r'C:\Users\user\.cython\inline\openpyfst.cp36 - win_amd64.pyd'
cython_compiler_directives = []
_cython_inline_default_context = create_context(('.', ))
cython_include_dirs = None
ctx = create_context(
tuple(cython_include_dirs
)) if cython_include_dirs else _cython_inline_default_context
# build include dirs and cflags.
#cflags = c_flags #['/MT'] # no debug symbols.
#lflags = l_flags # ['/verbose:lib']
lang = "c++" # c++
extension = Extension(name=module_name,
sources=[pyx_file],
include_dirs=include_dirs,
library_dirs=libraries_dirs,
libraries=libraries,
extra_compile_args=c_flags,
extra_link_args=l_flags,
language=lang)
build_extension.extensions = cythonize(
[extension],
include_path=cython_include_dirs or ['.'],
compiler_directives=cython_compiler_directives,
quiet=quiet)
build_extension.build_temp = lib_dir
build_extension.build_lib = lib_dir
build_extension.verbose = not quiet
build_extension.run()
print('done', build_extension)
def inline(code,
export=None,
name=None,
force=False,
quiet=True,
auto_flag=True,
fast_indexing=False,
directives={},
cimport_dirs=[],
cythonize_args={},
lib_dir=os.path.join(get_cython_cache_dir(), 'inline/lib'),
tmp_dir=os.path.join(get_cython_cache_dir(), 'inline/tmp'),
environ={},
**extension_args):
"""Compile a code snippet in string.
The contents of the code are written to a `.pyx` file in the
cython cache directory using a filename with the hash of the
code. This file is then cythonized and compiled.
Parameters
----------
code : str
The code to compile.
It can also be a file path, but must start with "./", "/", "X:", or "~",
and end with ".py" or ".pyx".
Strings like "import abc.pyx" or "a=1; b=a.pyx" will be treated as
code snippet.
export : dict
Export the variables from the compiled module to a dict.
`export=globals()` is equivalent to `from module import *`.
name : str, optional
Name of compiled module. If not given, it will be generated
automatically by hash of the code and options (recommended).
force : bool
Force the compilation of a new module, even if the source
has been previously compiled.
quiet : bool
Suppress compiler's outputs/warnings unless the compiling failed.
auto_flag : bool
If True, numpy and openmp will be auto-detected from the code.
fast_indexing : bool
If True, `boundscheck` and `wraparound` are turned off
for better array indexing performance (at cost of safety).
This setting can be overridden by `directives`.
directives : dict
Cython compiler directives, including
binding, boundscheck, wraparound, initializedcheck, nonecheck,
overflowcheck, overflowcheck.fold, embedsignature, cdivision, cdivision_warnings,
always_allow_keywords, profile, linetrace, infer_types, language_level, etc.
Ref http://docs.cython.org/en/latest/src/userguide/source_files_and_compilation.html#compiler-directives
This setting can be overridden by `cythonize_args['compiler_directives']`.
cimport_dirs : list of str
Directories for finding cimport modules (.pxd files).
This setting can be overridden by `cythonize_args['include_path']`.
cythonize_args : dict
Arguments for `Cython.Build.cythonize`, including
aliases, quiet, force, language, annotate, build_dir, output_file,
include_path, compiler_directives, etc.
Ref http://docs.cython.org/en/latest/src/userguide/source_files_and_compilation.html#cythonize-arguments
environ : dict
Temporary environment variables for compilation.
lib_dir : str
Directory to put the compiled module.
tmp_dir : str
Directory to put the temporary files.
**extension_args :
Arguments for `distutils.core.Extension`, including
name, sources, define_macros, undef_macros,
include_dirs, library_dirs, runtime_library_dirs,
libraries, extra_compile_args, extra_link_args,
extra_objects, export_symbols, depends, language
Ref https://docs.python.org/2/distutils/apiref.html#distutils.core.Extension
Examples
--------
Basic usage
import cyper
code = r'''
def func(x):
return 2.0 * x
'''
pyx = cyper.inline(code)
pyx.func(1)
# 2.0
Raw string is recommended to avoid breaking escape character.
It is convenient (though usually not encouraged) to export the variables from compiled module to the current namespace
cyper.inline(code, globals())
func(1)
Example of using Numpy array and external gsl library, assuming gsl installed at `/opt/gsl/`
code = r'''
import numpy as np
cdef extern from "gsl/gsl_math.h":
double gsl_pow_int (double x, int n)
def pow(double x, int n):
y = gsl_pow_int(x, n)
return y
def pow_array(double[:] x, int n):
cdef:
int i, m=len(x)
double[:] y=np.empty(m, dtype='f8')
for i in range(m):
y[i] = gsl_pow_int(x[i], n)
return y.base
'''
pyx = cyper.inline(
code,
include_dirs=['/opt/gsl/include/'],
library_dirs=['/opt/gsl/lib'],
libraries=['gsl', 'gslcblas']
)
pyx.pow(2, 6)
# 64.0
import numpy as np
pyx.pow_array(np.arange(5, dtype='f8'), 2)
# array([ 0., 1., 4., 9., 16.])
Get better performance (at your own risk) with arrays
cyper.inline(code, fast_indexing=True)
# or equivalently
cyper.inline(code, directives=dict(boundscheck=False, wraparound=False))
Advanced usage
Set the compiler options, e.g., compiling OpenMP codes with gcc
cyper.inline(openmpcode,
extra_compile_args=['-fopenmp'],
extra_link_args=['-fopenmp'],
)
# use '-openmp' or '-qopenmp' (>=15.0) for Intel
# use '/openmp' for Microsoft Visual C++ Compiler
# use '-fopenmp=libomp' for Clang
Or equivalently write this for short
cyper.inline(openmpcode, openmp='-fopenmp')
The cython `directives` and distutils `extension_args` can also be set in a directive comment at the top of the code snippet, e.g.,
code = r'''
# cython: boundscheck=False, wraparound=False, cdivision=True
# distutils: extra_compile_args = -fopenmp
# distutils: extra_link_args = -fopenmp
...<code>...
'''
cyper.inline(code)
Set environment variables, e.g., using icc to compile
cyper.inline(code, environ={'CC':'icc', 'LDSHARED':'icc -shared'})
See https://software.intel.com/en-us/articles/thread-parallelism-in-cython
Set directory for searching cimport (.pxd file)
cyper.inline(code, cimport_dirs=[custom_path]})
# or equivalently
cyper.inline(code, cythonize_args={'include_path': [custom_path]})
Try setting `cimport_dirs=sys.path` if Cython can not find the installed cimport modules.
See also
--------
https://github.com/cython/cython/blob/master/Cython/Build/IpythonMagic.py
https://github.com/cython/cython/blob/master/Cython/Build/Inline.py
"""
# get working directories
# assume all paths are relative to the directory of the caller's frame
cur_dir = get_frame_dir(depth=1) # where inline is called
lib_dir = join_path(cur_dir, lib_dir)
tmp_dir = join_path(cur_dir, tmp_dir)
if not os.path.isdir(lib_dir):
os.makedirs(lib_dir)
if not os.path.isdir(tmp_dir):
os.makedirs(tmp_dir)
# check if `code` is a code snippet or a .pyx/.py file
reg_pyx = re.compile(
r"^ ( ~ | [\.]? [/\\] | [a-zA-Z]:) .* \.pyx? $ | "
r"^ [^\s=;]+ \.pyx? $", re.X | re.S)
is_file = reg_pyx.match(code)
if is_file:
file = join_path(cur_dir, code)
code = io.open(file, 'r', encoding='utf-8').read()
if name is None:
name = get_basename(file)
# it might exist related .pyd file in the same directory
cimport_dirs = cimport_dirs + [os.path.dirname(file)]
else:
cimport_dirs = cimport_dirs + [cur_dir]
code = strip_common_indent(to_unicode(code))
# update arguments
directives = directives.copy()
if fast_indexing:
directives.setdefault('boundscheck', False)
directives.setdefault('wraparound', False)
directives.setdefault('embedsignature', True) # recommended setting
cythonize_args = cythonize_args.copy()
cythonize_args.setdefault('compiler_directives', directives)
cythonize_args.setdefault('include_path', cimport_dirs)
# if any extra dependencies
extra_depends = any(
extension_args.get(k, [])
for k in ['sources', 'extra_objects', 'depends'])
# module signature
key = (code, name, cythonize_args, extension_args, environ, os.environ,
sys.executable, sys.version_info, Cython.__version__)
key_bytes = u"{}".format(key).encode('utf-8') # for 2, 3 compatibility
signature = hashlib.md5(key_bytes).hexdigest()
# embed module signature?
# code = u"{}\n\n# added by cyper.inline\n{} = '{}'".format(
# code, '__cyper_signature__', signature)
# module name and path
pyx_name = "_cyper_{}".format(signature)
ext_name = pyx_name if name is None else name
pyx_file = os.path.join(tmp_dir, pyx_name + '.pyx') # path of source file
ext_file = os.path.join(lib_dir, ext_name + so_ext()) # path of extension
# write pyx file
if force or not os.path.isfile(pyx_file):
with io.open(pyx_file, 'w', encoding='utf-8') as f:
f.write(code)
if os.path.isfile(ext_file):
os.remove(ext_file) # dangerous?
# build
# if existing extra depends, let distutils to decide whether rebuild or not
if not os.path.isfile(ext_file) or extra_depends:
with set_env(**environ):
_update_flag(code, extension_args, auto_flag=auto_flag)
cython_build(ext_name,
file=pyx_file,
force=force,
quiet=quiet,
cythonize_args=cythonize_args,
lib_dir=lib_dir,
tmp_dir=tmp_dir,
**extension_args)
# import
module = load_dynamic(ext_name, ext_file)
# module.__pyx_file__ = pyx_file
if export is not None:
_export_all(module.__dict__, export)
return module
def cython_inline(code,
get_type=unsafe_type,
lib_dir=os.path.join(get_cython_cache_dir(), 'inline'),
cython_include_dirs=None,
force=False,
quiet=False,
local_sym=None,
global_sym=None,
**kwargs):
if get_type is None:
get_type = lambda x: 'object'
ctx = _create_context(tuple(cython_include_dirs)
) if cython_include_dirs else _inline_default_context
# Fast path if this has been called in this session.
_unbound_symbols = _inline_cache.get(code)
if _unbound_symbols is not None:
_populate_unbound(kwargs, _unbound_symbols, local_sym, global_sym)
args = sorted(kwargs.items())
arg_sigs = tuple([(get_type(value, ctx), arg) for arg, value in args])
invoke = _inline_cache.get((code, arg_sigs))
if invoke is not None:
arg_list = [arg[1] for arg in args]
return invoke(*arg_list)
orig_code = code
code, literals = strip_string_literals(code)
if local_sym is None:
local_sym = inspect.currentframe().f_back.f_back.f_locals
if global_sym is None:
global_sym = inspect.currentframe().f_back.f_back.f_globals
try:
_inline_cache[orig_code] = _unbound_symbols = unbound_symbols(code)
_populate_unbound(kwargs, _unbound_symbols, local_sym, global_sym)
except AssertionError:
if not quiet:
# Parsing from strings not fully supported (e.g. cimports).
print(
"Could not parse code as a string (to extract unbound symbols)."
)
cimports = []
for name, arg in list(kwargs.items()):
if arg is cython:
cimports.append('\ncimport cython as %s' % name)
del kwargs[name]
arg_names = sorted(kwargs)
arg_sigs = tuple([(get_type(kwargs[arg], ctx), arg) for arg in arg_names])
key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__
module_name = "_cython_inline_" + hashlib.md5(
str(key).encode('utf-8')).hexdigest()
if module_name in sys.modules:
_module = sys.modules[module_name]
else:
build_extension = None
global _so_ext
if _so_ext is None:
# Figure out and cache current extension suffix
build_extension = _get_build_extension()
_so_ext = build_extension.get_ext_filename('')
module_path = os.path.join(lib_dir, module_name + _so_ext)
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
if force or not os.path.isfile(module_path):
cflags = []
c_include_dirs = []
qualified = re.compile(r'([.\w]+)[.]')
for type, _ in arg_sigs:
m = qualified.match(type)
if m:
cimports.append('\ncimport %s' % m.groups()[0])
# one special case
if m.groups()[0] == 'numpy':
import numpy
c_include_dirs.append(numpy.get_include())
# cflags.append('-Wno-unused')
module_body, func_body = extract_func_code(code)
params = ', '.join(['%s %s' % a for a in arg_sigs])
module_code = '''
%(module_body)s
%(cimports)s
def __invoke(%(params)s):
%(func_body)s
return locals()
''' % {
'cimports': '\n'.join(cimports),
'module_body': module_body,
'params': params,
'func_body': func_body
}
for key, value in literals.items():
module_code = module_code.replace(key, value)
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
fh = open(pyx_file, 'w')
try:
fh.write(module_code)
finally:
fh.close()
extension = Extension(name=module_name,
sources=[pyx_file],
include_dirs=c_include_dirs,
extra_compile_args=cflags)
if build_extension is None:
build_extension = _get_build_extension()
build_extension.extensions = cythonize(
[extension],
include_path=cython_include_dirs or ['.'],
quiet=quiet)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
build_extension.run()
_module = imp.load_dynamic(module_name, module_path)
_inline_cache[orig_code, arg_sigs] = _module.__invoke
arg_list = [kwargs[arg] for arg in arg_names]
return _module.__invoke(*arg_list)
def cython_inline(
code,
get_type=unsafe_type,
lib_dir=os.path.join(get_cython_cache_dir(), "inline"),
cython_include_dirs=None,
force=False,
quiet=False,
locals=None,
globals=None,
**kwds
):
if cython_include_dirs is None:
cython_include_dirs = ["."]
if get_type is None:
get_type = lambda x: "object"
code = to_unicode(code)
orig_code = code
code, literals = strip_string_literals(code)
code = strip_common_indent(code)
ctx = _create_context(tuple(cython_include_dirs))
if locals is None:
locals = inspect.currentframe().f_back.f_back.f_locals
if globals is None:
globals = inspect.currentframe().f_back.f_back.f_globals
try:
for symbol in unbound_symbols(code):
if symbol in kwds:
continue
elif symbol in locals:
kwds[symbol] = locals[symbol]
elif symbol in globals:
kwds[symbol] = globals[symbol]
else:
print("Couldn't find ", symbol)
except AssertionError:
if not quiet:
# Parsing from strings not fully supported (e.g. cimports).
print("Could not parse code as a string (to extract unbound symbols).")
cimports = []
for name, arg in kwds.items():
if arg is cython_module:
cimports.append("\ncimport cython as %s" % name)
del kwds[name]
arg_names = kwds.keys()
arg_names.sort()
arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names])
key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__
module_name = "_cython_inline_" + hashlib.md5(str(key).encode("utf-8")).hexdigest()
if module_name in sys.modules:
module = sys.modules[module_name]
else:
build_extension = None
if cython_inline.so_ext is None:
# Figure out and cache current extension suffix
build_extension = _get_build_extension()
cython_inline.so_ext = build_extension.get_ext_filename("")
module_path = os.path.join(lib_dir, module_name + cython_inline.so_ext)
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
if force or not os.path.isfile(module_path):
cflags = []
c_include_dirs = []
qualified = re.compile(r"([.\w]+)[.]")
for type, _ in arg_sigs:
m = qualified.match(type)
if m:
cimports.append("\ncimport %s" % m.groups()[0])
# one special case
if m.groups()[0] == "numpy":
import numpy
c_include_dirs.append(numpy.get_include())
# cflags.append('-Wno-unused')
module_body, func_body = extract_func_code(code)
params = ", ".join(["%s %s" % a for a in arg_sigs])
module_code = """
%(module_body)s
%(cimports)s
def __invoke(%(params)s):
%(func_body)s
""" % {
"cimports": "\n".join(cimports),
"module_body": module_body,
"params": params,
"func_body": func_body,
}
for key, value in literals.items():
module_code = module_code.replace(key, value)
pyx_file = os.path.join(lib_dir, module_name + ".pyx")
fh = open(pyx_file, "w")
try:
fh.write(module_code)
finally:
fh.close()
extension = Extension(
name=module_name, sources=[pyx_file], include_dirs=c_include_dirs, extra_compile_args=cflags
)
if build_extension is None:
build_extension = _get_build_extension()
build_extension.extensions = cythonize([extension], include_path=cython_include_dirs, quiet=quiet)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
build_extension.run()
module = imp.load_dynamic(module_name, module_path)
arg_list = [kwds[arg] for arg in arg_names]
return module.__invoke(*arg_list)
