def finalize_options(self):
if self.distribution.ext_modules:
nthreads = getattr(self, 'parallel', None) # -j option in Py3.5+
nthreads = int(nthreads) if nthreads else None
from Cython.Build.Dependencies import cythonize
self.distribution.ext_modules[:] = cythonize(
self.distribution.ext_modules, nthreads=nthreads, force=self.force)
super(new_build_ext, self).finalize_options()
def cython(self, line, cell):
"""Compile and import everything from a Cython code cell.
The contents of the cell are written to a `.pyx` file in the
directory `IPYTHONDIR/cython` using a filename with the hash of the code.
This file is then cythonized and compiled. The resulting module
is imported and all of its symbols are injected into the user's
namespace. The usage is similar to that of `%%cython_pyximport` but
you don't have to pass a module name::
%%cython
def f(x):
return 2.0*x
"""
args = parse_argstring(self.cython, line)
code = cell if cell.endswith('\n') else cell+'\n'
lib_dir = os.path.join(self.shell.ipython_dir, 'cython')
cython_include_dirs = ['.']
force = args.force
quiet = True
ctx = Context(cython_include_dirs, default_options)
key = code, sys.version_info, sys.executable, Cython.__version__
module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest()
module_path = os.path.join(lib_dir, module_name+self.so_ext)
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
if force or not os.path.isfile(module_path):
c_include_dirs = args.include
if 'numpy' in code:
import numpy
c_include_dirs.append(numpy.get_include())
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
pyx_file = py3compat.cast_bytes_py2(pyx_file, encoding=sys.getfilesystemencoding())
with io.open(pyx_file, 'w', encoding='utf-8') as f:
f.write(code)
extension = Extension(
name = module_name,
sources = [pyx_file],
include_dirs = c_include_dirs,
extra_compile_args = args.compile_args,
libraries = args.lib,
)
build_extension = self._get_build_extension()
try:
build_extension.extensions = cythonize([extension], ctx=ctx, quiet=quiet)
except CompileError:
return
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
build_extension.run()
self._code_cache[key] = module_name
module = imp.load_dynamic(module_name, module_path)
self._import_all(module)
def cython_compile(path_pattern, options):
pool = None
paths = map(os.path.abspath, extended_iglob(path_pattern))
try:
for path in paths:
if options.build_inplace:
base_dir = path
while not os.path.isdir(base_dir) or is_package_dir(base_dir):
base_dir = os.path.dirname(base_dir)
else:
base_dir = None
if os.path.isdir(path):
# recursively compiling a package
paths = [os.path.join(path, '**', '*.%s' % ext)
for ext in ('py', 'pyx')]
else:
# assume it's a file(-like thing)
paths = [path]
cwd = os.getcwd()
try:
if base_dir:
os.chdir(base_dir)
ext_modules = cythonize(
paths,
nthreads=options.parallel,
exclude_failures=options.keep_going,
exclude=options.excludes,
compiler_directives=options.directives,
force=options.force,
quiet=options.quiet,
**options.options)
finally:
if base_dir:
os.chdir(cwd)
if ext_modules and options.build:
if len(ext_modules) > 1 and options.parallel > 1:
if pool is None:
try:
pool = multiprocessing.Pool(options.parallel)
except OSError:
pool = _FakePool()
pool.map_async(run_distutils, [
(base_dir, [ext]) for ext in ext_modules])
else:
run_distutils((base_dir, ext_modules))
except:
if pool is not None:
pool.terminate()
raise
else:
if pool is not None:
pool.close()
pool.join()
def build_extensions(self):
self.__reimport()
if profile:
from Cython.Compiler.Options import directive_defaults
directive_defaults['profile'] = True
print("Enabled profiling for the Cython binary modules")
from Cython.Build.Dependencies import cythonize
self.distribution.ext_modules[:] = cythonize(
self.distribution.ext_modules)
build_ext_orig.build_extensions(self)
def finalize_options(self):
# Generate files
from autogen import rebuild
rebuild()
import sys
# Check dependencies
try:
from Cython.Build.Dependencies import cythonize
except ImportError as E:
sys.stderr.write("Error: {0}\n".format(E))
sys.stderr.write("The installation of ppl requires Cython\n")
sys.exit(1)
# cysignals not yet integrated
# try:
# # We need the header files for cysignals at compile-time
# import cysignals
# except ImportError as E:
# sys.stderr.write("Error: {0}\n".format(E))
# sys.stderr.write("The installation of ppl requires cysignals\n")
# sys.exit(1)
compile_time_env = {}
if compile_from_sage:
version = sage.env.SAGE_VERSION.split('.')
major = int(version[0])
minor = int(version[1])
if len(version) == 3:
beta = int(version[2][4:])
else:
beta = -1
sage_version = (major,minor,beta)
# trac ticket #22970 changes datastructure of rational matrices
# merged in 8.0.beta9
compile_time_env['SAGE_RAT_MAT_ARE_FMPQ_T_MAT'] = (sage_version >= (8,0,9))
self.distribution.ext_modules[:] = cythonize(
self.distribution.ext_modules,
compile_time_env=compile_time_env,
include_path=sys.path)
_build_ext.finalize_options(self)
def cythonize(self, extensions):
return cythonize(extensions,
build_dir=cythonize_dir, include_path=["src", os.path.join(cythonize_dir, "src")])
def compile(self, walked_jitted_funcs=None): #compile and load c_func
if self.compile_state=='not_compiled':
#print self.func_name, 'compiling'
self.compile_state='compiling'
if walked_jitted_funcs==None:
self.walked_jitted_funcs=[]
py_src=get_func_source(self.py_func)
args=get_args(self.py_func)
body=get_body(py_src)
#print py_src
indent=get_indent(body)
func_def='cpdef'
func_ast=ast.parse(py_src)
jvisit(func_ast, args, self.func_globals, self.called_jitted_funcs, self.local_names, self.load_names)
#print called_jitted_funcs
for jitted_func, mode, extra in self.called_jitted_funcs:
if mode=='direct':
self.putln("from %s cimport %s__c"%(jitted_func.module_name, jitted_func.func_name))
body=re.sub(r'\b%s *\('%jitted_func.func_name, '%s__c('%(jitted_func.func_name), body)
#elif mode=='attribute':#fix
#self.putln('cimport %s'%(jitted_func.module_name,))
#attribute_func_name=extra
##
#body=re.sub('%s *\('%attribute_func_name, '%s.%s__c('%(jitted_func.module_name, jitted_func.func_name), body)
self.putln()
#process signature
formated_sig=self.sig.replace(' ','')
if formated_sig:
return_type, arg_types=parse_sig(formated_sig)
assert len(args)==len(arg_types), "the function %s has %s args but the signature has %s args "%(func_name, len(args), len(arg_types))
func_args=[]
for arg_name, arg_type in zip(args, arg_types):
func_args.append('%s %s'%(arg_type, arg_name))
func_args=", ".join(func_args)
else:
func_args=', '.join(args)
return_type=''
#directives
#print "in"
directive_decorates=[]
directives=['wraparound', 'boundscheck']
for directive in directives:
value=self.options.get(directive)
if value != None:
directive_decorates.append('@cython.%s(%s)\n'%(directive, value))
if directive_decorates:
self.putln('cimport cython')
for decorate in directive_decorates:
self.putln(decorate)
nogil='nogil' if self.options.get("nogil") else ''
#head
#add like 'nogil'
cfunc_head='cdef %s %s__c(%s) %s'%(return_type, self.func_name, func_args, nogil)
pyfunc_head='def %s__p(%s)'%(self.func_name, ', '.join(args))
#func_head_1='\n%s %s(%s)%s'%('cdef', func_name, func_args, '')
self.putln(cfunc_head,'pxd')
#self.pxd_buf.write(func_init_head+"\n")
if self.load_names:
init_func_head='def %s_init(f_globals)'%(self.func_name)
globals_name='_%s_GLOBALS'%self.func_name.upper()
self.putln('%s = None\n'%globals_name)
self.putln(init_func_head+':')
self.putln(indent+'global %s'%globals_name)
self.putln(indent+'%s = f_globals'%globals_name)
self.putln()
self.putln(cfunc_head+':')
#cdef vars
locals=self.options.get('locals')
if type(locals) is str:
self.putln(indent+"cdef:")
for line in locals.strip().split('\n'):
line=line.strip()
if line[-1]==';':
line=line[:-1]
self.putln(indent*2+line)
#if type(locals) is str:
# for type_name, vars_name in parse_cdeclare(locals):
# cdef=indent+'cdef %s %s\n'%(type_name, vars_name)
# pyx_buf.write(cdef)
#body
#self.pyx_buf.write(indent+'%s=sys._getframe(1).f_locals\n'%func_globals_name)
if self.load_names:
self.putln(indent+'#load globals')
for name in self.load_names:
self.putln(indent+"%s = %s['%s']"%(name, globals_name, name))
self.putln()
#self.putln(indent+'print "%s", %s["__file__"]'%(self.func_name, func_globals_name))
self.putln(body)
self.putln()
self.putln(pyfunc_head+':')
self.putln(indent+'return %s__c(%s)'%(self.func_name, ', '.join(args)))
self.pyx_src=self.pyx_buf.getvalue()
self.pxd_src=self.pxd_buf.getvalue()
file_name=self.func_globals.get('__file__')
if file_name:
func_file_dir=os.path.dirname(os.path.abspath(file_name))
else:
func_file_dir=os.getcwd()
cython_compile_dir=os.path.join(func_file_dir, '__cython_compile__')
if not os.path.exists(cython_compile_dir):
os.mkdir(cython_compile_dir)
key = self.pyx_src+str(sys.version_info)+sys.executable+cython.__version__
self.module_dir=os.path.join(cython_compile_dir, self.module_name+'__'+hashlib.md5(key.encode('utf-8')).hexdigest() )
if not os.path.exists(self.module_dir):
os.mkdir(self.module_dir)
pyx_file=os.path.join(self.module_dir, self.module_name+'.pyx')
pxd_file=os.path.join(self.module_dir, self.module_name+'.pxd')
so_ext='.pyd'
so_file=os.path.join(self.module_dir, self.module_name+so_ext)
init_file=os.path.join(self.module_dir, '__init__.py')
if not os.path.exists(so_file):
fw=open(pyx_file,"w")
fw.write(self.pyx_src)
fw.close()
fw=open(pxd_file,"w")
fw.write(self.pxd_src)
fw.close()
fw=open(init_file, "w")
fw.close()
for jitted_func,_,_ in self.called_jitted_funcs:
#print jitted_func.func_name, jitted_func.compile_state
if jitted_func.compile_state=='not_compiled':
if walked_jitted_funcs==None:
jitted_func.compile(self.walked_jitted_funcs)
else:
jitted_func.compile(walked_jitted_funcs)
#check
if not os.path.exists(so_file):
extension = Extension(name = self.module_name,
sources = [pyx_file],
#include_dirs = c_include_dirs,
#extra_compile_args = cflags
)
#find called_jitted_funcs's module path
cython_include_dirs=set()
for jitted_func, _, _ in self.called_jitted_funcs:
cython_include_dirs.add(jitted_func.module_dir)
print 'cython_include_dirs', self.func_name, cython_include_dirs
build_extension = _get_build_extension()
build_extension.extensions = cythonize([extension],
annotate=True,
include_path=list(cython_include_dirs),
#quiet=quiet
)
temp_dir=os.path.join(cython_compile_dir, "__build_temp__")
build_extension.build_temp = temp_dir
build_extension.build_lib = self.module_dir
#print "build"
build_extension.run()
self.compile_state='compiled'
self.so_file=so_file
if walked_jitted_funcs==None:
self.walked_jitted_funcs.append(self)
else:
walked_jitted_funcs.append(self)
def cython_inline(code,
types='aggressive',
lib_dir=os.path.expanduser('~/.cython/inline'),
include_dirs=['.'],
locals=None,
globals=None,
**kwds):
code, literals = strip_string_literals(code)
code = strip_common_indent(code)
ctx = Context(include_dirs, default_options)
if locals is None:
locals = inspect.currentframe().f_back.f_back.f_locals
if globals is None:
globals = inspect.currentframe().f_back.f_back.f_globals
try:
for symbol in unbound_symbols(code):
if symbol in kwds:
continue
elif symbol in locals:
kwds[symbol] = locals[symbol]
elif symbol in globals:
kwds[symbol] = globals[symbol]
else:
print "Couldn't find ", symbol
except AssertionError:
# Parsing from strings not fully supported (e.g. cimports).
print "Could not parse code as a string (to extract unbound symbols)."
arg_names = kwds.keys()
arg_names.sort()
arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names])
key = code, arg_sigs, sys.version_info, sys.executable, Cython.__version__
module_name = "_cython_inline_" + hashlib.md5(str(key)).hexdigest()
# # TODO: Does this cover all the platforms?
# if (not os.path.exists(os.path.join(lib_dir, module_name + ".so")) and
# not os.path.exists(os.path.join(lib_dir, module_name + ".dll"))):
try:
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
if lib_dir not in sys.path:
sys.path.append(lib_dir)
__import__(module_name)
except ImportError:
c_include_dirs = []
cimports = []
qualified = re.compile(r'([.\w]+)[.]')
for type, _ in arg_sigs:
m = qualified.match(type)
if m:
cimports.append('\ncimport %s' % m.groups()[0])
# one special case
if m.groups()[0] == 'numpy':
import numpy
c_include_dirs.append(numpy.get_include())
module_body, func_body = extract_func_code(code)
params = ', '.join(['%s %s' % a for a in arg_sigs])
module_code = """
%(module_body)s
%(cimports)s
def __invoke(%(params)s):
%(func_body)s
""" % {'cimports': '\n'.join(cimports), 'module_body': module_body, 'params': params, 'func_body': func_body }
for key, value in literals.items():
module_code = module_code.replace(key, value)
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
open(pyx_file, 'w').write(module_code)
extension = Extension(
name = module_name,
sources = [pyx_file],
include_dirs = c_include_dirs)
build_extension = build_ext(Distribution())
build_extension.finalize_options()
build_extension.extensions = cythonize([extension])
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
build_extension.run()
_code_cache[key] = module_name
arg_list = [kwds[arg] for arg in arg_names]
return __import__(module_name).__invoke(*arg_list)
include_dirs=[numpy.get_include()],
libraries=[fftwlib])
srft = Extension("fjlt.SubsampledRandomizedFourrierTransform1d",
sources=["fjlt/SubsampledRandomizedFourrierTransform1d.pyx", "fjlt/SubsampledRandomizedFourrierTransform1d.pxd"],
include_dirs=[numpy.get_include()],
libraries=[fftwlib])
demo = Extension("fjlt.demo_cython",
sources=["demo_cython.pyx"],
include_dirs=[numpy.get_include()],
libraries=[fftwlib])
# setup(ext_modules=[random_projection, srft],
# cmdclass={'build_ext': build_ext})
exec(open('fjlt/version.py').read())
setup(name='FJLT',
version=__version__, # @UndefinedVariable
description='Fast Johnson Lindenstrauss Transform',
author='Gabriel Krummenacher',
author_email='*****@*****.**',
url='http://people.inf.ethz.ch/kgabriel/software.html',
packages=['fjlt'],
# py_modules=['demo'],
requires=['numpy'],
ext_modules=cythonize([random_projection, srft, demo]),
cmdclass={'build_ext': build_ext}
)
libdir + '/include/OGRE',
np.get_include()
]
extensions = []
for i in glob.glob('./*.pyx'):
e = Extension(os.path.basename(i)[:-4], [i],
define_macros=[('BOOST_SYSTEM_NO_DEPRECATED', None)],
include_dirs=includedirs,
libraries=libraries,
extra_compile_args=extra_compile_args,
compiler_directives=compiledirs,
language='c++')
extensions.append(e)
ext = cythonize(extensions, include_path=['.', '../renderer'])
# HORRIBLE KLUDGE: need to get around problem of Cython generating code which relies on CPython having been compiled with --with-fpectl
# The macros PyFTE_*_PROTECT invoke symbols which don't exist with Anaconda builds so they need to be removed which appears to be safe.
# See https://github.com/numpy/numpy/issues/8415 http://www.psf.upfronthosting.co.za/issue29137
for cppfile in glob.glob('*.cpp'):
sys.stdout.write('Processing %s\n' % cppfile)
sys.stdout.flush()
cpplines = open(cppfile).readlines()
with open(cppfile, 'w') as o:
for line in cpplines:
if 'PyFPE_START_PROTECT' not in line and 'PyFPE_END_PROTECT' not in line: # remove the symbol lines from the source
o.write(line)
setup(ext_modules=ext)
def wrap(py_func):
#clear buf
pyx_buf.seek(0)
pyx_buf.truncate()
pxd_buf.seek(0)
pxd_buf.truncate()
called_jitted_funcs = set()
local_names = set() #in py_func
load_names = set() #in py_func
func_name = py_func.__name__
func_globals_name = 'func_%s_globals' % func_name
#func_globals=sys._getframe(1).f_locals #get caller's scope
func_globals = py_func.func_globals
py_src = get_func_source(py_func)
args = get_args(py_func)
body = get_body(py_src)
#print py_src
indent = get_indent(body)
func_def = 'cpdef'
func_ast = ast.parse(py_src)
jvisit(func_ast, func_globals, called_jitted_funcs, local_names,
load_names)
#print called_jitted_funcs
for jitted_func, mode, extra in called_jitted_funcs:
if mode == 'direct':
pyx_buf.write("from %s cimport %s\n" %
(jitted_func.module_name, jitted_func.func_name))
elif mode == 'attribute':
pyx_buf.write('cimport %s\n' % (jitted_func.module_name, ))
attribute_func_name = extra
#
body = re.sub(
'%s *\(' % attribute_func_name, '%s.%s(' %
(jitted_func.module_name, jitted_func.func_name), body)
pyx_buf.write('\n')
pyx_buf.write('''
import sys
%s=sys._getframe(1).f_locals
''' % func_globals_name)
for name in load_names:
pyx_buf.write("%s=%s.get('%s')\n" %
(name, func_globals_name, name))
pyx_buf.write('\n')
#process signature
formated_sig = sig.replace(' ', '')
if formated_sig:
return_type, arg_types = parse_sig(formated_sig)
assert len(args) == len(
arg_types
), "the function %s has %s args but the signature has %s args " % (
func_name, len(args), len(arg_types))
func_args = []
for arg_name, arg_type in zip(args, arg_types):
func_args.append('%s %s' % (arg_type, arg_name))
func_args = ", ".join(func_args)
else:
func_args = ', '.join(args)
return_type = ''
#directives
#print "in"
directive_decorates = []
directives = ['wraparound', 'boundscheck']
for directive in directives:
value = kwds.get(directive)
if value != None:
directive_decorates.append('@cython.%s(%s)\n' %
(directive, value))
if directive_decorates:
pyx_buf.write('cimport cython\n\n')
for decorate in directive_decorates:
pyx_buf.write(decorate)
nogil = 'nogil' if kwds.get("nogil") else ''
#head
#add like 'nogil'
func_head = '%s %s %s(%s) %s' % (func_def, return_type, func_name,
func_args, nogil)
#func_head_1='\n%s %s(%s)%s'%('cdef', func_name, func_args, '')
pxd_buf.write(func_head + "\n")
pyx_buf.write(func_head + ':\n')
#cdef vars
locals = kwds.get('locals')
if type(locals) is str:
pyx_buf.write(indent + "cdef:\n")
for line in locals.strip().split('\n'):
line = line.strip()
if line[-1] == ';':
line = line[:-1]
pyx_buf.write(indent * 2 + line + '\n')
#if type(locals) is str:
# for type_name, vars_name in parse_cdeclare(locals):
# cdef=indent+'cdef %s %s\n'%(type_name, vars_name)
# pyx_buf.write(cdef)
#body
pyx_buf.write(body + '\n')
#build
pyx_src = pyx_buf.getvalue()
pxd_src = pxd_buf.getvalue()
base_name = os.path.basename(func_globals['__file__'])
file_name = os.path.splitext(base_name)[0]
key = pyx_src + str(
sys.version_info) + sys.executable + cython.__version__
hashed_module_name = file_name + '_' + func_name + '__' + hashlib.md5(
key.encode('utf-8')).hexdigest()
caller_file_dir = os.path.dirname(
os.path.abspath(func_globals['__file__']))
#print 'caller_file_dir', caller_file_dir
module_dir = os.path.join(caller_file_dir, '__cython_compile__')
if not os.path.exists(module_dir):
os.mkdir(module_dir)
pyx_file = os.path.join(module_dir, hashed_module_name + '.pyx')
pxd_file = os.path.join(module_dir, hashed_module_name + '.pxd')
so_ext = '.pyd'
so_file = os.path.join(module_dir, hashed_module_name + so_ext)
init_file = os.path.join(module_dir, '__init__.py')
#check
if not os.path.exists(so_file):
fw = open(pyx_file, "w")
fw.write(pyx_src)
fw.close()
fw = open(pxd_file, "w")
fw.write(pxd_src)
fw.close()
fw = open(init_file, "w")
fw.close()
extension = Extension(
name=hashed_module_name,
sources=[pyx_file],
#include_dirs = c_include_dirs,
#extra_compile_args = cflags
)
#find called_jitted_funcs's module path
cython_include_dirs = []
for jitted_func, _, _ in called_jitted_funcs:
cython_include_dirs.append(jitted_func.module_dir)
#print 'cython_include_dirs', cython_include_dirs
build_extension = _get_build_extension()
build_extension.extensions = cythonize(
[extension],
annotate=True,
include_path=cython_include_dirs,
#quiet=quiet
)
temp_dir = os.path.join(module_dir, "__build_temp__")
build_extension.build_temp = temp_dir
build_extension.build_lib = module_dir
#print "build"
build_extension.run()
compiled_module = imp.load_dynamic(hashed_module_name, so_file)
c_func = getattr(compiled_module, func_name)
return JittedFunc(c_func, py_func, func_name, hashed_module_name,
module_dir, pyx_src)
def compile(self, walked_jitted_funcs=None): #compile and load c_func
if self.compile_state == 'not_compiled':
#print self.func_name, 'compiling'
self.compile_state = 'compiling'
if walked_jitted_funcs == None:
self.walked_jitted_funcs = []
py_src = get_func_source(self.py_func)
args = get_args(self.py_func)
body = get_body(py_src)
#print py_src
indent = get_indent(body)
func_def = 'cpdef'
func_ast = ast.parse(py_src)
jvisit(func_ast, args, self.func_globals, self.called_jitted_funcs,
self.local_names, self.load_names)
#print called_jitted_funcs
for jitted_func, mode, extra in self.called_jitted_funcs:
if mode == 'direct':
self.putln(
"from %s cimport %s__c" %
(jitted_func.module_name, jitted_func.func_name))
body = re.sub(r'\b%s *\(' % jitted_func.func_name,
'%s__c(' % (jitted_func.func_name), body)
#elif mode=='attribute':#fix
#self.putln('cimport %s'%(jitted_func.module_name,))
#attribute_func_name=extra
##
#body=re.sub('%s *\('%attribute_func_name, '%s.%s__c('%(jitted_func.module_name, jitted_func.func_name), body)
self.putln()
#process signature
formated_sig = self.sig.replace(' ', '')
if formated_sig:
return_type, arg_types = parse_sig(formated_sig)
assert len(args) == len(
arg_types
), "the function %s has %s args but the signature has %s args " % (
func_name, len(args), len(arg_types))
func_args = []
for arg_name, arg_type in zip(args, arg_types):
func_args.append('%s %s' % (arg_type, arg_name))
func_args = ", ".join(func_args)
else:
func_args = ', '.join(args)
return_type = ''
#directives
#print "in"
directive_decorates = []
directives = ['wraparound', 'boundscheck']
for directive in directives:
value = self.options.get(directive)
if value != None:
directive_decorates.append('@cython.%s(%s)\n' %
(directive, value))
if directive_decorates:
self.putln('cimport cython')
for decorate in directive_decorates:
self.putln(decorate)
nogil = 'nogil' if self.options.get("nogil") else ''
#head
#add like 'nogil'
cfunc_head = 'cdef %s %s__c(%s) %s' % (return_type, self.func_name,
func_args, nogil)
pyfunc_head = 'def %s__p(%s)' % (self.func_name, ', '.join(args))
#func_head_1='\n%s %s(%s)%s'%('cdef', func_name, func_args, '')
self.putln(cfunc_head, 'pxd')
#self.pxd_buf.write(func_init_head+"\n")
if self.load_names:
init_func_head = 'def %s_init(f_globals)' % (self.func_name)
globals_name = '_%s_GLOBALS' % self.func_name.upper()
self.putln('%s = None\n' % globals_name)
self.putln(init_func_head + ':')
self.putln(indent + 'global %s' % globals_name)
self.putln(indent + '%s = f_globals' % globals_name)
self.putln()
self.putln(cfunc_head + ':')
#cdef vars
locals = self.options.get('locals')
if type(locals) is str:
self.putln(indent + "cdef:")
for line in locals.strip().split('\n'):
line = line.strip()
if line[-1] == ';':
line = line[:-1]
self.putln(indent * 2 + line)
#if type(locals) is str:
# for type_name, vars_name in parse_cdeclare(locals):
# cdef=indent+'cdef %s %s\n'%(type_name, vars_name)
# pyx_buf.write(cdef)
#body
#self.pyx_buf.write(indent+'%s=sys._getframe(1).f_locals\n'%func_globals_name)
if self.load_names:
self.putln(indent + '#load globals')
for name in self.load_names:
self.putln(indent + "%s = %s['%s']" %
(name, globals_name, name))
self.putln()
#self.putln(indent+'print "%s", %s["__file__"]'%(self.func_name, func_globals_name))
self.putln(body)
self.putln()
self.putln(pyfunc_head + ':')
self.putln(indent + 'return %s__c(%s)' %
(self.func_name, ', '.join(args)))
self.pyx_src = self.pyx_buf.getvalue()
self.pxd_src = self.pxd_buf.getvalue()
file_name = self.func_globals.get('__file__')
if file_name:
func_file_dir = os.path.dirname(os.path.abspath(file_name))
else:
func_file_dir = os.getcwd()
cython_compile_dir = os.path.join(func_file_dir,
'__cython_compile__')
if not os.path.exists(cython_compile_dir):
os.mkdir(cython_compile_dir)
key = self.pyx_src + str(
sys.version_info) + sys.executable + cython.__version__
self.module_dir = os.path.join(
cython_compile_dir, self.module_name + '__' +
hashlib.md5(key.encode('utf-8')).hexdigest())
if not os.path.exists(self.module_dir):
os.mkdir(self.module_dir)
pyx_file = os.path.join(self.module_dir, self.module_name + '.pyx')
pxd_file = os.path.join(self.module_dir, self.module_name + '.pxd')
so_ext = '.pyd'
so_file = os.path.join(self.module_dir, self.module_name + so_ext)
init_file = os.path.join(self.module_dir, '__init__.py')
if not os.path.exists(so_file):
fw = open(pyx_file, "w")
fw.write(self.pyx_src)
fw.close()
fw = open(pxd_file, "w")
fw.write(self.pxd_src)
fw.close()
fw = open(init_file, "w")
fw.close()
for jitted_func, _, _ in self.called_jitted_funcs:
#print jitted_func.func_name, jitted_func.compile_state
if jitted_func.compile_state == 'not_compiled':
if walked_jitted_funcs == None:
jitted_func.compile(self.walked_jitted_funcs)
else:
jitted_func.compile(walked_jitted_funcs)
#check
if not os.path.exists(so_file):
extension = Extension(
name=self.module_name,
sources=[pyx_file],
#include_dirs = c_include_dirs,
#extra_compile_args = cflags
)
#find called_jitted_funcs's module path
cython_include_dirs = set()
for jitted_func, _, _ in self.called_jitted_funcs:
cython_include_dirs.add(jitted_func.module_dir)
print 'cython_include_dirs', self.func_name, cython_include_dirs
build_extension = _get_build_extension()
build_extension.extensions = cythonize(
[extension],
annotate=True,
include_path=list(cython_include_dirs),
#quiet=quiet
)
temp_dir = os.path.join(cython_compile_dir, "__build_temp__")
build_extension.build_temp = temp_dir
build_extension.build_lib = self.module_dir
#print "build"
build_extension.run()
self.compile_state = 'compiled'
self.so_file = so_file
if walked_jitted_funcs == None:
self.walked_jitted_funcs.append(self)
else:
walked_jitted_funcs.append(self)
def __init__(self,
file=None,
charset='utf-8',
model_name="anon_model",
model_code=None,
stanc_ret=None,
boost_lib=None,
eigen_lib=None,
verbose=False,
obfuscate_model_name=True):
if stanc_ret is None:
stanc_ret = pystan.api.stanc(
file=file,
charset=charset,
model_code=model_code,
model_name=model_name,
verbose=verbose,
obfuscate_model_name=obfuscate_model_name)
if not isinstance(stanc_ret, dict):
raise ValueError("stanc_ret must be an object returned by stanc.")
stanc_ret_keys = {
'status', 'model_code', 'model_cppname', 'cppcode', 'model_name'
}
if not all(n in stanc_ret_keys for n in stanc_ret):
raise ValueError("stanc_ret lacks one or more of the keys: "
"{}".format(str(stanc_ret_keys)))
elif stanc_ret['status'] != 0: # success == 0
raise ValueError("stanc_ret is not a successfully returned "
"dictionary from stanc.")
self.model_cppname = stanc_ret['model_cppname']
self.model_name = stanc_ret['model_name']
self.model_code = stanc_ret['model_code']
self.model_cppcode = stanc_ret['cppcode']
msg = "COMPILING THE C++ CODE FOR MODEL {} NOW."
logger.info(msg.format(self.model_name))
if verbose:
msg = "OS: {}, Python: {}, Cython {}".format(
sys.platform, sys.version, Cython.__version__)
logger.info(msg)
if boost_lib is not None:
# FIXME: allow boost_lib, eigen_lib to be specified
raise NotImplementedError
if eigen_lib is not None:
raise NotImplementedError
key = tuple([
self.model_code, self.model_cppcode, sys.version_info,
sys.executable
])
module_name = ("stanfit4" + self.model_name + '_' +
hashlib.md5(str(key).encode('utf-8')).hexdigest())
self._temp_dir = temp_dir = tempfile.mkdtemp()
lib_dir = os.path.join(temp_dir, 'pystan')
pystan_dir = os.path.dirname(__file__)
include_dirs = [
lib_dir,
pystan_dir,
os.path.join(pystan_dir, "stan", "src"),
os.path.join(pystan_dir, "stan", "lib", "stan_math_2.8.0"),
os.path.join(pystan_dir, "stan", "lib", "stan_math_2.8.0", "lib",
"eigen_3.2.4"),
os.path.join(pystan_dir, "stan", "lib", "stan_math_2.8.0", "lib",
"boost_1.58.0"),
np.get_include(),
]
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp')
with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile:
outfile.write(self.model_cppcode)
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx')
with io.open(pyx_template_file, 'r', encoding='utf-8') as infile:
s = infile.read()
template = string.Template(s)
with io.open(pyx_file, 'w', encoding='utf-8') as outfile:
s = template.safe_substitute(model_cppname=self.model_cppname)
outfile.write(s)
stan_macros = [
('BOOST_RESULT_OF_USE_TR1', None),
('BOOST_NO_DECLTYPE', None),
('BOOST_DISABLE_ASSERTS', None),
('EIGEN_NO_DEBUG', None),
]
extra_compile_args = [
'-O0',
'-ftemplate-depth-256',
'-Wno-unused-function',
'-Wno-uninitialized',
]
distutils.log.set_verbosity(verbose)
extension = Extension(name=module_name,
language="c++",
sources=[pyx_file],
define_macros=stan_macros,
include_dirs=include_dirs,
extra_compile_args=extra_compile_args)
cython_include_dirs = ['.', pystan_dir]
build_extension = _get_build_extension()
build_extension.extensions = cythonize(
[extension], include_path=cython_include_dirs, quiet=not verbose)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr)
if redirect_stderr:
# silence stderr for compilation
orig_stderr = pystan.misc._redirect_stderr()
try:
build_extension.run()
finally:
if redirect_stderr:
# restore stderr
os.dup2(orig_stderr, sys.stderr.fileno())
self.module = load_module(module_name, lib_dir)
self.fit_class = getattr(self.module, "StanFit4Model")
def cython_inline(
code,
get_type=unsafe_type,
lib_dir=os.path.join(get_cython_cache_dir(), "inline"),
cython_include_dirs=None,
force=False,
quiet=False,
locals=None,
globals=None,
**kwds
):
if cython_include_dirs is None:
cython_include_dirs = ["."]
if get_type is None:
get_type = lambda x: "object"
code = to_unicode(code)
orig_code = code
code, literals = strip_string_literals(code)
code = strip_common_indent(code)
ctx = _create_context(tuple(cython_include_dirs))
if locals is None:
locals = inspect.currentframe().f_back.f_back.f_locals
if globals is None:
globals = inspect.currentframe().f_back.f_back.f_globals
try:
for symbol in unbound_symbols(code):
if symbol in kwds:
continue
elif symbol in locals:
kwds[symbol] = locals[symbol]
elif symbol in globals:
kwds[symbol] = globals[symbol]
else:
print("Couldn't find ", symbol)
except AssertionError:
if not quiet:
# Parsing from strings not fully supported (e.g. cimports).
print("Could not parse code as a string (to extract unbound symbols).")
cimports = []
for name, arg in kwds.items():
if arg is cython_module:
cimports.append("\ncimport cython as %s" % name)
del kwds[name]
arg_names = kwds.keys()
arg_names.sort()
arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names])
key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__
module_name = "_cython_inline_" + hashlib.md5(str(key).encode("utf-8")).hexdigest()
if module_name in sys.modules:
module = sys.modules[module_name]
else:
build_extension = None
if cython_inline.so_ext is None:
# Figure out and cache current extension suffix
build_extension = _get_build_extension()
cython_inline.so_ext = build_extension.get_ext_filename("")
module_path = os.path.join(lib_dir, module_name + cython_inline.so_ext)
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
if force or not os.path.isfile(module_path):
cflags = []
c_include_dirs = []
qualified = re.compile(r"([.\w]+)[.]")
for type, _ in arg_sigs:
m = qualified.match(type)
if m:
cimports.append("\ncimport %s" % m.groups()[0])
# one special case
if m.groups()[0] == "numpy":
import numpy
c_include_dirs.append(numpy.get_include())
# cflags.append('-Wno-unused')
module_body, func_body = extract_func_code(code)
params = ", ".join(["%s %s" % a for a in arg_sigs])
module_code = """
%(module_body)s
%(cimports)s
def __invoke(%(params)s):
%(func_body)s
""" % {
"cimports": "\n".join(cimports),
"module_body": module_body,
"params": params,
"func_body": func_body,
}
for key, value in literals.items():
module_code = module_code.replace(key, value)
pyx_file = os.path.join(lib_dir, module_name + ".pyx")
fh = open(pyx_file, "w")
try:
fh.write(module_code)
finally:
fh.close()
extension = Extension(
name=module_name, sources=[pyx_file], include_dirs=c_include_dirs, extra_compile_args=cflags
)
if build_extension is None:
build_extension = _get_build_extension()
build_extension.extensions = cythonize([extension], include_path=cython_include_dirs, quiet=quiet)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
build_extension.run()
module = imp.load_dynamic(module_name, module_path)
arg_list = [kwds[arg] for arg in arg_names]
return module.__invoke(*arg_list)
def create_fake_model(module_name):
"""Creates a fake model with specific module name.
Parameters
----------
module_name : str
Returns
-------
StanModel
Dummy StanModel object with specific module name.
"""
# reverse engineer the name
model_name, nonce = module_name.replace("stanfit4", "").rsplit("_", 1)
# create minimal code
model_code = """generated quantities {real fake_parameter=1;}"""
stanc_ret = pystan.api.stanc(model_code=model_code,
model_name=model_name,
obfuscate_model_name=False)
model_cppname = stanc_ret['model_cppname']
model_cppcode = stanc_ret['cppcode']
original_module_name = module_name
module_name = 'stanfit4{}_{}'.format(model_name, nonce)
# module name should be same
assert original_module_name == module_name
lib_dir = tempfile.mkdtemp()
pystan_dir = os.path.dirname(pystan.__file__)
include_dirs = [
lib_dir,
pystan_dir,
os.path.join(pystan_dir, "stan", "src"),
os.path.join(pystan_dir, "stan", "lib", "stan_math"),
os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib", "eigen_3.3.3"),
os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib", "boost_1.69.0"),
os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib", "sundials_4.1.0", "include"),
np.get_include(),
]
model_cpp_file = os.path.join(lib_dir, model_cppname + '.hpp')
with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile:
outfile.write(model_cppcode)
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx')
with io.open(pyx_template_file, 'r', encoding='utf-8') as infile:
s = infile.read()
template = string.Template(s)
with io.open(pyx_file, 'w', encoding='utf-8') as outfile:
s = template.safe_substitute(model_cppname=model_cppname)
outfile.write(s)
stan_macros = [
('BOOST_RESULT_OF_USE_TR1', None),
('BOOST_NO_DECLTYPE', None),
('BOOST_DISABLE_ASSERTS', None),
]
build_extension = _get_build_extension()
extra_compile_args = []
if platform.platform().startswith('Win'):
if build_extension.compiler in (None, 'msvc'):
logger.warning("MSVC compiler is not supported")
extra_compile_args = [
'/EHsc',
'-DBOOST_DATE_TIME_NO_LIB',
'/std:c++14',
] + extra_compile_args
else:
# Windows, but not msvc, likely mingw
# fix bug in MingW-W64
# use posix threads
extra_compile_args = [
'-O1',
'-ftemplate-depth-256',
'-Wno-unused-function',
'-Wno-uninitialized',
'-std=c++1y',
"-D_hypot=hypot",
"-pthread",
"-fexceptions",
] + extra_compile_args
else:
# linux or macOS
extra_compile_args = [
'-O1',
'-ftemplate-depth-256',
'-Wno-unused-function',
'-Wno-uninitialized',
'-std=c++1y',
] + extra_compile_args
extension = Extension(name=module_name,
language="c++",
sources=[pyx_file],
define_macros=stan_macros,
include_dirs=include_dirs,
extra_compile_args=extra_compile_args)
cython_include_dirs = ['.', pystan_dir]
build_extension.extensions = cythonize([extension],
include_path=cython_include_dirs,
quiet=True)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
build_extension.run()
module = load_module(module_name, lib_dir)
shutil.rmtree(lib_dir, ignore_errors=True)
return module
def __init__(self, file=None, charset='utf-8', model_name="anon_model",
model_code=None, stanc_ret=None, boost_lib=None,
eigen_lib=None, save_dso=True, verbose=False, **kwargs):
if stanc_ret is None:
stanc_ret = pystan.api.stanc(file=file,
model_code=model_code,
model_name=model_name,
verbose=verbose, **kwargs)
if not isinstance(stanc_ret, dict):
raise ValueError("stanc_ret must be an object returned by stanc.")
stanc_ret_keys = {'status', 'model_code', 'model_cppname',
'cppcode', 'model_name'}
if not all(n in stanc_ret_keys for n in stanc_ret):
raise ValueError("stanc_ret lacks one or more of the keys: "
"{}".format(str(stanc_ret_keys)))
elif stanc_ret['status'] != 0: # success == 0
if (file==None and os.path.exists(model_code)):
raise ValueError("stanc_ret is not a successfully returned "
"dictionary from stanc. Have you passed "
"a stan code filename as model_code?")
else:
raise ValueError("stanc_ret is not a successfully returned "
"dictionary from stanc.")
self.model_cppname = stanc_ret['model_cppname']
self.model_name = stanc_ret['model_name']
self.model_code = model_code
self.model_cppcode = stanc_ret['cppcode']
self.save_dso = save_dso
msg = "COMPILING THE C++ CODE FOR MODEL {} NOW."
logger.warning(msg.format(self.model_name))
if verbose:
msg = "OS: {}, Python: {}, Cython {}".format(sys.platform,
sys.version,
Cython.__version__)
logger.info(msg)
if boost_lib is not None:
# FIXME: allow boost_lib, eigen_lib to be specified
raise NotImplementedError
if eigen_lib is not None:
raise NotImplementedError
key = tuple([self.model_code, self.model_cppcode, sys.version_info,
sys.executable])
module_name = ("stanfit4" + self.model_name + '_' +
hashlib.md5(str(key).encode('utf-8')).hexdigest())
temp_dir = tempfile.mkdtemp()
lib_dir = os.path.join(temp_dir, 'pystan')
pystan_dir = os.path.dirname(__file__)
include_dirs = [lib_dir,
pystan_dir,
os.path.join(pystan_dir, "stan/src"),
os.path.join(pystan_dir, "stan/lib/eigen_3.2.0"),
os.path.join(pystan_dir, "stan/lib/boost_1.54.0")]
library_dirs = [os.path.join(pystan_dir, "bin")]
libraries = ['stan']
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp')
with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile:
outfile.write(self.model_cppcode)
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx')
with io.open(pyx_template_file, 'r', encoding='utf-8') as infile:
s = infile.read()
template = string.Template(s)
with io.open(pyx_file, 'w', encoding='utf-8') as outfile:
s = template.safe_substitute(model_cppname=self.model_cppname)
outfile.write(s)
extra_compile_args = ['-O3', '-ftemplate-depth-256']
extension = Extension(name=module_name,
language="c++",
sources=[pyx_file],
include_dirs=include_dirs,
library_dirs=library_dirs,
libraries=libraries,
extra_compile_args=extra_compile_args)
cython_include_dirs = ['.', pystan_dir]
build_extension = _get_build_extension()
build_extension.extensions = cythonize([extension],
include_path=cython_include_dirs,
quiet=not verbose)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr)
if redirect_stderr:
# silence stderr for compilation
orig_stderr = pystan.misc._redirect_stderr()
try:
build_extension.run()
finally:
if redirect_stderr:
# restore stderr
os.dup2(orig_stderr, sys.stderr.fileno())
self.module = load_module(module_name, lib_dir)
self.fit_class = getattr(self.module, "StanFit4" + self.model_cppname)
from Cython.Distutils import build_ext
cythonize_switch = True
del sys.argv[sys.argv.index('--cythonize')]
ext = 'pyx'
directives = {}
directives['linetrace'] = False
directives['binding'] = False
directives['profile'] = False
cmdclass = {'build_ext': build_ext}
else:
from setuptools.command.build_ext import build_ext # @NoMove @Reimport
cythonize_switch = False
ext = 'c'
cmdclass = {'build_ext': build_ext}
ext_modules = [Extension('cydatemap.cydatemap',
[os.path.join('cydatemap',
'cydatemap.%s' % ext)])]
setup(name='cydatemap',
version=versioneer.get_version(),
cmdclass=versioneer.get_cmdclass(cmdclass),
author='Jason Rudy',
author_email='*****@*****.**',
url='https://github.com/jcrudy/cydatemap',
packages=find_packages(),
package_data={'cydatemap': ['cydatemap.pxd', 'cydatemap.pyx']},
ext_modules = cythonize(ext_modules, compiler_directives=directives) if cythonize_switch else ext_modules,
install_requires=[]
)
)
try:
from Cython.Build.Dependencies import cythonize
_proesmans_lib_extension = Extension(
str("pysteps.motion._proesmans"),
sources=[str("pysteps/motion/_proesmans.pyx")],
**_proesmans_extension_arguments)
_vet_lib_extension = Extension(str("pysteps.motion._vet"),
sources=[str("pysteps/motion/_vet.pyx")],
**_vet_extension_arguments)
external_modules = cythonize(
[_proesmans_lib_extension, _vet_lib_extension],
force=True,
language_level=3)
except ImportError:
_proesmans_lib_extension = Extension(
str("pysteps.motion._proesmans"),
sources=[str("pysteps/motion/_proesmans.c")],
**_proesmans_extension_arguments)
_vet_lib_extension = Extension(str("pysteps.motion._vet"),
sources=[str("pysteps/motion/_vet.c")],
**_vet_extension_arguments)
external_modules = [_proesmans_lib_extension, _vet_lib_extension]
requirements = [
"numpy", "attrdict", "jsmin", "scipy", "matplotlib", "jsonschema"
]
def cython(self, line, cell):
"""Compile and import everything from a Cython code cell.
The contents of the cell are written to a `.pyx` file in the
directory `IPYTHONDIR/cython` using a filename with the hash of the
code. This file is then cythonized and compiled. The resulting module
is imported and all of its symbols are injected into the user's
namespace. The usage is similar to that of `%%cython_pyximport` but
you don't have to pass a module name::
%%cython
def f(x):
return 2.0*x
To compile OpenMP codes, pass the required `--compile-args`
and `--link-args`. For example with gcc::
%%cython --compile-args=-fopenmp --link-args=-fopenmp
...
"""
args = magic_arguments.parse_argstring(self.cython, line)
code = cell if cell.endswith('\n') else cell+'\n'
lib_dir = os.path.join(get_ipython_cache_dir(), 'cython')
quiet = True
key = code, sys.version_info, sys.executable, Cython.__version__
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
if args.force:
# Force a new module name by adding the current time to the
# key which is hashed to determine the module name.
key += time.time(),
if args.name:
module_name = py3compat.unicode_to_str(args.name)
else:
module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest()
module_path = os.path.join(lib_dir, module_name + self.so_ext)
have_module = os.path.isfile(module_path)
need_cythonize = not have_module
if args.annotate:
html_file = os.path.join(lib_dir, module_name + '.html')
if not os.path.isfile(html_file):
need_cythonize = True
if need_cythonize:
c_include_dirs = args.include
if 'numpy' in code:
import numpy
c_include_dirs.append(numpy.get_include())
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
pyx_file = py3compat.cast_bytes_py2(pyx_file, encoding=sys.getfilesystemencoding())
with io.open(pyx_file, 'w', encoding='utf-8') as f:
f.write(code)
extension = Extension(
name = module_name,
sources = [pyx_file],
include_dirs = c_include_dirs,
library_dirs = args.library_dirs,
extra_compile_args = args.compile_args,
extra_link_args = args.link_args,
libraries = args.lib,
language = 'c++' if args.cplus else 'c',
)
build_extension = self._get_build_extension()
try:
opts = dict(
quiet=quiet,
annotate = args.annotate,
force = True,
)
build_extension.extensions = cythonize([extension], **opts)
except CompileError:
return
if not have_module:
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
build_extension.run()
self._code_cache[key] = module_name
module = imp.load_dynamic(module_name, module_path)
self._import_all(module)
if args.annotate:
try:
with io.open(html_file, encoding='utf-8') as f:
annotated_html = f.read()
except IOError as e:
# File could not be opened. Most likely the user has a version
# of Cython before 0.15.1 (when `cythonize` learned the
# `force` keyword argument) and has already compiled this
# exact source without annotation.
print('Cython completed successfully but the annotated '
'source could not be read.', file=sys.stderr)
print(e, file=sys.stderr)
else:
return display.HTML(self.clean_annotated_html(annotated_html))
def __init__(self,
file=None,
charset='utf-8',
model_name="anon_model",
model_code=None,
stanc_ret=None,
boost_lib=None,
eigen_lib=None,
save_dso=True,
verbose=False,
**kwargs):
if stanc_ret is None:
stanc_ret = pystan.api.stanc(file=file,
model_code=model_code,
model_name=model_name,
verbose=verbose,
**kwargs)
if not isinstance(stanc_ret, dict):
raise ValueError("stanc_ret must be an object returned by stanc.")
stanc_ret_keys = {
'status', 'model_code', 'model_cppname', 'cppcode', 'model_name'
}
if not all(n in stanc_ret_keys for n in stanc_ret):
raise ValueError("stanc_ret lacks one or more of the keys: "
"{}".format(str(stanc_ret_keys)))
elif stanc_ret['status'] != 0: # success == 0
raise ValueError("stanc_ret is not a successfully returned "
"dictionary from stanc.")
self.model_cppname = stanc_ret['model_cppname']
self.model_name = stanc_ret['model_name']
self.model_code = model_code
self.model_cppcode = stanc_ret['cppcode']
msg = "COMPILING THE C++ CODE FOR MODEL {} NOW."
logging.warn(msg.format(self.model_name))
if verbose:
msg = "OS: {}, Python: {}, Cython {}".format(
sys.platform, sys.version, Cython.__version__)
logging.info(msg)
if boost_lib is not None:
# FIXME: allow boost_lib, eigen_lib to be specified
raise NotImplementedError
if eigen_lib is not None:
raise NotImplementedError
key = tuple([
self.model_code, self.model_cppcode, sys.version_info,
sys.executable
])
module_name = ("stanfit4" + self.model_name + '_' +
hashlib.md5(str(key).encode('utf-8')).hexdigest())
temp_dir = tempfile.mkdtemp()
lib_dir = os.path.join(temp_dir, 'pystan')
pystan_dir = os.path.dirname(__file__)
include_dirs = [
lib_dir, pystan_dir,
os.path.join(pystan_dir, "stan/src"),
os.path.join(pystan_dir, "stan/lib/eigen_3.2.0"),
os.path.join(pystan_dir, "stan/lib/boost_1.54.0")
]
library_dirs = [os.path.join(pystan_dir, "bin")]
libraries = ['stan']
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp')
with open(model_cpp_file, 'w') as outfile:
outfile.write(self.model_cppcode)
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx')
with open(pyx_template_file) as infile:
s = infile.read()
template = string.Template(s)
with open(pyx_file, 'w') as outfile:
s = template.safe_substitute(model_cppname=self.model_cppname)
outfile.write(s)
extension = Extension(name=module_name,
language="c++",
sources=[pyx_file],
include_dirs=include_dirs,
library_dirs=library_dirs,
libraries=libraries,
extra_compile_args=['-O3'])
cython_include_dirs = ['.', pystan_dir]
build_extension = _get_build_extension()
build_extension.extensions = cythonize(
[extension], include_path=cython_include_dirs, quiet=not verbose)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr)
if redirect_stderr:
# silence stderr for compilation
orig_stderr = pystan.misc._redirect_stderr()
try:
build_extension.run()
finally:
if redirect_stderr:
# restore stderr
os.dup2(orig_stderr, sys.stderr.fileno())
module_path = lib_dir
self.module = load_module(module_name, module_path)
self.fit_class = getattr(self.module, "StanFit4" + self.model_cppname)
def build(force_rebuild=False):
import sys, os
import shutil
import glob
import numpy
import fcntl
from distutils.core import Distribution, Extension
from distutils.command.build_ext import build_ext
from distutils.sysconfig import get_config_vars, get_python_inc
import Cython
from Cython.Compiler.Main import Context, default_options
from Cython.Build.Dependencies import cythonize
source_dir = os.path.dirname(os.path.abspath(__file__))
build_dir = os.path.join(source_dir, 'bld')
source_dir = os.path.join(source_dir, 'pymalis')
source_files = [
os.path.join(source_dir, 'frontend.pyx'),
os.path.join(source_dir, 'malis_loss_layer.hpp'),
os.path.join(source_dir, 'malis_loss_layer.cpp'),
os.path.join(source_dir, 'c_frontend.h'),
]
source_files.sort()
module_name = '_pymalis'
# since this could be called concurrently, there is no good way to check
# whether the directory already exists
try:
os.makedirs(build_dir)
except:
pass
# make sure the same module is not build concurrently
with open(os.path.join(build_dir, module_name + '.lock'),
'w') as lock_file:
fcntl.lockf(lock_file, fcntl.LOCK_EX)
try:
if build_dir not in sys.path:
sys.path.append(build_dir)
if force_rebuild:
raise ImportError
else:
__import__(module_name)
print("Re-using already compiled pymalis version")
except ImportError:
print("Compiling pymalis in " + str(build_dir))
cython_include_dirs = ['.']
ctx = Context(cython_include_dirs, default_options)
scoring_function_include_dir = os.path.join(build_dir, module_name)
if not os.path.exists(scoring_function_include_dir):
os.makedirs(scoring_function_include_dir)
include_dirs = [
source_dir,
os.path.dirname(get_python_inc()),
numpy.get_include(),
]
# cython requires that the pyx file has the same name as the module
shutil.copy(os.path.join(source_dir, 'frontend.pyx'),
os.path.join(build_dir, module_name + '.pyx'))
shutil.copy(
os.path.join(source_dir, 'malis_loss_layer.cpp'),
os.path.join(build_dir, module_name + '_malis_loss_layer.cpp'))
# Remove the "-Wstrict-prototypes" compiler option, which isn't valid
# for C++.
cfg_vars = get_config_vars()
if "CFLAGS" in cfg_vars:
cfg_vars["CFLAGS"] = cfg_vars["CFLAGS"].replace(
"-Wstrict-prototypes", "")
extension = Extension(
module_name,
sources=[
os.path.join(build_dir, module_name + '.pyx'),
os.path.join(build_dir,
module_name + '_malis_loss_layer.cpp')
],
include_dirs=include_dirs,
language='c++',
extra_link_args=['-std=c++11'],
extra_compile_args=['-std=c++11', '-w'])
build_extension = build_ext(Distribution())
build_extension.finalize_options()
build_extension.extensions = cythonize([extension],
quiet=True,
nthreads=2)
build_extension.build_temp = build_dir
build_extension.build_lib = build_dir
build_extension.run()
def cython_inline(code,
get_type=unsafe_type,
lib_dir=os.path.join(get_cython_cache_dir(), 'inline'),
cython_include_dirs=['.'],
force=False,
quiet=False,
locals=None,
globals=None,
**kwds):
if get_type is None:
get_type = lambda x: 'object'
code = to_unicode(code)
orig_code = code
code, literals = strip_string_literals(code)
code = strip_common_indent(code)
ctx = _create_context(tuple(cython_include_dirs))
if locals is None:
locals = inspect.currentframe().f_back.f_back.f_locals
if globals is None:
globals = inspect.currentframe().f_back.f_back.f_globals
try:
for symbol in unbound_symbols(code):
if symbol in kwds:
continue
elif symbol in locals:
kwds[symbol] = locals[symbol]
elif symbol in globals:
kwds[symbol] = globals[symbol]
else:
print("Couldn't find ", symbol)
except AssertionError:
if not quiet:
# Parsing from strings not fully supported (e.g. cimports).
print(
"Could not parse code as a string (to extract unbound symbols)."
)
cimports = []
for name, arg in kwds.items():
if arg is cython_module:
cimports.append('\ncimport cython as %s' % name)
del kwds[name]
arg_names = kwds.keys()
arg_names.sort()
arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names])
key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__
module_name = "_cython_inline_" + hashlib.md5(
str(key).encode('utf-8')).hexdigest()
if module_name in sys.modules:
module = sys.modules[module_name]
else:
build_extension = None
if cython_inline.so_ext is None:
# Figure out and cache current extension suffix
build_extension = _get_build_extension()
cython_inline.so_ext = build_extension.get_ext_filename('')
module_path = os.path.join(lib_dir, module_name + cython_inline.so_ext)
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
if force or not os.path.isfile(module_path):
cflags = []
c_include_dirs = []
qualified = re.compile(r'([.\w]+)[.]')
for type, _ in arg_sigs:
m = qualified.match(type)
if m:
cimports.append('\ncimport %s' % m.groups()[0])
# one special case
if m.groups()[0] == 'numpy':
import numpy
c_include_dirs.append(numpy.get_include())
# cflags.append('-Wno-unused')
module_body, func_body = extract_func_code(code)
params = ', '.join(['%s %s' % a for a in arg_sigs])
module_code = """
%(module_body)s
%(cimports)s
def __invoke(%(params)s):
%(func_body)s
return locals()
""" % {
'cimports': '\n'.join(cimports),
'module_body': module_body,
'params': params,
'func_body': func_body
}
for key, value in literals.items():
module_code = module_code.replace(key, value)
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
fh = open(pyx_file, 'w')
try:
fh.write(module_code)
finally:
fh.close()
extension = Extension(name=module_name,
sources=[pyx_file],
include_dirs=c_include_dirs,
extra_compile_args=cflags)
if build_extension is None:
build_extension = _get_build_extension()
build_extension.extensions = cythonize(
[extension], include_path=cython_include_dirs, quiet=quiet)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
build_extension.run()
module = imp.load_dynamic(module_name, module_path)
arg_list = [kwds[arg] for arg in arg_names]
return module.__invoke(*arg_list)
def __init__(
self,
file=None,
charset="utf-8",
model_name="anon_model",
model_code=None,
stanc_ret=None,
boost_lib=None,
eigen_lib=None,
save_dso=True,
verbose=False,
obfuscate_model_name=True,
):
if stanc_ret is None:
stanc_ret = pystan.api.stanc(
file=file,
charset=charset,
model_code=model_code,
model_name=model_name,
verbose=verbose,
obfuscate_model_name=obfuscate_model_name,
)
if not isinstance(stanc_ret, dict):
raise ValueError("stanc_ret must be an object returned by stanc.")
stanc_ret_keys = {"status", "model_code", "model_cppname", "cppcode", "model_name"}
if not all(n in stanc_ret_keys for n in stanc_ret):
raise ValueError("stanc_ret lacks one or more of the keys: " "{}".format(str(stanc_ret_keys)))
elif stanc_ret["status"] != 0: # success == 0
raise ValueError("stanc_ret is not a successfully returned " "dictionary from stanc.")
self.model_cppname = stanc_ret["model_cppname"]
self.model_name = stanc_ret["model_name"]
self.model_code = stanc_ret["model_code"]
self.model_cppcode = stanc_ret["cppcode"]
self.save_dso = save_dso
msg = "COMPILING THE C++ CODE FOR MODEL {} NOW."
logger.info(msg.format(self.model_name))
if verbose:
msg = "OS: {}, Python: {}, Cython {}".format(sys.platform, sys.version, Cython.__version__)
logger.info(msg)
if boost_lib is not None:
# FIXME: allow boost_lib, eigen_lib to be specified
raise NotImplementedError
if eigen_lib is not None:
raise NotImplementedError
key = tuple([self.model_code, self.model_cppcode, sys.version_info, sys.executable])
module_name = "stanfit4" + self.model_name + "_" + hashlib.md5(str(key).encode("utf-8")).hexdigest()
self._temp_dir = temp_dir = tempfile.mkdtemp()
lib_dir = os.path.join(temp_dir, "pystan")
pystan_dir = os.path.dirname(__file__)
include_dirs = [
lib_dir,
pystan_dir,
os.path.join(pystan_dir, "stan/src"),
os.path.join(pystan_dir, "math"),
os.path.join(pystan_dir, "stan/lib/eigen_3.2.4"),
os.path.join(pystan_dir, "stan/lib/boost_1.58.0"),
np.get_include(),
]
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
model_cpp_file = os.path.join(lib_dir, self.model_cppname + ".hpp")
with io.open(model_cpp_file, "w", encoding="utf-8") as outfile:
outfile.write(self.model_cppcode)
pyx_file = os.path.join(lib_dir, module_name + ".pyx")
pyx_template_file = os.path.join(pystan_dir, "stanfit4model.pyx")
with io.open(pyx_template_file, "r", encoding="utf-8") as infile:
s = infile.read()
template = string.Template(s)
with io.open(pyx_file, "w", encoding="utf-8") as outfile:
s = template.safe_substitute(model_cppname=self.model_cppname)
outfile.write(s)
stan_macros = [
("BOOST_RESULT_OF_USE_TR1", None),
("BOOST_NO_DECLTYPE", None),
("BOOST_DISABLE_ASSERTS", None),
("EIGEN_NO_DEBUG", None),
]
extra_compile_args = ["-O0", "-ftemplate-depth-256", "-Wno-unused-function", "-Wno-uninitialized"]
distutils.log.set_verbosity(verbose)
extension = Extension(
name=module_name,
language="c++",
sources=[pyx_file],
define_macros=stan_macros,
include_dirs=include_dirs,
extra_compile_args=extra_compile_args,
)
cython_include_dirs = [".", pystan_dir]
build_extension = _get_build_extension()
build_extension.extensions = cythonize([extension], include_path=cython_include_dirs, quiet=not verbose)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr)
if redirect_stderr:
# silence stderr for compilation
orig_stderr = pystan.misc._redirect_stderr()
try:
build_extension.run()
finally:
if redirect_stderr:
# restore stderr
os.dup2(orig_stderr, sys.stderr.fileno())
self.module = load_module(module_name, lib_dir)
self.fit_class = getattr(self.module, "StanFit4Model")
def finalize_options(self):
from Cython.Build.Dependencies import cythonize
self.distribution.ext_modules[:] = cythonize(
self.distribution.ext_modules,
compile_time_env=_CYTHON_COMPILE_TIME_ENV)
super(my_build_ext, self).finalize_options()
def finalize_options(self):
if self.distribution.ext_modules:
from Cython.Build.Dependencies import cythonize
self.distribution.ext_modules[:] = cythonize(
self.distribution.ext_modules)
super(build_ext, self).finalize_options()
cythonize_switch = True
del sys.argv[sys.argv.index('--cythonize')]
ext = 'pyx'
directives = {}
directives['linetrace'] = False
directives['binding'] = False
directives['profile'] = False
cmdclass = {'build_ext': build_ext}
else:
from setuptools.command.build_ext import build_ext # @NoMove @Reimport
cythonize_switch = False
ext = 'c'
cmdclass = {'build_ext': build_ext}
ext_modules = [
Extension('sklearntools.interpolation',
[os.path.join('sklearntools', 'interpolation.%s' % ext)])
]
setup(name='sklearntools',
version=versioneer.get_version(),
cmdclass=versioneer.get_cmdclass(),
description='Tools for sklearn models',
author='Jason Rudy',
author_email='*****@*****.**',
url='https://gitlab.com/jcrudy/sklearntools',
packages=find_packages(),
package_data={'': ['sym/resources/*']},
ext_modules=cythonize(ext_modules, compiler_directives=directives)
if cythonize_switch else ext_modules,
install_requires=['scikit-learn>=0.17.0', 'pandas', 'numpy'])
def cython_inline(code,
get_type=unsafe_type,
lib_dir=os.path.join(get_cython_cache_dir(), 'inline'),
cython_include_dirs=['.'],
force=False,
quiet=False,
locals=None,
globals=None,
**kwds):
if get_type is None:
get_type = lambda x: 'object'
code = to_unicode(code)
orig_code = code
code, literals = strip_string_literals(code)
code = strip_common_indent(code)
ctx = Context(cython_include_dirs, default_options)
if locals is None:
locals = inspect.currentframe().f_back.f_back.f_locals
if globals is None:
globals = inspect.currentframe().f_back.f_back.f_globals
try:
for symbol in unbound_symbols(code):
if symbol in kwds:
continue
elif symbol in locals:
kwds[symbol] = locals[symbol]
elif symbol in globals:
kwds[symbol] = globals[symbol]
else:
print("Couldn't find ", symbol)
except AssertionError:
if not quiet:
# Parsing from strings not fully supported (e.g. cimports).
print("Could not parse code as a string (to extract unbound symbols).")
cimports = []
for name, arg in kwds.items():
if arg is cython_module:
cimports.append('\ncimport cython as %s' % name)
del kwds[name]
arg_names = kwds.keys()
arg_names.sort()
arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names])
key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__
module_name = "_cython_inline_" + hashlib.md5(str(key).encode('utf-8')).hexdigest()
so_ext = [ ext for ext,_,mod_type in imp.get_suffixes() if mod_type == imp.C_EXTENSION ][0]
module_path = os.path.join(lib_dir, module_name+so_ext)
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
if force or not os.path.isfile(module_path):
cflags = []
c_include_dirs = []
qualified = re.compile(r'([.\w]+)[.]')
for type, _ in arg_sigs:
m = qualified.match(type)
if m:
cimports.append('\ncimport %s' % m.groups()[0])
# one special case
if m.groups()[0] == 'numpy':
import numpy
c_include_dirs.append(numpy.get_include())
# cflags.append('-Wno-unused')
module_body, func_body = extract_func_code(code)
params = ', '.join(['%s %s' % a for a in arg_sigs])
module_code = """
%(module_body)s
%(cimports)s
def __invoke(%(params)s):
%(func_body)s
""" % {'cimports': '\n'.join(cimports), 'module_body': module_body, 'params': params, 'func_body': func_body }
for key, value in literals.items():
module_code = module_code.replace(key, value)
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
fh = open(pyx_file, 'w')
try:
fh.write(module_code)
finally:
fh.close()
extension = Extension(
name = module_name,
sources = [pyx_file],
include_dirs = c_include_dirs,
extra_compile_args = cflags)
dist = Distribution()
# Ensure the build respects distutils configuration by parsing
# the configuration files
config_files = dist.find_config_files()
dist.parse_config_files(config_files)
build_extension = build_ext(dist)
build_extension.finalize_options()
build_extension.extensions = cythonize([extension], ctx=ctx, quiet=quiet)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
build_extension.run()
_code_cache[key] = module_name
module = imp.load_dynamic(module_name, module_path)
arg_list = [kwds[arg] for arg in arg_names]
return module.__invoke(*arg_list)
def finalize_options(self):
if self.distribution.ext_modules:
from Cython.Build.Dependencies import cythonize
self.distribution.ext_modules[:] = cythonize(
self.distribution.ext_modules)
super(new_build_ext, self).finalize_options()
def wrap(py_func):
#clear buf
pyx_buf.seek(0)
pyx_buf.truncate()
pxd_buf.seek(0)
pxd_buf.truncate()
called_jitted_funcs=set()
local_names=set() #in py_func
load_names=set() #in py_func
func_name=py_func.__name__
func_globals_name='func_%s_globals'%func_name
#func_globals=sys._getframe(1).f_locals #get caller's scope
func_globals=py_func.func_globals
py_src=get_func_source(py_func)
args=get_args(py_func)
body=get_body(py_src)
#print py_src
indent=get_indent(body)
func_def='cpdef'
func_ast=ast.parse(py_src)
jvisit(func_ast, func_globals, called_jitted_funcs, local_names, load_names)
#print called_jitted_funcs
for jitted_func, mode, extra in called_jitted_funcs:
if mode=='direct':
pyx_buf.write("from %s cimport %s\n"%(jitted_func.module_name, jitted_func.func_name))
elif mode=='attribute':
pyx_buf.write('cimport %s\n'%(jitted_func.module_name,))
attribute_func_name=extra
#
body=re.sub('%s *\('%attribute_func_name, '%s.%s('%(jitted_func.module_name, jitted_func.func_name), body)
pyx_buf.write('\n')
pyx_buf.write('''
import sys
%s=sys._getframe(1).f_locals
'''%func_globals_name)
for name in load_names:
pyx_buf.write("%s=%s.get('%s')\n"%(name, func_globals_name, name))
pyx_buf.write('\n')
#process signature
formated_sig=sig.replace(' ','')
if formated_sig:
return_type, arg_types=parse_sig(formated_sig)
assert len(args)==len(arg_types), "the function %s has %s args but the signature has %s args "%(func_name, len(args), len(arg_types))
func_args=[]
for arg_name, arg_type in zip(args, arg_types):
func_args.append('%s %s'%(arg_type, arg_name))
func_args=", ".join(func_args)
else:
func_args=', '.join(args)
return_type=''
#directives
#print "in"
directive_decorates=[]
directives=['wraparound', 'boundscheck']
for directive in directives:
value=kwds.get(directive)
if value != None:
directive_decorates.append('@cython.%s(%s)\n'%(directive, value))
if directive_decorates:
pyx_buf.write('cimport cython\n\n')
for decorate in directive_decorates:
pyx_buf.write(decorate)
nogil='nogil' if kwds.get("nogil") else ''
#head
#add like 'nogil'
func_head='%s %s %s(%s) %s'%(func_def, return_type, func_name, func_args, nogil)
#func_head_1='\n%s %s(%s)%s'%('cdef', func_name, func_args, '')
pxd_buf.write(func_head+"\n")
pyx_buf.write(func_head+':\n')
#cdef vars
locals=kwds.get('locals')
if type(locals) is str:
pyx_buf.write(indent+"cdef:\n")
for line in locals.strip().split('\n'):
line=line.strip()
if line[-1]==';':
line=line[:-1]
pyx_buf.write(indent*2+line+'\n')
#if type(locals) is str:
# for type_name, vars_name in parse_cdeclare(locals):
# cdef=indent+'cdef %s %s\n'%(type_name, vars_name)
# pyx_buf.write(cdef)
#body
pyx_buf.write(body+'\n')
#build
pyx_src=pyx_buf.getvalue()
pxd_src=pxd_buf.getvalue()
base_name=os.path.basename(func_globals['__file__'])
file_name=os.path.splitext(base_name)[0]
key = pyx_src+str(sys.version_info)+sys.executable+cython.__version__
hashed_module_name = file_name+'_'+func_name+'__'+hashlib.md5(key.encode('utf-8')).hexdigest()
caller_file_dir=os.path.dirname(os.path.abspath(func_globals['__file__']))
#print 'caller_file_dir', caller_file_dir
module_dir=os.path.join(caller_file_dir, '__cython_compile__')
if not os.path.exists(module_dir):
os.mkdir(module_dir)
pyx_file=os.path.join(module_dir, hashed_module_name+'.pyx')
pxd_file=os.path.join(module_dir, hashed_module_name+'.pxd')
so_ext='.pyd'
so_file=os.path.join(module_dir, hashed_module_name+so_ext)
init_file=os.path.join(module_dir, '__init__.py')
#check
if not os.path.exists(so_file):
fw=open(pyx_file,"w")
fw.write(pyx_src)
fw.close()
fw=open(pxd_file,"w")
fw.write(pxd_src)
fw.close()
fw=open(init_file, "w")
fw.close()
extension = Extension(name = hashed_module_name,
sources = [pyx_file],
#include_dirs = c_include_dirs,
#extra_compile_args = cflags
)
#find called_jitted_funcs's module path
cython_include_dirs=[]
for jitted_func, _, _ in called_jitted_funcs:
cython_include_dirs.append(jitted_func.module_dir)
#print 'cython_include_dirs', cython_include_dirs
build_extension = _get_build_extension()
build_extension.extensions = cythonize([extension],
annotate=True,
include_path=cython_include_dirs,
#quiet=quiet
)
temp_dir=os.path.join(module_dir, "__build_temp__")
build_extension.build_temp = temp_dir
build_extension.build_lib = module_dir
#print "build"
build_extension.run()
compiled_module=imp.load_dynamic(hashed_module_name, so_file)
c_func=getattr(compiled_module, func_name)
return JittedFunc(c_func,
py_func,
func_name,
hashed_module_name,
module_dir,
pyx_src)
def __init__(self, file=None, charset='utf-8', model_name="anon_model",
model_code=None, stanc_ret=None, boost_lib=None,
eigen_lib=None, verbose=False, obfuscate_model_name=True,
extra_compile_args=None):
if stanc_ret is None:
stanc_ret = pystan.api.stanc(file=file,
charset=charset,
model_code=model_code,
model_name=model_name,
verbose=verbose,
obfuscate_model_name=obfuscate_model_name)
if not isinstance(stanc_ret, dict):
raise ValueError("stanc_ret must be an object returned by stanc.")
stanc_ret_keys = {'status', 'model_code', 'model_cppname',
'cppcode', 'model_name'}
if not all(n in stanc_ret_keys for n in stanc_ret):
raise ValueError("stanc_ret lacks one or more of the keys: "
"{}".format(str(stanc_ret_keys)))
elif stanc_ret['status'] != 0: # success == 0
raise ValueError("stanc_ret is not a successfully returned "
"dictionary from stanc.")
self.model_cppname = stanc_ret['model_cppname']
self.model_name = stanc_ret['model_name']
self.model_code = stanc_ret['model_code']
self.model_cppcode = stanc_ret['cppcode']
msg = "COMPILING THE C++ CODE FOR MODEL {} NOW."
logger.info(msg.format(self.model_name))
if verbose:
msg = "OS: {}, Python: {}, Cython {}".format(sys.platform,
sys.version,
Cython.__version__)
logger.info(msg)
if boost_lib is not None:
# FIXME: allow boost_lib, eigen_lib to be specified
raise NotImplementedError
if eigen_lib is not None:
raise NotImplementedError
# module_name needs to be unique so that each model instance has its own module
nonce = abs(hash((self.model_name, time.time())))
self.module_name = 'stanfit4{}_{}'.format(self.model_name, nonce)
lib_dir = tempfile.mkdtemp()
pystan_dir = os.path.dirname(__file__)
include_dirs = [
lib_dir,
pystan_dir,
os.path.join(pystan_dir, "stan", "src"),
os.path.join(pystan_dir, "stan", "lib", "stan_math_2.14.0"),
os.path.join(pystan_dir, "stan", "lib", "stan_math_2.14.0", "lib", "eigen_3.2.9"),
os.path.join(pystan_dir, "stan", "lib", "stan_math_2.14.0", "lib", "boost_1.62.0"),
os.path.join(pystan_dir, "stan", "lib", "stan_math_2.14.0", "lib", "cvodes_2.9.0", "include"),
np.get_include(),
]
model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp')
with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile:
outfile.write(self.model_cppcode)
pyx_file = os.path.join(lib_dir, self.module_name + '.pyx')
pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx')
with io.open(pyx_template_file, 'r', encoding='utf-8') as infile:
s = infile.read()
template = string.Template(s)
with io.open(pyx_file, 'w', encoding='utf-8') as outfile:
s = template.safe_substitute(model_cppname=self.model_cppname)
outfile.write(s)
stan_macros = [
('BOOST_RESULT_OF_USE_TR1', None),
('BOOST_NO_DECLTYPE', None),
('BOOST_DISABLE_ASSERTS', None),
]
# compile stan models with optimization (-O2)
# (stanc is compiled without optimization (-O0) currently, see #33)
if extra_compile_args is None:
extra_compile_args = [
'-O2',
'-ftemplate-depth-256',
'-Wno-unused-function',
'-Wno-uninitialized',
]
if platform.platform().startswith('Win'):
extra_compile_args = ['/EHsc', '-DBOOST_DATE_TIME_NO_LIB']
distutils.log.set_verbosity(verbose)
extension = Extension(name=self.module_name,
language="c++",
sources=[pyx_file],
define_macros=stan_macros,
include_dirs=include_dirs,
extra_compile_args=extra_compile_args)
cython_include_dirs = ['.', pystan_dir]
build_extension = _get_build_extension()
build_extension.extensions = cythonize([extension],
include_path=cython_include_dirs,
quiet=not verbose)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr)
if redirect_stderr:
# silence stderr for compilation
orig_stderr = pystan.misc._redirect_stderr()
try:
build_extension.run()
finally:
if redirect_stderr:
# restore stderr
os.dup2(orig_stderr, sys.stderr.fileno())
self.module = load_module(self.module_name, lib_dir)
self.module_filename = os.path.basename(self.module.__file__)
# once the module is in memory, we no longer need the file on disk
# but we do need a copy of the file for pickling and the module name
with io.open(os.path.join(lib_dir, self.module_filename), 'rb') as f:
self.module_bytes = f.read()
shutil.rmtree(lib_dir, ignore_errors=True)
self.fit_class = getattr(self.module, "StanFit4Model")
def __init__(self, file=None, charset='utf-8', model_name="anon_model",
model_code=None, stanc_ret=None, boost_lib=None,
eigen_lib=None, verbose=False, obfuscate_model_name=True,
extra_compile_args=None):
if stanc_ret is None:
stanc_ret = pystan.api.stanc(file=file,
charset=charset,
model_code=model_code,
model_name=model_name,
verbose=verbose,
obfuscate_model_name=obfuscate_model_name)
if not isinstance(stanc_ret, dict):
raise ValueError("stanc_ret must be an object returned by stanc.")
stanc_ret_keys = {'status', 'model_code', 'model_cppname',
'cppcode', 'model_name'}
if not all(n in stanc_ret_keys for n in stanc_ret):
raise ValueError("stanc_ret lacks one or more of the keys: "
"{}".format(str(stanc_ret_keys)))
elif stanc_ret['status'] != 0: # success == 0
raise ValueError("stanc_ret is not a successfully returned "
"dictionary from stanc.")
self.model_cppname = stanc_ret['model_cppname']
self.model_name = stanc_ret['model_name']
self.model_code = stanc_ret['model_code']
self.model_cppcode = stanc_ret['cppcode']
msg = "COMPILING THE C++ CODE FOR MODEL {} NOW."
logger.info(msg.format(self.model_name))
if verbose:
msg = "OS: {}, Python: {}, Cython {}".format(sys.platform,
sys.version,
Cython.__version__)
logger.info(msg)
if boost_lib is not None:
# FIXME: allow boost_lib, eigen_lib to be specified
raise NotImplementedError
if eigen_lib is not None:
raise NotImplementedError
# module_name needs to be unique so that each model instance has its own module
nonce = abs(hash((self.model_name, time.time())))
self.module_name = 'stanfit4{}_{}'.format(self.model_name, nonce)
lib_dir = tempfile.mkdtemp()
pystan_dir = os.path.dirname(__file__)
include_dirs = [
lib_dir,
pystan_dir,
os.path.join(pystan_dir, "stan", "src"),
os.path.join(pystan_dir, "stan", "lib", "stan_math_2.10.0"),
os.path.join(pystan_dir, "stan", "lib", "stan_math_2.10.0", "lib", "eigen_3.2.8"),
os.path.join(pystan_dir, "stan", "lib", "stan_math_2.10.0", "lib", "boost_1.60.0"),
os.path.join(pystan_dir, "stan", "lib", "stan_math_2.10.0", "lib", "cvodes_2.8.2", "include"),
np.get_include(),
]
model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp')
with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile:
outfile.write(self.model_cppcode)
pyx_file = os.path.join(lib_dir, self.module_name + '.pyx')
pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx')
with io.open(pyx_template_file, 'r', encoding='utf-8') as infile:
s = infile.read()
template = string.Template(s)
with io.open(pyx_file, 'w', encoding='utf-8') as outfile:
s = template.safe_substitute(model_cppname=self.model_cppname)
outfile.write(s)
stan_macros = [
('BOOST_RESULT_OF_USE_TR1', None),
('BOOST_NO_DECLTYPE', None),
('BOOST_DISABLE_ASSERTS', None),
]
# compile stan models with optimization (-O2)
# (stanc is compiled without optimization (-O0) currently, see #33)
if extra_compile_args is None:
extra_compile_args = [
'-O2',
'-ftemplate-depth-256',
'-Wno-unused-function',
'-Wno-uninitialized',
]
if platform.platform().startswith('Win'):
extra_compile_args = ['/EHsc', '-DBOOST_DATE_TIME_NO_LIB']
distutils.log.set_verbosity(verbose)
extension = Extension(name=self.module_name,
language="c++",
sources=[pyx_file],
define_macros=stan_macros,
include_dirs=include_dirs,
extra_compile_args=extra_compile_args)
cython_include_dirs = ['.', pystan_dir]
build_extension = _get_build_extension()
build_extension.extensions = cythonize([extension],
include_path=cython_include_dirs,
quiet=not verbose)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr)
if redirect_stderr:
# silence stderr for compilation
orig_stderr = pystan.misc._redirect_stderr()
try:
build_extension.run()
finally:
if redirect_stderr:
# restore stderr
os.dup2(orig_stderr, sys.stderr.fileno())
self.module = load_module(self.module_name, lib_dir)
self.module_filename = os.path.basename(self.module.__file__)
# once the module is in memory, we no longer need the file on disk
# but we do need a copy of the file for pickling and the module name
with io.open(os.path.join(lib_dir, self.module_filename), 'rb') as f:
self.module_bytes = f.read()
shutil.rmtree(lib_dir, ignore_errors=True)
self.fit_class = getattr(self.module, "StanFit4Model")
def __compile(
scoring_function='OneMinus<MeanAffinity<RegionGraphType, ScoreValue>>',
discretize_queue=0,
force_rebuild=False):
import sys
import os
import shutil
import glob
import numpy
import fcntl
try:
import hashlib
except ImportError:
import md5 as hashlib
from distutils.core import Distribution, Extension
from distutils.command.build_ext import build_ext
from distutils.sysconfig import get_config_vars, get_python_inc
import Cython
from Cython.Compiler.Main import Context, default_options
from Cython.Build.Dependencies import cythonize
# compile frontend.pyx for given scoring function
source_dir = os.path.dirname(os.path.abspath(__file__))
source_files = [
os.path.join(source_dir, 'agglomerate.pyx'),
os.path.join(source_dir, 'frontend_agglomerate.h'),
os.path.join(source_dir, 'frontend_agglomerate.cpp')
]
source_files += glob.glob(source_dir + '/backend/*.hpp')
source_files.sort()
source_files_hashes = [
hashlib.md5(open(f, 'r').read().encode('utf-8')).hexdigest()
for f in source_files
]
key = scoring_function, discretize_queue, source_files_hashes, sys.version_info, sys.executable, Cython.__version__
module_name = 'waterz_' + hashlib.md5(str(key).encode('utf-8')).hexdigest()
lib_dir = os.path.expanduser('~/.cython/inline')
# since this could be called concurrently, there is no good way to check
# whether the directory already exists
try:
os.makedirs(lib_dir)
except:
pass
# make sure the same module is not build concurrently
with open(
os.path.join(
lib_dir,
module_name + '_%s.lock' % str(numpy.random.random(1)[0])[2:]),
'w') as lock_file:
fcntl.lockf(lock_file, fcntl.LOCK_EX)
try:
if lib_dir not in sys.path:
sys.path.append(lib_dir)
if force_rebuild:
raise ImportError
else:
print("Re-using already compiled waterz version")
return module_name
except ImportError:
print("Compiling waterz in " + str(lib_dir))
cython_include_dirs = ['.']
ctx = Context(cython_include_dirs, default_options)
include_dir = os.path.join(lib_dir, module_name)
if not os.path.exists(include_dir):
os.makedirs(include_dir)
include_dirs = [
source_dir,
include_dir,
os.path.join(source_dir, 'backend'),
os.path.dirname(get_python_inc()),
numpy.get_include(),
]
scoring_function_header = os.path.join(include_dir,
'ScoringFunction.h')
with open(scoring_function_header, 'w') as f:
f.write('typedef %s ScoringFunctionType;' % scoring_function)
queue_header = os.path.join(include_dir, 'Queue.h')
with open(queue_header, 'w') as f:
if discretize_queue == 0:
f.write(
'template<typename T, typename S> using QueueType = PriorityQueue<T, S>;'
)
else:
f.write(
'template<typename T, typename S> using QueueType = BinQueue<T, S, %d>;'
% discretize_queue)
# cython requires that the pyx file has the same name as the module
shutil.copy(os.path.join(source_dir, 'agglomerate.pyx'),
os.path.join(lib_dir, module_name + '.pyx'))
shutil.copy(
os.path.join(source_dir, 'frontend_agglomerate.cpp'),
os.path.join(lib_dir,
module_name + '_frontend_agglomerate.cpp'))
# Remove the "-Wstrict-prototypes" compiler option, which isn't valid
# for C++.
cfg_vars = get_config_vars()
if "CFLAGS" in cfg_vars:
cfg_vars["CFLAGS"] = cfg_vars["CFLAGS"].replace(
"-Wstrict-prototypes", "")
extension = Extension(
module_name,
sources=[
os.path.join(lib_dir, module_name + '.pyx'),
os.path.join(lib_dir,
module_name + '_frontend_agglomerate.cpp')
],
include_dirs=include_dirs,
language='c++',
extra_link_args=['-std=c++11'],
extra_compile_args=['-std=c++11', '-w'])
build_extension = build_ext(Distribution())
build_extension.finalize_options()
build_extension.extensions = cythonize([extension],
quiet=True,
nthreads=1,
language_level="3")
build_extension.build_temp = lib_dir
build_extension.build_lib = lib_dir
build_extension.run()
return module_name
def cython(self, line, cell):
"""Compile and import everything from a Cython code cell.
The contents of the cell are written to a `.pyx` file in the
directory `IPYTHONDIR/cython` using a filename with the hash of the
code. This file is then cythonized and compiled. The resulting module
is imported and all of its symbols are injected into the user's
namespace. The usage is similar to that of `%%cython_pyximport` but
you don't have to pass a module name::
%%cython
def f(x):
return 2.0*x
"""
args = magic_arguments.parse_argstring(self.cython, line)
code = cell if cell.endswith('\n') else cell+'\n'
lib_dir = os.path.join(self.shell.ipython_dir, 'cython')
quiet = True
key = code, sys.version_info, sys.executable, Cython.__version__
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
if args.force:
# Force a new module name by adding the current time to the
# key which is hashed to determine the module name.
key += time.time(),
module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest()
module_path = os.path.join(lib_dir, module_name + self.so_ext)
have_module = os.path.isfile(module_path)
need_cythonize = not have_module
if args.annotate:
html_file = os.path.join(lib_dir, module_name + '.html')
if not os.path.isfile(html_file):
need_cythonize = True
if need_cythonize:
c_include_dirs = args.include
if 'numpy' in code:
import numpy
c_include_dirs.append(numpy.get_include())
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
pyx_file = py3compat.cast_bytes_py2(pyx_file, encoding=sys.getfilesystemencoding())
with io.open(pyx_file, 'w', encoding='utf-8') as f:
f.write(code)
extension = Extension(
name = module_name,
sources = [pyx_file],
include_dirs = c_include_dirs,
extra_compile_args = args.compile_args,
extra_link_args = args.link_args,
libraries = args.lib,
language = 'c++' if args.cplus else 'c',
)
build_extension = self._get_build_extension()
try:
opts = dict(
quiet=quiet,
annotate = args.annotate,
force = True,
)
build_extension.extensions = cythonize([extension], **opts)
except CompileError:
return
if not have_module:
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
build_extension.run()
self._code_cache[key] = module_name
module = imp.load_dynamic(module_name, module_path)
self._import_all(module)
if args.annotate:
try:
with io.open(html_file, encoding='utf-8') as f:
annotated_html = f.read()
except IOError as e:
# File could not be opened. Most likely the user has a version
# of Cython before 0.15.1 (when `cythonize` learned the
# `force` keyword argument) and has already compiled this
# exact source without annotation.
print('Cython completed successfully but the annotated '
'source could not be read.', file=sys.stderr)
print(e, file=sys.stderr)
else:
return display.HTML(self.clean_annotated_html(annotated_html))
def cythonize(self, extensions):
from Cython.Build.Dependencies import cythonize
return cythonize(extensions)
