def finalize_options(self): if self.distribution.ext_modules: nthreads = getattr(self, 'parallel', None) # -j option in Py3.5+ nthreads = int(nthreads) if nthreads else None from Cython.Build.Dependencies import cythonize self.distribution.ext_modules[:] = cythonize( self.distribution.ext_modules, nthreads=nthreads, force=self.force) super(new_build_ext, self).finalize_options()
def cython(self, line, cell): """Compile and import everything from a Cython code cell. The contents of the cell are written to a `.pyx` file in the directory `IPYTHONDIR/cython` using a filename with the hash of the code. This file is then cythonized and compiled. The resulting module is imported and all of its symbols are injected into the user's namespace. The usage is similar to that of `%%cython_pyximport` but you don't have to pass a module name:: %%cython def f(x): return 2.0*x """ args = parse_argstring(self.cython, line) code = cell if cell.endswith('\n') else cell+'\n' lib_dir = os.path.join(self.shell.ipython_dir, 'cython') cython_include_dirs = ['.'] force = args.force quiet = True ctx = Context(cython_include_dirs, default_options) key = code, sys.version_info, sys.executable, Cython.__version__ module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest() module_path = os.path.join(lib_dir, module_name+self.so_ext) if not os.path.exists(lib_dir): os.makedirs(lib_dir) if force or not os.path.isfile(module_path): c_include_dirs = args.include if 'numpy' in code: import numpy c_include_dirs.append(numpy.get_include()) pyx_file = os.path.join(lib_dir, module_name + '.pyx') pyx_file = py3compat.cast_bytes_py2(pyx_file, encoding=sys.getfilesystemencoding()) with io.open(pyx_file, 'w', encoding='utf-8') as f: f.write(code) extension = Extension( name = module_name, sources = [pyx_file], include_dirs = c_include_dirs, extra_compile_args = args.compile_args, libraries = args.lib, ) build_extension = self._get_build_extension() try: build_extension.extensions = cythonize([extension], ctx=ctx, quiet=quiet) except CompileError: return build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() self._code_cache[key] = module_name module = imp.load_dynamic(module_name, module_path) self._import_all(module)
def cython_compile(path_pattern, options): pool = None paths = map(os.path.abspath, extended_iglob(path_pattern)) try: for path in paths: if options.build_inplace: base_dir = path while not os.path.isdir(base_dir) or is_package_dir(base_dir): base_dir = os.path.dirname(base_dir) else: base_dir = None if os.path.isdir(path): # recursively compiling a package paths = [os.path.join(path, '**', '*.%s' % ext) for ext in ('py', 'pyx')] else: # assume it's a file(-like thing) paths = [path] cwd = os.getcwd() try: if base_dir: os.chdir(base_dir) ext_modules = cythonize( paths, nthreads=options.parallel, exclude_failures=options.keep_going, exclude=options.excludes, compiler_directives=options.directives, force=options.force, quiet=options.quiet, **options.options) finally: if base_dir: os.chdir(cwd) if ext_modules and options.build: if len(ext_modules) > 1 and options.parallel > 1: if pool is None: try: pool = multiprocessing.Pool(options.parallel) except OSError: pool = _FakePool() pool.map_async(run_distutils, [ (base_dir, [ext]) for ext in ext_modules]) else: run_distutils((base_dir, ext_modules)) except: if pool is not None: pool.terminate() raise else: if pool is not None: pool.close() pool.join()
def build_extensions(self): self.__reimport() if profile: from Cython.Compiler.Options import directive_defaults directive_defaults['profile'] = True print("Enabled profiling for the Cython binary modules") from Cython.Build.Dependencies import cythonize self.distribution.ext_modules[:] = cythonize( self.distribution.ext_modules) build_ext_orig.build_extensions(self)
def finalize_options(self): # Generate files from autogen import rebuild rebuild() import sys # Check dependencies try: from Cython.Build.Dependencies import cythonize except ImportError as E: sys.stderr.write("Error: {0}\n".format(E)) sys.stderr.write("The installation of ppl requires Cython\n") sys.exit(1) # cysignals not yet integrated # try: # # We need the header files for cysignals at compile-time # import cysignals # except ImportError as E: # sys.stderr.write("Error: {0}\n".format(E)) # sys.stderr.write("The installation of ppl requires cysignals\n") # sys.exit(1) compile_time_env = {} if compile_from_sage: version = sage.env.SAGE_VERSION.split('.') major = int(version[0]) minor = int(version[1]) if len(version) == 3: beta = int(version[2][4:]) else: beta = -1 sage_version = (major,minor,beta) # trac ticket #22970 changes datastructure of rational matrices # merged in 8.0.beta9 compile_time_env['SAGE_RAT_MAT_ARE_FMPQ_T_MAT'] = (sage_version >= (8,0,9)) self.distribution.ext_modules[:] = cythonize( self.distribution.ext_modules, compile_time_env=compile_time_env, include_path=sys.path) _build_ext.finalize_options(self)
def cythonize(self, extensions): return cythonize(extensions, build_dir=cythonize_dir, include_path=["src", os.path.join(cythonize_dir, "src")])
def compile(self, walked_jitted_funcs=None): #compile and load c_func if self.compile_state=='not_compiled': #print self.func_name, 'compiling' self.compile_state='compiling' if walked_jitted_funcs==None: self.walked_jitted_funcs=[] py_src=get_func_source(self.py_func) args=get_args(self.py_func) body=get_body(py_src) #print py_src indent=get_indent(body) func_def='cpdef' func_ast=ast.parse(py_src) jvisit(func_ast, args, self.func_globals, self.called_jitted_funcs, self.local_names, self.load_names) #print called_jitted_funcs for jitted_func, mode, extra in self.called_jitted_funcs: if mode=='direct': self.putln("from %s cimport %s__c"%(jitted_func.module_name, jitted_func.func_name)) body=re.sub(r'\b%s *\('%jitted_func.func_name, '%s__c('%(jitted_func.func_name), body) #elif mode=='attribute':#fix #self.putln('cimport %s'%(jitted_func.module_name,)) #attribute_func_name=extra ## #body=re.sub('%s *\('%attribute_func_name, '%s.%s__c('%(jitted_func.module_name, jitted_func.func_name), body) self.putln() #process signature formated_sig=self.sig.replace(' ','') if formated_sig: return_type, arg_types=parse_sig(formated_sig) assert len(args)==len(arg_types), "the function %s has %s args but the signature has %s args "%(func_name, len(args), len(arg_types)) func_args=[] for arg_name, arg_type in zip(args, arg_types): func_args.append('%s %s'%(arg_type, arg_name)) func_args=", ".join(func_args) else: func_args=', '.join(args) return_type='' #directives #print "in" directive_decorates=[] directives=['wraparound', 'boundscheck'] for directive in directives: value=self.options.get(directive) if value != None: directive_decorates.append('@cython.%s(%s)\n'%(directive, value)) if directive_decorates: self.putln('cimport cython') for decorate in directive_decorates: self.putln(decorate) nogil='nogil' if self.options.get("nogil") else '' #head #add like 'nogil' cfunc_head='cdef %s %s__c(%s) %s'%(return_type, self.func_name, func_args, nogil) pyfunc_head='def %s__p(%s)'%(self.func_name, ', '.join(args)) #func_head_1='\n%s %s(%s)%s'%('cdef', func_name, func_args, '') self.putln(cfunc_head,'pxd') #self.pxd_buf.write(func_init_head+"\n") if self.load_names: init_func_head='def %s_init(f_globals)'%(self.func_name) globals_name='_%s_GLOBALS'%self.func_name.upper() self.putln('%s = None\n'%globals_name) self.putln(init_func_head+':') self.putln(indent+'global %s'%globals_name) self.putln(indent+'%s = f_globals'%globals_name) self.putln() self.putln(cfunc_head+':') #cdef vars locals=self.options.get('locals') if type(locals) is str: self.putln(indent+"cdef:") for line in locals.strip().split('\n'): line=line.strip() if line[-1]==';': line=line[:-1] self.putln(indent*2+line) #if type(locals) is str: # for type_name, vars_name in parse_cdeclare(locals): # cdef=indent+'cdef %s %s\n'%(type_name, vars_name) # pyx_buf.write(cdef) #body #self.pyx_buf.write(indent+'%s=sys._getframe(1).f_locals\n'%func_globals_name) if self.load_names: self.putln(indent+'#load globals') for name in self.load_names: self.putln(indent+"%s = %s['%s']"%(name, globals_name, name)) self.putln() #self.putln(indent+'print "%s", %s["__file__"]'%(self.func_name, func_globals_name)) self.putln(body) self.putln() self.putln(pyfunc_head+':') self.putln(indent+'return %s__c(%s)'%(self.func_name, ', '.join(args))) self.pyx_src=self.pyx_buf.getvalue() self.pxd_src=self.pxd_buf.getvalue() file_name=self.func_globals.get('__file__') if file_name: func_file_dir=os.path.dirname(os.path.abspath(file_name)) else: func_file_dir=os.getcwd() cython_compile_dir=os.path.join(func_file_dir, '__cython_compile__') if not os.path.exists(cython_compile_dir): os.mkdir(cython_compile_dir) key = self.pyx_src+str(sys.version_info)+sys.executable+cython.__version__ self.module_dir=os.path.join(cython_compile_dir, self.module_name+'__'+hashlib.md5(key.encode('utf-8')).hexdigest() ) if not os.path.exists(self.module_dir): os.mkdir(self.module_dir) pyx_file=os.path.join(self.module_dir, self.module_name+'.pyx') pxd_file=os.path.join(self.module_dir, self.module_name+'.pxd') so_ext='.pyd' so_file=os.path.join(self.module_dir, self.module_name+so_ext) init_file=os.path.join(self.module_dir, '__init__.py') if not os.path.exists(so_file): fw=open(pyx_file,"w") fw.write(self.pyx_src) fw.close() fw=open(pxd_file,"w") fw.write(self.pxd_src) fw.close() fw=open(init_file, "w") fw.close() for jitted_func,_,_ in self.called_jitted_funcs: #print jitted_func.func_name, jitted_func.compile_state if jitted_func.compile_state=='not_compiled': if walked_jitted_funcs==None: jitted_func.compile(self.walked_jitted_funcs) else: jitted_func.compile(walked_jitted_funcs) #check if not os.path.exists(so_file): extension = Extension(name = self.module_name, sources = [pyx_file], #include_dirs = c_include_dirs, #extra_compile_args = cflags ) #find called_jitted_funcs's module path cython_include_dirs=set() for jitted_func, _, _ in self.called_jitted_funcs: cython_include_dirs.add(jitted_func.module_dir) print 'cython_include_dirs', self.func_name, cython_include_dirs build_extension = _get_build_extension() build_extension.extensions = cythonize([extension], annotate=True, include_path=list(cython_include_dirs), #quiet=quiet ) temp_dir=os.path.join(cython_compile_dir, "__build_temp__") build_extension.build_temp = temp_dir build_extension.build_lib = self.module_dir #print "build" build_extension.run() self.compile_state='compiled' self.so_file=so_file if walked_jitted_funcs==None: self.walked_jitted_funcs.append(self) else: walked_jitted_funcs.append(self)
def cython_inline(code, types='aggressive', lib_dir=os.path.expanduser('~/.cython/inline'), include_dirs=['.'], locals=None, globals=None, **kwds): code, literals = strip_string_literals(code) code = strip_common_indent(code) ctx = Context(include_dirs, default_options) if locals is None: locals = inspect.currentframe().f_back.f_back.f_locals if globals is None: globals = inspect.currentframe().f_back.f_back.f_globals try: for symbol in unbound_symbols(code): if symbol in kwds: continue elif symbol in locals: kwds[symbol] = locals[symbol] elif symbol in globals: kwds[symbol] = globals[symbol] else: print "Couldn't find ", symbol except AssertionError: # Parsing from strings not fully supported (e.g. cimports). print "Could not parse code as a string (to extract unbound symbols)." arg_names = kwds.keys() arg_names.sort() arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names]) key = code, arg_sigs, sys.version_info, sys.executable, Cython.__version__ module_name = "_cython_inline_" + hashlib.md5(str(key)).hexdigest() # # TODO: Does this cover all the platforms? # if (not os.path.exists(os.path.join(lib_dir, module_name + ".so")) and # not os.path.exists(os.path.join(lib_dir, module_name + ".dll"))): try: if not os.path.exists(lib_dir): os.makedirs(lib_dir) if lib_dir not in sys.path: sys.path.append(lib_dir) __import__(module_name) except ImportError: c_include_dirs = [] cimports = [] qualified = re.compile(r'([.\w]+)[.]') for type, _ in arg_sigs: m = qualified.match(type) if m: cimports.append('\ncimport %s' % m.groups()[0]) # one special case if m.groups()[0] == 'numpy': import numpy c_include_dirs.append(numpy.get_include()) module_body, func_body = extract_func_code(code) params = ', '.join(['%s %s' % a for a in arg_sigs]) module_code = """ %(module_body)s %(cimports)s def __invoke(%(params)s): %(func_body)s """ % {'cimports': '\n'.join(cimports), 'module_body': module_body, 'params': params, 'func_body': func_body } for key, value in literals.items(): module_code = module_code.replace(key, value) pyx_file = os.path.join(lib_dir, module_name + '.pyx') open(pyx_file, 'w').write(module_code) extension = Extension( name = module_name, sources = [pyx_file], include_dirs = c_include_dirs) build_extension = build_ext(Distribution()) build_extension.finalize_options() build_extension.extensions = cythonize([extension]) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() _code_cache[key] = module_name arg_list = [kwds[arg] for arg in arg_names] return __import__(module_name).__invoke(*arg_list)
include_dirs=[numpy.get_include()], libraries=[fftwlib]) srft = Extension("fjlt.SubsampledRandomizedFourrierTransform1d", sources=["fjlt/SubsampledRandomizedFourrierTransform1d.pyx", "fjlt/SubsampledRandomizedFourrierTransform1d.pxd"], include_dirs=[numpy.get_include()], libraries=[fftwlib]) demo = Extension("fjlt.demo_cython", sources=["demo_cython.pyx"], include_dirs=[numpy.get_include()], libraries=[fftwlib]) # setup(ext_modules=[random_projection, srft], # cmdclass={'build_ext': build_ext}) exec(open('fjlt/version.py').read()) setup(name='FJLT', version=__version__, # @UndefinedVariable description='Fast Johnson Lindenstrauss Transform', author='Gabriel Krummenacher', author_email='*****@*****.**', url='http://people.inf.ethz.ch/kgabriel/software.html', packages=['fjlt'], # py_modules=['demo'], requires=['numpy'], ext_modules=cythonize([random_projection, srft, demo]), cmdclass={'build_ext': build_ext} )
libdir + '/include/OGRE', np.get_include() ] extensions = [] for i in glob.glob('./*.pyx'): e = Extension(os.path.basename(i)[:-4], [i], define_macros=[('BOOST_SYSTEM_NO_DEPRECATED', None)], include_dirs=includedirs, libraries=libraries, extra_compile_args=extra_compile_args, compiler_directives=compiledirs, language='c++') extensions.append(e) ext = cythonize(extensions, include_path=['.', '../renderer']) # HORRIBLE KLUDGE: need to get around problem of Cython generating code which relies on CPython having been compiled with --with-fpectl # The macros PyFTE_*_PROTECT invoke symbols which don't exist with Anaconda builds so they need to be removed which appears to be safe. # See https://github.com/numpy/numpy/issues/8415 http://www.psf.upfronthosting.co.za/issue29137 for cppfile in glob.glob('*.cpp'): sys.stdout.write('Processing %s\n' % cppfile) sys.stdout.flush() cpplines = open(cppfile).readlines() with open(cppfile, 'w') as o: for line in cpplines: if 'PyFPE_START_PROTECT' not in line and 'PyFPE_END_PROTECT' not in line: # remove the symbol lines from the source o.write(line) setup(ext_modules=ext)
def wrap(py_func): #clear buf pyx_buf.seek(0) pyx_buf.truncate() pxd_buf.seek(0) pxd_buf.truncate() called_jitted_funcs = set() local_names = set() #in py_func load_names = set() #in py_func func_name = py_func.__name__ func_globals_name = 'func_%s_globals' % func_name #func_globals=sys._getframe(1).f_locals #get caller's scope func_globals = py_func.func_globals py_src = get_func_source(py_func) args = get_args(py_func) body = get_body(py_src) #print py_src indent = get_indent(body) func_def = 'cpdef' func_ast = ast.parse(py_src) jvisit(func_ast, func_globals, called_jitted_funcs, local_names, load_names) #print called_jitted_funcs for jitted_func, mode, extra in called_jitted_funcs: if mode == 'direct': pyx_buf.write("from %s cimport %s\n" % (jitted_func.module_name, jitted_func.func_name)) elif mode == 'attribute': pyx_buf.write('cimport %s\n' % (jitted_func.module_name, )) attribute_func_name = extra # body = re.sub( '%s *\(' % attribute_func_name, '%s.%s(' % (jitted_func.module_name, jitted_func.func_name), body) pyx_buf.write('\n') pyx_buf.write(''' import sys %s=sys._getframe(1).f_locals ''' % func_globals_name) for name in load_names: pyx_buf.write("%s=%s.get('%s')\n" % (name, func_globals_name, name)) pyx_buf.write('\n') #process signature formated_sig = sig.replace(' ', '') if formated_sig: return_type, arg_types = parse_sig(formated_sig) assert len(args) == len( arg_types ), "the function %s has %s args but the signature has %s args " % ( func_name, len(args), len(arg_types)) func_args = [] for arg_name, arg_type in zip(args, arg_types): func_args.append('%s %s' % (arg_type, arg_name)) func_args = ", ".join(func_args) else: func_args = ', '.join(args) return_type = '' #directives #print "in" directive_decorates = [] directives = ['wraparound', 'boundscheck'] for directive in directives: value = kwds.get(directive) if value != None: directive_decorates.append('@cython.%s(%s)\n' % (directive, value)) if directive_decorates: pyx_buf.write('cimport cython\n\n') for decorate in directive_decorates: pyx_buf.write(decorate) nogil = 'nogil' if kwds.get("nogil") else '' #head #add like 'nogil' func_head = '%s %s %s(%s) %s' % (func_def, return_type, func_name, func_args, nogil) #func_head_1='\n%s %s(%s)%s'%('cdef', func_name, func_args, '') pxd_buf.write(func_head + "\n") pyx_buf.write(func_head + ':\n') #cdef vars locals = kwds.get('locals') if type(locals) is str: pyx_buf.write(indent + "cdef:\n") for line in locals.strip().split('\n'): line = line.strip() if line[-1] == ';': line = line[:-1] pyx_buf.write(indent * 2 + line + '\n') #if type(locals) is str: # for type_name, vars_name in parse_cdeclare(locals): # cdef=indent+'cdef %s %s\n'%(type_name, vars_name) # pyx_buf.write(cdef) #body pyx_buf.write(body + '\n') #build pyx_src = pyx_buf.getvalue() pxd_src = pxd_buf.getvalue() base_name = os.path.basename(func_globals['__file__']) file_name = os.path.splitext(base_name)[0] key = pyx_src + str( sys.version_info) + sys.executable + cython.__version__ hashed_module_name = file_name + '_' + func_name + '__' + hashlib.md5( key.encode('utf-8')).hexdigest() caller_file_dir = os.path.dirname( os.path.abspath(func_globals['__file__'])) #print 'caller_file_dir', caller_file_dir module_dir = os.path.join(caller_file_dir, '__cython_compile__') if not os.path.exists(module_dir): os.mkdir(module_dir) pyx_file = os.path.join(module_dir, hashed_module_name + '.pyx') pxd_file = os.path.join(module_dir, hashed_module_name + '.pxd') so_ext = '.pyd' so_file = os.path.join(module_dir, hashed_module_name + so_ext) init_file = os.path.join(module_dir, '__init__.py') #check if not os.path.exists(so_file): fw = open(pyx_file, "w") fw.write(pyx_src) fw.close() fw = open(pxd_file, "w") fw.write(pxd_src) fw.close() fw = open(init_file, "w") fw.close() extension = Extension( name=hashed_module_name, sources=[pyx_file], #include_dirs = c_include_dirs, #extra_compile_args = cflags ) #find called_jitted_funcs's module path cython_include_dirs = [] for jitted_func, _, _ in called_jitted_funcs: cython_include_dirs.append(jitted_func.module_dir) #print 'cython_include_dirs', cython_include_dirs build_extension = _get_build_extension() build_extension.extensions = cythonize( [extension], annotate=True, include_path=cython_include_dirs, #quiet=quiet ) temp_dir = os.path.join(module_dir, "__build_temp__") build_extension.build_temp = temp_dir build_extension.build_lib = module_dir #print "build" build_extension.run() compiled_module = imp.load_dynamic(hashed_module_name, so_file) c_func = getattr(compiled_module, func_name) return JittedFunc(c_func, py_func, func_name, hashed_module_name, module_dir, pyx_src)
def compile(self, walked_jitted_funcs=None): #compile and load c_func if self.compile_state == 'not_compiled': #print self.func_name, 'compiling' self.compile_state = 'compiling' if walked_jitted_funcs == None: self.walked_jitted_funcs = [] py_src = get_func_source(self.py_func) args = get_args(self.py_func) body = get_body(py_src) #print py_src indent = get_indent(body) func_def = 'cpdef' func_ast = ast.parse(py_src) jvisit(func_ast, args, self.func_globals, self.called_jitted_funcs, self.local_names, self.load_names) #print called_jitted_funcs for jitted_func, mode, extra in self.called_jitted_funcs: if mode == 'direct': self.putln( "from %s cimport %s__c" % (jitted_func.module_name, jitted_func.func_name)) body = re.sub(r'\b%s *\(' % jitted_func.func_name, '%s__c(' % (jitted_func.func_name), body) #elif mode=='attribute':#fix #self.putln('cimport %s'%(jitted_func.module_name,)) #attribute_func_name=extra ## #body=re.sub('%s *\('%attribute_func_name, '%s.%s__c('%(jitted_func.module_name, jitted_func.func_name), body) self.putln() #process signature formated_sig = self.sig.replace(' ', '') if formated_sig: return_type, arg_types = parse_sig(formated_sig) assert len(args) == len( arg_types ), "the function %s has %s args but the signature has %s args " % ( func_name, len(args), len(arg_types)) func_args = [] for arg_name, arg_type in zip(args, arg_types): func_args.append('%s %s' % (arg_type, arg_name)) func_args = ", ".join(func_args) else: func_args = ', '.join(args) return_type = '' #directives #print "in" directive_decorates = [] directives = ['wraparound', 'boundscheck'] for directive in directives: value = self.options.get(directive) if value != None: directive_decorates.append('@cython.%s(%s)\n' % (directive, value)) if directive_decorates: self.putln('cimport cython') for decorate in directive_decorates: self.putln(decorate) nogil = 'nogil' if self.options.get("nogil") else '' #head #add like 'nogil' cfunc_head = 'cdef %s %s__c(%s) %s' % (return_type, self.func_name, func_args, nogil) pyfunc_head = 'def %s__p(%s)' % (self.func_name, ', '.join(args)) #func_head_1='\n%s %s(%s)%s'%('cdef', func_name, func_args, '') self.putln(cfunc_head, 'pxd') #self.pxd_buf.write(func_init_head+"\n") if self.load_names: init_func_head = 'def %s_init(f_globals)' % (self.func_name) globals_name = '_%s_GLOBALS' % self.func_name.upper() self.putln('%s = None\n' % globals_name) self.putln(init_func_head + ':') self.putln(indent + 'global %s' % globals_name) self.putln(indent + '%s = f_globals' % globals_name) self.putln() self.putln(cfunc_head + ':') #cdef vars locals = self.options.get('locals') if type(locals) is str: self.putln(indent + "cdef:") for line in locals.strip().split('\n'): line = line.strip() if line[-1] == ';': line = line[:-1] self.putln(indent * 2 + line) #if type(locals) is str: # for type_name, vars_name in parse_cdeclare(locals): # cdef=indent+'cdef %s %s\n'%(type_name, vars_name) # pyx_buf.write(cdef) #body #self.pyx_buf.write(indent+'%s=sys._getframe(1).f_locals\n'%func_globals_name) if self.load_names: self.putln(indent + '#load globals') for name in self.load_names: self.putln(indent + "%s = %s['%s']" % (name, globals_name, name)) self.putln() #self.putln(indent+'print "%s", %s["__file__"]'%(self.func_name, func_globals_name)) self.putln(body) self.putln() self.putln(pyfunc_head + ':') self.putln(indent + 'return %s__c(%s)' % (self.func_name, ', '.join(args))) self.pyx_src = self.pyx_buf.getvalue() self.pxd_src = self.pxd_buf.getvalue() file_name = self.func_globals.get('__file__') if file_name: func_file_dir = os.path.dirname(os.path.abspath(file_name)) else: func_file_dir = os.getcwd() cython_compile_dir = os.path.join(func_file_dir, '__cython_compile__') if not os.path.exists(cython_compile_dir): os.mkdir(cython_compile_dir) key = self.pyx_src + str( sys.version_info) + sys.executable + cython.__version__ self.module_dir = os.path.join( cython_compile_dir, self.module_name + '__' + hashlib.md5(key.encode('utf-8')).hexdigest()) if not os.path.exists(self.module_dir): os.mkdir(self.module_dir) pyx_file = os.path.join(self.module_dir, self.module_name + '.pyx') pxd_file = os.path.join(self.module_dir, self.module_name + '.pxd') so_ext = '.pyd' so_file = os.path.join(self.module_dir, self.module_name + so_ext) init_file = os.path.join(self.module_dir, '__init__.py') if not os.path.exists(so_file): fw = open(pyx_file, "w") fw.write(self.pyx_src) fw.close() fw = open(pxd_file, "w") fw.write(self.pxd_src) fw.close() fw = open(init_file, "w") fw.close() for jitted_func, _, _ in self.called_jitted_funcs: #print jitted_func.func_name, jitted_func.compile_state if jitted_func.compile_state == 'not_compiled': if walked_jitted_funcs == None: jitted_func.compile(self.walked_jitted_funcs) else: jitted_func.compile(walked_jitted_funcs) #check if not os.path.exists(so_file): extension = Extension( name=self.module_name, sources=[pyx_file], #include_dirs = c_include_dirs, #extra_compile_args = cflags ) #find called_jitted_funcs's module path cython_include_dirs = set() for jitted_func, _, _ in self.called_jitted_funcs: cython_include_dirs.add(jitted_func.module_dir) print 'cython_include_dirs', self.func_name, cython_include_dirs build_extension = _get_build_extension() build_extension.extensions = cythonize( [extension], annotate=True, include_path=list(cython_include_dirs), #quiet=quiet ) temp_dir = os.path.join(cython_compile_dir, "__build_temp__") build_extension.build_temp = temp_dir build_extension.build_lib = self.module_dir #print "build" build_extension.run() self.compile_state = 'compiled' self.so_file = so_file if walked_jitted_funcs == None: self.walked_jitted_funcs.append(self) else: walked_jitted_funcs.append(self)
def __init__(self, file=None, charset='utf-8', model_name="anon_model", model_code=None, stanc_ret=None, boost_lib=None, eigen_lib=None, verbose=False, obfuscate_model_name=True): if stanc_ret is None: stanc_ret = pystan.api.stanc( file=file, charset=charset, model_code=model_code, model_name=model_name, verbose=verbose, obfuscate_model_name=obfuscate_model_name) if not isinstance(stanc_ret, dict): raise ValueError("stanc_ret must be an object returned by stanc.") stanc_ret_keys = { 'status', 'model_code', 'model_cppname', 'cppcode', 'model_name' } if not all(n in stanc_ret_keys for n in stanc_ret): raise ValueError("stanc_ret lacks one or more of the keys: " "{}".format(str(stanc_ret_keys))) elif stanc_ret['status'] != 0: # success == 0 raise ValueError("stanc_ret is not a successfully returned " "dictionary from stanc.") self.model_cppname = stanc_ret['model_cppname'] self.model_name = stanc_ret['model_name'] self.model_code = stanc_ret['model_code'] self.model_cppcode = stanc_ret['cppcode'] msg = "COMPILING THE C++ CODE FOR MODEL {} NOW." logger.info(msg.format(self.model_name)) if verbose: msg = "OS: {}, Python: {}, Cython {}".format( sys.platform, sys.version, Cython.__version__) logger.info(msg) if boost_lib is not None: # FIXME: allow boost_lib, eigen_lib to be specified raise NotImplementedError if eigen_lib is not None: raise NotImplementedError key = tuple([ self.model_code, self.model_cppcode, sys.version_info, sys.executable ]) module_name = ("stanfit4" + self.model_name + '_' + hashlib.md5(str(key).encode('utf-8')).hexdigest()) self._temp_dir = temp_dir = tempfile.mkdtemp() lib_dir = os.path.join(temp_dir, 'pystan') pystan_dir = os.path.dirname(__file__) include_dirs = [ lib_dir, pystan_dir, os.path.join(pystan_dir, "stan", "src"), os.path.join(pystan_dir, "stan", "lib", "stan_math_2.8.0"), os.path.join(pystan_dir, "stan", "lib", "stan_math_2.8.0", "lib", "eigen_3.2.4"), os.path.join(pystan_dir, "stan", "lib", "stan_math_2.8.0", "lib", "boost_1.58.0"), np.get_include(), ] if not os.path.exists(lib_dir): os.makedirs(lib_dir) model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp') with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile: outfile.write(self.model_cppcode) pyx_file = os.path.join(lib_dir, module_name + '.pyx') pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx') with io.open(pyx_template_file, 'r', encoding='utf-8') as infile: s = infile.read() template = string.Template(s) with io.open(pyx_file, 'w', encoding='utf-8') as outfile: s = template.safe_substitute(model_cppname=self.model_cppname) outfile.write(s) stan_macros = [ ('BOOST_RESULT_OF_USE_TR1', None), ('BOOST_NO_DECLTYPE', None), ('BOOST_DISABLE_ASSERTS', None), ('EIGEN_NO_DEBUG', None), ] extra_compile_args = [ '-O0', '-ftemplate-depth-256', '-Wno-unused-function', '-Wno-uninitialized', ] distutils.log.set_verbosity(verbose) extension = Extension(name=module_name, language="c++", sources=[pyx_file], define_macros=stan_macros, include_dirs=include_dirs, extra_compile_args=extra_compile_args) cython_include_dirs = ['.', pystan_dir] build_extension = _get_build_extension() build_extension.extensions = cythonize( [extension], include_path=cython_include_dirs, quiet=not verbose) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr) if redirect_stderr: # silence stderr for compilation orig_stderr = pystan.misc._redirect_stderr() try: build_extension.run() finally: if redirect_stderr: # restore stderr os.dup2(orig_stderr, sys.stderr.fileno()) self.module = load_module(module_name, lib_dir) self.fit_class = getattr(self.module, "StanFit4Model")
def cython_inline( code, get_type=unsafe_type, lib_dir=os.path.join(get_cython_cache_dir(), "inline"), cython_include_dirs=None, force=False, quiet=False, locals=None, globals=None, **kwds ): if cython_include_dirs is None: cython_include_dirs = ["."] if get_type is None: get_type = lambda x: "object" code = to_unicode(code) orig_code = code code, literals = strip_string_literals(code) code = strip_common_indent(code) ctx = _create_context(tuple(cython_include_dirs)) if locals is None: locals = inspect.currentframe().f_back.f_back.f_locals if globals is None: globals = inspect.currentframe().f_back.f_back.f_globals try: for symbol in unbound_symbols(code): if symbol in kwds: continue elif symbol in locals: kwds[symbol] = locals[symbol] elif symbol in globals: kwds[symbol] = globals[symbol] else: print("Couldn't find ", symbol) except AssertionError: if not quiet: # Parsing from strings not fully supported (e.g. cimports). print("Could not parse code as a string (to extract unbound symbols).") cimports = [] for name, arg in kwds.items(): if arg is cython_module: cimports.append("\ncimport cython as %s" % name) del kwds[name] arg_names = kwds.keys() arg_names.sort() arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names]) key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__ module_name = "_cython_inline_" + hashlib.md5(str(key).encode("utf-8")).hexdigest() if module_name in sys.modules: module = sys.modules[module_name] else: build_extension = None if cython_inline.so_ext is None: # Figure out and cache current extension suffix build_extension = _get_build_extension() cython_inline.so_ext = build_extension.get_ext_filename("") module_path = os.path.join(lib_dir, module_name + cython_inline.so_ext) if not os.path.exists(lib_dir): os.makedirs(lib_dir) if force or not os.path.isfile(module_path): cflags = [] c_include_dirs = [] qualified = re.compile(r"([.\w]+)[.]") for type, _ in arg_sigs: m = qualified.match(type) if m: cimports.append("\ncimport %s" % m.groups()[0]) # one special case if m.groups()[0] == "numpy": import numpy c_include_dirs.append(numpy.get_include()) # cflags.append('-Wno-unused') module_body, func_body = extract_func_code(code) params = ", ".join(["%s %s" % a for a in arg_sigs]) module_code = """ %(module_body)s %(cimports)s def __invoke(%(params)s): %(func_body)s """ % { "cimports": "\n".join(cimports), "module_body": module_body, "params": params, "func_body": func_body, } for key, value in literals.items(): module_code = module_code.replace(key, value) pyx_file = os.path.join(lib_dir, module_name + ".pyx") fh = open(pyx_file, "w") try: fh.write(module_code) finally: fh.close() extension = Extension( name=module_name, sources=[pyx_file], include_dirs=c_include_dirs, extra_compile_args=cflags ) if build_extension is None: build_extension = _get_build_extension() build_extension.extensions = cythonize([extension], include_path=cython_include_dirs, quiet=quiet) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() module = imp.load_dynamic(module_name, module_path) arg_list = [kwds[arg] for arg in arg_names] return module.__invoke(*arg_list)
def create_fake_model(module_name): """Creates a fake model with specific module name. Parameters ---------- module_name : str Returns ------- StanModel Dummy StanModel object with specific module name. """ # reverse engineer the name model_name, nonce = module_name.replace("stanfit4", "").rsplit("_", 1) # create minimal code model_code = """generated quantities {real fake_parameter=1;}""" stanc_ret = pystan.api.stanc(model_code=model_code, model_name=model_name, obfuscate_model_name=False) model_cppname = stanc_ret['model_cppname'] model_cppcode = stanc_ret['cppcode'] original_module_name = module_name module_name = 'stanfit4{}_{}'.format(model_name, nonce) # module name should be same assert original_module_name == module_name lib_dir = tempfile.mkdtemp() pystan_dir = os.path.dirname(pystan.__file__) include_dirs = [ lib_dir, pystan_dir, os.path.join(pystan_dir, "stan", "src"), os.path.join(pystan_dir, "stan", "lib", "stan_math"), os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib", "eigen_3.3.3"), os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib", "boost_1.69.0"), os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib", "sundials_4.1.0", "include"), np.get_include(), ] model_cpp_file = os.path.join(lib_dir, model_cppname + '.hpp') with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile: outfile.write(model_cppcode) pyx_file = os.path.join(lib_dir, module_name + '.pyx') pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx') with io.open(pyx_template_file, 'r', encoding='utf-8') as infile: s = infile.read() template = string.Template(s) with io.open(pyx_file, 'w', encoding='utf-8') as outfile: s = template.safe_substitute(model_cppname=model_cppname) outfile.write(s) stan_macros = [ ('BOOST_RESULT_OF_USE_TR1', None), ('BOOST_NO_DECLTYPE', None), ('BOOST_DISABLE_ASSERTS', None), ] build_extension = _get_build_extension() extra_compile_args = [] if platform.platform().startswith('Win'): if build_extension.compiler in (None, 'msvc'): logger.warning("MSVC compiler is not supported") extra_compile_args = [ '/EHsc', '-DBOOST_DATE_TIME_NO_LIB', '/std:c++14', ] + extra_compile_args else: # Windows, but not msvc, likely mingw # fix bug in MingW-W64 # use posix threads extra_compile_args = [ '-O1', '-ftemplate-depth-256', '-Wno-unused-function', '-Wno-uninitialized', '-std=c++1y', "-D_hypot=hypot", "-pthread", "-fexceptions", ] + extra_compile_args else: # linux or macOS extra_compile_args = [ '-O1', '-ftemplate-depth-256', '-Wno-unused-function', '-Wno-uninitialized', '-std=c++1y', ] + extra_compile_args extension = Extension(name=module_name, language="c++", sources=[pyx_file], define_macros=stan_macros, include_dirs=include_dirs, extra_compile_args=extra_compile_args) cython_include_dirs = ['.', pystan_dir] build_extension.extensions = cythonize([extension], include_path=cython_include_dirs, quiet=True) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() module = load_module(module_name, lib_dir) shutil.rmtree(lib_dir, ignore_errors=True) return module
def __init__(self, file=None, charset='utf-8', model_name="anon_model", model_code=None, stanc_ret=None, boost_lib=None, eigen_lib=None, save_dso=True, verbose=False, **kwargs): if stanc_ret is None: stanc_ret = pystan.api.stanc(file=file, model_code=model_code, model_name=model_name, verbose=verbose, **kwargs) if not isinstance(stanc_ret, dict): raise ValueError("stanc_ret must be an object returned by stanc.") stanc_ret_keys = {'status', 'model_code', 'model_cppname', 'cppcode', 'model_name'} if not all(n in stanc_ret_keys for n in stanc_ret): raise ValueError("stanc_ret lacks one or more of the keys: " "{}".format(str(stanc_ret_keys))) elif stanc_ret['status'] != 0: # success == 0 if (file==None and os.path.exists(model_code)): raise ValueError("stanc_ret is not a successfully returned " "dictionary from stanc. Have you passed " "a stan code filename as model_code?") else: raise ValueError("stanc_ret is not a successfully returned " "dictionary from stanc.") self.model_cppname = stanc_ret['model_cppname'] self.model_name = stanc_ret['model_name'] self.model_code = model_code self.model_cppcode = stanc_ret['cppcode'] self.save_dso = save_dso msg = "COMPILING THE C++ CODE FOR MODEL {} NOW." logger.warning(msg.format(self.model_name)) if verbose: msg = "OS: {}, Python: {}, Cython {}".format(sys.platform, sys.version, Cython.__version__) logger.info(msg) if boost_lib is not None: # FIXME: allow boost_lib, eigen_lib to be specified raise NotImplementedError if eigen_lib is not None: raise NotImplementedError key = tuple([self.model_code, self.model_cppcode, sys.version_info, sys.executable]) module_name = ("stanfit4" + self.model_name + '_' + hashlib.md5(str(key).encode('utf-8')).hexdigest()) temp_dir = tempfile.mkdtemp() lib_dir = os.path.join(temp_dir, 'pystan') pystan_dir = os.path.dirname(__file__) include_dirs = [lib_dir, pystan_dir, os.path.join(pystan_dir, "stan/src"), os.path.join(pystan_dir, "stan/lib/eigen_3.2.0"), os.path.join(pystan_dir, "stan/lib/boost_1.54.0")] library_dirs = [os.path.join(pystan_dir, "bin")] libraries = ['stan'] if not os.path.exists(lib_dir): os.makedirs(lib_dir) model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp') with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile: outfile.write(self.model_cppcode) pyx_file = os.path.join(lib_dir, module_name + '.pyx') pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx') with io.open(pyx_template_file, 'r', encoding='utf-8') as infile: s = infile.read() template = string.Template(s) with io.open(pyx_file, 'w', encoding='utf-8') as outfile: s = template.safe_substitute(model_cppname=self.model_cppname) outfile.write(s) extra_compile_args = ['-O3', '-ftemplate-depth-256'] extension = Extension(name=module_name, language="c++", sources=[pyx_file], include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries, extra_compile_args=extra_compile_args) cython_include_dirs = ['.', pystan_dir] build_extension = _get_build_extension() build_extension.extensions = cythonize([extension], include_path=cython_include_dirs, quiet=not verbose) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr) if redirect_stderr: # silence stderr for compilation orig_stderr = pystan.misc._redirect_stderr() try: build_extension.run() finally: if redirect_stderr: # restore stderr os.dup2(orig_stderr, sys.stderr.fileno()) self.module = load_module(module_name, lib_dir) self.fit_class = getattr(self.module, "StanFit4" + self.model_cppname)
from Cython.Distutils import build_ext cythonize_switch = True del sys.argv[sys.argv.index('--cythonize')] ext = 'pyx' directives = {} directives['linetrace'] = False directives['binding'] = False directives['profile'] = False cmdclass = {'build_ext': build_ext} else: from setuptools.command.build_ext import build_ext # @NoMove @Reimport cythonize_switch = False ext = 'c' cmdclass = {'build_ext': build_ext} ext_modules = [Extension('cydatemap.cydatemap', [os.path.join('cydatemap', 'cydatemap.%s' % ext)])] setup(name='cydatemap', version=versioneer.get_version(), cmdclass=versioneer.get_cmdclass(cmdclass), author='Jason Rudy', author_email='*****@*****.**', url='https://github.com/jcrudy/cydatemap', packages=find_packages(), package_data={'cydatemap': ['cydatemap.pxd', 'cydatemap.pyx']}, ext_modules = cythonize(ext_modules, compiler_directives=directives) if cythonize_switch else ext_modules, install_requires=[] )
) try: from Cython.Build.Dependencies import cythonize _proesmans_lib_extension = Extension( str("pysteps.motion._proesmans"), sources=[str("pysteps/motion/_proesmans.pyx")], **_proesmans_extension_arguments) _vet_lib_extension = Extension(str("pysteps.motion._vet"), sources=[str("pysteps/motion/_vet.pyx")], **_vet_extension_arguments) external_modules = cythonize( [_proesmans_lib_extension, _vet_lib_extension], force=True, language_level=3) except ImportError: _proesmans_lib_extension = Extension( str("pysteps.motion._proesmans"), sources=[str("pysteps/motion/_proesmans.c")], **_proesmans_extension_arguments) _vet_lib_extension = Extension(str("pysteps.motion._vet"), sources=[str("pysteps/motion/_vet.c")], **_vet_extension_arguments) external_modules = [_proesmans_lib_extension, _vet_lib_extension] requirements = [ "numpy", "attrdict", "jsmin", "scipy", "matplotlib", "jsonschema" ]
def cython(self, line, cell): """Compile and import everything from a Cython code cell. The contents of the cell are written to a `.pyx` file in the directory `IPYTHONDIR/cython` using a filename with the hash of the code. This file is then cythonized and compiled. The resulting module is imported and all of its symbols are injected into the user's namespace. The usage is similar to that of `%%cython_pyximport` but you don't have to pass a module name:: %%cython def f(x): return 2.0*x To compile OpenMP codes, pass the required `--compile-args` and `--link-args`. For example with gcc:: %%cython --compile-args=-fopenmp --link-args=-fopenmp ... """ args = magic_arguments.parse_argstring(self.cython, line) code = cell if cell.endswith('\n') else cell+'\n' lib_dir = os.path.join(get_ipython_cache_dir(), 'cython') quiet = True key = code, sys.version_info, sys.executable, Cython.__version__ if not os.path.exists(lib_dir): os.makedirs(lib_dir) if args.force: # Force a new module name by adding the current time to the # key which is hashed to determine the module name. key += time.time(), if args.name: module_name = py3compat.unicode_to_str(args.name) else: module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest() module_path = os.path.join(lib_dir, module_name + self.so_ext) have_module = os.path.isfile(module_path) need_cythonize = not have_module if args.annotate: html_file = os.path.join(lib_dir, module_name + '.html') if not os.path.isfile(html_file): need_cythonize = True if need_cythonize: c_include_dirs = args.include if 'numpy' in code: import numpy c_include_dirs.append(numpy.get_include()) pyx_file = os.path.join(lib_dir, module_name + '.pyx') pyx_file = py3compat.cast_bytes_py2(pyx_file, encoding=sys.getfilesystemencoding()) with io.open(pyx_file, 'w', encoding='utf-8') as f: f.write(code) extension = Extension( name = module_name, sources = [pyx_file], include_dirs = c_include_dirs, library_dirs = args.library_dirs, extra_compile_args = args.compile_args, extra_link_args = args.link_args, libraries = args.lib, language = 'c++' if args.cplus else 'c', ) build_extension = self._get_build_extension() try: opts = dict( quiet=quiet, annotate = args.annotate, force = True, ) build_extension.extensions = cythonize([extension], **opts) except CompileError: return if not have_module: build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() self._code_cache[key] = module_name module = imp.load_dynamic(module_name, module_path) self._import_all(module) if args.annotate: try: with io.open(html_file, encoding='utf-8') as f: annotated_html = f.read() except IOError as e: # File could not be opened. Most likely the user has a version # of Cython before 0.15.1 (when `cythonize` learned the # `force` keyword argument) and has already compiled this # exact source without annotation. print('Cython completed successfully but the annotated ' 'source could not be read.', file=sys.stderr) print(e, file=sys.stderr) else: return display.HTML(self.clean_annotated_html(annotated_html))
def __init__(self, file=None, charset='utf-8', model_name="anon_model", model_code=None, stanc_ret=None, boost_lib=None, eigen_lib=None, save_dso=True, verbose=False, **kwargs): if stanc_ret is None: stanc_ret = pystan.api.stanc(file=file, model_code=model_code, model_name=model_name, verbose=verbose, **kwargs) if not isinstance(stanc_ret, dict): raise ValueError("stanc_ret must be an object returned by stanc.") stanc_ret_keys = { 'status', 'model_code', 'model_cppname', 'cppcode', 'model_name' } if not all(n in stanc_ret_keys for n in stanc_ret): raise ValueError("stanc_ret lacks one or more of the keys: " "{}".format(str(stanc_ret_keys))) elif stanc_ret['status'] != 0: # success == 0 raise ValueError("stanc_ret is not a successfully returned " "dictionary from stanc.") self.model_cppname = stanc_ret['model_cppname'] self.model_name = stanc_ret['model_name'] self.model_code = model_code self.model_cppcode = stanc_ret['cppcode'] msg = "COMPILING THE C++ CODE FOR MODEL {} NOW." logging.warn(msg.format(self.model_name)) if verbose: msg = "OS: {}, Python: {}, Cython {}".format( sys.platform, sys.version, Cython.__version__) logging.info(msg) if boost_lib is not None: # FIXME: allow boost_lib, eigen_lib to be specified raise NotImplementedError if eigen_lib is not None: raise NotImplementedError key = tuple([ self.model_code, self.model_cppcode, sys.version_info, sys.executable ]) module_name = ("stanfit4" + self.model_name + '_' + hashlib.md5(str(key).encode('utf-8')).hexdigest()) temp_dir = tempfile.mkdtemp() lib_dir = os.path.join(temp_dir, 'pystan') pystan_dir = os.path.dirname(__file__) include_dirs = [ lib_dir, pystan_dir, os.path.join(pystan_dir, "stan/src"), os.path.join(pystan_dir, "stan/lib/eigen_3.2.0"), os.path.join(pystan_dir, "stan/lib/boost_1.54.0") ] library_dirs = [os.path.join(pystan_dir, "bin")] libraries = ['stan'] if not os.path.exists(lib_dir): os.makedirs(lib_dir) model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp') with open(model_cpp_file, 'w') as outfile: outfile.write(self.model_cppcode) pyx_file = os.path.join(lib_dir, module_name + '.pyx') pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx') with open(pyx_template_file) as infile: s = infile.read() template = string.Template(s) with open(pyx_file, 'w') as outfile: s = template.safe_substitute(model_cppname=self.model_cppname) outfile.write(s) extension = Extension(name=module_name, language="c++", sources=[pyx_file], include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries, extra_compile_args=['-O3']) cython_include_dirs = ['.', pystan_dir] build_extension = _get_build_extension() build_extension.extensions = cythonize( [extension], include_path=cython_include_dirs, quiet=not verbose) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr) if redirect_stderr: # silence stderr for compilation orig_stderr = pystan.misc._redirect_stderr() try: build_extension.run() finally: if redirect_stderr: # restore stderr os.dup2(orig_stderr, sys.stderr.fileno()) module_path = lib_dir self.module = load_module(module_name, module_path) self.fit_class = getattr(self.module, "StanFit4" + self.model_cppname)
def build(force_rebuild=False): import sys, os import shutil import glob import numpy import fcntl from distutils.core import Distribution, Extension from distutils.command.build_ext import build_ext from distutils.sysconfig import get_config_vars, get_python_inc import Cython from Cython.Compiler.Main import Context, default_options from Cython.Build.Dependencies import cythonize source_dir = os.path.dirname(os.path.abspath(__file__)) build_dir = os.path.join(source_dir, 'bld') source_dir = os.path.join(source_dir, 'pymalis') source_files = [ os.path.join(source_dir, 'frontend.pyx'), os.path.join(source_dir, 'malis_loss_layer.hpp'), os.path.join(source_dir, 'malis_loss_layer.cpp'), os.path.join(source_dir, 'c_frontend.h'), ] source_files.sort() module_name = '_pymalis' # since this could be called concurrently, there is no good way to check # whether the directory already exists try: os.makedirs(build_dir) except: pass # make sure the same module is not build concurrently with open(os.path.join(build_dir, module_name + '.lock'), 'w') as lock_file: fcntl.lockf(lock_file, fcntl.LOCK_EX) try: if build_dir not in sys.path: sys.path.append(build_dir) if force_rebuild: raise ImportError else: __import__(module_name) print("Re-using already compiled pymalis version") except ImportError: print("Compiling pymalis in " + str(build_dir)) cython_include_dirs = ['.'] ctx = Context(cython_include_dirs, default_options) scoring_function_include_dir = os.path.join(build_dir, module_name) if not os.path.exists(scoring_function_include_dir): os.makedirs(scoring_function_include_dir) include_dirs = [ source_dir, os.path.dirname(get_python_inc()), numpy.get_include(), ] # cython requires that the pyx file has the same name as the module shutil.copy(os.path.join(source_dir, 'frontend.pyx'), os.path.join(build_dir, module_name + '.pyx')) shutil.copy( os.path.join(source_dir, 'malis_loss_layer.cpp'), os.path.join(build_dir, module_name + '_malis_loss_layer.cpp')) # Remove the "-Wstrict-prototypes" compiler option, which isn't valid # for C++. cfg_vars = get_config_vars() if "CFLAGS" in cfg_vars: cfg_vars["CFLAGS"] = cfg_vars["CFLAGS"].replace( "-Wstrict-prototypes", "") extension = Extension( module_name, sources=[ os.path.join(build_dir, module_name + '.pyx'), os.path.join(build_dir, module_name + '_malis_loss_layer.cpp') ], include_dirs=include_dirs, language='c++', extra_link_args=['-std=c++11'], extra_compile_args=['-std=c++11', '-w']) build_extension = build_ext(Distribution()) build_extension.finalize_options() build_extension.extensions = cythonize([extension], quiet=True, nthreads=2) build_extension.build_temp = build_dir build_extension.build_lib = build_dir build_extension.run()
def cython_inline(code, get_type=unsafe_type, lib_dir=os.path.join(get_cython_cache_dir(), 'inline'), cython_include_dirs=['.'], force=False, quiet=False, locals=None, globals=None, **kwds): if get_type is None: get_type = lambda x: 'object' code = to_unicode(code) orig_code = code code, literals = strip_string_literals(code) code = strip_common_indent(code) ctx = _create_context(tuple(cython_include_dirs)) if locals is None: locals = inspect.currentframe().f_back.f_back.f_locals if globals is None: globals = inspect.currentframe().f_back.f_back.f_globals try: for symbol in unbound_symbols(code): if symbol in kwds: continue elif symbol in locals: kwds[symbol] = locals[symbol] elif symbol in globals: kwds[symbol] = globals[symbol] else: print("Couldn't find ", symbol) except AssertionError: if not quiet: # Parsing from strings not fully supported (e.g. cimports). print( "Could not parse code as a string (to extract unbound symbols)." ) cimports = [] for name, arg in kwds.items(): if arg is cython_module: cimports.append('\ncimport cython as %s' % name) del kwds[name] arg_names = kwds.keys() arg_names.sort() arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names]) key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__ module_name = "_cython_inline_" + hashlib.md5( str(key).encode('utf-8')).hexdigest() if module_name in sys.modules: module = sys.modules[module_name] else: build_extension = None if cython_inline.so_ext is None: # Figure out and cache current extension suffix build_extension = _get_build_extension() cython_inline.so_ext = build_extension.get_ext_filename('') module_path = os.path.join(lib_dir, module_name + cython_inline.so_ext) if not os.path.exists(lib_dir): os.makedirs(lib_dir) if force or not os.path.isfile(module_path): cflags = [] c_include_dirs = [] qualified = re.compile(r'([.\w]+)[.]') for type, _ in arg_sigs: m = qualified.match(type) if m: cimports.append('\ncimport %s' % m.groups()[0]) # one special case if m.groups()[0] == 'numpy': import numpy c_include_dirs.append(numpy.get_include()) # cflags.append('-Wno-unused') module_body, func_body = extract_func_code(code) params = ', '.join(['%s %s' % a for a in arg_sigs]) module_code = """ %(module_body)s %(cimports)s def __invoke(%(params)s): %(func_body)s return locals() """ % { 'cimports': '\n'.join(cimports), 'module_body': module_body, 'params': params, 'func_body': func_body } for key, value in literals.items(): module_code = module_code.replace(key, value) pyx_file = os.path.join(lib_dir, module_name + '.pyx') fh = open(pyx_file, 'w') try: fh.write(module_code) finally: fh.close() extension = Extension(name=module_name, sources=[pyx_file], include_dirs=c_include_dirs, extra_compile_args=cflags) if build_extension is None: build_extension = _get_build_extension() build_extension.extensions = cythonize( [extension], include_path=cython_include_dirs, quiet=quiet) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() module = imp.load_dynamic(module_name, module_path) arg_list = [kwds[arg] for arg in arg_names] return module.__invoke(*arg_list)
def __init__( self, file=None, charset="utf-8", model_name="anon_model", model_code=None, stanc_ret=None, boost_lib=None, eigen_lib=None, save_dso=True, verbose=False, obfuscate_model_name=True, ): if stanc_ret is None: stanc_ret = pystan.api.stanc( file=file, charset=charset, model_code=model_code, model_name=model_name, verbose=verbose, obfuscate_model_name=obfuscate_model_name, ) if not isinstance(stanc_ret, dict): raise ValueError("stanc_ret must be an object returned by stanc.") stanc_ret_keys = {"status", "model_code", "model_cppname", "cppcode", "model_name"} if not all(n in stanc_ret_keys for n in stanc_ret): raise ValueError("stanc_ret lacks one or more of the keys: " "{}".format(str(stanc_ret_keys))) elif stanc_ret["status"] != 0: # success == 0 raise ValueError("stanc_ret is not a successfully returned " "dictionary from stanc.") self.model_cppname = stanc_ret["model_cppname"] self.model_name = stanc_ret["model_name"] self.model_code = stanc_ret["model_code"] self.model_cppcode = stanc_ret["cppcode"] self.save_dso = save_dso msg = "COMPILING THE C++ CODE FOR MODEL {} NOW." logger.info(msg.format(self.model_name)) if verbose: msg = "OS: {}, Python: {}, Cython {}".format(sys.platform, sys.version, Cython.__version__) logger.info(msg) if boost_lib is not None: # FIXME: allow boost_lib, eigen_lib to be specified raise NotImplementedError if eigen_lib is not None: raise NotImplementedError key = tuple([self.model_code, self.model_cppcode, sys.version_info, sys.executable]) module_name = "stanfit4" + self.model_name + "_" + hashlib.md5(str(key).encode("utf-8")).hexdigest() self._temp_dir = temp_dir = tempfile.mkdtemp() lib_dir = os.path.join(temp_dir, "pystan") pystan_dir = os.path.dirname(__file__) include_dirs = [ lib_dir, pystan_dir, os.path.join(pystan_dir, "stan/src"), os.path.join(pystan_dir, "math"), os.path.join(pystan_dir, "stan/lib/eigen_3.2.4"), os.path.join(pystan_dir, "stan/lib/boost_1.58.0"), np.get_include(), ] if not os.path.exists(lib_dir): os.makedirs(lib_dir) model_cpp_file = os.path.join(lib_dir, self.model_cppname + ".hpp") with io.open(model_cpp_file, "w", encoding="utf-8") as outfile: outfile.write(self.model_cppcode) pyx_file = os.path.join(lib_dir, module_name + ".pyx") pyx_template_file = os.path.join(pystan_dir, "stanfit4model.pyx") with io.open(pyx_template_file, "r", encoding="utf-8") as infile: s = infile.read() template = string.Template(s) with io.open(pyx_file, "w", encoding="utf-8") as outfile: s = template.safe_substitute(model_cppname=self.model_cppname) outfile.write(s) stan_macros = [ ("BOOST_RESULT_OF_USE_TR1", None), ("BOOST_NO_DECLTYPE", None), ("BOOST_DISABLE_ASSERTS", None), ("EIGEN_NO_DEBUG", None), ] extra_compile_args = ["-O0", "-ftemplate-depth-256", "-Wno-unused-function", "-Wno-uninitialized"] distutils.log.set_verbosity(verbose) extension = Extension( name=module_name, language="c++", sources=[pyx_file], define_macros=stan_macros, include_dirs=include_dirs, extra_compile_args=extra_compile_args, ) cython_include_dirs = [".", pystan_dir] build_extension = _get_build_extension() build_extension.extensions = cythonize([extension], include_path=cython_include_dirs, quiet=not verbose) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr) if redirect_stderr: # silence stderr for compilation orig_stderr = pystan.misc._redirect_stderr() try: build_extension.run() finally: if redirect_stderr: # restore stderr os.dup2(orig_stderr, sys.stderr.fileno()) self.module = load_module(module_name, lib_dir) self.fit_class = getattr(self.module, "StanFit4Model")
def finalize_options(self): from Cython.Build.Dependencies import cythonize self.distribution.ext_modules[:] = cythonize( self.distribution.ext_modules, compile_time_env=_CYTHON_COMPILE_TIME_ENV) super(my_build_ext, self).finalize_options()
def finalize_options(self): if self.distribution.ext_modules: from Cython.Build.Dependencies import cythonize self.distribution.ext_modules[:] = cythonize( self.distribution.ext_modules) super(build_ext, self).finalize_options()
cythonize_switch = True del sys.argv[sys.argv.index('--cythonize')] ext = 'pyx' directives = {} directives['linetrace'] = False directives['binding'] = False directives['profile'] = False cmdclass = {'build_ext': build_ext} else: from setuptools.command.build_ext import build_ext # @NoMove @Reimport cythonize_switch = False ext = 'c' cmdclass = {'build_ext': build_ext} ext_modules = [ Extension('sklearntools.interpolation', [os.path.join('sklearntools', 'interpolation.%s' % ext)]) ] setup(name='sklearntools', version=versioneer.get_version(), cmdclass=versioneer.get_cmdclass(), description='Tools for sklearn models', author='Jason Rudy', author_email='*****@*****.**', url='https://gitlab.com/jcrudy/sklearntools', packages=find_packages(), package_data={'': ['sym/resources/*']}, ext_modules=cythonize(ext_modules, compiler_directives=directives) if cythonize_switch else ext_modules, install_requires=['scikit-learn>=0.17.0', 'pandas', 'numpy'])
def cython_inline(code, get_type=unsafe_type, lib_dir=os.path.join(get_cython_cache_dir(), 'inline'), cython_include_dirs=['.'], force=False, quiet=False, locals=None, globals=None, **kwds): if get_type is None: get_type = lambda x: 'object' code = to_unicode(code) orig_code = code code, literals = strip_string_literals(code) code = strip_common_indent(code) ctx = Context(cython_include_dirs, default_options) if locals is None: locals = inspect.currentframe().f_back.f_back.f_locals if globals is None: globals = inspect.currentframe().f_back.f_back.f_globals try: for symbol in unbound_symbols(code): if symbol in kwds: continue elif symbol in locals: kwds[symbol] = locals[symbol] elif symbol in globals: kwds[symbol] = globals[symbol] else: print("Couldn't find ", symbol) except AssertionError: if not quiet: # Parsing from strings not fully supported (e.g. cimports). print("Could not parse code as a string (to extract unbound symbols).") cimports = [] for name, arg in kwds.items(): if arg is cython_module: cimports.append('\ncimport cython as %s' % name) del kwds[name] arg_names = kwds.keys() arg_names.sort() arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names]) key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__ module_name = "_cython_inline_" + hashlib.md5(str(key).encode('utf-8')).hexdigest() so_ext = [ ext for ext,_,mod_type in imp.get_suffixes() if mod_type == imp.C_EXTENSION ][0] module_path = os.path.join(lib_dir, module_name+so_ext) if not os.path.exists(lib_dir): os.makedirs(lib_dir) if force or not os.path.isfile(module_path): cflags = [] c_include_dirs = [] qualified = re.compile(r'([.\w]+)[.]') for type, _ in arg_sigs: m = qualified.match(type) if m: cimports.append('\ncimport %s' % m.groups()[0]) # one special case if m.groups()[0] == 'numpy': import numpy c_include_dirs.append(numpy.get_include()) # cflags.append('-Wno-unused') module_body, func_body = extract_func_code(code) params = ', '.join(['%s %s' % a for a in arg_sigs]) module_code = """ %(module_body)s %(cimports)s def __invoke(%(params)s): %(func_body)s """ % {'cimports': '\n'.join(cimports), 'module_body': module_body, 'params': params, 'func_body': func_body } for key, value in literals.items(): module_code = module_code.replace(key, value) pyx_file = os.path.join(lib_dir, module_name + '.pyx') fh = open(pyx_file, 'w') try: fh.write(module_code) finally: fh.close() extension = Extension( name = module_name, sources = [pyx_file], include_dirs = c_include_dirs, extra_compile_args = cflags) dist = Distribution() # Ensure the build respects distutils configuration by parsing # the configuration files config_files = dist.find_config_files() dist.parse_config_files(config_files) build_extension = build_ext(dist) build_extension.finalize_options() build_extension.extensions = cythonize([extension], ctx=ctx, quiet=quiet) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() _code_cache[key] = module_name module = imp.load_dynamic(module_name, module_path) arg_list = [kwds[arg] for arg in arg_names] return module.__invoke(*arg_list)
def finalize_options(self): if self.distribution.ext_modules: from Cython.Build.Dependencies import cythonize self.distribution.ext_modules[:] = cythonize( self.distribution.ext_modules) super(new_build_ext, self).finalize_options()
def wrap(py_func): #clear buf pyx_buf.seek(0) pyx_buf.truncate() pxd_buf.seek(0) pxd_buf.truncate() called_jitted_funcs=set() local_names=set() #in py_func load_names=set() #in py_func func_name=py_func.__name__ func_globals_name='func_%s_globals'%func_name #func_globals=sys._getframe(1).f_locals #get caller's scope func_globals=py_func.func_globals py_src=get_func_source(py_func) args=get_args(py_func) body=get_body(py_src) #print py_src indent=get_indent(body) func_def='cpdef' func_ast=ast.parse(py_src) jvisit(func_ast, func_globals, called_jitted_funcs, local_names, load_names) #print called_jitted_funcs for jitted_func, mode, extra in called_jitted_funcs: if mode=='direct': pyx_buf.write("from %s cimport %s\n"%(jitted_func.module_name, jitted_func.func_name)) elif mode=='attribute': pyx_buf.write('cimport %s\n'%(jitted_func.module_name,)) attribute_func_name=extra # body=re.sub('%s *\('%attribute_func_name, '%s.%s('%(jitted_func.module_name, jitted_func.func_name), body) pyx_buf.write('\n') pyx_buf.write(''' import sys %s=sys._getframe(1).f_locals '''%func_globals_name) for name in load_names: pyx_buf.write("%s=%s.get('%s')\n"%(name, func_globals_name, name)) pyx_buf.write('\n') #process signature formated_sig=sig.replace(' ','') if formated_sig: return_type, arg_types=parse_sig(formated_sig) assert len(args)==len(arg_types), "the function %s has %s args but the signature has %s args "%(func_name, len(args), len(arg_types)) func_args=[] for arg_name, arg_type in zip(args, arg_types): func_args.append('%s %s'%(arg_type, arg_name)) func_args=", ".join(func_args) else: func_args=', '.join(args) return_type='' #directives #print "in" directive_decorates=[] directives=['wraparound', 'boundscheck'] for directive in directives: value=kwds.get(directive) if value != None: directive_decorates.append('@cython.%s(%s)\n'%(directive, value)) if directive_decorates: pyx_buf.write('cimport cython\n\n') for decorate in directive_decorates: pyx_buf.write(decorate) nogil='nogil' if kwds.get("nogil") else '' #head #add like 'nogil' func_head='%s %s %s(%s) %s'%(func_def, return_type, func_name, func_args, nogil) #func_head_1='\n%s %s(%s)%s'%('cdef', func_name, func_args, '') pxd_buf.write(func_head+"\n") pyx_buf.write(func_head+':\n') #cdef vars locals=kwds.get('locals') if type(locals) is str: pyx_buf.write(indent+"cdef:\n") for line in locals.strip().split('\n'): line=line.strip() if line[-1]==';': line=line[:-1] pyx_buf.write(indent*2+line+'\n') #if type(locals) is str: # for type_name, vars_name in parse_cdeclare(locals): # cdef=indent+'cdef %s %s\n'%(type_name, vars_name) # pyx_buf.write(cdef) #body pyx_buf.write(body+'\n') #build pyx_src=pyx_buf.getvalue() pxd_src=pxd_buf.getvalue() base_name=os.path.basename(func_globals['__file__']) file_name=os.path.splitext(base_name)[0] key = pyx_src+str(sys.version_info)+sys.executable+cython.__version__ hashed_module_name = file_name+'_'+func_name+'__'+hashlib.md5(key.encode('utf-8')).hexdigest() caller_file_dir=os.path.dirname(os.path.abspath(func_globals['__file__'])) #print 'caller_file_dir', caller_file_dir module_dir=os.path.join(caller_file_dir, '__cython_compile__') if not os.path.exists(module_dir): os.mkdir(module_dir) pyx_file=os.path.join(module_dir, hashed_module_name+'.pyx') pxd_file=os.path.join(module_dir, hashed_module_name+'.pxd') so_ext='.pyd' so_file=os.path.join(module_dir, hashed_module_name+so_ext) init_file=os.path.join(module_dir, '__init__.py') #check if not os.path.exists(so_file): fw=open(pyx_file,"w") fw.write(pyx_src) fw.close() fw=open(pxd_file,"w") fw.write(pxd_src) fw.close() fw=open(init_file, "w") fw.close() extension = Extension(name = hashed_module_name, sources = [pyx_file], #include_dirs = c_include_dirs, #extra_compile_args = cflags ) #find called_jitted_funcs's module path cython_include_dirs=[] for jitted_func, _, _ in called_jitted_funcs: cython_include_dirs.append(jitted_func.module_dir) #print 'cython_include_dirs', cython_include_dirs build_extension = _get_build_extension() build_extension.extensions = cythonize([extension], annotate=True, include_path=cython_include_dirs, #quiet=quiet ) temp_dir=os.path.join(module_dir, "__build_temp__") build_extension.build_temp = temp_dir build_extension.build_lib = module_dir #print "build" build_extension.run() compiled_module=imp.load_dynamic(hashed_module_name, so_file) c_func=getattr(compiled_module, func_name) return JittedFunc(c_func, py_func, func_name, hashed_module_name, module_dir, pyx_src)
def __init__(self, file=None, charset='utf-8', model_name="anon_model", model_code=None, stanc_ret=None, boost_lib=None, eigen_lib=None, verbose=False, obfuscate_model_name=True, extra_compile_args=None): if stanc_ret is None: stanc_ret = pystan.api.stanc(file=file, charset=charset, model_code=model_code, model_name=model_name, verbose=verbose, obfuscate_model_name=obfuscate_model_name) if not isinstance(stanc_ret, dict): raise ValueError("stanc_ret must be an object returned by stanc.") stanc_ret_keys = {'status', 'model_code', 'model_cppname', 'cppcode', 'model_name'} if not all(n in stanc_ret_keys for n in stanc_ret): raise ValueError("stanc_ret lacks one or more of the keys: " "{}".format(str(stanc_ret_keys))) elif stanc_ret['status'] != 0: # success == 0 raise ValueError("stanc_ret is not a successfully returned " "dictionary from stanc.") self.model_cppname = stanc_ret['model_cppname'] self.model_name = stanc_ret['model_name'] self.model_code = stanc_ret['model_code'] self.model_cppcode = stanc_ret['cppcode'] msg = "COMPILING THE C++ CODE FOR MODEL {} NOW." logger.info(msg.format(self.model_name)) if verbose: msg = "OS: {}, Python: {}, Cython {}".format(sys.platform, sys.version, Cython.__version__) logger.info(msg) if boost_lib is not None: # FIXME: allow boost_lib, eigen_lib to be specified raise NotImplementedError if eigen_lib is not None: raise NotImplementedError # module_name needs to be unique so that each model instance has its own module nonce = abs(hash((self.model_name, time.time()))) self.module_name = 'stanfit4{}_{}'.format(self.model_name, nonce) lib_dir = tempfile.mkdtemp() pystan_dir = os.path.dirname(__file__) include_dirs = [ lib_dir, pystan_dir, os.path.join(pystan_dir, "stan", "src"), os.path.join(pystan_dir, "stan", "lib", "stan_math_2.14.0"), os.path.join(pystan_dir, "stan", "lib", "stan_math_2.14.0", "lib", "eigen_3.2.9"), os.path.join(pystan_dir, "stan", "lib", "stan_math_2.14.0", "lib", "boost_1.62.0"), os.path.join(pystan_dir, "stan", "lib", "stan_math_2.14.0", "lib", "cvodes_2.9.0", "include"), np.get_include(), ] model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp') with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile: outfile.write(self.model_cppcode) pyx_file = os.path.join(lib_dir, self.module_name + '.pyx') pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx') with io.open(pyx_template_file, 'r', encoding='utf-8') as infile: s = infile.read() template = string.Template(s) with io.open(pyx_file, 'w', encoding='utf-8') as outfile: s = template.safe_substitute(model_cppname=self.model_cppname) outfile.write(s) stan_macros = [ ('BOOST_RESULT_OF_USE_TR1', None), ('BOOST_NO_DECLTYPE', None), ('BOOST_DISABLE_ASSERTS', None), ] # compile stan models with optimization (-O2) # (stanc is compiled without optimization (-O0) currently, see #33) if extra_compile_args is None: extra_compile_args = [ '-O2', '-ftemplate-depth-256', '-Wno-unused-function', '-Wno-uninitialized', ] if platform.platform().startswith('Win'): extra_compile_args = ['/EHsc', '-DBOOST_DATE_TIME_NO_LIB'] distutils.log.set_verbosity(verbose) extension = Extension(name=self.module_name, language="c++", sources=[pyx_file], define_macros=stan_macros, include_dirs=include_dirs, extra_compile_args=extra_compile_args) cython_include_dirs = ['.', pystan_dir] build_extension = _get_build_extension() build_extension.extensions = cythonize([extension], include_path=cython_include_dirs, quiet=not verbose) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr) if redirect_stderr: # silence stderr for compilation orig_stderr = pystan.misc._redirect_stderr() try: build_extension.run() finally: if redirect_stderr: # restore stderr os.dup2(orig_stderr, sys.stderr.fileno()) self.module = load_module(self.module_name, lib_dir) self.module_filename = os.path.basename(self.module.__file__) # once the module is in memory, we no longer need the file on disk # but we do need a copy of the file for pickling and the module name with io.open(os.path.join(lib_dir, self.module_filename), 'rb') as f: self.module_bytes = f.read() shutil.rmtree(lib_dir, ignore_errors=True) self.fit_class = getattr(self.module, "StanFit4Model")
def __init__(self, file=None, charset='utf-8', model_name="anon_model", model_code=None, stanc_ret=None, boost_lib=None, eigen_lib=None, verbose=False, obfuscate_model_name=True, extra_compile_args=None): if stanc_ret is None: stanc_ret = pystan.api.stanc(file=file, charset=charset, model_code=model_code, model_name=model_name, verbose=verbose, obfuscate_model_name=obfuscate_model_name) if not isinstance(stanc_ret, dict): raise ValueError("stanc_ret must be an object returned by stanc.") stanc_ret_keys = {'status', 'model_code', 'model_cppname', 'cppcode', 'model_name'} if not all(n in stanc_ret_keys for n in stanc_ret): raise ValueError("stanc_ret lacks one or more of the keys: " "{}".format(str(stanc_ret_keys))) elif stanc_ret['status'] != 0: # success == 0 raise ValueError("stanc_ret is not a successfully returned " "dictionary from stanc.") self.model_cppname = stanc_ret['model_cppname'] self.model_name = stanc_ret['model_name'] self.model_code = stanc_ret['model_code'] self.model_cppcode = stanc_ret['cppcode'] msg = "COMPILING THE C++ CODE FOR MODEL {} NOW." logger.info(msg.format(self.model_name)) if verbose: msg = "OS: {}, Python: {}, Cython {}".format(sys.platform, sys.version, Cython.__version__) logger.info(msg) if boost_lib is not None: # FIXME: allow boost_lib, eigen_lib to be specified raise NotImplementedError if eigen_lib is not None: raise NotImplementedError # module_name needs to be unique so that each model instance has its own module nonce = abs(hash((self.model_name, time.time()))) self.module_name = 'stanfit4{}_{}'.format(self.model_name, nonce) lib_dir = tempfile.mkdtemp() pystan_dir = os.path.dirname(__file__) include_dirs = [ lib_dir, pystan_dir, os.path.join(pystan_dir, "stan", "src"), os.path.join(pystan_dir, "stan", "lib", "stan_math_2.10.0"), os.path.join(pystan_dir, "stan", "lib", "stan_math_2.10.0", "lib", "eigen_3.2.8"), os.path.join(pystan_dir, "stan", "lib", "stan_math_2.10.0", "lib", "boost_1.60.0"), os.path.join(pystan_dir, "stan", "lib", "stan_math_2.10.0", "lib", "cvodes_2.8.2", "include"), np.get_include(), ] model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp') with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile: outfile.write(self.model_cppcode) pyx_file = os.path.join(lib_dir, self.module_name + '.pyx') pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx') with io.open(pyx_template_file, 'r', encoding='utf-8') as infile: s = infile.read() template = string.Template(s) with io.open(pyx_file, 'w', encoding='utf-8') as outfile: s = template.safe_substitute(model_cppname=self.model_cppname) outfile.write(s) stan_macros = [ ('BOOST_RESULT_OF_USE_TR1', None), ('BOOST_NO_DECLTYPE', None), ('BOOST_DISABLE_ASSERTS', None), ] # compile stan models with optimization (-O2) # (stanc is compiled without optimization (-O0) currently, see #33) if extra_compile_args is None: extra_compile_args = [ '-O2', '-ftemplate-depth-256', '-Wno-unused-function', '-Wno-uninitialized', ] if platform.platform().startswith('Win'): extra_compile_args = ['/EHsc', '-DBOOST_DATE_TIME_NO_LIB'] distutils.log.set_verbosity(verbose) extension = Extension(name=self.module_name, language="c++", sources=[pyx_file], define_macros=stan_macros, include_dirs=include_dirs, extra_compile_args=extra_compile_args) cython_include_dirs = ['.', pystan_dir] build_extension = _get_build_extension() build_extension.extensions = cythonize([extension], include_path=cython_include_dirs, quiet=not verbose) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr) if redirect_stderr: # silence stderr for compilation orig_stderr = pystan.misc._redirect_stderr() try: build_extension.run() finally: if redirect_stderr: # restore stderr os.dup2(orig_stderr, sys.stderr.fileno()) self.module = load_module(self.module_name, lib_dir) self.module_filename = os.path.basename(self.module.__file__) # once the module is in memory, we no longer need the file on disk # but we do need a copy of the file for pickling and the module name with io.open(os.path.join(lib_dir, self.module_filename), 'rb') as f: self.module_bytes = f.read() shutil.rmtree(lib_dir, ignore_errors=True) self.fit_class = getattr(self.module, "StanFit4Model")
def __compile( scoring_function='OneMinus<MeanAffinity<RegionGraphType, ScoreValue>>', discretize_queue=0, force_rebuild=False): import sys import os import shutil import glob import numpy import fcntl try: import hashlib except ImportError: import md5 as hashlib from distutils.core import Distribution, Extension from distutils.command.build_ext import build_ext from distutils.sysconfig import get_config_vars, get_python_inc import Cython from Cython.Compiler.Main import Context, default_options from Cython.Build.Dependencies import cythonize # compile frontend.pyx for given scoring function source_dir = os.path.dirname(os.path.abspath(__file__)) source_files = [ os.path.join(source_dir, 'agglomerate.pyx'), os.path.join(source_dir, 'frontend_agglomerate.h'), os.path.join(source_dir, 'frontend_agglomerate.cpp') ] source_files += glob.glob(source_dir + '/backend/*.hpp') source_files.sort() source_files_hashes = [ hashlib.md5(open(f, 'r').read().encode('utf-8')).hexdigest() for f in source_files ] key = scoring_function, discretize_queue, source_files_hashes, sys.version_info, sys.executable, Cython.__version__ module_name = 'waterz_' + hashlib.md5(str(key).encode('utf-8')).hexdigest() lib_dir = os.path.expanduser('~/.cython/inline') # since this could be called concurrently, there is no good way to check # whether the directory already exists try: os.makedirs(lib_dir) except: pass # make sure the same module is not build concurrently with open( os.path.join( lib_dir, module_name + '_%s.lock' % str(numpy.random.random(1)[0])[2:]), 'w') as lock_file: fcntl.lockf(lock_file, fcntl.LOCK_EX) try: if lib_dir not in sys.path: sys.path.append(lib_dir) if force_rebuild: raise ImportError else: print("Re-using already compiled waterz version") return module_name except ImportError: print("Compiling waterz in " + str(lib_dir)) cython_include_dirs = ['.'] ctx = Context(cython_include_dirs, default_options) include_dir = os.path.join(lib_dir, module_name) if not os.path.exists(include_dir): os.makedirs(include_dir) include_dirs = [ source_dir, include_dir, os.path.join(source_dir, 'backend'), os.path.dirname(get_python_inc()), numpy.get_include(), ] scoring_function_header = os.path.join(include_dir, 'ScoringFunction.h') with open(scoring_function_header, 'w') as f: f.write('typedef %s ScoringFunctionType;' % scoring_function) queue_header = os.path.join(include_dir, 'Queue.h') with open(queue_header, 'w') as f: if discretize_queue == 0: f.write( 'template<typename T, typename S> using QueueType = PriorityQueue<T, S>;' ) else: f.write( 'template<typename T, typename S> using QueueType = BinQueue<T, S, %d>;' % discretize_queue) # cython requires that the pyx file has the same name as the module shutil.copy(os.path.join(source_dir, 'agglomerate.pyx'), os.path.join(lib_dir, module_name + '.pyx')) shutil.copy( os.path.join(source_dir, 'frontend_agglomerate.cpp'), os.path.join(lib_dir, module_name + '_frontend_agglomerate.cpp')) # Remove the "-Wstrict-prototypes" compiler option, which isn't valid # for C++. cfg_vars = get_config_vars() if "CFLAGS" in cfg_vars: cfg_vars["CFLAGS"] = cfg_vars["CFLAGS"].replace( "-Wstrict-prototypes", "") extension = Extension( module_name, sources=[ os.path.join(lib_dir, module_name + '.pyx'), os.path.join(lib_dir, module_name + '_frontend_agglomerate.cpp') ], include_dirs=include_dirs, language='c++', extra_link_args=['-std=c++11'], extra_compile_args=['-std=c++11', '-w']) build_extension = build_ext(Distribution()) build_extension.finalize_options() build_extension.extensions = cythonize([extension], quiet=True, nthreads=1, language_level="3") build_extension.build_temp = lib_dir build_extension.build_lib = lib_dir build_extension.run() return module_name
def cython(self, line, cell): """Compile and import everything from a Cython code cell. The contents of the cell are written to a `.pyx` file in the directory `IPYTHONDIR/cython` using a filename with the hash of the code. This file is then cythonized and compiled. The resulting module is imported and all of its symbols are injected into the user's namespace. The usage is similar to that of `%%cython_pyximport` but you don't have to pass a module name:: %%cython def f(x): return 2.0*x """ args = magic_arguments.parse_argstring(self.cython, line) code = cell if cell.endswith('\n') else cell+'\n' lib_dir = os.path.join(self.shell.ipython_dir, 'cython') quiet = True key = code, sys.version_info, sys.executable, Cython.__version__ if not os.path.exists(lib_dir): os.makedirs(lib_dir) if args.force: # Force a new module name by adding the current time to the # key which is hashed to determine the module name. key += time.time(), module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest() module_path = os.path.join(lib_dir, module_name + self.so_ext) have_module = os.path.isfile(module_path) need_cythonize = not have_module if args.annotate: html_file = os.path.join(lib_dir, module_name + '.html') if not os.path.isfile(html_file): need_cythonize = True if need_cythonize: c_include_dirs = args.include if 'numpy' in code: import numpy c_include_dirs.append(numpy.get_include()) pyx_file = os.path.join(lib_dir, module_name + '.pyx') pyx_file = py3compat.cast_bytes_py2(pyx_file, encoding=sys.getfilesystemencoding()) with io.open(pyx_file, 'w', encoding='utf-8') as f: f.write(code) extension = Extension( name = module_name, sources = [pyx_file], include_dirs = c_include_dirs, extra_compile_args = args.compile_args, extra_link_args = args.link_args, libraries = args.lib, language = 'c++' if args.cplus else 'c', ) build_extension = self._get_build_extension() try: opts = dict( quiet=quiet, annotate = args.annotate, force = True, ) build_extension.extensions = cythonize([extension], **opts) except CompileError: return if not have_module: build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() self._code_cache[key] = module_name module = imp.load_dynamic(module_name, module_path) self._import_all(module) if args.annotate: try: with io.open(html_file, encoding='utf-8') as f: annotated_html = f.read() except IOError as e: # File could not be opened. Most likely the user has a version # of Cython before 0.15.1 (when `cythonize` learned the # `force` keyword argument) and has already compiled this # exact source without annotation. print('Cython completed successfully but the annotated ' 'source could not be read.', file=sys.stderr) print(e, file=sys.stderr) else: return display.HTML(self.clean_annotated_html(annotated_html))
def cythonize(self, extensions): from Cython.Build.Dependencies import cythonize return cythonize(extensions)