def _compile_cython_module_nocache(src, modname, verbose=False): modfile = os.path.join(MODDIR, modname + '.pyx') with open(modfile, 'w+') as f: f.write(src) Cython.Compiler.Options.cimport_from_pyx = True include_dirs = [numpy.get_include()] extra_compile_args = [ '-O3', '-march=native', '-ffast-math', '-fopenmp', '-g1' ] extension = Extension(name=modname, sources=[modfile], include_dirs=include_dirs, extra_compile_args=extra_compile_args) cython_include_dirs = [PYIGAPATH] build_extension = _get_build_extension() build_extension.extensions = cythonize( [extension], include_path=cython_include_dirs, compiler_directives={'language_level': 3}, quiet=False) build_extension.build_temp = MODDIR build_extension.build_lib = MODDIR distutils.log.set_verbosity(verbose) build_extension.run() return importlib.import_module(modname)
def cython_build(name, file=None, force=False, quiet=True, cythonize_args={}, lib_dir=os.path.join(get_cython_cache_dir(), 'inline/lib'), tmp_dir=os.path.join(get_cython_cache_dir(), 'inline/tmp'), **extension_args): """Build a cython extension. """ if file is not None: _append_args(extension_args, 'sources', file) with _suppress_output(quiet=quiet): extension = Extension(name, **extension_args) extensions = cythonize([extension], force=force, **cythonize_args) build_extension = _get_build_extension() build_extension.extensions = extensions build_extension.build_lib = lib_dir build_extension.build_temp = tmp_dir build_extension.run()
('BOOST_DISABLE_ASSERTS', None), ('BOOST_NO_DECLTYPE', None), ('BOOST_PHOENIX_NO_VARIADIC_EXPRESSION', None), # needed for stanc ('BOOST_RESULT_OF_USE_TR1', None), ('FUSION_MAX_VECTOR_SIZE', 12), # for parser, stan-dev/pystan#222 ] extra_compile_args = [ '-Os', '-ftemplate-depth-256', '-Wno-unused-function', '-Wno-uninitialized', ] if platform.platform().startswith('Win'): from Cython.Build.Inline import _get_build_extension if _get_build_extension().compiler in (None, 'msvc'): extra_compile_args = [ '/EHsc', '-DBOOST_DATE_TIME_NO_LIB', ] stanc_sources = [ "pystan/stan/src/stan/lang/ast_def.cpp", "pystan/stan/src/stan/lang/grammars/bare_type_grammar_inst.cpp", "pystan/stan/src/stan/lang/grammars/expression07_grammar_inst.cpp", "pystan/stan/src/stan/lang/grammars/expression_grammar_inst.cpp", "pystan/stan/src/stan/lang/grammars/functions_grammar_inst.cpp", "pystan/stan/src/stan/lang/grammars/indexes_grammar_inst.cpp", "pystan/stan/src/stan/lang/grammars/program_grammar_inst.cpp", "pystan/stan/src/stan/lang/grammars/semantic_actions_def.cpp", "pystan/stan/src/stan/lang/grammars/statement_2_grammar_inst.cpp",
def __init__(self, file=None, charset='utf-8', model_name="anon_model", model_code=None, stanc_ret=None, boost_lib=None, eigen_lib=None, verbose=False, obfuscate_model_name=True): if stanc_ret is None: stanc_ret = pystan.api.stanc( file=file, charset=charset, model_code=model_code, model_name=model_name, verbose=verbose, obfuscate_model_name=obfuscate_model_name) if not isinstance(stanc_ret, dict): raise ValueError("stanc_ret must be an object returned by stanc.") stanc_ret_keys = { 'status', 'model_code', 'model_cppname', 'cppcode', 'model_name' } if not all(n in stanc_ret_keys for n in stanc_ret): raise ValueError("stanc_ret lacks one or more of the keys: " "{}".format(str(stanc_ret_keys))) elif stanc_ret['status'] != 0: # success == 0 raise ValueError("stanc_ret is not a successfully returned " "dictionary from stanc.") self.model_cppname = stanc_ret['model_cppname'] self.model_name = stanc_ret['model_name'] self.model_code = stanc_ret['model_code'] self.model_cppcode = stanc_ret['cppcode'] msg = "COMPILING THE C++ CODE FOR MODEL {} NOW." logger.info(msg.format(self.model_name)) if verbose: msg = "OS: {}, Python: {}, Cython {}".format( sys.platform, sys.version, Cython.__version__) logger.info(msg) if boost_lib is not None: # FIXME: allow boost_lib, eigen_lib to be specified raise NotImplementedError if eigen_lib is not None: raise NotImplementedError # module_name needs to be unique so that each model instance has its own module nonce = abs(hash((self.model_name, time.time()))) self.module_name = 'stanfit4{}_{}'.format(self.model_name, nonce) lib_dir = tempfile.mkdtemp() pystan_dir = os.path.dirname(__file__) include_dirs = [ lib_dir, pystan_dir, os.path.join(pystan_dir, "stan", "src"), os.path.join(pystan_dir, "stan", "lib", "stan_math"), os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib", "eigen_3.2.4"), os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib", "boost_1.58.0"), np.get_include(), ] model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp') with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile: outfile.write(self.model_cppcode) pyx_file = os.path.join(lib_dir, self.module_name + '.pyx') pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx') with io.open(pyx_template_file, 'r', encoding='utf-8') as infile: s = infile.read() template = string.Template(s) with io.open(pyx_file, 'w', encoding='utf-8') as outfile: s = template.safe_substitute(model_cppname=self.model_cppname) outfile.write(s) stan_macros = [ ('BOOST_RESULT_OF_USE_TR1', None), ('BOOST_NO_DECLTYPE', None), ('BOOST_DISABLE_ASSERTS', None), ('EIGEN_NO_DEBUG', None), ] # compile stan models with optimization (-O2) # (stanc is compiled without optimization (-O0) currently, see #33) extra_compile_args = [ '-O2', '-ftemplate-depth-256', '-Wno-unused-function', '-Wno-uninitialized', ] if platform.platform().startswith('Win'): extra_compile_args = [ '/EHsc', '-DBOOST_DATE_TIME_NO_LIB', ] distutils.log.set_verbosity(verbose) extension = Extension(name=self.module_name, language="c++", sources=[pyx_file], define_macros=stan_macros, include_dirs=include_dirs, extra_compile_args=extra_compile_args) cython_include_dirs = ['.', pystan_dir] build_extension = _get_build_extension() build_extension.extensions = cythonize( [extension], include_path=cython_include_dirs, quiet=not verbose) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr) if redirect_stderr: # silence stderr for compilation orig_stderr = pystan.misc._redirect_stderr() try: build_extension.run() finally: if redirect_stderr: # restore stderr os.dup2(orig_stderr, sys.stderr.fileno()) self.module = load_module(self.module_name, lib_dir) self.module_filename = os.path.basename(self.module.__file__) # once the module is in memory, we no longer need the file on disk # but we do need a copy of the file for pickling and the module name with io.open(os.path.join(lib_dir, self.module_filename), 'rb') as f: self.module_bytes = f.read() shutil.rmtree(lib_dir, ignore_errors=True) self.fit_class = getattr(self.module, "StanFit4Model")
def __init__(self, file=None, charset='utf-8', model_name="anon_model", model_code=None, stanc_ret=None, boost_lib=None, eigen_lib=None, save_dso=True, verbose=False, **kwargs): if stanc_ret is None: stanc_ret = pystan.api.stanc(file=file, model_code=model_code, model_name=model_name, verbose=verbose, **kwargs) if not isinstance(stanc_ret, dict): raise ValueError("stanc_ret must be an object returned by stanc.") stanc_ret_keys = {'status', 'model_code', 'model_cppname', 'cppcode', 'model_name'} if not all(n in stanc_ret_keys for n in stanc_ret): raise ValueError("stanc_ret lacks one or more of the keys: " "{}".format(str(stanc_ret_keys))) elif stanc_ret['status'] != 0: # success == 0 if (file==None and os.path.exists(model_code)): raise ValueError("stanc_ret is not a successfully returned " "dictionary from stanc. Have you passed " "a stan code filename as model_code?") else: raise ValueError("stanc_ret is not a successfully returned " "dictionary from stanc.") self.model_cppname = stanc_ret['model_cppname'] self.model_name = stanc_ret['model_name'] self.model_code = model_code self.model_cppcode = stanc_ret['cppcode'] self.save_dso = save_dso msg = "COMPILING THE C++ CODE FOR MODEL {} NOW." logger.warning(msg.format(self.model_name)) if verbose: msg = "OS: {}, Python: {}, Cython {}".format(sys.platform, sys.version, Cython.__version__) logger.info(msg) if boost_lib is not None: # FIXME: allow boost_lib, eigen_lib to be specified raise NotImplementedError if eigen_lib is not None: raise NotImplementedError key = tuple([self.model_code, self.model_cppcode, sys.version_info, sys.executable]) module_name = ("stanfit4" + self.model_name + '_' + hashlib.md5(str(key).encode('utf-8')).hexdigest()) temp_dir = tempfile.mkdtemp() lib_dir = os.path.join(temp_dir, 'pystan') pystan_dir = os.path.dirname(__file__) include_dirs = [lib_dir, pystan_dir, os.path.join(pystan_dir, "stan/src"), os.path.join(pystan_dir, "stan/lib/eigen_3.2.0"), os.path.join(pystan_dir, "stan/lib/boost_1.54.0")] library_dirs = [os.path.join(pystan_dir, "bin")] libraries = ['stan'] if not os.path.exists(lib_dir): os.makedirs(lib_dir) model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp') with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile: outfile.write(self.model_cppcode) pyx_file = os.path.join(lib_dir, module_name + '.pyx') pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx') with io.open(pyx_template_file, 'r', encoding='utf-8') as infile: s = infile.read() template = string.Template(s) with io.open(pyx_file, 'w', encoding='utf-8') as outfile: s = template.safe_substitute(model_cppname=self.model_cppname) outfile.write(s) extra_compile_args = ['-O3', '-ftemplate-depth-256'] extension = Extension(name=module_name, language="c++", sources=[pyx_file], include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries, extra_compile_args=extra_compile_args) cython_include_dirs = ['.', pystan_dir] build_extension = _get_build_extension() build_extension.extensions = cythonize([extension], include_path=cython_include_dirs, quiet=not verbose) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr) if redirect_stderr: # silence stderr for compilation orig_stderr = pystan.misc._redirect_stderr() try: build_extension.run() finally: if redirect_stderr: # restore stderr os.dup2(orig_stderr, sys.stderr.fileno()) self.module = load_module(module_name, lib_dir) self.fit_class = getattr(self.module, "StanFit4" + self.model_cppname)
def __init__(self, file=None, charset='utf-8', model_name="anon_model", model_code=None, stanc_ret=None, boost_lib=None, eigen_lib=None, verbose=False, obfuscate_model_name=True, extra_compile_args=None): if stanc_ret is None: stanc_ret = pystan.api.stanc(file=file, charset=charset, model_code=model_code, model_name=model_name, verbose=verbose, obfuscate_model_name=obfuscate_model_name) if not isinstance(stanc_ret, dict): raise ValueError("stanc_ret must be an object returned by stanc.") stanc_ret_keys = {'status', 'model_code', 'model_cppname', 'cppcode', 'model_name'} if not all(n in stanc_ret_keys for n in stanc_ret): raise ValueError("stanc_ret lacks one or more of the keys: " "{}".format(str(stanc_ret_keys))) elif stanc_ret['status'] != 0: # success == 0 raise ValueError("stanc_ret is not a successfully returned " "dictionary from stanc.") self.model_cppname = stanc_ret['model_cppname'] self.model_name = stanc_ret['model_name'] self.model_code = stanc_ret['model_code'] self.model_cppcode = stanc_ret['cppcode'] msg = "COMPILING THE C++ CODE FOR MODEL {} NOW." logger.info(msg.format(self.model_name)) if verbose: msg = "OS: {}, Python: {}, Cython {}".format(sys.platform, sys.version, Cython.__version__) logger.info(msg) if boost_lib is not None: # FIXME: allow boost_lib, eigen_lib to be specified raise NotImplementedError if eigen_lib is not None: raise NotImplementedError # module_name needs to be unique so that each model instance has its own module nonce = abs(hash((self.model_name, time.time()))) self.module_name = 'stanfit4{}_{}'.format(self.model_name, nonce) lib_dir = tempfile.mkdtemp() pystan_dir = os.path.dirname(__file__) include_dirs = [ lib_dir, pystan_dir, os.path.join(pystan_dir, "stan", "src"), os.path.join(pystan_dir, "stan", "lib", "stan_math_2.10.0"), os.path.join(pystan_dir, "stan", "lib", "stan_math_2.10.0", "lib", "eigen_3.2.8"), os.path.join(pystan_dir, "stan", "lib", "stan_math_2.10.0", "lib", "boost_1.60.0"), os.path.join(pystan_dir, "stan", "lib", "stan_math_2.10.0", "lib", "cvodes_2.8.2", "include"), np.get_include(), ] model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp') with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile: outfile.write(self.model_cppcode) pyx_file = os.path.join(lib_dir, self.module_name + '.pyx') pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx') with io.open(pyx_template_file, 'r', encoding='utf-8') as infile: s = infile.read() template = string.Template(s) with io.open(pyx_file, 'w', encoding='utf-8') as outfile: s = template.safe_substitute(model_cppname=self.model_cppname) outfile.write(s) stan_macros = [ ('BOOST_RESULT_OF_USE_TR1', None), ('BOOST_NO_DECLTYPE', None), ('BOOST_DISABLE_ASSERTS', None), ] # compile stan models with optimization (-O2) # (stanc is compiled without optimization (-O0) currently, see #33) if extra_compile_args is None: extra_compile_args = [ '-O2', '-ftemplate-depth-256', '-Wno-unused-function', '-Wno-uninitialized', ] if platform.platform().startswith('Win'): extra_compile_args = ['/EHsc', '-DBOOST_DATE_TIME_NO_LIB'] distutils.log.set_verbosity(verbose) extension = Extension(name=self.module_name, language="c++", sources=[pyx_file], define_macros=stan_macros, include_dirs=include_dirs, extra_compile_args=extra_compile_args) cython_include_dirs = ['.', pystan_dir] build_extension = _get_build_extension() build_extension.extensions = cythonize([extension], include_path=cython_include_dirs, quiet=not verbose) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr) if redirect_stderr: # silence stderr for compilation orig_stderr = pystan.misc._redirect_stderr() try: build_extension.run() finally: if redirect_stderr: # restore stderr os.dup2(orig_stderr, sys.stderr.fileno()) self.module = load_module(self.module_name, lib_dir) self.module_filename = os.path.basename(self.module.__file__) # once the module is in memory, we no longer need the file on disk # but we do need a copy of the file for pickling and the module name with io.open(os.path.join(lib_dir, self.module_filename), 'rb') as f: self.module_bytes = f.read() shutil.rmtree(lib_dir, ignore_errors=True) self.fit_class = getattr(self.module, "StanFit4Model")
def create_fake_model(module_name): """Creates a fake model with specific module name. Parameters ---------- module_name : str Returns ------- StanModel Dummy StanModel object with specific module name. """ # reverse engineer the name model_name, nonce = module_name.replace("stanfit4", "").rsplit("_", 1) # create minimal code model_code = """generated quantities {real fake_parameter=1;}""" stanc_ret = pystan.api.stanc(model_code=model_code, model_name=model_name, obfuscate_model_name=False) model_cppname = stanc_ret['model_cppname'] model_cppcode = stanc_ret['cppcode'] original_module_name = module_name module_name = 'stanfit4{}_{}'.format(model_name, nonce) # module name should be same assert original_module_name == module_name lib_dir = tempfile.mkdtemp() pystan_dir = os.path.dirname(pystan.__file__) include_dirs = [ lib_dir, pystan_dir, os.path.join(pystan_dir, "stan", "src"), os.path.join(pystan_dir, "stan", "lib", "stan_math"), os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib", "eigen_3.3.3"), os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib", "boost_1.69.0"), os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib", "sundials_4.1.0", "include"), np.get_include(), ] model_cpp_file = os.path.join(lib_dir, model_cppname + '.hpp') with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile: outfile.write(model_cppcode) pyx_file = os.path.join(lib_dir, module_name + '.pyx') pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx') with io.open(pyx_template_file, 'r', encoding='utf-8') as infile: s = infile.read() template = string.Template(s) with io.open(pyx_file, 'w', encoding='utf-8') as outfile: s = template.safe_substitute(model_cppname=model_cppname) outfile.write(s) stan_macros = [ ('BOOST_RESULT_OF_USE_TR1', None), ('BOOST_NO_DECLTYPE', None), ('BOOST_DISABLE_ASSERTS', None), ] build_extension = _get_build_extension() extra_compile_args = [] if platform.platform().startswith('Win'): if build_extension.compiler in (None, 'msvc'): logger.warning("MSVC compiler is not supported") extra_compile_args = [ '/EHsc', '-DBOOST_DATE_TIME_NO_LIB', '/std:c++14', ] + extra_compile_args else: # Windows, but not msvc, likely mingw # fix bug in MingW-W64 # use posix threads extra_compile_args = [ '-O1', '-ftemplate-depth-256', '-Wno-unused-function', '-Wno-uninitialized', '-std=c++1y', "-D_hypot=hypot", "-pthread", "-fexceptions", ] + extra_compile_args else: # linux or macOS extra_compile_args = [ '-O1', '-ftemplate-depth-256', '-Wno-unused-function', '-Wno-uninitialized', '-std=c++1y', ] + extra_compile_args extension = Extension(name=module_name, language="c++", sources=[pyx_file], define_macros=stan_macros, include_dirs=include_dirs, extra_compile_args=extra_compile_args) cython_include_dirs = ['.', pystan_dir] build_extension.extensions = cythonize([extension], include_path=cython_include_dirs, quiet=True) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() module = load_module(module_name, lib_dir) shutil.rmtree(lib_dir, ignore_errors=True) return module
def so_ext(): """Get extension for the compiled library. """ if not hasattr(so_ext, 'ext'): so_ext.ext = _get_build_extension().get_ext_filename('') return so_ext.ext
('BOOST_DISABLE_ASSERTS', None), ('BOOST_NO_DECLTYPE', None), ('BOOST_PHOENIX_NO_VARIADIC_EXPRESSION', None), # needed for stanc ('BOOST_RESULT_OF_USE_TR1', None), ('FUSION_MAX_VECTOR_SIZE', 12), # for parser, stan-dev/pystan#222 ] extra_compile_args = [ '-Os', '-ftemplate-depth-256', '-Wno-unused-function', '-Wno-uninitialized', ] if platform.platform().startswith('Win'): from Cython.Build.Inline import _get_build_extension if _get_build_extension().compiler in (None, 'msvc'): extra_compile_args = [ '/EHsc', '-DBOOST_DATE_TIME_NO_LIB', ] stanc_sources = [ "pystan/stan/src/stan/lang/ast_def.cpp", "pystan/stan/src/stan/lang/grammars/bare_type_grammar_inst.cpp", "pystan/stan/src/stan/lang/grammars/expression07_grammar_inst.cpp", "pystan/stan/src/stan/lang/grammars/expression_grammar_inst.cpp", "pystan/stan/src/stan/lang/grammars/functions_grammar_inst.cpp", "pystan/stan/src/stan/lang/grammars/indexes_grammar_inst.cpp", "pystan/stan/src/stan/lang/grammars/program_grammar_inst.cpp", "pystan/stan/src/stan/lang/grammars/semantic_actions_def.cpp",
def __init__(self, file=None, charset='utf-8', model_name="anon_model", model_code=None, stanc_ret=None, include_paths=None, boost_lib=None, eigen_lib=None, verbose=False, obfuscate_model_name=True, extra_compile_args=None, allow_undefined=False, include_dirs=None, includes=None): if stanc_ret is None: stanc_ret = pystan.api.stanc( file=file, charset=charset, model_code=model_code, model_name=model_name, verbose=verbose, include_paths=include_paths, obfuscate_model_name=obfuscate_model_name, allow_undefined=allow_undefined) if not isinstance(stanc_ret, dict): raise ValueError("stanc_ret must be an object returned by stanc.") stanc_ret_keys = { 'status', 'model_code', 'model_cppname', 'cppcode', 'model_name', 'include_paths' } if not all(n in stanc_ret_keys for n in stanc_ret): raise ValueError("stanc_ret lacks one or more of the keys: " "{}".format(str(stanc_ret_keys))) elif stanc_ret['status'] != 0: # success == 0 raise ValueError("stanc_ret is not a successfully returned " "dictionary from stanc.") self.model_cppname = stanc_ret['model_cppname'] self.model_name = stanc_ret['model_name'] self.model_code = stanc_ret['model_code'] self.model_cppcode = stanc_ret['cppcode'] self.model_include_paths = stanc_ret['include_paths'] if allow_undefined or include_dirs or includes: logger.warning( "External C++ interface is an experimental feature. Be careful." ) msg = "COMPILING THE C++ CODE FOR MODEL {} NOW." logger.info(msg.format(self.model_name)) if verbose: msg = "OS: {}, Python: {}, Cython {}".format( sys.platform, sys.version, Cython.__version__) logger.info(msg) if boost_lib is not None: # FIXME: allow boost_lib, eigen_lib to be specified raise NotImplementedError if eigen_lib is not None: raise NotImplementedError # module_name needs to be unique so that each model instance has its own module nonce = abs(hash((self.model_name, time.time()))) self.module_name = 'stanfit4{}_{}'.format(self.model_name, nonce) lib_dir = tempfile.mkdtemp(prefix='pystan_') pystan_dir = os.path.dirname(__file__) if include_dirs is None: include_dirs = [] elif not isinstance(include_dirs, list): raise TypeError( "'include_dirs' needs to be a list: type={}".format( type(include_dirs))) include_dirs += [ lib_dir, pystan_dir, os.path.join(pystan_dir, "stan", "src"), os.path.join(pystan_dir, "stan", "lib", "stan_math"), os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib", "eigen_3.3.3"), os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib", "boost_1.69.0"), os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib", "sundials_4.1.0", "include"), np.get_include(), ] model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp') if includes is not None: code = "" for fn in includes: code += '#include "{0}"\n'.format(fn) ind = self.model_cppcode.index( "static int current_statement_begin__;") self.model_cppcode = "\n".join( [self.model_cppcode[:ind], code, self.model_cppcode[ind:]]) with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile: outfile.write(self.model_cppcode) pyx_file = os.path.join(lib_dir, self.module_name + '.pyx') pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx') with io.open(pyx_template_file, 'r', encoding='utf-8') as infile: s = infile.read() template = string.Template(s) with io.open(pyx_file, 'w', encoding='utf-8') as outfile: s = template.safe_substitute(model_cppname=self.model_cppname) outfile.write(s) stan_macros = [ ('BOOST_RESULT_OF_USE_TR1', None), ('BOOST_NO_DECLTYPE', None), ('BOOST_DISABLE_ASSERTS', None), ] build_extension = _get_build_extension() # compile stan models with optimization (-O2) # (stanc is compiled without optimization (-O0) currently, see #33) if extra_compile_args is None: extra_compile_args = [] if platform.platform().startswith('Win'): if build_extension.compiler in (None, 'msvc'): logger.warning("MSVC compiler is not supported") extra_compile_args = [ '/EHsc', '-DBOOST_DATE_TIME_NO_LIB', '/std:c++14', ] + extra_compile_args else: # Windows, but not msvc, likely mingw # fix bug in MingW-W64 # use posix threads extra_compile_args = [ '-O2', '-ftemplate-depth-256', '-Wno-unused-function', '-Wno-uninitialized', '-std=c++1y', "-D_hypot=hypot", "-pthread", "-fexceptions", ] + extra_compile_args else: # linux or macOS extra_compile_args = [ '-O2', '-ftemplate-depth-256', '-Wno-unused-function', '-Wno-uninitialized', '-std=c++1y', ] + extra_compile_args distutils.log.set_verbosity(verbose) extension = Extension(name=self.module_name, language="c++", sources=[pyx_file], define_macros=stan_macros, include_dirs=include_dirs, extra_compile_args=extra_compile_args) cython_include_dirs = ['.', pystan_dir] build_extension.extensions = cythonize( [extension], include_path=cython_include_dirs, quiet=not verbose) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr) if redirect_stderr: # silence stderr for compilation orig_stderr = pystan.misc._redirect_stderr() try: build_extension.run() finally: if redirect_stderr: # restore stderr os.dup2(orig_stderr, sys.stderr.fileno()) self.module = load_module(self.module_name, lib_dir) self.module_filename = os.path.basename(self.module.__file__) # once the module is in memory, we no longer need the file on disk # but we do need a copy of the file for pickling and the module name with io.open(os.path.join(lib_dir, self.module_filename), 'rb') as f: self.module_bytes = f.read() shutil.rmtree(lib_dir, ignore_errors=True) self.fit_class = getattr(self.module, "StanFit4Model")
def __init__(self, file=None, charset='utf-8', model_name="anon_model", model_code=None, stanc_ret=None, boost_lib=None, eigen_lib=None, save_dso=True, verbose=False, obfuscate_model_name=True): if stanc_ret is None: stanc_ret = pystan.api.stanc( file=file, charset=charset, model_code=model_code, model_name=model_name, verbose=verbose, obfuscate_model_name=obfuscate_model_name) if not isinstance(stanc_ret, dict): raise ValueError("stanc_ret must be an object returned by stanc.") stanc_ret_keys = { 'status', 'model_code', 'model_cppname', 'cppcode', 'model_name' } if not all(n in stanc_ret_keys for n in stanc_ret): raise ValueError("stanc_ret lacks one or more of the keys: " "{}".format(str(stanc_ret_keys))) elif stanc_ret['status'] != 0: # success == 0 raise ValueError("stanc_ret is not a successfully returned " "dictionary from stanc.") self.model_cppname = stanc_ret['model_cppname'] self.model_name = stanc_ret['model_name'] self.model_code = stanc_ret['model_code'] self.model_cppcode = stanc_ret['cppcode'] self.save_dso = save_dso msg = "COMPILING THE C++ CODE FOR MODEL {} NOW." logger.info(msg.format(self.model_name)) if verbose: msg = "OS: {}, Python: {}, Cython {}".format( sys.platform, sys.version, Cython.__version__) logger.info(msg) if boost_lib is not None: # FIXME: allow boost_lib, eigen_lib to be specified raise NotImplementedError if eigen_lib is not None: raise NotImplementedError key = tuple([ self.model_code, self.model_cppcode, sys.version_info, sys.executable ]) module_name = ("stanfit4" + self.model_name + '_' + hashlib.md5(str(key).encode('utf-8')).hexdigest()) self._temp_dir = temp_dir = tempfile.mkdtemp() lib_dir = os.path.join(temp_dir, 'pystan') pystan_dir = os.path.dirname(__file__) include_dirs = [ lib_dir, pystan_dir, os.path.join(pystan_dir, "stan/src"), os.path.join(pystan_dir, "stan/lib/eigen_3.2.2"), os.path.join(pystan_dir, "stan/lib/boost_1.55.0"), np.get_include(), ] if not os.path.exists(lib_dir): os.makedirs(lib_dir) model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp') with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile: outfile.write(self.model_cppcode) pyx_file = os.path.join(lib_dir, module_name + '.pyx') pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx') with io.open(pyx_template_file, 'r', encoding='utf-8') as infile: s = infile.read() template = string.Template(s) with io.open(pyx_file, 'w', encoding='utf-8') as outfile: s = template.safe_substitute(model_cppname=self.model_cppname) outfile.write(s) extra_compile_args = ['-O3', '-ftemplate-depth-256'] distutils.log.set_verbosity(verbose) extension = Extension(name=module_name, language="c++", sources=[pyx_file], include_dirs=include_dirs, extra_compile_args=extra_compile_args) cython_include_dirs = ['.', pystan_dir] build_extension = _get_build_extension() build_extension.extensions = cythonize( [extension], include_path=cython_include_dirs, quiet=not verbose) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr) if redirect_stderr: # silence stderr for compilation orig_stderr = pystan.misc._redirect_stderr() try: build_extension.run() finally: if redirect_stderr: # restore stderr os.dup2(orig_stderr, sys.stderr.fileno()) self.module = load_module(module_name, lib_dir) self.fit_class = getattr(self.module, "StanFit4Model")
def __init__( self, file=None, charset="utf-8", model_name="anon_model", model_code=None, stanc_ret=None, boost_lib=None, eigen_lib=None, save_dso=True, verbose=False, obfuscate_model_name=True, ): if stanc_ret is None: stanc_ret = pystan.api.stanc( file=file, charset=charset, model_code=model_code, model_name=model_name, verbose=verbose, obfuscate_model_name=obfuscate_model_name, ) if not isinstance(stanc_ret, dict): raise ValueError("stanc_ret must be an object returned by stanc.") stanc_ret_keys = {"status", "model_code", "model_cppname", "cppcode", "model_name"} if not all(n in stanc_ret_keys for n in stanc_ret): raise ValueError("stanc_ret lacks one or more of the keys: " "{}".format(str(stanc_ret_keys))) elif stanc_ret["status"] != 0: # success == 0 raise ValueError("stanc_ret is not a successfully returned " "dictionary from stanc.") self.model_cppname = stanc_ret["model_cppname"] self.model_name = stanc_ret["model_name"] self.model_code = stanc_ret["model_code"] self.model_cppcode = stanc_ret["cppcode"] self.save_dso = save_dso msg = "COMPILING THE C++ CODE FOR MODEL {} NOW." logger.info(msg.format(self.model_name)) if verbose: msg = "OS: {}, Python: {}, Cython {}".format(sys.platform, sys.version, Cython.__version__) logger.info(msg) if boost_lib is not None: # FIXME: allow boost_lib, eigen_lib to be specified raise NotImplementedError if eigen_lib is not None: raise NotImplementedError key = tuple([self.model_code, self.model_cppcode, sys.version_info, sys.executable]) module_name = "stanfit4" + self.model_name + "_" + hashlib.md5(str(key).encode("utf-8")).hexdigest() self._temp_dir = temp_dir = tempfile.mkdtemp() lib_dir = os.path.join(temp_dir, "pystan") pystan_dir = os.path.dirname(__file__) include_dirs = [ lib_dir, pystan_dir, os.path.join(pystan_dir, "stan/src"), os.path.join(pystan_dir, "math"), os.path.join(pystan_dir, "stan/lib/eigen_3.2.4"), os.path.join(pystan_dir, "stan/lib/boost_1.58.0"), np.get_include(), ] if not os.path.exists(lib_dir): os.makedirs(lib_dir) model_cpp_file = os.path.join(lib_dir, self.model_cppname + ".hpp") with io.open(model_cpp_file, "w", encoding="utf-8") as outfile: outfile.write(self.model_cppcode) pyx_file = os.path.join(lib_dir, module_name + ".pyx") pyx_template_file = os.path.join(pystan_dir, "stanfit4model.pyx") with io.open(pyx_template_file, "r", encoding="utf-8") as infile: s = infile.read() template = string.Template(s) with io.open(pyx_file, "w", encoding="utf-8") as outfile: s = template.safe_substitute(model_cppname=self.model_cppname) outfile.write(s) stan_macros = [ ("BOOST_RESULT_OF_USE_TR1", None), ("BOOST_NO_DECLTYPE", None), ("BOOST_DISABLE_ASSERTS", None), ("EIGEN_NO_DEBUG", None), ] extra_compile_args = ["-O0", "-ftemplate-depth-256", "-Wno-unused-function", "-Wno-uninitialized"] distutils.log.set_verbosity(verbose) extension = Extension( name=module_name, language="c++", sources=[pyx_file], define_macros=stan_macros, include_dirs=include_dirs, extra_compile_args=extra_compile_args, ) cython_include_dirs = [".", pystan_dir] build_extension = _get_build_extension() build_extension.extensions = cythonize([extension], include_path=cython_include_dirs, quiet=not verbose) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr) if redirect_stderr: # silence stderr for compilation orig_stderr = pystan.misc._redirect_stderr() try: build_extension.run() finally: if redirect_stderr: # restore stderr os.dup2(orig_stderr, sys.stderr.fileno()) self.module = load_module(module_name, lib_dir) self.fit_class = getattr(self.module, "StanFit4Model")
def cython_inline(code, get_type=unsafe_type, lib_dir=os.path.join(get_cython_cache_dir(), 'inline'), cython_include_dirs=None, force=False, quiet=False, local_sym=None, global_sym=None, **kwargs): if get_type is None: get_type = lambda x: 'object' ctx = _create_context(tuple(cython_include_dirs) ) if cython_include_dirs else _inline_default_context # Fast path if this has been called in this session. _unbound_symbols = _inline_cache.get(code) if _unbound_symbols is not None: _populate_unbound(kwargs, _unbound_symbols, local_sym, global_sym) args = sorted(kwargs.items()) arg_sigs = tuple([(get_type(value, ctx), arg) for arg, value in args]) invoke = _inline_cache.get((code, arg_sigs)) if invoke is not None: arg_list = [arg[1] for arg in args] return invoke(*arg_list) orig_code = code code, literals = strip_string_literals(code) if local_sym is None: local_sym = inspect.currentframe().f_back.f_back.f_locals if global_sym is None: global_sym = inspect.currentframe().f_back.f_back.f_globals try: _inline_cache[orig_code] = _unbound_symbols = unbound_symbols(code) _populate_unbound(kwargs, _unbound_symbols, local_sym, global_sym) except AssertionError: if not quiet: # Parsing from strings not fully supported (e.g. cimports). print( "Could not parse code as a string (to extract unbound symbols)." ) cimports = [] for name, arg in list(kwargs.items()): if arg is cython: cimports.append('\ncimport cython as %s' % name) del kwargs[name] arg_names = sorted(kwargs) arg_sigs = tuple([(get_type(kwargs[arg], ctx), arg) for arg in arg_names]) key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__ module_name = "_cython_inline_" + hashlib.md5( str(key).encode('utf-8')).hexdigest() if module_name in sys.modules: _module = sys.modules[module_name] else: build_extension = None global _so_ext if _so_ext is None: # Figure out and cache current extension suffix build_extension = _get_build_extension() _so_ext = build_extension.get_ext_filename('') module_path = os.path.join(lib_dir, module_name + _so_ext) if not os.path.exists(lib_dir): os.makedirs(lib_dir) if force or not os.path.isfile(module_path): cflags = [] c_include_dirs = [] qualified = re.compile(r'([.\w]+)[.]') for type, _ in arg_sigs: m = qualified.match(type) if m: cimports.append('\ncimport %s' % m.groups()[0]) # one special case if m.groups()[0] == 'numpy': import numpy c_include_dirs.append(numpy.get_include()) # cflags.append('-Wno-unused') module_body, func_body = extract_func_code(code) params = ', '.join(['%s %s' % a for a in arg_sigs]) module_code = ''' %(module_body)s %(cimports)s def __invoke(%(params)s): %(func_body)s return locals() ''' % { 'cimports': '\n'.join(cimports), 'module_body': module_body, 'params': params, 'func_body': func_body } for key, value in literals.items(): module_code = module_code.replace(key, value) pyx_file = os.path.join(lib_dir, module_name + '.pyx') fh = open(pyx_file, 'w') try: fh.write(module_code) finally: fh.close() extension = Extension(name=module_name, sources=[pyx_file], include_dirs=c_include_dirs, extra_compile_args=cflags) if build_extension is None: build_extension = _get_build_extension() build_extension.extensions = cythonize( [extension], include_path=cython_include_dirs or ['.'], quiet=quiet) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() _module = imp.load_dynamic(module_name, module_path) _inline_cache[orig_code, arg_sigs] = _module.__invoke arg_list = [kwargs[arg] for arg in arg_names] return _module.__invoke(*arg_list)