def _compile_cython_module_nocache(src, modname, verbose=False):
modfile = os.path.join(MODDIR, modname + '.pyx')
with open(modfile, 'w+') as f:
f.write(src)
Cython.Compiler.Options.cimport_from_pyx = True
include_dirs = [numpy.get_include()]
extra_compile_args = [
'-O3', '-march=native', '-ffast-math', '-fopenmp', '-g1'
]
extension = Extension(name=modname,
sources=[modfile],
include_dirs=include_dirs,
extra_compile_args=extra_compile_args)
cython_include_dirs = [PYIGAPATH]
build_extension = _get_build_extension()
build_extension.extensions = cythonize(
[extension],
include_path=cython_include_dirs,
compiler_directives={'language_level': 3},
quiet=False)
build_extension.build_temp = MODDIR
build_extension.build_lib = MODDIR
distutils.log.set_verbosity(verbose)
build_extension.run()
return importlib.import_module(modname)
def cython_build(name, file=None, force=False, quiet=True, cythonize_args={},
lib_dir=os.path.join(get_cython_cache_dir(), 'inline/lib'),
tmp_dir=os.path.join(get_cython_cache_dir(), 'inline/tmp'),
**extension_args):
"""Build a cython extension.
"""
if file is not None:
_append_args(extension_args, 'sources', file)
with _suppress_output(quiet=quiet):
extension = Extension(name, **extension_args)
extensions = cythonize([extension], force=force, **cythonize_args)
build_extension = _get_build_extension()
build_extension.extensions = extensions
build_extension.build_lib = lib_dir
build_extension.build_temp = tmp_dir
build_extension.run()
def cython_build(name,
file=None,
force=False,
quiet=True,
cythonize_args={},
lib_dir=os.path.join(get_cython_cache_dir(), 'inline/lib'),
tmp_dir=os.path.join(get_cython_cache_dir(), 'inline/tmp'),
**extension_args):
"""Build a cython extension.
"""
if file is not None:
_append_args(extension_args, 'sources', file)
with _suppress_output(quiet=quiet):
extension = Extension(name, **extension_args)
extensions = cythonize([extension], force=force, **cythonize_args)
build_extension = _get_build_extension()
build_extension.extensions = extensions
build_extension.build_lib = lib_dir
build_extension.build_temp = tmp_dir
build_extension.run()
('BOOST_DISABLE_ASSERTS', None),
('BOOST_NO_DECLTYPE', None),
('BOOST_PHOENIX_NO_VARIADIC_EXPRESSION', None), # needed for stanc
('BOOST_RESULT_OF_USE_TR1', None),
('FUSION_MAX_VECTOR_SIZE', 12), # for parser, stan-dev/pystan#222
]
extra_compile_args = [
'-Os',
'-ftemplate-depth-256',
'-Wno-unused-function',
'-Wno-uninitialized',
]
if platform.platform().startswith('Win'):
from Cython.Build.Inline import _get_build_extension
if _get_build_extension().compiler in (None, 'msvc'):
extra_compile_args = [
'/EHsc',
'-DBOOST_DATE_TIME_NO_LIB',
]
stanc_sources = [
"pystan/stan/src/stan/lang/ast_def.cpp",
"pystan/stan/src/stan/lang/grammars/bare_type_grammar_inst.cpp",
"pystan/stan/src/stan/lang/grammars/expression07_grammar_inst.cpp",
"pystan/stan/src/stan/lang/grammars/expression_grammar_inst.cpp",
"pystan/stan/src/stan/lang/grammars/functions_grammar_inst.cpp",
"pystan/stan/src/stan/lang/grammars/indexes_grammar_inst.cpp",
"pystan/stan/src/stan/lang/grammars/program_grammar_inst.cpp",
"pystan/stan/src/stan/lang/grammars/semantic_actions_def.cpp",
"pystan/stan/src/stan/lang/grammars/statement_2_grammar_inst.cpp",
def __init__(self,
file=None,
charset='utf-8',
model_name="anon_model",
model_code=None,
stanc_ret=None,
boost_lib=None,
eigen_lib=None,
verbose=False,
obfuscate_model_name=True):
if stanc_ret is None:
stanc_ret = pystan.api.stanc(
file=file,
charset=charset,
model_code=model_code,
model_name=model_name,
verbose=verbose,
obfuscate_model_name=obfuscate_model_name)
if not isinstance(stanc_ret, dict):
raise ValueError("stanc_ret must be an object returned by stanc.")
stanc_ret_keys = {
'status', 'model_code', 'model_cppname', 'cppcode', 'model_name'
}
if not all(n in stanc_ret_keys for n in stanc_ret):
raise ValueError("stanc_ret lacks one or more of the keys: "
"{}".format(str(stanc_ret_keys)))
elif stanc_ret['status'] != 0: # success == 0
raise ValueError("stanc_ret is not a successfully returned "
"dictionary from stanc.")
self.model_cppname = stanc_ret['model_cppname']
self.model_name = stanc_ret['model_name']
self.model_code = stanc_ret['model_code']
self.model_cppcode = stanc_ret['cppcode']
msg = "COMPILING THE C++ CODE FOR MODEL {} NOW."
logger.info(msg.format(self.model_name))
if verbose:
msg = "OS: {}, Python: {}, Cython {}".format(
sys.platform, sys.version, Cython.__version__)
logger.info(msg)
if boost_lib is not None:
# FIXME: allow boost_lib, eigen_lib to be specified
raise NotImplementedError
if eigen_lib is not None:
raise NotImplementedError
# module_name needs to be unique so that each model instance has its own module
nonce = abs(hash((self.model_name, time.time())))
self.module_name = 'stanfit4{}_{}'.format(self.model_name, nonce)
lib_dir = tempfile.mkdtemp()
pystan_dir = os.path.dirname(__file__)
include_dirs = [
lib_dir,
pystan_dir,
os.path.join(pystan_dir, "stan", "src"),
os.path.join(pystan_dir, "stan", "lib", "stan_math"),
os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib",
"eigen_3.2.4"),
os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib",
"boost_1.58.0"),
np.get_include(),
]
model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp')
with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile:
outfile.write(self.model_cppcode)
pyx_file = os.path.join(lib_dir, self.module_name + '.pyx')
pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx')
with io.open(pyx_template_file, 'r', encoding='utf-8') as infile:
s = infile.read()
template = string.Template(s)
with io.open(pyx_file, 'w', encoding='utf-8') as outfile:
s = template.safe_substitute(model_cppname=self.model_cppname)
outfile.write(s)
stan_macros = [
('BOOST_RESULT_OF_USE_TR1', None),
('BOOST_NO_DECLTYPE', None),
('BOOST_DISABLE_ASSERTS', None),
('EIGEN_NO_DEBUG', None),
]
# compile stan models with optimization (-O2)
# (stanc is compiled without optimization (-O0) currently, see #33)
extra_compile_args = [
'-O2',
'-ftemplate-depth-256',
'-Wno-unused-function',
'-Wno-uninitialized',
]
if platform.platform().startswith('Win'):
extra_compile_args = [
'/EHsc',
'-DBOOST_DATE_TIME_NO_LIB',
]
distutils.log.set_verbosity(verbose)
extension = Extension(name=self.module_name,
language="c++",
sources=[pyx_file],
define_macros=stan_macros,
include_dirs=include_dirs,
extra_compile_args=extra_compile_args)
cython_include_dirs = ['.', pystan_dir]
build_extension = _get_build_extension()
build_extension.extensions = cythonize(
[extension], include_path=cython_include_dirs, quiet=not verbose)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr)
if redirect_stderr:
# silence stderr for compilation
orig_stderr = pystan.misc._redirect_stderr()
try:
build_extension.run()
finally:
if redirect_stderr:
# restore stderr
os.dup2(orig_stderr, sys.stderr.fileno())
self.module = load_module(self.module_name, lib_dir)
self.module_filename = os.path.basename(self.module.__file__)
# once the module is in memory, we no longer need the file on disk
# but we do need a copy of the file for pickling and the module name
with io.open(os.path.join(lib_dir, self.module_filename), 'rb') as f:
self.module_bytes = f.read()
shutil.rmtree(lib_dir, ignore_errors=True)
self.fit_class = getattr(self.module, "StanFit4Model")
def __init__(self, file=None, charset='utf-8', model_name="anon_model",
model_code=None, stanc_ret=None, boost_lib=None,
eigen_lib=None, save_dso=True, verbose=False, **kwargs):
if stanc_ret is None:
stanc_ret = pystan.api.stanc(file=file,
model_code=model_code,
model_name=model_name,
verbose=verbose, **kwargs)
if not isinstance(stanc_ret, dict):
raise ValueError("stanc_ret must be an object returned by stanc.")
stanc_ret_keys = {'status', 'model_code', 'model_cppname',
'cppcode', 'model_name'}
if not all(n in stanc_ret_keys for n in stanc_ret):
raise ValueError("stanc_ret lacks one or more of the keys: "
"{}".format(str(stanc_ret_keys)))
elif stanc_ret['status'] != 0: # success == 0
if (file==None and os.path.exists(model_code)):
raise ValueError("stanc_ret is not a successfully returned "
"dictionary from stanc. Have you passed "
"a stan code filename as model_code?")
else:
raise ValueError("stanc_ret is not a successfully returned "
"dictionary from stanc.")
self.model_cppname = stanc_ret['model_cppname']
self.model_name = stanc_ret['model_name']
self.model_code = model_code
self.model_cppcode = stanc_ret['cppcode']
self.save_dso = save_dso
msg = "COMPILING THE C++ CODE FOR MODEL {} NOW."
logger.warning(msg.format(self.model_name))
if verbose:
msg = "OS: {}, Python: {}, Cython {}".format(sys.platform,
sys.version,
Cython.__version__)
logger.info(msg)
if boost_lib is not None:
# FIXME: allow boost_lib, eigen_lib to be specified
raise NotImplementedError
if eigen_lib is not None:
raise NotImplementedError
key = tuple([self.model_code, self.model_cppcode, sys.version_info,
sys.executable])
module_name = ("stanfit4" + self.model_name + '_' +
hashlib.md5(str(key).encode('utf-8')).hexdigest())
temp_dir = tempfile.mkdtemp()
lib_dir = os.path.join(temp_dir, 'pystan')
pystan_dir = os.path.dirname(__file__)
include_dirs = [lib_dir,
pystan_dir,
os.path.join(pystan_dir, "stan/src"),
os.path.join(pystan_dir, "stan/lib/eigen_3.2.0"),
os.path.join(pystan_dir, "stan/lib/boost_1.54.0")]
library_dirs = [os.path.join(pystan_dir, "bin")]
libraries = ['stan']
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp')
with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile:
outfile.write(self.model_cppcode)
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx')
with io.open(pyx_template_file, 'r', encoding='utf-8') as infile:
s = infile.read()
template = string.Template(s)
with io.open(pyx_file, 'w', encoding='utf-8') as outfile:
s = template.safe_substitute(model_cppname=self.model_cppname)
outfile.write(s)
extra_compile_args = ['-O3', '-ftemplate-depth-256']
extension = Extension(name=module_name,
language="c++",
sources=[pyx_file],
include_dirs=include_dirs,
library_dirs=library_dirs,
libraries=libraries,
extra_compile_args=extra_compile_args)
cython_include_dirs = ['.', pystan_dir]
build_extension = _get_build_extension()
build_extension.extensions = cythonize([extension],
include_path=cython_include_dirs,
quiet=not verbose)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr)
if redirect_stderr:
# silence stderr for compilation
orig_stderr = pystan.misc._redirect_stderr()
try:
build_extension.run()
finally:
if redirect_stderr:
# restore stderr
os.dup2(orig_stderr, sys.stderr.fileno())
self.module = load_module(module_name, lib_dir)
self.fit_class = getattr(self.module, "StanFit4" + self.model_cppname)
def __init__(self, file=None, charset='utf-8', model_name="anon_model",
model_code=None, stanc_ret=None, boost_lib=None,
eigen_lib=None, verbose=False, obfuscate_model_name=True,
extra_compile_args=None):
if stanc_ret is None:
stanc_ret = pystan.api.stanc(file=file,
charset=charset,
model_code=model_code,
model_name=model_name,
verbose=verbose,
obfuscate_model_name=obfuscate_model_name)
if not isinstance(stanc_ret, dict):
raise ValueError("stanc_ret must be an object returned by stanc.")
stanc_ret_keys = {'status', 'model_code', 'model_cppname',
'cppcode', 'model_name'}
if not all(n in stanc_ret_keys for n in stanc_ret):
raise ValueError("stanc_ret lacks one or more of the keys: "
"{}".format(str(stanc_ret_keys)))
elif stanc_ret['status'] != 0: # success == 0
raise ValueError("stanc_ret is not a successfully returned "
"dictionary from stanc.")
self.model_cppname = stanc_ret['model_cppname']
self.model_name = stanc_ret['model_name']
self.model_code = stanc_ret['model_code']
self.model_cppcode = stanc_ret['cppcode']
msg = "COMPILING THE C++ CODE FOR MODEL {} NOW."
logger.info(msg.format(self.model_name))
if verbose:
msg = "OS: {}, Python: {}, Cython {}".format(sys.platform,
sys.version,
Cython.__version__)
logger.info(msg)
if boost_lib is not None:
# FIXME: allow boost_lib, eigen_lib to be specified
raise NotImplementedError
if eigen_lib is not None:
raise NotImplementedError
# module_name needs to be unique so that each model instance has its own module
nonce = abs(hash((self.model_name, time.time())))
self.module_name = 'stanfit4{}_{}'.format(self.model_name, nonce)
lib_dir = tempfile.mkdtemp()
pystan_dir = os.path.dirname(__file__)
include_dirs = [
lib_dir,
pystan_dir,
os.path.join(pystan_dir, "stan", "src"),
os.path.join(pystan_dir, "stan", "lib", "stan_math_2.10.0"),
os.path.join(pystan_dir, "stan", "lib", "stan_math_2.10.0", "lib", "eigen_3.2.8"),
os.path.join(pystan_dir, "stan", "lib", "stan_math_2.10.0", "lib", "boost_1.60.0"),
os.path.join(pystan_dir, "stan", "lib", "stan_math_2.10.0", "lib", "cvodes_2.8.2", "include"),
np.get_include(),
]
model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp')
with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile:
outfile.write(self.model_cppcode)
pyx_file = os.path.join(lib_dir, self.module_name + '.pyx')
pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx')
with io.open(pyx_template_file, 'r', encoding='utf-8') as infile:
s = infile.read()
template = string.Template(s)
with io.open(pyx_file, 'w', encoding='utf-8') as outfile:
s = template.safe_substitute(model_cppname=self.model_cppname)
outfile.write(s)
stan_macros = [
('BOOST_RESULT_OF_USE_TR1', None),
('BOOST_NO_DECLTYPE', None),
('BOOST_DISABLE_ASSERTS', None),
]
# compile stan models with optimization (-O2)
# (stanc is compiled without optimization (-O0) currently, see #33)
if extra_compile_args is None:
extra_compile_args = [
'-O2',
'-ftemplate-depth-256',
'-Wno-unused-function',
'-Wno-uninitialized',
]
if platform.platform().startswith('Win'):
extra_compile_args = ['/EHsc', '-DBOOST_DATE_TIME_NO_LIB']
distutils.log.set_verbosity(verbose)
extension = Extension(name=self.module_name,
language="c++",
sources=[pyx_file],
define_macros=stan_macros,
include_dirs=include_dirs,
extra_compile_args=extra_compile_args)
cython_include_dirs = ['.', pystan_dir]
build_extension = _get_build_extension()
build_extension.extensions = cythonize([extension],
include_path=cython_include_dirs,
quiet=not verbose)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr)
if redirect_stderr:
# silence stderr for compilation
orig_stderr = pystan.misc._redirect_stderr()
try:
build_extension.run()
finally:
if redirect_stderr:
# restore stderr
os.dup2(orig_stderr, sys.stderr.fileno())
self.module = load_module(self.module_name, lib_dir)
self.module_filename = os.path.basename(self.module.__file__)
# once the module is in memory, we no longer need the file on disk
# but we do need a copy of the file for pickling and the module name
with io.open(os.path.join(lib_dir, self.module_filename), 'rb') as f:
self.module_bytes = f.read()
shutil.rmtree(lib_dir, ignore_errors=True)
self.fit_class = getattr(self.module, "StanFit4Model")
def create_fake_model(module_name):
"""Creates a fake model with specific module name.
Parameters
----------
module_name : str
Returns
-------
StanModel
Dummy StanModel object with specific module name.
"""
# reverse engineer the name
model_name, nonce = module_name.replace("stanfit4", "").rsplit("_", 1)
# create minimal code
model_code = """generated quantities {real fake_parameter=1;}"""
stanc_ret = pystan.api.stanc(model_code=model_code,
model_name=model_name,
obfuscate_model_name=False)
model_cppname = stanc_ret['model_cppname']
model_cppcode = stanc_ret['cppcode']
original_module_name = module_name
module_name = 'stanfit4{}_{}'.format(model_name, nonce)
# module name should be same
assert original_module_name == module_name
lib_dir = tempfile.mkdtemp()
pystan_dir = os.path.dirname(pystan.__file__)
include_dirs = [
lib_dir,
pystan_dir,
os.path.join(pystan_dir, "stan", "src"),
os.path.join(pystan_dir, "stan", "lib", "stan_math"),
os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib", "eigen_3.3.3"),
os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib", "boost_1.69.0"),
os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib", "sundials_4.1.0", "include"),
np.get_include(),
]
model_cpp_file = os.path.join(lib_dir, model_cppname + '.hpp')
with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile:
outfile.write(model_cppcode)
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx')
with io.open(pyx_template_file, 'r', encoding='utf-8') as infile:
s = infile.read()
template = string.Template(s)
with io.open(pyx_file, 'w', encoding='utf-8') as outfile:
s = template.safe_substitute(model_cppname=model_cppname)
outfile.write(s)
stan_macros = [
('BOOST_RESULT_OF_USE_TR1', None),
('BOOST_NO_DECLTYPE', None),
('BOOST_DISABLE_ASSERTS', None),
]
build_extension = _get_build_extension()
extra_compile_args = []
if platform.platform().startswith('Win'):
if build_extension.compiler in (None, 'msvc'):
logger.warning("MSVC compiler is not supported")
extra_compile_args = [
'/EHsc',
'-DBOOST_DATE_TIME_NO_LIB',
'/std:c++14',
] + extra_compile_args
else:
# Windows, but not msvc, likely mingw
# fix bug in MingW-W64
# use posix threads
extra_compile_args = [
'-O1',
'-ftemplate-depth-256',
'-Wno-unused-function',
'-Wno-uninitialized',
'-std=c++1y',
"-D_hypot=hypot",
"-pthread",
"-fexceptions",
] + extra_compile_args
else:
# linux or macOS
extra_compile_args = [
'-O1',
'-ftemplate-depth-256',
'-Wno-unused-function',
'-Wno-uninitialized',
'-std=c++1y',
] + extra_compile_args
extension = Extension(name=module_name,
language="c++",
sources=[pyx_file],
define_macros=stan_macros,
include_dirs=include_dirs,
extra_compile_args=extra_compile_args)
cython_include_dirs = ['.', pystan_dir]
build_extension.extensions = cythonize([extension],
include_path=cython_include_dirs,
quiet=True)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
build_extension.run()
module = load_module(module_name, lib_dir)
shutil.rmtree(lib_dir, ignore_errors=True)
return module
def so_ext():
"""Get extension for the compiled library.
"""
if not hasattr(so_ext, 'ext'):
so_ext.ext = _get_build_extension().get_ext_filename('')
return so_ext.ext
('BOOST_DISABLE_ASSERTS', None),
('BOOST_NO_DECLTYPE', None),
('BOOST_PHOENIX_NO_VARIADIC_EXPRESSION', None), # needed for stanc
('BOOST_RESULT_OF_USE_TR1', None),
('FUSION_MAX_VECTOR_SIZE', 12), # for parser, stan-dev/pystan#222
]
extra_compile_args = [
'-Os',
'-ftemplate-depth-256',
'-Wno-unused-function',
'-Wno-uninitialized',
]
if platform.platform().startswith('Win'):
from Cython.Build.Inline import _get_build_extension
if _get_build_extension().compiler in (None, 'msvc'):
extra_compile_args = [
'/EHsc',
'-DBOOST_DATE_TIME_NO_LIB',
]
stanc_sources = [
"pystan/stan/src/stan/lang/ast_def.cpp",
"pystan/stan/src/stan/lang/grammars/bare_type_grammar_inst.cpp",
"pystan/stan/src/stan/lang/grammars/expression07_grammar_inst.cpp",
"pystan/stan/src/stan/lang/grammars/expression_grammar_inst.cpp",
"pystan/stan/src/stan/lang/grammars/functions_grammar_inst.cpp",
"pystan/stan/src/stan/lang/grammars/indexes_grammar_inst.cpp",
"pystan/stan/src/stan/lang/grammars/program_grammar_inst.cpp",
"pystan/stan/src/stan/lang/grammars/semantic_actions_def.cpp",
def __init__(self,
file=None,
charset='utf-8',
model_name="anon_model",
model_code=None,
stanc_ret=None,
include_paths=None,
boost_lib=None,
eigen_lib=None,
verbose=False,
obfuscate_model_name=True,
extra_compile_args=None,
allow_undefined=False,
include_dirs=None,
includes=None):
if stanc_ret is None:
stanc_ret = pystan.api.stanc(
file=file,
charset=charset,
model_code=model_code,
model_name=model_name,
verbose=verbose,
include_paths=include_paths,
obfuscate_model_name=obfuscate_model_name,
allow_undefined=allow_undefined)
if not isinstance(stanc_ret, dict):
raise ValueError("stanc_ret must be an object returned by stanc.")
stanc_ret_keys = {
'status', 'model_code', 'model_cppname', 'cppcode', 'model_name',
'include_paths'
}
if not all(n in stanc_ret_keys for n in stanc_ret):
raise ValueError("stanc_ret lacks one or more of the keys: "
"{}".format(str(stanc_ret_keys)))
elif stanc_ret['status'] != 0: # success == 0
raise ValueError("stanc_ret is not a successfully returned "
"dictionary from stanc.")
self.model_cppname = stanc_ret['model_cppname']
self.model_name = stanc_ret['model_name']
self.model_code = stanc_ret['model_code']
self.model_cppcode = stanc_ret['cppcode']
self.model_include_paths = stanc_ret['include_paths']
if allow_undefined or include_dirs or includes:
logger.warning(
"External C++ interface is an experimental feature. Be careful."
)
msg = "COMPILING THE C++ CODE FOR MODEL {} NOW."
logger.info(msg.format(self.model_name))
if verbose:
msg = "OS: {}, Python: {}, Cython {}".format(
sys.platform, sys.version, Cython.__version__)
logger.info(msg)
if boost_lib is not None:
# FIXME: allow boost_lib, eigen_lib to be specified
raise NotImplementedError
if eigen_lib is not None:
raise NotImplementedError
# module_name needs to be unique so that each model instance has its own module
nonce = abs(hash((self.model_name, time.time())))
self.module_name = 'stanfit4{}_{}'.format(self.model_name, nonce)
lib_dir = tempfile.mkdtemp(prefix='pystan_')
pystan_dir = os.path.dirname(__file__)
if include_dirs is None:
include_dirs = []
elif not isinstance(include_dirs, list):
raise TypeError(
"'include_dirs' needs to be a list: type={}".format(
type(include_dirs)))
include_dirs += [
lib_dir,
pystan_dir,
os.path.join(pystan_dir, "stan", "src"),
os.path.join(pystan_dir, "stan", "lib", "stan_math"),
os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib",
"eigen_3.3.3"),
os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib",
"boost_1.69.0"),
os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib",
"sundials_4.1.0", "include"),
np.get_include(),
]
model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp')
if includes is not None:
code = ""
for fn in includes:
code += '#include "{0}"\n'.format(fn)
ind = self.model_cppcode.index(
"static int current_statement_begin__;")
self.model_cppcode = "\n".join(
[self.model_cppcode[:ind], code, self.model_cppcode[ind:]])
with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile:
outfile.write(self.model_cppcode)
pyx_file = os.path.join(lib_dir, self.module_name + '.pyx')
pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx')
with io.open(pyx_template_file, 'r', encoding='utf-8') as infile:
s = infile.read()
template = string.Template(s)
with io.open(pyx_file, 'w', encoding='utf-8') as outfile:
s = template.safe_substitute(model_cppname=self.model_cppname)
outfile.write(s)
stan_macros = [
('BOOST_RESULT_OF_USE_TR1', None),
('BOOST_NO_DECLTYPE', None),
('BOOST_DISABLE_ASSERTS', None),
]
build_extension = _get_build_extension()
# compile stan models with optimization (-O2)
# (stanc is compiled without optimization (-O0) currently, see #33)
if extra_compile_args is None:
extra_compile_args = []
if platform.platform().startswith('Win'):
if build_extension.compiler in (None, 'msvc'):
logger.warning("MSVC compiler is not supported")
extra_compile_args = [
'/EHsc',
'-DBOOST_DATE_TIME_NO_LIB',
'/std:c++14',
] + extra_compile_args
else:
# Windows, but not msvc, likely mingw
# fix bug in MingW-W64
# use posix threads
extra_compile_args = [
'-O2',
'-ftemplate-depth-256',
'-Wno-unused-function',
'-Wno-uninitialized',
'-std=c++1y',
"-D_hypot=hypot",
"-pthread",
"-fexceptions",
] + extra_compile_args
else:
# linux or macOS
extra_compile_args = [
'-O2',
'-ftemplate-depth-256',
'-Wno-unused-function',
'-Wno-uninitialized',
'-std=c++1y',
] + extra_compile_args
distutils.log.set_verbosity(verbose)
extension = Extension(name=self.module_name,
language="c++",
sources=[pyx_file],
define_macros=stan_macros,
include_dirs=include_dirs,
extra_compile_args=extra_compile_args)
cython_include_dirs = ['.', pystan_dir]
build_extension.extensions = cythonize(
[extension], include_path=cython_include_dirs, quiet=not verbose)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr)
if redirect_stderr:
# silence stderr for compilation
orig_stderr = pystan.misc._redirect_stderr()
try:
build_extension.run()
finally:
if redirect_stderr:
# restore stderr
os.dup2(orig_stderr, sys.stderr.fileno())
self.module = load_module(self.module_name, lib_dir)
self.module_filename = os.path.basename(self.module.__file__)
# once the module is in memory, we no longer need the file on disk
# but we do need a copy of the file for pickling and the module name
with io.open(os.path.join(lib_dir, self.module_filename), 'rb') as f:
self.module_bytes = f.read()
shutil.rmtree(lib_dir, ignore_errors=True)
self.fit_class = getattr(self.module, "StanFit4Model")
def __init__(self,
file=None,
charset='utf-8',
model_name="anon_model",
model_code=None,
stanc_ret=None,
boost_lib=None,
eigen_lib=None,
save_dso=True,
verbose=False,
obfuscate_model_name=True):
if stanc_ret is None:
stanc_ret = pystan.api.stanc(
file=file,
charset=charset,
model_code=model_code,
model_name=model_name,
verbose=verbose,
obfuscate_model_name=obfuscate_model_name)
if not isinstance(stanc_ret, dict):
raise ValueError("stanc_ret must be an object returned by stanc.")
stanc_ret_keys = {
'status', 'model_code', 'model_cppname', 'cppcode', 'model_name'
}
if not all(n in stanc_ret_keys for n in stanc_ret):
raise ValueError("stanc_ret lacks one or more of the keys: "
"{}".format(str(stanc_ret_keys)))
elif stanc_ret['status'] != 0: # success == 0
raise ValueError("stanc_ret is not a successfully returned "
"dictionary from stanc.")
self.model_cppname = stanc_ret['model_cppname']
self.model_name = stanc_ret['model_name']
self.model_code = stanc_ret['model_code']
self.model_cppcode = stanc_ret['cppcode']
self.save_dso = save_dso
msg = "COMPILING THE C++ CODE FOR MODEL {} NOW."
logger.info(msg.format(self.model_name))
if verbose:
msg = "OS: {}, Python: {}, Cython {}".format(
sys.platform, sys.version, Cython.__version__)
logger.info(msg)
if boost_lib is not None:
# FIXME: allow boost_lib, eigen_lib to be specified
raise NotImplementedError
if eigen_lib is not None:
raise NotImplementedError
key = tuple([
self.model_code, self.model_cppcode, sys.version_info,
sys.executable
])
module_name = ("stanfit4" + self.model_name + '_' +
hashlib.md5(str(key).encode('utf-8')).hexdigest())
self._temp_dir = temp_dir = tempfile.mkdtemp()
lib_dir = os.path.join(temp_dir, 'pystan')
pystan_dir = os.path.dirname(__file__)
include_dirs = [
lib_dir,
pystan_dir,
os.path.join(pystan_dir, "stan/src"),
os.path.join(pystan_dir, "stan/lib/eigen_3.2.2"),
os.path.join(pystan_dir, "stan/lib/boost_1.55.0"),
np.get_include(),
]
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
model_cpp_file = os.path.join(lib_dir, self.model_cppname + '.hpp')
with io.open(model_cpp_file, 'w', encoding='utf-8') as outfile:
outfile.write(self.model_cppcode)
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
pyx_template_file = os.path.join(pystan_dir, 'stanfit4model.pyx')
with io.open(pyx_template_file, 'r', encoding='utf-8') as infile:
s = infile.read()
template = string.Template(s)
with io.open(pyx_file, 'w', encoding='utf-8') as outfile:
s = template.safe_substitute(model_cppname=self.model_cppname)
outfile.write(s)
extra_compile_args = ['-O3', '-ftemplate-depth-256']
distutils.log.set_verbosity(verbose)
extension = Extension(name=module_name,
language="c++",
sources=[pyx_file],
include_dirs=include_dirs,
extra_compile_args=extra_compile_args)
cython_include_dirs = ['.', pystan_dir]
build_extension = _get_build_extension()
build_extension.extensions = cythonize(
[extension], include_path=cython_include_dirs, quiet=not verbose)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr)
if redirect_stderr:
# silence stderr for compilation
orig_stderr = pystan.misc._redirect_stderr()
try:
build_extension.run()
finally:
if redirect_stderr:
# restore stderr
os.dup2(orig_stderr, sys.stderr.fileno())
self.module = load_module(module_name, lib_dir)
self.fit_class = getattr(self.module, "StanFit4Model")
def __init__(
self,
file=None,
charset="utf-8",
model_name="anon_model",
model_code=None,
stanc_ret=None,
boost_lib=None,
eigen_lib=None,
save_dso=True,
verbose=False,
obfuscate_model_name=True,
):
if stanc_ret is None:
stanc_ret = pystan.api.stanc(
file=file,
charset=charset,
model_code=model_code,
model_name=model_name,
verbose=verbose,
obfuscate_model_name=obfuscate_model_name,
)
if not isinstance(stanc_ret, dict):
raise ValueError("stanc_ret must be an object returned by stanc.")
stanc_ret_keys = {"status", "model_code", "model_cppname", "cppcode", "model_name"}
if not all(n in stanc_ret_keys for n in stanc_ret):
raise ValueError("stanc_ret lacks one or more of the keys: " "{}".format(str(stanc_ret_keys)))
elif stanc_ret["status"] != 0: # success == 0
raise ValueError("stanc_ret is not a successfully returned " "dictionary from stanc.")
self.model_cppname = stanc_ret["model_cppname"]
self.model_name = stanc_ret["model_name"]
self.model_code = stanc_ret["model_code"]
self.model_cppcode = stanc_ret["cppcode"]
self.save_dso = save_dso
msg = "COMPILING THE C++ CODE FOR MODEL {} NOW."
logger.info(msg.format(self.model_name))
if verbose:
msg = "OS: {}, Python: {}, Cython {}".format(sys.platform, sys.version, Cython.__version__)
logger.info(msg)
if boost_lib is not None:
# FIXME: allow boost_lib, eigen_lib to be specified
raise NotImplementedError
if eigen_lib is not None:
raise NotImplementedError
key = tuple([self.model_code, self.model_cppcode, sys.version_info, sys.executable])
module_name = "stanfit4" + self.model_name + "_" + hashlib.md5(str(key).encode("utf-8")).hexdigest()
self._temp_dir = temp_dir = tempfile.mkdtemp()
lib_dir = os.path.join(temp_dir, "pystan")
pystan_dir = os.path.dirname(__file__)
include_dirs = [
lib_dir,
pystan_dir,
os.path.join(pystan_dir, "stan/src"),
os.path.join(pystan_dir, "math"),
os.path.join(pystan_dir, "stan/lib/eigen_3.2.4"),
os.path.join(pystan_dir, "stan/lib/boost_1.58.0"),
np.get_include(),
]
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
model_cpp_file = os.path.join(lib_dir, self.model_cppname + ".hpp")
with io.open(model_cpp_file, "w", encoding="utf-8") as outfile:
outfile.write(self.model_cppcode)
pyx_file = os.path.join(lib_dir, module_name + ".pyx")
pyx_template_file = os.path.join(pystan_dir, "stanfit4model.pyx")
with io.open(pyx_template_file, "r", encoding="utf-8") as infile:
s = infile.read()
template = string.Template(s)
with io.open(pyx_file, "w", encoding="utf-8") as outfile:
s = template.safe_substitute(model_cppname=self.model_cppname)
outfile.write(s)
stan_macros = [
("BOOST_RESULT_OF_USE_TR1", None),
("BOOST_NO_DECLTYPE", None),
("BOOST_DISABLE_ASSERTS", None),
("EIGEN_NO_DEBUG", None),
]
extra_compile_args = ["-O0", "-ftemplate-depth-256", "-Wno-unused-function", "-Wno-uninitialized"]
distutils.log.set_verbosity(verbose)
extension = Extension(
name=module_name,
language="c++",
sources=[pyx_file],
define_macros=stan_macros,
include_dirs=include_dirs,
extra_compile_args=extra_compile_args,
)
cython_include_dirs = [".", pystan_dir]
build_extension = _get_build_extension()
build_extension.extensions = cythonize([extension], include_path=cython_include_dirs, quiet=not verbose)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
redirect_stderr = not verbose and pystan.misc._has_fileno(sys.stderr)
if redirect_stderr:
# silence stderr for compilation
orig_stderr = pystan.misc._redirect_stderr()
try:
build_extension.run()
finally:
if redirect_stderr:
# restore stderr
os.dup2(orig_stderr, sys.stderr.fileno())
self.module = load_module(module_name, lib_dir)
self.fit_class = getattr(self.module, "StanFit4Model")
def so_ext():
"""Get extension for the compiled library.
"""
if not hasattr(so_ext, 'ext'):
so_ext.ext = _get_build_extension().get_ext_filename('')
return so_ext.ext
def cython_inline(code,
get_type=unsafe_type,
lib_dir=os.path.join(get_cython_cache_dir(), 'inline'),
cython_include_dirs=None,
force=False,
quiet=False,
local_sym=None,
global_sym=None,
**kwargs):
if get_type is None:
get_type = lambda x: 'object'
ctx = _create_context(tuple(cython_include_dirs)
) if cython_include_dirs else _inline_default_context
# Fast path if this has been called in this session.
_unbound_symbols = _inline_cache.get(code)
if _unbound_symbols is not None:
_populate_unbound(kwargs, _unbound_symbols, local_sym, global_sym)
args = sorted(kwargs.items())
arg_sigs = tuple([(get_type(value, ctx), arg) for arg, value in args])
invoke = _inline_cache.get((code, arg_sigs))
if invoke is not None:
arg_list = [arg[1] for arg in args]
return invoke(*arg_list)
orig_code = code
code, literals = strip_string_literals(code)
if local_sym is None:
local_sym = inspect.currentframe().f_back.f_back.f_locals
if global_sym is None:
global_sym = inspect.currentframe().f_back.f_back.f_globals
try:
_inline_cache[orig_code] = _unbound_symbols = unbound_symbols(code)
_populate_unbound(kwargs, _unbound_symbols, local_sym, global_sym)
except AssertionError:
if not quiet:
# Parsing from strings not fully supported (e.g. cimports).
print(
"Could not parse code as a string (to extract unbound symbols)."
)
cimports = []
for name, arg in list(kwargs.items()):
if arg is cython:
cimports.append('\ncimport cython as %s' % name)
del kwargs[name]
arg_names = sorted(kwargs)
arg_sigs = tuple([(get_type(kwargs[arg], ctx), arg) for arg in arg_names])
key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__
module_name = "_cython_inline_" + hashlib.md5(
str(key).encode('utf-8')).hexdigest()
if module_name in sys.modules:
_module = sys.modules[module_name]
else:
build_extension = None
global _so_ext
if _so_ext is None:
# Figure out and cache current extension suffix
build_extension = _get_build_extension()
_so_ext = build_extension.get_ext_filename('')
module_path = os.path.join(lib_dir, module_name + _so_ext)
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)
if force or not os.path.isfile(module_path):
cflags = []
c_include_dirs = []
qualified = re.compile(r'([.\w]+)[.]')
for type, _ in arg_sigs:
m = qualified.match(type)
if m:
cimports.append('\ncimport %s' % m.groups()[0])
# one special case
if m.groups()[0] == 'numpy':
import numpy
c_include_dirs.append(numpy.get_include())
# cflags.append('-Wno-unused')
module_body, func_body = extract_func_code(code)
params = ', '.join(['%s %s' % a for a in arg_sigs])
module_code = '''
%(module_body)s
%(cimports)s
def __invoke(%(params)s):
%(func_body)s
return locals()
''' % {
'cimports': '\n'.join(cimports),
'module_body': module_body,
'params': params,
'func_body': func_body
}
for key, value in literals.items():
module_code = module_code.replace(key, value)
pyx_file = os.path.join(lib_dir, module_name + '.pyx')
fh = open(pyx_file, 'w')
try:
fh.write(module_code)
finally:
fh.close()
extension = Extension(name=module_name,
sources=[pyx_file],
include_dirs=c_include_dirs,
extra_compile_args=cflags)
if build_extension is None:
build_extension = _get_build_extension()
build_extension.extensions = cythonize(
[extension],
include_path=cython_include_dirs or ['.'],
quiet=quiet)
build_extension.build_temp = os.path.dirname(pyx_file)
build_extension.build_lib = lib_dir
build_extension.run()
_module = imp.load_dynamic(module_name, module_path)
_inline_cache[orig_code, arg_sigs] = _module.__invoke
arg_list = [kwargs[arg] for arg in arg_names]
return _module.__invoke(*arg_list)
