def call(self, fun, arg=list()):
'''
Call a function in the load path.
'''
name = fun[:fun.rindex('.')]
try:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(name, fn_, path, desc)
except ImportError:
if self.opts.get('cython_enable', True) is True:
# The module was not found, try to find a cython module
try:
import pyximport
pyximport.install()
for mod_dir in self.module_dirs:
for fn_ in os.listdir(mod_dir):
if name == fn_[:fn_.rindex('.')]:
# Found it, load the mod and break the loop
mod = pyximport.load_module(
name, os.path.join(mod_dir, fn_)
)
return getattr(
mod, fun[fun.rindex('.') + 1:])(*arg)
except ImportError:
log.info("Cython is enabled in options though it's not "
"present in the system path. Skipping Cython "
"modules.")
return getattr(mod, fun[fun.rindex('.') + 1:])(*arg)
def call(self, fun, arg=None):
'''
Call a function in the load path.
'''
if arg is None:
arg = []
name = fun[:fun.rindex('.')]
try:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(name, fn_, path, desc)
except ImportError:
if self.opts.get('cython_enable', True) is True:
# The module was not found, try to find a cython module
try:
import pyximport
pyximport.install()
for mod_dir in self.module_dirs:
for fn_ in os.listdir(mod_dir):
if name == fn_[:fn_.rindex('.')]:
# Found it, load the mod and break the loop
mod = pyximport.load_module(
name, os.path.join(mod_dir, fn_))
return getattr(mod,
fun[fun.rindex('.') + 1:])(*arg)
except ImportError:
log.info('Cython is enabled in options though it\'s not '
'present in the system path. Skipping Cython '
'modules.')
return getattr(mod, fun[fun.rindex('.') + 1:])(*arg)
def scanfuncs(filename, prefixes, cython=False):
""" Return list of function names from ``filename`` that begin with prefix.
This *does not* import the Python file, so this is safe to use, but
functionality is limited to retrieving names of basic functions defined
within global scope of the file.
This *does*, however, import Cython files (if applicable).
"""
# Used by: findarenas, findbenchmarks
path, name = os.path.split(filename)
name, ext = os.path.splitext(name)
# Should `cython` be a keyword argument, or should we just infer it?
cython = cython or ext == '.pyx'
if not cython:
funcs = pyclbr.readmodule_ex(name, [path])
funcnames = []
for key, val in funcs.items():
if (any(key.startswith(prefix) for prefix in prefixes) and
val.file == filename):
funcnames.append(key)
return funcnames
# Scan Cython file. We need to import it.
import pyximport
pyximport.install()
sys.dont_write_bytecode = True
# Make sure local imports work for the given file
sys.path.insert(0, path)
pyximport.build_module(name, filename)
try:
mod = pyximport.load_module(name, filename)
except ImportError:
# There is most likely a '*.py' file that shares the same
# base name as the '*.pyx' file we are trying to import.
# Removing the directory from sys.path should fix this,
# but will disable local importing.
sys.path.pop(0)
mod = pyximport.load_module(name, filename)
# Undo making local imports work
if sys.path[0] == path:
sys.path.pop(0)
funcnames = []
for funcname in mod.__dict__:
if any(funcname.startswith(prefix) for prefix in prefixes):
funcnames.append(funcname)
return funcnames
def gen_functions(self):
'''
Return a dict of functions found in the defined module_dirs
'''
names = {}
modules = []
funcs = {}
for mod_dir in self.module_dirs:
if not mod_dir.startswith('/'):
continue
if not os.path.isdir(mod_dir):
continue
for fn_ in os.listdir(mod_dir):
if fn_.startswith('_'):
continue
if fn_.endswith('.py')\
or fn_.endswith('.pyc')\
or fn_.endswith('.pyo')\
or fn_.endswith('.so')\
or fn_.endswith('.pyx'):
names[fn_[:fn_.rindex('.')]] = os.path.join(mod_dir, fn_)
for name in names:
try:
if names[name].endswith('.pyx'):
mod = pyximport.load_module(name, names[name], '/tmp')
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(name, fn_, path, desc)
except ImportError:
continue
modules.append(mod)
for mod in modules:
virtual = ''
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
if hasattr(mod, '__virtual__'):
if callable(mod.__virtual__):
virtual = mod.__virtual__()
for attr in dir(mod):
if attr.startswith('_'):
continue
if callable(getattr(mod, attr)):
if virtual:
funcs[virtual + '.' + attr] = getattr(mod, attr)
elif virtual == False:
pass
else:
funcs[mod.__name__ + '.' + attr] = getattr(mod, attr)
return funcs
def __init__(self, template_string):
self.module = None
src = tempfile.NamedTemporaryFile(suffix='.pyx')
try:
src.write(Compiler().visit(Template(template_string)))
src.flush()
self.module = pyximport.load_module('template', src.name)
finally:
src.close()
def cython(modname, path, parent):
'''
Import cython module
'''
if not HAS_CYTHON:
try:
raise ImportError('Cython not available')
except ImportError as exc:
return exc
try:
return pyximport.load_module(modname, path, tempfile.gettempdir())
except Exception as exc: # pylint: disable=broad-except
return LoadError('Failed to load cython module {} at path {}'.format(
modname, path),
exception=exc,
traceback=stdlib_traceback.format_exc())
def call(self, fun, arg=[]):
'''
Call a function in the load path.
'''
name = fun[:fun.rindex('.')]
try:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(name, fn_, path, desc)
except ImportError:
# The module was not found, try to find a cython module
for mod_dir in self.module_dirs:
for fn_ in os.listdir(mod_dir):
if name == fn_[:fn_.rindex('.')]:
# Found it, load the mod and break the loop
mod = pyximport.load_module(name, os.path.join(mod_dir, fn_))
return getattr(mod, fun[fun.rindex('.') + 1:])(*arg)
return getattr(mod, fun[fun.rindex('.') + 1:])(*arg)
def runtime_ncells(setup_primordial_network, setup_solver_options):
density = 1.0e10
temperature = 2000.0
h2frac = 1.0e-3
solver_name = setup_solver_options["solver_name"]
niters = setup_solver_options["niters"]
reltol = setup_solver_options["reltol"]
write_network(setup_primordial_network,
solver_options=setup_solver_options)
output_dir = setup_solver_options["output_dir"]
if not os.path.exists(output_dir):
os.mkdir(output_dir)
os.chdir(output_dir)
pyximport.install(setup_args={"include_dirs": np.get_include()},
reload_support=True,
inplace=True)
_solver_run = pyximport.load_module(
"{}_solver_run".format(solver_name),
"{}_solver_run.pyx".format(solver_name),
build_inplace=True,
pyxbuild_dir="_dengo_temp")
ncells = 1
init_values = setup_initial_conditions(setup_primordial_network, density,
temperature, h2frac, ncells)
dtf = freefall_time(init_values["density"])
from timeit import default_timer as timer
for ncells in [2**(i * 2) for i in range(6)]:
init_values = setup_initial_conditions(setup_primordial_network,
density, temperature, h2frac,
ncells)
start = timer()
rv, rv_int = eval(
"_solver_run.run_{}(init_values, dtf, niter={}, reltol = {}, floor_value = 1.0e-16)"
.format(solver_name, niters, reltol))
end = timer()
print("ncells = {:d}, with {:0.2e} s\n".format(ncells,
(end - start) / ncells))
#TestConservation(setup_primordial_network, rv_int, density)
os.chdir("../")
def create_bechem_solver(init, primordial, solver_name, cooling):
# name of the solver
pyximport.install(setup_args={"include_dirs": np.get_include()},
reload_support=True,
inplace=True)
# write the network
primordial.write_solver(solver_name,
output_dir=".",
solver_template="rates_and_rate_tables",
ode_solver_source="BE_chem_solve.C",
init_values=init,
input_is_number=False)
# import the pyx module
bechem_run = pyximport.load_module("{}_run".format(solver_name),
"{}_solver_run.pyx".format(solver_name),
build_inplace=True,
pyxbuild_dir="_dengo_temp")
return bechem_run
def run_model(init_values, reltol=1.0e-5, make_plot=True, dtf=5.0e1):
if not os.path.exists(output_dir):
os.mkdir(output_dir)
os.chdir(output_dir)
print(os.getcwd())
pyximport.install(setup_args={"include_dirs": np.get_include()},
reload_support=True,
inplace=True,
language_level=3)
_solver_run = pyximport.load_module(
"{}_solver_run".format(solver_name),
"{}_solver_run.pyx".format(solver_name),
build_inplace=True,
pyxbuild_dir="_dengo_temp")
print("finished loading!")
rv, rv_int = _solver_run.run_predator_prey(init_values,
dtf,
niter=5e2,
adaptive_step=False,
reltol=reltol)
mask = rv_int['successful']
for name in sorted(rv_int):
if len(rv_int[name].shape) == 1:
rv_int[name] = rv_int[name][mask]
else:
rv_int[name] = rv_int[name][0, mask]
if make_plot:
plt.clf()
skip = ('successful', 'dt', 't', 'ge', "T", "dead_predator",
"dead_prey")
for n, v in sorted(rv_int.items()):
if n in skip:
continue
plt.plot(rv_int['t'], v, label=n)
plt.xlabel("time [s]")
plt.legend(loc='best', fontsize='small')
plt.savefig("prey_predator.png")
os.chdir("../")
return rv_int
def gen_functions(self, pack=None):
"""
Return a dict of functions found in the defined module_dirs
"""
names = {}
modules = []
funcs = {}
cython_enabled = False
if self.opts.get("cython_enable", True) is True:
try:
import pyximport
pyximport.install()
cython_enabled = True
except ImportError:
log.info(
"Cython is enabled in options though it's not present "
"in the system path. Skipping Cython modules."
)
for mod_dir in self.module_dirs:
if not mod_dir.startswith("/"):
continue
if not os.path.isdir(mod_dir):
continue
for fn_ in os.listdir(mod_dir):
if fn_.startswith("_"):
continue
if (
fn_.endswith(".py")
or fn_.endswith(".pyc")
or fn_.endswith(".pyo")
or fn_.endswith(".so")
or (cython_enabled and fn_.endswith(".pyx"))
):
names[fn_[: fn_.rindex(".")]] = os.path.join(mod_dir, fn_)
for name in names:
try:
if names[name].endswith(".pyx"):
# If there's a name which ends in .pyx it means the above
# cython_enabled is True. Continue...
mod = pyximport.load_module(name, names[name], "/tmp")
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(name, fn_, path, desc)
except ImportError:
continue
modules.append(mod)
for mod in modules:
virtual = ""
if hasattr(mod, "__opts__"):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
if pack:
if type(pack) == type(list()):
for chunk in pack:
setattr(mod, chunk["name"], chunk["value"])
else:
setattr(mod, pack["name"], pack["value"])
if hasattr(mod, "__virtual__"):
if callable(mod.__virtual__):
virtual = mod.__virtual__()
for attr in dir(mod):
if attr.startswith("_"):
continue
if callable(getattr(mod, attr)):
if virtual:
funcs[virtual + "." + attr] = getattr(mod, attr)
elif virtual == False:
pass
else:
funcs[mod.__name__ + "." + attr] = getattr(mod, attr)
return funcs
def gen_module(self, name, functions, pack=None):
'''
Load a single module and pack it with the functions passed
'''
full = ''
mod = None
for mod_dir in self.module_dirs:
if not os.path.isabs(mod_dir):
continue
if not os.path.isdir(mod_dir):
continue
fn_ = os.path.join(mod_dir, name)
if os.path.isdir(fn_):
full = fn_
break
else:
for ext in ('.py', '.pyo', '.pyc', '.so'):
full_test = '{0}{1}'.format(fn_, ext)
if os.path.isfile(full_test):
full = full_test
break
if full:
break
if not full:
return None
cython_enabled = False
if self.opts.get('cython_enable', True) is True:
try:
import pyximport
pyximport.install()
cython_enabled = True
except ImportError:
log.info('Cython is enabled in the options but not present '
'in the system path. Skipping Cython modules.')
try:
if full.endswith('.pyx') and cython_enabled:
# If there's a name which ends in .pyx it means the above
# cython_enabled is True. Continue...
mod = pyximport.load_module(name, full, tempfile.gettempdir())
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(
'{0}.{1}.{2}.{3}'.format(self.loaded_base_name,
self.mod_type_check(path),
self.tag, name), fn_, path, desc)
except ImportError:
log.debug('Failed to import {0} {1}:\n'.format(self.tag, name),
exc_info=True)
return mod
except Exception:
log.warning(
'Failed to import {0} {1}, this is due most likely to a '
'syntax error:\n'.format(self.tag, name),
exc_info=True)
return mod
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
if pack:
if isinstance(pack, list):
for chunk in pack:
try:
setattr(mod, chunk['name'], chunk['value'])
except KeyError:
pass
else:
setattr(mod, pack['name'], pack['value'])
# Call a module's initialization method if it exists
if hasattr(mod, '__init__'):
if callable(mod.__init__):
try:
mod.__init__(self.opts)
except TypeError:
pass
funcs = {}
module_name = mod.__name__[mod.__name__.rindex('.') + 1:]
if getattr(mod, '__load__', False) is not False:
log.info(
'The functions from module {0!r} are being loaded from the '
'provided __load__ attribute'.format(module_name))
for attr in getattr(mod, '__load__', dir(mod)):
if attr.startswith('_'):
# private functions are skipped
continue
if callable(getattr(mod, attr)):
func = getattr(mod, attr)
if hasattr(func, '__bases__'):
if 'BaseException' in func.__bases__:
# the callable object is an exception, don't load it
continue
# Let's get the function name.
# If the module has the __func_alias__ attribute, it must be a
# dictionary mapping in the form of(key -> value):
# <real-func-name> -> <desired-func-name>
#
# It default's of course to the found callable attribute name
# if no alias is defined.
funcname = getattr(mod, '__func_alias__', {}).get(attr, attr)
funcs['{0}.{1}'.format(module_name, funcname)] = func
self._apply_outputter(func, mod)
if not hasattr(mod, '__salt__'):
mod.__salt__ = functions
try:
context = sys.modules[functions[functions.keys()
[0]].__module__].__context__
except (AttributeError, IndexError):
context = {}
mod.__context__ = context
return funcs
def gen_functions(self, pack=None, virtual_enable=True):
"""
Return a dict of functions found in the defined module_dirs
"""
names = {}
modules = []
funcs = {}
cython_enabled = False
if self.opts.get("cython_enable", True) is True:
try:
import pyximport
pyximport.install()
cython_enabled = True
except ImportError:
log.info(
"Cython is enabled in the options but not present " "in the system path. Skipping Cython modules."
)
for mod_dir in self.module_dirs:
if not os.path.isabs(mod_dir):
continue
if not os.path.isdir(mod_dir):
continue
for fn_ in os.listdir(mod_dir):
if fn_.startswith("_"):
continue
if fn_.endswith((".py", ".pyc", ".pyo", ".so")) or (cython_enabled and fn_.endswith(".pyx")):
names[fn_[: fn_.rindex(".")]] = os.path.join(mod_dir, fn_)
for name in names:
try:
if names[name].endswith(".pyx"):
# If there's a name which ends in .pyx it means the above
# cython_enabled is True. Continue...
mod = pyximport.load_module("{0}_{1}".format(name, self.tag), names[name], "/tmp")
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module("{0}_{1}".format(name, self.tag), fn_, path, desc)
except ImportError as exc:
log.debug(("Failed to import module {0}, this is most likely" " NOT a problem: {1}").format(name, exc))
continue
except Exception as exc:
log.warning(
("Failed to import module {0}, this is due most" " likely to a syntax error: {1}").format(name, exc)
)
continue
modules.append(mod)
for mod in modules:
virtual = ""
if hasattr(mod, "__opts__"):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
mod.__pillar__ = self.pillar
if pack:
if isinstance(pack, list):
for chunk in pack:
setattr(mod, chunk["name"], chunk["value"])
else:
setattr(mod, pack["name"], pack["value"])
# Call a module's initialization method if it exists
if hasattr(mod, "__init__"):
if callable(mod.__init__):
try:
mod.__init__()
except TypeError:
pass
if virtual_enable:
if hasattr(mod, "__virtual__"):
if callable(mod.__virtual__):
virtual = mod.__virtual__()
for attr in dir(mod):
if attr.startswith("_"):
continue
if callable(getattr(mod, attr)):
func = getattr(mod, attr)
if isinstance(func, type):
if any(["Error" in func.__name__, "Exception" in func.__name__]):
continue
if virtual:
funcs["{0}.{1}".format(virtual, attr)] = func
self._apply_outputter(func, mod)
elif virtual is False:
pass
else:
funcs["{0}.{1}".format(mod.__name__[: mod.__name__.rindex("_")], attr)] = func
self._apply_outputter(func, mod)
for mod in modules:
if not hasattr(mod, "__salt__"):
mod.__salt__ = funcs
return funcs
def import_module(module):
"""This will try to import the passed module. This will return
the module if it was imported, or will return 'None' if
it should not be imported.
Parameters
----------
module: str
The name of the module to import
"""
from ._console import Console
try:
import importlib
m = importlib.import_module(module)
except SyntaxError as e:
Console.error(f"\nSyntax error when importing {module}\n"
f"{e.__class__.__name__}:{e}\n"
f"Line {e.lineno}.{e.offset}:{(e.offset-1)*' '} |\n"
f"Line {e.lineno}.{e.offset}:{(e.offset-1)*' '}\\|/\n"
f"Line {e.lineno}.{e.offset}: {e.text}")
m = None
except ImportError:
# this is ok and expected if the module is in a python file
# that will be loaded below
m = None
except Exception:
# something else went wrong
m = None
if m is None:
try:
import os
if os.path.exists(module):
pyfile = module
elif os.path.exists(f"{module}.py"):
pyfile = f"{module}.py"
elif os.path.exists(f"{module}.pyx"):
pyfile = f"{module}.pyx"
else:
pyfile = None
if pyfile:
from pathlib import Path as _Path
module = _Path(pyfile).stem
if pyfile.endswith(".pyx"):
try:
import pyximport
pyfile = _clean_cython(pyfile)
Console.print(f"Compiling cython plugin from {pyfile}")
pyximport.install(language_level=3)
m = pyximport.load_module(module,
pyfile,
language_level=3)
Console.print(f"Loaded cython {module} from {pyfile}")
except Exception as e:
Console.error(
f"Cannot compile and load the cython plugin "
f"{pyfile}. Error is {e.__class__}: {e}")
else:
import importlib.util
spec = importlib.util.spec_from_file_location(
module, pyfile)
if spec is None:
raise ImportError(
f"Cannot build a spec for the module from "
f"the file {pyfile}")
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
Console.print(f"Loaded {module} from {pyfile}")
except SyntaxError as e:
Console.error(
f"\nSyntax error when reading {pyfile}\n"
f"{e.__class__.__name__}:{e}\n"
f"Line {e.lineno}.{e.offset}:{(e.offset-1)*' '} |\n"
f"Line {e.lineno}.{e.offset}:{(e.offset-1)*' '}\\|/\n"
f"Line {e.lineno}.{e.offset}: {e.text}")
except Exception as e:
Console.error(f"\nError when importing {module}\n"
f"{e.__class__.__name__}: {e}\n")
m = None
if m is not None:
Console.print(f"IMPORT {m}")
return m
def gen_functions(self, pack=None):
'''
Return a dict of functions found in the defined module_dirs
'''
names = {}
modules = []
funcs = {}
cython_enabled = False
if self.opts.get('cython_enable', True) is True:
try:
import pyximport
pyximport.install()
cython_enabled = True
except ImportError:
log.info('Cython is enabled in options put not present '
'on the system path. Skipping Cython modules.')
for mod_dir in self.module_dirs:
if not mod_dir.startswith('/'):
continue
if not os.path.isdir(mod_dir):
continue
for fn_ in os.listdir(mod_dir):
if fn_.startswith('_'):
continue
if fn_.endswith('.py')\
or fn_.endswith('.pyc')\
or fn_.endswith('.pyo')\
or fn_.endswith('.so')\
or (cython_enabled and fn_.endswith('.pyx')):
names[fn_[:fn_.rindex('.')]] = os.path.join(mod_dir, fn_)
for name in names:
try:
if names[name].endswith('.pyx'):
# If there's a name which ends in .pyx it means the above
# cython_enabled is True. Continue...
mod = pyximport.load_module(name, names[name], '/tmp')
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(name, fn_, path, desc)
except ImportError:
continue
modules.append(mod)
for mod in modules:
virtual = ''
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
if pack:
if type(pack) == type(list()):
for chunk in pack:
setattr(mod, chunk['name'], chunk['value'])
else:
setattr(mod, pack['name'], pack['value'])
if hasattr(mod, '__virtual__'):
if callable(mod.__virtual__):
virtual = mod.__virtual__()
for attr in dir(mod):
if attr.startswith('_'):
continue
if callable(getattr(mod, attr)):
if virtual:
func = getattr(mod, attr)
funcs[virtual + '.' + attr] = func
self._apply_outputter(func, mod)
elif virtual == False:
pass
else:
func = getattr(mod, attr)
funcs[mod.__name__ + '.' + attr] = func
self._apply_outputter(func, mod)
for mod in modules:
if not hasattr(mod, '__salt__'):
mod.__salt__ = funcs
return funcs
tornado.web.URLSpec(r'/images/(?P<uid>[a-z0-9-]+)', ImageHandler, name='image'),
tornado.web.URLSpec(r'/algorithms/([^/]+)(.*)', AlgorithmHandler, name='algorithm'),
(r'/algorithms', AlgorithmHandler),
(r'/', IndexHandler),
])
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--logging', help='Set log level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'],
default='INFO')
parser.add_argument('-c', '--cython', help='Compile algorithms with cython', action='store_true')
parser.add_argument('-p', '--port', help='Set port', type=int, default=8887)
args = parser.parse_args()
logging.basicConfig(level=args.logging)
if args.cython:
import pyximport
pyximport.install()
geometry.kcenter = pyximport.load_module('geometry.kcenter', 'geometry/kcenter.py')
graph.kcenter = pyximport.load_module('graph.kcenter', 'graph/kcenter.py')
application.listen(args.port)
print('Press strg-c to exit')
try:
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt:
print('Shut down...')
def run_solver(init_values,
dtf=None,
make_plot=True,
intermediate=True,
solver_name="temp",
output_dir=".",
niters=1e3,
reltol=1.0e-3,
adaptive_step=True,
**kwargs):
print("########### from run solver", os.getcwd())
#solver_name = solver_options["solver_name"]
#solver_dir = solver_options["output_dir"]
#niters = solver_options["niters"]
#reltol = solver_options["reltol"]
pyximport.install(setup_args={"include_dirs": np.get_include()},
reload_support=True,
inplace=True)
_solver_run = pyximport.load_module(
"{}_solver_run".format(solver_name),
"{}_solver_run.pyx".format(solver_name),
build_inplace=True,
pyxbuild_dir="_dengo_temp")
if dtf is None:
dtf = freefall_time(init_values["density"])
rv, rv_int = eval("_solver_run.run_{}(init_values, dtf, \
niter={}, reltol = {}, floor_value = 1.0e-20)".format(
solver_name, niters, reltol))
mask = rv_int['successful']
for name in sorted(rv_int):
if len(rv_int[name].shape) == 1:
rv_int[name] = rv_int[name][mask]
else:
rv_int[name] = rv_int[name][0, mask]
if make_plot:
plt.clf()
skip = ('successful', 'dt', 't', 'ge')
for n, v in sorted(rv_int.items()):
if n in skip:
continue
plt.loglog(rv_int['t'], v, label=n)
print(n, "{0:0.5g}".format(v[-1]))
# plt.ylim(density * 1e-20, density * 100)
plt.xlabel("time [s]")
plt.legend(loc='best', fontsize='xx-small')
plt.savefig("plot_{}_network.png".format(solver_name))
if intermediate:
return rv_int
else:
try:
for n, v in sorted(rv_int.items()):
rv_int[n] = np.array([v[-1]])
return rv_int
except BaseException:
return rv_int
def gen_functions(self, pack=None, virtual_enable=True):
'''
Return a dict of functions found in the defined module_dirs
'''
names = {}
modules = []
funcs = {}
disable = set(self.opts.get('disable_{0}s'.format(self.tag), []))
cython_enabled = False
if self.opts.get('cython_enable', True) is True:
try:
import pyximport
pyximport.install()
cython_enabled = True
except ImportError:
log.info('Cython is enabled in the options but not present '
'in the system path. Skipping Cython modules.')
for mod_dir in self.module_dirs:
if not os.path.isabs(mod_dir):
continue
if not os.path.isdir(mod_dir):
continue
for fn_ in os.listdir(mod_dir):
if fn_.startswith('_'):
continue
if fn_.split('.')[0] in disable:
continue
if (fn_.endswith(('.py', '.pyc', '.pyo', '.so'))
or (cython_enabled and fn_.endswith('.pyx'))
or os.path.isdir(os.path.join(mod_dir, fn_))):
extpos = fn_.rfind('.')
if extpos > 0:
_name = fn_[:extpos]
else:
_name = fn_
names[_name] = os.path.join(mod_dir, fn_)
for name in names:
try:
if names[name].endswith('.pyx'):
# If there's a name which ends in .pyx it means the above
# cython_enabled is True. Continue...
mod = pyximport.load_module(
'{0}.{1}.{2}.{3}'.format(
loaded_base_name,
_mod_type(names[name]),
self.tag,
name
), names[name], tempfile.gettempdir()
)
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(
'{0}.{1}.{2}.{3}'.format(
loaded_base_name, _mod_type(path), self.tag, name
), fn_, path, desc
)
# reload all submodules if necessary
submodules = [
getattr(mod, sname) for sname in dir(mod) if
type(getattr(mod, sname))==type(mod)
]
# reload only custom "sub"modules i.e is a submodule in
# parent module that are still available on disk (i.e. not
# removed during sync_modules)
for submodule in submodules:
try:
smname = '{0}.{1}.{2}'.format(loaded_base_name, self.tag, name)
smfile = os.path.splitext(submodule.__file__)[0] + ".py"
if submodule.__name__.startswith(smname) and os.path.isfile(smfile):
reload(submodule)
except AttributeError:
continue
except ImportError as exc:
log.debug('Failed to import module {0}, this is most likely '
'NOT a problem: {1}'.format(name, exc))
continue
except Exception as exc:
trb = traceback.format_exc()
log.warning('Failed to import module {0}, this is due most '
'likely to a syntax error: {1}'.format(name, trb))
continue
modules.append(mod)
for mod in modules:
virtual = ''
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
mod.__pillar__ = self.pillar
if pack:
if isinstance(pack, list):
for chunk in pack:
setattr(mod, chunk['name'], chunk['value'])
else:
setattr(mod, pack['name'], pack['value'])
# Call a module's initialization method if it exists
if hasattr(mod, '__init__'):
if callable(mod.__init__):
try:
mod.__init__(self.opts)
except TypeError:
pass
if virtual_enable:
if hasattr(mod, '__virtual__'):
if callable(mod.__virtual__):
virtual = mod.__virtual__()
for attr in dir(mod):
if attr.startswith('_'):
continue
if callable(getattr(mod, attr)):
func = getattr(mod, attr)
if isinstance(func, type):
if any([
'Error' in func.__name__,
'Exception' in func.__name__]):
continue
if virtual:
funcs['{0}.{1}'.format(virtual, attr)] = func
self._apply_outputter(func, mod)
elif virtual is False:
pass
else:
funcs[
'{0}.{1}'.format(
mod.__name__[mod.__name__.rindex('.')+1:],
attr
)
] = func
self._apply_outputter(func, mod)
for mod in modules:
if not hasattr(mod, '__salt__'):
mod.__salt__ = funcs
return funcs
f.write('''
def score(candidate, solution="Grab your towel and DON'T PANIC!"):
"""Evaluates a string.
Usage: score(candidate, solution)
Coded as: sum(c1 == c2 for c1, c2 in zip(candidate, solution))
The objective is to find the default string for solution.
>>> score("poo", solution="foo")
2
"""
return sum(c1 == c2 for c1, c2 in zip(candidate, solution))
''')
__data = pyximport.load_module("data", __file, os.path.dirname(__file))
score = __data.score
## Problème des villes
cities = """
Amsterdam Athens Barcelona Berlin Bucarest Budapest Brussels Copenhagen
Dublin Edinburgh Gibraltar Helsinki Istanbul Kiev Lisbon London
Madrid Milan Moscow Munich Nantes Oslo Paris Prague
Reykjavik Riga Rome Sofia Stockholm Toulouse Vilnius Warsaw
"""
cities = cities.split()
distances_base64 = \
def gen_module(self, name, functions, pack=None):
'''
Load a single module and pack it with the functions passed
'''
full = ''
mod = None
for mod_dir in self.module_dirs:
if not os.path.isabs(mod_dir):
continue
if not os.path.isdir(mod_dir):
continue
fn_ = os.path.join(mod_dir, name)
if os.path.isdir(fn_):
full = fn_
else:
for ext in ('.py', '.pyo', '.pyc', '.so'):
full_test = '{0}{1}'.format(fn_, ext)
if os.path.isfile(full_test):
full = full_test
if not full:
return None
cython_enabled = False
if self.opts.get('cython_enable', True) is True:
try:
import pyximport
pyximport.install()
cython_enabled = True
except ImportError:
log.info('Cython is enabled in the options but not present '
'in the system path. Skipping Cython modules.')
try:
if full.endswith('.pyx') and cython_enabled:
# If there's a name which ends in .pyx it means the above
# cython_enabled is True. Continue...
mod = pyximport.load_module(name, full, tempfile.gettempdir())
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(
'{0}.{1}.{2}.{3}'.format(
loaded_base_name, _mod_type(path), self.tag, name
), fn_, path, desc
)
except ImportError as exc:
log.debug('Failed to import module {0}: {1}'.format(name, exc))
return mod
except Exception:
trb = traceback.format_exc()
log.warning('Failed to import module {0}, this is due most likely '
'to a syntax error: {1}'.format(name, trb))
return mod
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
if pack:
if isinstance(pack, list):
for chunk in pack:
try:
setattr(mod, chunk['name'], chunk['value'])
except KeyError:
pass
else:
setattr(mod, pack['name'], pack['value'])
# Call a module's initialization method if it exists
if hasattr(mod, '__init__'):
if callable(mod.__init__):
try:
mod.__init__(self.opts)
except TypeError:
pass
funcs = {}
for attr in dir(mod):
if attr.startswith('_'):
continue
if callable(getattr(mod, attr)):
func = getattr(mod, attr)
if hasattr(func, '__bases__'):
if 'BaseException' in func.__bases__:
# the callable object is an exception, don't load it
continue
funcs[
'{0}.{1}'.format(
mod.__name__[mod.__name__.rindex('.') + 1:], attr
)
] = func
self._apply_outputter(func, mod)
if not hasattr(mod, '__salt__'):
mod.__salt__ = functions
return funcs
def gen_functions(self, pack=None, virtual_enable=True):
'''
Return a dict of functions found in the defined module_dirs
'''
log.debug('loading {0} in {1}'.format(self.tag, self.module_dirs))
names = {}
modules = []
funcs = {}
disable = set(self.opts.get('disable_{0}s'.format(self.tag), []))
cython_enabled = False
if self.opts.get('cython_enable', True) is True:
try:
import pyximport
pyximport.install()
cython_enabled = True
except ImportError:
log.info('Cython is enabled in the options but not present '
'in the system path. Skipping Cython modules.')
for mod_dir in self.module_dirs:
if not os.path.isabs(mod_dir):
log.debug(('Skipping {0}, it is not an abosolute '
'path').format(mod_dir))
continue
if not os.path.isdir(mod_dir):
log.debug(('Skipping {0}, it is not a '
'directory').format(mod_dir))
continue
for fn_ in os.listdir(mod_dir):
if fn_.startswith('_'):
# skip private modules
# log messages omitted for obviousness
continue
if fn_.split('.')[0] in disable:
log.debug(('Skipping {0}, it is disabled by '
'configuration').format(fn_))
continue
if (fn_.endswith(('.py', '.pyc', '.pyo', '.so'))
or (cython_enabled and fn_.endswith('.pyx'))
or os.path.isdir(os.path.join(mod_dir, fn_))):
extpos = fn_.rfind('.')
if extpos > 0:
_name = fn_[:extpos]
else:
_name = fn_
names[_name] = os.path.join(mod_dir, fn_)
else:
log.debug(('Skipping {0}, it does not end with an '
'expected extension').format(fn_))
for name in names:
try:
if names[name].endswith('.pyx'):
# If there's a name which ends in .pyx it means the above
# cython_enabled is True. Continue...
mod = pyximport.load_module(
'{0}.{1}.{2}.{3}'.format(
loaded_base_name,
_mod_type(names[name]),
self.tag,
name
), names[name], tempfile.gettempdir()
)
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(
'{0}.{1}.{2}.{3}'.format(
loaded_base_name, _mod_type(path), self.tag, name
), fn_, path, desc
)
# reload all submodules if necessary
submodules = [
getattr(mod, sname) for sname in dir(mod) if
isinstance(getattr(mod, sname), mod.__class__)
]
# reload only custom "sub"modules i.e is a submodule in
# parent module that are still available on disk (i.e. not
# removed during sync_modules)
for submodule in submodules:
try:
smname = '{0}.{1}.{2}'.format(loaded_base_name, self.tag, name)
smfile = os.path.splitext(submodule.__file__)[0] + ".py"
if submodule.__name__.startswith(smname) and os.path.isfile(smfile):
reload(submodule)
except AttributeError:
continue
except ImportError as exc:
log.debug('Failed to import module {0}, this is most likely '
'NOT a problem: {1}'.format(name, exc))
continue
except Exception:
trb = traceback.format_exc()
log.warning('Failed to import module {0}, this is due most '
'likely to a syntax error: {1}'.format(name, trb))
continue
modules.append(mod)
for mod in modules:
virtual = ''
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
mod.__pillar__ = self.pillar
if pack:
if isinstance(pack, list):
for chunk in pack:
if not isinstance(chunk, dict):
continue
try:
setattr(mod, chunk['name'], chunk['value'])
except KeyError:
pass
else:
setattr(mod, pack['name'], pack['value'])
# Call a module's initialization method if it exists
if hasattr(mod, '__init__'):
if callable(mod.__init__):
try:
mod.__init__(self.opts)
except TypeError:
pass
# Trim the full pathname to just the module
# this will be the short name that other salt modules and state
# will refer to it as.
module_name = mod.__name__.rsplit('.', 1)[-1]
if virtual_enable:
# if virtual modules are enabled, we need to look for the
# __virtual__() function inside that module and run it.
# This function will return either a new name for the module
# or False. This allows us to have things like the pkg module
# working on all platforms under the name 'pkg'. It also allows
# for modules like augeas_cfg to be referred to as 'augeas',
# which would otherwise have namespace collisions. And finally
# it allows modules to return False if they are not intended
# to run on the given platform or are missing dependencies.
try:
if hasattr(mod, '__virtual__'):
if callable(mod.__virtual__):
virtual = mod.__virtual__()
if virtual is False:
# if __virtual__() returns false then the
# module wasn't meant for this platform or it's
# not supposed to load for some other reason.
continue
if module_name != virtual:
# update the module name with the new name
log.debug(
'Loaded {0} as virtual {1}'.format(
module_name, virtual
)
)
module_name = virtual
except Exception:
# If the module throws an exception during __virtual__()
# then log the information and continue to the next.
log.exception(('Failed to read the virtual function for '
'module: {0}').format(module_name))
continue
for attr in dir(mod):
# functions are namespaced with their module name
attr_name = '{0}.{1}'.format(module_name, attr)
if attr.startswith('_'):
# skip private attributes
# log messages omitted for obviousness
continue
if callable(getattr(mod, attr)):
# check to make sure this is callable
func = getattr(mod, attr)
if isinstance(func, type):
# skip callables that might be exceptions
if any(['Error' in func.__name__,
'Exception' in func.__name__]):
continue
# now that callable passes all the checks, add it to the
# library of available functions of this type
funcs[attr_name] = func
log.trace('Added {0} to {1}'.format(attr_name, self.tag))
self._apply_outputter(func, mod)
# now that all the functions have been collected, iterate back over
# the available modules and inject the special __salt__ namespace that
# contains these functions.
for mod in modules:
if not hasattr(mod, '__salt__'):
mod.__salt__ = funcs
elif not in_pack(pack, '__salt__'):
mod.__salt__.update(funcs)
return funcs
def gen_module(self, name, functions, pack=None):
'''
Load a single module and pack it with the functions passed
'''
full = ''
mod = None
for mod_dir in self.module_dirs:
if not os.path.isabs(mod_dir):
continue
if not os.path.isdir(mod_dir):
continue
fn_ = os.path.join(mod_dir, name)
for ext in ('.py', '.pyo', '.pyc', '.so'):
full_test = '{0}{1}'.format(fn_, ext)
if os.path.isfile(full_test):
full = full_test
if not full:
return None
try:
if full.endswith('.pyx') and self.opts['cython_enable']:
mod = pyximport.load_module(name, full, '/tmp')
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module('{0}_{1}'.format(name, self.tag), fn_,
path, desc)
except ImportError as exc:
log.debug(('Failed to import module {0}: {1}').format(name, exc))
return mod
except Exception as exc:
log.warning(('Failed to import module {0}, this is due most'
' likely to a syntax error: {1}').format(name, exc))
return mod
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
if pack:
if isinstance(pack, list):
for chunk in pack:
setattr(mod, chunk['name'], chunk['value'])
else:
setattr(mod, pack['name'], pack['value'])
# Call a module's initialization method if it exists
if hasattr(mod, '__init__'):
if callable(mod.__init__):
try:
mod.__init__()
except TypeError:
pass
funcs = {}
for attr in dir(mod):
if attr.startswith('_'):
continue
if callable(getattr(mod, attr)):
func = getattr(mod, attr)
if hasattr(func, '__bases__'):
if 'BaseException' in func.__bases__:
# the callable object is an exception, don't load it
continue
funcs['{0}.{1}'.format(mod.__name__[:mod.__name__.rindex('_')],
attr)] = func
self._apply_outputter(func, mod)
if not hasattr(mod, '__salt__'):
mod.__salt__ = functions
return funcs
def run_dengo_freefall(update_options):
solver_options = {
"output_dir": "temp_freefall",
"solver_name": "test_freefall",
"use_omp": True,
"use_cvode": True,
"use_suitesparse": True,
"niters": 1,
"NCELLS": 1,
"reltol": 1e-5
}
solver_options.update(update_options)
# Initial conditions
temperature = 20000.0 # K
density = 1.0e-1 # cm^-3
h2frac = 1.0e-3
solver_name = solver_options["solver_name"]
network = setup_primordial_network()
write_network(network, solver_options=solver_options)
os.chdir(pytest_dir)
os.chdir(solver_options["output_dir"])
pyximport.install(setup_args={"include_dirs": np.get_include()},
reload_support=True,
inplace=True)
chemistry_run = pyximport.load_module(
"{}_solver_run".format(solver_name),
"{}_solver_run.pyx".format(solver_name),
build_inplace=True,
pyxbuild_dir="_dengo_temp")
ncells = 1
init_values = setup_initial_conditions(network, density, temperature,
h2frac, ncells)
total_t = 0.0
final_density = 1.0e12 * 1.00794
density_array = numpy.array([init_values['density']])
pressure_array = numpy.array([])
ttt = []
run_time = []
current_density = density_array[-1]
all_data = {}
for key in init_values.keys():
all_data[key] = []
all_data['force_factor'] = []
import h5py
print("os.getcwd()")
print(os.getcwd())
dir_ff_grackle = "examples/test/freefall_solution/freefall.h5"
if "TRAVIS_BUILD_DIR" in os.environ:
dir_ff_grackle = os.path.join(os.environ["TRAVIS_BUILD_DIR"],
dir_ff_grackle)
else:
dir_ff_grackle = os.path.join(os.environ["DENGO_PATH"], dir_ff_grackle)
f = h5py.File(dir_ff_grackle)
fdata = f['data']
grackle_init = convert_from_grackle_to_dengo(fdata)
new_init = setup_initial_conditions(network, density, temperature, h2frac,
ncells)
# new_init, primordial = Init_values(np.array([temperature]),
# np.array([density]) ,
# n_species = 9)
for i in new_init.keys():
if i not in ['density', 'ge']:
#print(i, grackle_init[i])
new_init[i] = numpy.array([grackle_init[i]])
f.close()
new_init['de'] = network.calculate_free_electrons(new_init)
new_init['ge'] = calculate_energy(new_init, network)
rv, rv_int = chemistry_run.run_primordial(new_init, 1e-4, niter=1e0)
count = 0
time0 = time.time()
while current_density < final_density:
# keep track of time in here
new_init = generate_init_from_results(rv_int, network, new_init)
init, pressure_array, density_array, dt, force_factor = update_initial_condition(
new_init,
network,
pressure_array,
density_array,
safety_factor=1.0e-2)
#print("\t time on generating init = {}".format(toc - tic))
tic = time.time()
rv, rv_int = chemistry_run.run_primordial(init,
dt,
niter=1,
intermediate=1)
toc = time.time()
total_t += dt
ttt.append(float(total_t))
run_time.append(toc - tic)
flag = rv_int['successful']
for key in init.keys():
if key not in ['density']:
data = rv_int[key][0][flag][-1]
all_data[key].append(data)
all_data['force_factor'].append(float(force_factor))
current_density = density_array[-1]
if count % 500 == 0:
current_time = time.time()
print("Time Lapsed: {}".format(current_time - time0))
print("density = {0:.2E}, percentage: {1:0.2g}".format(
current_density, current_density / final_density))
count += 1
# if count > 10:
# break
all_data['density'] = density_array
all_data['run_time'] = run_time
for k, v in all_data.items():
all_data[k] = np.array(v)
save_obj(all_data, 'freefall_dengo')
os.chdir("../")
def gen_module(self, name, functions, pack=None):
'''
Load a single module and pack it with the functions passed
'''
full = ''
mod = None
for mod_dir in self.module_dirs:
if not os.path.isabs(mod_dir):
continue
if not os.path.isdir(mod_dir):
continue
fn_ = os.path.join(mod_dir, name)
if os.path.isdir(fn_):
full = fn_
else:
for ext in ('.py', '.pyo', '.pyc', '.so'):
full_test = '{0}{1}'.format(fn_, ext)
if os.path.isfile(full_test):
full = full_test
if not full:
return None
cython_enabled = False
if self.opts.get('cython_enable', True) is True:
try:
import pyximport
pyximport.install()
cython_enabled = True
except ImportError:
log.info('Cython is enabled in the options but not present '
'in the system path. Skipping Cython modules.')
try:
if full.endswith('.pyx') and cython_enabled:
# If there's a name which ends in .pyx it means the above
# cython_enabled is True. Continue...
mod = pyximport.load_module(name, full, tempfile.gettempdir())
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(
'{0}.{1}.{2}.{3}'.format(
self.loaded_base_name,
self.mod_type_check(path),
self.tag,
name
), fn_, path, desc
)
except ImportError:
log.debug(
'Failed to import {0} {1}:\n'.format(
self.tag, name
),
exc_info=True
)
return mod
except Exception:
log.warning(
'Failed to import {0} {1}, this is due most likely to a '
'syntax error:\n'.format(
self.tag, name
),
exc_info=True
)
return mod
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
if pack:
if isinstance(pack, list):
for chunk in pack:
try:
setattr(mod, chunk['name'], chunk['value'])
except KeyError:
pass
else:
setattr(mod, pack['name'], pack['value'])
# Call a module's initialization method if it exists
if hasattr(mod, '__init__'):
if callable(mod.__init__):
try:
mod.__init__(self.opts)
except TypeError:
pass
funcs = {}
module_name = mod.__name__[mod.__name__.rindex('.') + 1:]
if getattr(mod, '__load__', False) is not False:
log.info(
'The functions from module {0!r} are being loaded from the '
'provided __load__ attribute'.format(
module_name
)
)
for attr in getattr(mod, '__load__', dir(mod)):
if attr.startswith('_'):
# private functions are skipped
continue
if callable(getattr(mod, attr)):
func = getattr(mod, attr)
if hasattr(func, '__bases__'):
if 'BaseException' in func.__bases__:
# the callable object is an exception, don't load it
continue
# Let's get the function name.
# If the module has the __func_alias__ attribute, it must be a
# dictionary mapping in the form of(key -> value):
# <real-func-name> -> <desired-func-name>
#
# It default's of course to the found callable attribute name
# if no alias is defined.
funcname = getattr(mod, '__func_alias__', {}).get(attr, attr)
funcs['{0}.{1}'.format(module_name, funcname)] = func
self._apply_outputter(func, mod)
if not hasattr(mod, '__salt__'):
mod.__salt__ = functions
try:
context = sys.modules[
functions[functions.keys()[0]].__module__
].__context__
except AttributeError:
context = {}
mod.__context__ = context
return funcs
def gen_functions(self, pack=None):
'''
Return a dict of functions found in the defined module_dirs
'''
names = {}
modules = []
funcs = {}
cython_enabled = False
if self.opts.get('cython_enable', True) is True:
try:
import pyximport
pyximport.install()
cython_enabled = True
except ImportError:
log.info("Cython is enabled in options though it's not present "
"in the system path. Skipping Cython modules.")
for mod_dir in self.module_dirs:
if not mod_dir.startswith('/'):
continue
if not os.path.isdir(mod_dir):
continue
for fn_ in os.listdir(mod_dir):
if fn_.startswith('_'):
continue
if fn_.endswith('.py')\
or fn_.endswith('.pyc')\
or fn_.endswith('.pyo')\
or fn_.endswith('.so')\
or (cython_enabled and fn_.endswith('.pyx')):
names[fn_[:fn_.rindex('.')]] = os.path.join(mod_dir, fn_)
for name in names:
try:
if names[name].endswith('.pyx'):
# If there's a name which ends in .pyx it means the above
# cython_enabled is True. Continue...
mod = pyximport.load_module(name, names[name], '/tmp')
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(name, fn_, path, desc)
except ImportError:
continue
modules.append(mod)
for mod in modules:
virtual = ''
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
if pack:
if type(pack) == type(list()):
for chunk in pack:
setattr(mod, chunk['name'], chunk['value'])
else:
setattr(mod, pack['name'], pack['value'])
if hasattr(mod, '__virtual__'):
if callable(mod.__virtual__):
virtual = mod.__virtual__()
for attr in dir(mod):
if attr.startswith('_'):
continue
if callable(getattr(mod, attr)):
if virtual:
func = getattr(mod, attr)
funcs[virtual + '.' + attr] = func
self._apply_outputter(func, mod)
elif virtual == False:
pass
else:
func = getattr(mod, attr)
funcs[mod.__name__ + '.' + attr] = func
self._apply_outputter(func, mod)
for mod in modules:
if not hasattr(mod, '__salt__'):
mod.__salt__ = funcs
return funcs
library_path["DENGO_INSTALL_PATH"] = "/home/kwoksun2/dengo_install"
primordial.write_solver("primordial",
output_dir=".",
solver_template="cv_omp/sundials_CVDls",
ode_solver_source="initialize_cvode_solver.C",
library_path=library_path)
import pyximport
pyximport.install(setup_args={"include_dirs": np.get_include()},
reload_support=True,
inplace=True)
primordial_solver_run = pyximport.load_module("primordial_solver_run",
"primordial_solver_run.pyx",
build_inplace=True,
pyxbuild_dir="_dengo_temp")
rv, rv_int = primordial_solver_run.run_primordial(init_values, 1e10, niter=100)
import pylab
pylab.clf()
mask = rv_int['successful']
for name in sorted(rv_int):
if len(rv_int[name].shape) == 1:
rv_int[name] = rv_int[name][mask]
else:
rv_int[name] = rv_int[name][0, mask]
skip = ('successful', 'dt', 't', 'ge')
def gen_functions(self, pack=None, virtual_enable=True):
'''
Return a dict of functions found in the defined module_dirs
'''
names = {}
modules = []
funcs = {}
cython_enabled = False
if self.opts.get('cython_enable', True) is True:
try:
import pyximport
pyximport.install()
cython_enabled = True
except ImportError:
log.info('Cython is enabled in the options but not present '
'in the system path. Skipping Cython modules.')
for mod_dir in self.module_dirs:
if not os.path.isabs(mod_dir):
continue
if not os.path.isdir(mod_dir):
continue
for fn_ in os.listdir(mod_dir):
if fn_.startswith('_'):
continue
if (fn_.endswith(('.py', '.pyc', '.pyo', '.so'))
or (cython_enabled and fn_.endswith('.pyx'))):
names[fn_[:fn_.rindex('.')]] = os.path.join(mod_dir, fn_)
for name in names:
try:
if names[name].endswith('.pyx'):
# If there's a name which ends in .pyx it means the above
# cython_enabled is True. Continue...
mod = pyximport.load_module(
'{0}_{1}'.format(name, self.tag), names[name], '/tmp')
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module('{0}_{1}'.format(name, self.tag),
fn_, path, desc)
except ImportError as exc:
log.debug(('Failed to import module {0}, this is most likely'
' NOT a problem: {1}').format(name, exc))
continue
except Exception as exc:
log.warning(
('Failed to import module {0}, this is due most'
' likely to a syntax error: {1}').format(name, exc))
continue
modules.append(mod)
for mod in modules:
virtual = ''
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
mod.__pillar__ = self.pillar
if pack:
if isinstance(pack, list):
for chunk in pack:
setattr(mod, chunk['name'], chunk['value'])
else:
setattr(mod, pack['name'], pack['value'])
# Call a module's initialization method if it exists
if hasattr(mod, '__init__'):
if callable(mod.__init__):
try:
mod.__init__()
except TypeError:
pass
if virtual_enable:
if hasattr(mod, '__virtual__'):
if callable(mod.__virtual__):
virtual = mod.__virtual__()
for attr in dir(mod):
if attr.startswith('_'):
continue
if callable(getattr(mod, attr)):
func = getattr(mod, attr)
if isinstance(func, type):
if any([
'Error' in func.__name__, 'Exception'
in func.__name__
]):
continue
if virtual:
funcs['{0}.{1}'.format(virtual, attr)] = func
self._apply_outputter(func, mod)
elif virtual is False:
pass
else:
funcs['{0}.{1}'.format(
mod.__name__[:mod.__name__.rindex('_')],
attr)] = func
self._apply_outputter(func, mod)
for mod in modules:
if not hasattr(mod, '__salt__'):
mod.__salt__ = funcs
return funcs
def gen_module(self, name, functions, pack=None):
"""
Load a single module and pack it with the functions passed
"""
full = ""
mod = None
for mod_dir in self.module_dirs:
if not os.path.isabs(mod_dir):
continue
if not os.path.isdir(mod_dir):
continue
fn_ = os.path.join(mod_dir, name)
for ext in (".py", ".pyo", ".pyc", ".so"):
full_test = "{0}{1}".format(fn_, ext)
if os.path.isfile(full_test):
full = full_test
if not full:
return None
try:
if full.endswith(".pyx") and self.opts["cython_enable"]:
mod = pyximport.load_module(name, full, "/tmp")
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module("{0}_{1}".format(name, self.tag), fn_, path, desc)
except ImportError as exc:
log.debug(("Failed to import module {0}: {1}").format(name, exc))
return mod
except Exception as exc:
log.warning(
("Failed to import module {0}, this is due most" " likely to a syntax error: {1}").format(name, exc)
)
return mod
if hasattr(mod, "__opts__"):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
if pack:
if isinstance(pack, list):
for chunk in pack:
setattr(mod, chunk["name"], chunk["value"])
else:
setattr(mod, pack["name"], pack["value"])
# Call a module's initialization method if it exists
if hasattr(mod, "__init__"):
if callable(mod.__init__):
try:
mod.__init__()
except TypeError:
pass
funcs = {}
for attr in dir(mod):
if attr.startswith("_"):
continue
if callable(getattr(mod, attr)):
func = getattr(mod, attr)
if hasattr(func, "__bases__"):
if "BaseException" in func.__bases__:
# the callable object is an exception, don't load it
continue
funcs["{0}.{1}".format(mod.__name__[: mod.__name__.rindex("_")], attr)] = func
self._apply_outputter(func, mod)
if not hasattr(mod, "__salt__"):
mod.__salt__ = functions
return funcs
# Write the initial conditions file
ion_by_ion.write_solver("ion_by_ion", output_dir = ".",
init_values=init_values,
input_is_number=False,
solver_template = "cv_omp/sundials_CVDls",
ode_solver_source = "initialize_cvode_solver.C",
library_path = library_path)
import pyximport
pyximport.install(setup_args={"include_dirs":np.get_include()},
reload_support=True, inplace=True)
ion_by_ion_solver_run = pyximport.load_module("ion_by_ion_solver_run",
"ion_by_ion_solver_run.pyx",
build_inplace = True, pyxbuild_dir = "_dengo_temp")
rv, rv_int = ion_by_ion_solver_run.run_ion_by_ion(init_values, 1e16, 100000,
z = 0.0)
import pylab
pylab.clf()
mask = rv_int['successful']
for name in sorted(rv_int):
if len(rv_int[name].shape) == 1:
rv_int[name] = rv_int[name][mask]
else:
rv_int[name] = rv_int[name][0, mask]
skip = ('successful', 'dt', 't', 'ge')
for n, v in sorted(rv_int.items()):
if n in skip: continue
def gen_module(self, name, functions, pack=None):
'''
Load a single module and pack it with the functions passed
'''
full = ''
mod = None
for mod_dir in self.module_dirs:
if not os.path.isabs(mod_dir):
continue
if not os.path.isdir(mod_dir):
continue
fn_ = os.path.join(mod_dir, name)
if os.path.isdir(fn_):
full = fn_
else:
for ext in ('.py', '.pyo', '.pyc', '.so'):
full_test = '{0}{1}'.format(fn_, ext)
if os.path.isfile(full_test):
full = full_test
if not full:
return None
cython_enabled = False
if self.opts.get('cython_enable', True) is True:
try:
import pyximport
pyximport.install()
cython_enabled = True
except ImportError:
log.info('Cython is enabled in the options but not present '
'in the system path. Skipping Cython modules.')
try:
if full.endswith('.pyx') and cython_enabled:
# If there's a name which ends in .pyx it means the above
# cython_enabled is True. Continue...
mod = pyximport.load_module(name, full, tempfile.gettempdir())
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(
'{0}.{1}.{2}.{3}'.format(loaded_base_name, _mod_type(path),
self.tag, name), fn_, path, desc)
except ImportError as exc:
log.debug('Failed to import module {0}: {1}'.format(name, exc))
return mod
except Exception:
trb = traceback.format_exc()
log.warning('Failed to import module {0}, this is due most likely '
'to a syntax error: {1}'.format(name, trb))
return mod
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
if pack:
if isinstance(pack, list):
for chunk in pack:
try:
setattr(mod, chunk['name'], chunk['value'])
except KeyError:
pass
else:
setattr(mod, pack['name'], pack['value'])
# Call a module's initialization method if it exists
if hasattr(mod, '__init__'):
if callable(mod.__init__):
try:
mod.__init__(self.opts)
except TypeError:
pass
funcs = {}
for attr in dir(mod):
if attr.startswith('_'):
continue
if callable(getattr(mod, attr)):
func = getattr(mod, attr)
if hasattr(func, '__bases__'):
if 'BaseException' in func.__bases__:
# the callable object is an exception, don't load it
continue
funcs['{0}.{1}'.format(
mod.__name__[mod.__name__.rindex('.') + 1:], attr)] = func
self._apply_outputter(func, mod)
if not hasattr(mod, '__salt__'):
mod.__salt__ = functions
return funcs
def gen_functions(self, pack=None, virtual_enable=True, whitelist=None):
'''
Return a dict of functions found in the defined module_dirs
'''
log.debug('loading {0} in {1}'.format(self.tag, self.module_dirs))
names = {}
modules = []
funcs = {}
disable = set(self.opts.get('disable_{0}s'.format(self.tag), []))
cython_enabled = False
if self.opts.get('cython_enable', True) is True:
try:
import pyximport
pyximport.install()
cython_enabled = True
except ImportError:
log.info('Cython is enabled in the options but not present '
'in the system path. Skipping Cython modules.')
for mod_dir in self.module_dirs:
if not os.path.isabs(mod_dir):
log.debug(
'Skipping {0}, it is not an absolute path'.format(
mod_dir
)
)
continue
if not os.path.isdir(mod_dir):
log.debug(
'Skipping {0}, it is not a directory'.format(
mod_dir
)
)
continue
for fn_ in os.listdir(mod_dir):
if fn_.startswith('_'):
# skip private modules
# log messages omitted for obviousness
continue
if fn_.split('.')[0] in disable:
log.debug(
'Skipping {0}, it is disabled by configuration'.format(
fn_
)
)
continue
if (fn_.endswith(('.py', '.pyc', '.pyo', '.so'))
or (cython_enabled and fn_.endswith('.pyx'))
or os.path.isdir(os.path.join(mod_dir, fn_))):
extpos = fn_.rfind('.')
if extpos > 0:
_name = fn_[:extpos]
else:
_name = fn_
names[_name] = os.path.join(mod_dir, fn_)
else:
log.debug(
'Skipping {0}, it does not end with an expected '
'extension'.format(
fn_
)
)
for name in names:
try:
if names[name].endswith('.pyx'):
# If there's a name which ends in .pyx it means the above
# cython_enabled is True. Continue...
mod = pyximport.load_module(
'{0}.{1}.{2}.{3}'.format(
self.loaded_base_name,
self.mod_type_check(names[name]),
self.tag,
name
), names[name], tempfile.gettempdir()
)
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(
'{0}.{1}.{2}.{3}'.format(
self.loaded_base_name,
self.mod_type_check(path),
self.tag,
name
), fn_, path, desc
)
# reload all submodules if necessary
submodules = [
getattr(mod, sname) for sname in dir(mod) if
isinstance(getattr(mod, sname), mod.__class__)
]
# reload only custom "sub"modules i.e is a submodule in
# parent module that are still available on disk (i.e. not
# removed during sync_modules)
for submodule in submodules:
try:
smname = '{0}.{1}.{2}'.format(
self.loaded_base_name,
self.tag,
name
)
smfile = '{0}.py'.format(
os.path.splitext(submodule.__file__)[0]
)
if submodule.__name__.startswith(smname) and \
os.path.isfile(smfile):
reload(submodule)
except AttributeError:
continue
except ImportError:
log.debug(
'Failed to import {0} {1}, this is most likely NOT a '
'problem:\n'.format(
self.tag, name
),
exc_info=True
)
continue
except Exception:
log.warning(
'Failed to import {0} {1}, this is due most likely to a '
'syntax error. Traceback raised:\n'.format(
self.tag, name
),
exc_info=True
)
continue
modules.append(mod)
for mod in modules:
virtual = ''
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
mod.__pillar__ = self.pillar
if pack:
if isinstance(pack, list):
for chunk in pack:
if not isinstance(chunk, dict):
continue
try:
setattr(mod, chunk['name'], chunk['value'])
except KeyError:
pass
else:
setattr(mod, pack['name'], pack['value'])
# Call a module's initialization method if it exists
if hasattr(mod, '__init__'):
if callable(mod.__init__):
try:
mod.__init__(self.opts)
except TypeError:
pass
# Trim the full pathname to just the module
# this will be the short name that other salt modules and state
# will refer to it as.
module_name = mod.__name__.rsplit('.', 1)[-1]
if virtual_enable:
# if virtual modules are enabled, we need to look for the
# __virtual__() function inside that module and run it.
# This function will return either a new name for the module,
# an empty string(won't be loaded but you just need to check
# against the same python type, a string) or False.
# This allows us to have things like the pkg module working on
# all platforms under the name 'pkg'. It also allows for
# modules like augeas_cfg to be referred to as 'augeas', which
# would otherwise have namespace collisions. And finally it
# allows modules to return False if they are not intended to
# run on the given platform or are missing dependencies.
try:
if hasattr(mod, '__virtual__'):
if callable(mod.__virtual__):
virtual = mod.__virtual__()
if not virtual:
# if __virtual__() evaluates to false then the
# module wasn't meant for this platform or it's
# not supposed to load for some other reason.
# Some modules might accidentally return None
# and are improperly loaded
if virtual is None:
log.warning(
'{0}.__virtual__() is wrongly '
'returning `None`. It should either '
'return `True`, `False` or a new '
'name. If you\'re the developer '
'of the module {1!r}, please fix '
'this.'.format(
mod.__name__,
module_name
)
)
continue
if virtual is not True and module_name != virtual:
# If __virtual__ returned True the module will
# be loaded with the same name, if it returned
# other value than `True`, it should be a new
# name for the module.
# Update the module name with the new name
log.debug(
'Loaded {0} as virtual {1}'.format(
module_name, virtual
)
)
module_name = virtual
except KeyError:
# Key errors come out of the virtual function when passing
# in incomplete grains sets, these can be safely ignored
# and logged to debug, still, it includes the traceback to
# help debugging.
log.debug(
'KeyError when loading {0}'.format(module_name),
exc_info=True
)
except Exception:
# If the module throws an exception during __virtual__()
# then log the information and continue to the next.
log.exception(
'Failed to read the virtual function for '
'{0}: {1}'.format(
self.tag, module_name
)
)
continue
if whitelist:
# If a whitelist is defined then only load the module if it is
# in the whitelist
if module_name not in whitelist:
continue
if getattr(mod, '__load__', False) is not False:
log.info(
'The functions from module {0!r} are being loaded from '
'the provided __load__ attribute'.format(
module_name
)
)
for attr in getattr(mod, '__load__', dir(mod)):
if attr.startswith('_'):
# skip private attributes
# log messages omitted for obviousness
continue
if callable(getattr(mod, attr)):
# check to make sure this is callable
func = getattr(mod, attr)
if isinstance(func, type):
# skip callables that might be exceptions
if any(['Error' in func.__name__,
'Exception' in func.__name__]):
continue
# now that callable passes all the checks, add it to the
# library of available functions of this type
# Let's get the function name.
# If the module has the __func_alias__ attribute, it must
# be a dictionary mapping in the form of(key -> value):
# <real-func-name> -> <desired-func-name>
#
# It default's of course to the found callable attribute
# name if no alias is defined.
funcname = getattr(mod, '__func_alias__', {}).get(
attr, attr
)
# functions are namespaced with their module name
module_func_name = '{0}.{1}'.format(module_name, funcname)
funcs[module_func_name] = func
log.trace(
'Added {0} to {1}'.format(module_func_name, self.tag)
)
self._apply_outputter(func, mod)
# now that all the functions have been collected, iterate back over
# the available modules and inject the special __salt__ namespace that
# contains these functions.
for mod in modules:
if not hasattr(mod, '__salt__'):
mod.__salt__ = funcs
elif not in_pack(pack, '__salt__'):
mod.__salt__.update(funcs)
return funcs
def gen_functions(self, pack=None, virtual_enable=True):
'''
Return a dict of functions found in the defined module_dirs
'''
log.debug('loading {0} in {1}'.format(self.tag, self.module_dirs))
names = {}
modules = []
funcs = {}
disable = set(self.opts.get('disable_{0}s'.format(self.tag), []))
cython_enabled = False
if self.opts.get('cython_enable', True) is True:
try:
import pyximport
pyximport.install()
cython_enabled = True
except ImportError:
log.info('Cython is enabled in the options but not present '
'in the system path. Skipping Cython modules.')
for mod_dir in self.module_dirs:
if not os.path.isabs(mod_dir):
log.debug(('Skipping {0}, it is not an abosolute '
'path').format(mod_dir))
continue
if not os.path.isdir(mod_dir):
log.debug(('Skipping {0}, it is not a '
'directory').format(mod_dir))
continue
for fn_ in os.listdir(mod_dir):
if fn_.startswith('_'):
# skip private modules
# log messages omitted for obviousness
continue
if fn_.split('.')[0] in disable:
log.debug(('Skipping {0}, it is disabled by '
'configuration').format(fn_))
continue
if (fn_.endswith(('.py', '.pyc', '.pyo', '.so'))
or (cython_enabled and fn_.endswith('.pyx'))
or os.path.isdir(os.path.join(mod_dir, fn_))):
extpos = fn_.rfind('.')
if extpos > 0:
_name = fn_[:extpos]
else:
_name = fn_
names[_name] = os.path.join(mod_dir, fn_)
else:
log.debug(('Skipping {0}, it does not end with an '
'expected extension').format(fn_))
for name in names:
try:
if names[name].endswith('.pyx'):
# If there's a name which ends in .pyx it means the above
# cython_enabled is True. Continue...
mod = pyximport.load_module(
'{0}.{1}.{2}.{3}'.format(loaded_base_name,
_mod_type(names[name]),
self.tag, name), names[name],
tempfile.gettempdir())
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(
'{0}.{1}.{2}.{3}'.format(loaded_base_name,
_mod_type(path), self.tag,
name), fn_, path, desc)
# reload all submodules if necessary
submodules = [
getattr(mod, sname) for sname in dir(mod)
if isinstance(getattr(mod, sname), mod.__class__)
]
# reload only custom "sub"modules i.e is a submodule in
# parent module that are still available on disk (i.e. not
# removed during sync_modules)
for submodule in submodules:
try:
smname = '{0}.{1}.{2}'.format(
loaded_base_name, self.tag, name)
smfile = os.path.splitext(
submodule.__file__)[0] + ".py"
if submodule.__name__.startswith(
smname) and os.path.isfile(smfile):
reload(submodule)
except AttributeError:
continue
except ImportError as exc:
log.debug('Failed to import module {0}, this is most likely '
'NOT a problem: {1}'.format(name, exc))
continue
except Exception:
trb = traceback.format_exc()
log.warning('Failed to import module {0}, this is due most '
'likely to a syntax error: {1}'.format(name, trb))
continue
modules.append(mod)
for mod in modules:
virtual = ''
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
mod.__pillar__ = self.pillar
if pack:
if isinstance(pack, list):
for chunk in pack:
if not isinstance(chunk, dict):
continue
try:
setattr(mod, chunk['name'], chunk['value'])
except KeyError:
pass
else:
setattr(mod, pack['name'], pack['value'])
# Call a module's initialization method if it exists
if hasattr(mod, '__init__'):
if callable(mod.__init__):
try:
mod.__init__(self.opts)
except TypeError:
pass
# Trim the full pathname to just the module
# this will be the short name that other salt modules and state
# will refer to it as.
module_name = mod.__name__.rsplit('.', 1)[-1]
if virtual_enable:
# if virtual modules are enabled, we need to look for the
# __virtual__() function inside that module and run it.
# This function will return either a new name for the module
# or False. This allows us to have things like the pkg module
# working on all platforms under the name 'pkg'. It also allows
# for modules like augeas_cfg to be referred to as 'augeas',
# which would otherwise have namespace collisions. And finally
# it allows modules to return False if they are not intended
# to run on the given platform or are missing dependencies.
try:
if hasattr(mod, '__virtual__'):
if callable(mod.__virtual__):
virtual = mod.__virtual__()
if virtual is False:
# if __virtual__() returns false then the
# module wasn't meant for this platform or it's
# not supposed to load for some other reason.
continue
if module_name != virtual:
# update the module name with the new name
log.debug('Loaded {0} as virtual {1}'.format(
module_name, virtual))
module_name = virtual
except Exception:
# If the module throws an exception during __virtual__()
# then log the information and continue to the next.
log.exception(('Failed to read the virtual function for '
'module: {0}').format(module_name))
continue
for attr in dir(mod):
# functions are namespaced with their module name
attr_name = '{0}.{1}'.format(module_name, attr)
if attr.startswith('_'):
# skip private attributes
# log messages omitted for obviousness
continue
if callable(getattr(mod, attr)):
# check to make sure this is callable
func = getattr(mod, attr)
if isinstance(func, type):
# skip callables that might be exceptions
if any([
'Error' in func.__name__, 'Exception'
in func.__name__
]):
continue
# now that callable passes all the checks, add it to the
# library of available functions of this type
funcs[attr_name] = func
log.trace('Added {0} to {1}'.format(attr_name, self.tag))
self._apply_outputter(func, mod)
# now that all the functions have been collected, iterate back over
# the available modules and inject the special __salt__ namespace that
# contains these functions.
for mod in modules:
if not hasattr(mod, '__salt__'):
mod.__salt__ = funcs
elif not in_pack(pack, '__salt__'):
mod.__salt__.update(funcs)
return funcs
def load_modules(self):
'''
Loads all of the modules from module_dirs and returns a list of them
'''
self.modules = []
log.trace('loading {0} in {1}'.format(self.tag, self.module_dirs))
names = {}
disable = set(self.opts.get('disable_{0}s'.format(self.tag), []))
cython_enabled = False
if self.opts.get('cython_enable', True) is True:
try:
import pyximport
pyximport.install()
cython_enabled = True
except ImportError:
log.info('Cython is enabled in the options but not present '
'in the system path. Skipping Cython modules.')
for mod_dir in self.module_dirs:
if not os.path.isabs(mod_dir):
log.trace(
'Skipping {0}, it is not an absolute path'.format(
mod_dir
)
)
continue
if not os.path.isdir(mod_dir):
log.trace(
'Skipping {0}, it is not a directory'.format(
mod_dir
)
)
continue
for fn_ in os.listdir(mod_dir):
if fn_.startswith('_'):
# skip private modules
# log messages omitted for obviousness
continue
if fn_.split('.')[0] in disable:
log.trace(
'Skipping {0}, it is disabled by configuration'.format(
fn_
)
)
continue
if (fn_.endswith(('.py', '.pyc', '.pyo', '.so'))
or (cython_enabled and fn_.endswith('.pyx'))
or os.path.isdir(os.path.join(mod_dir, fn_))):
extpos = fn_.rfind('.')
if extpos > 0:
_name = fn_[:extpos]
else:
_name = fn_
names[_name] = os.path.join(mod_dir, fn_)
else:
log.trace(
'Skipping {0}, it does not end with an expected '
'extension'.format(
fn_
)
)
for name in names:
try:
if names[name].endswith('.pyx'):
# If there's a name which ends in .pyx it means the above
# cython_enabled is True. Continue...
mod = pyximport.load_module(
'{0}.{1}.{2}.{3}'.format(
self.loaded_base_name,
self.mod_type_check(names[name]),
self.tag,
name
), names[name], tempfile.gettempdir()
)
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(
'{0}.{1}.{2}.{3}'.format(
self.loaded_base_name,
self.mod_type_check(path),
self.tag,
name
), fn_, path, desc
)
# reload all submodules if necessary
submodules = [
getattr(mod, sname) for sname in dir(mod) if
isinstance(getattr(mod, sname), mod.__class__)
]
# reload only custom "sub"modules i.e is a submodule in
# parent module that are still available on disk (i.e. not
# removed during sync_modules)
for submodule in submodules:
try:
smname = '{0}.{1}.{2}'.format(
self.loaded_base_name,
self.tag,
name
)
smfile = '{0}.py'.format(
os.path.splitext(submodule.__file__)[0]
)
if submodule.__name__.startswith(smname) and \
os.path.isfile(smfile):
reload(submodule)
except AttributeError:
continue
except ImportError:
log.debug(
'Failed to import {0} {1}, this is most likely NOT a '
'problem:\n'.format(
self.tag, name
),
exc_info=True
)
continue
except Exception:
log.warning(
'Failed to import {0} {1}, this is due most likely to a '
'syntax error. Traceback raised:\n'.format(
self.tag, name
),
exc_info=True
)
continue
self.modules.append(mod)
def gen_functions(self, pack=None, virtual_enable=True):
'''
Return a dict of functions found in the defined module_dirs
'''
names = {}
modules = []
funcs = {}
disable = set(self.opts.get('disable_{0}s'.format(self.tag), []))
cython_enabled = False
if self.opts.get('cython_enable', True) is True:
try:
import pyximport
pyximport.install()
cython_enabled = True
except ImportError:
log.info('Cython is enabled in the options but not present '
'in the system path. Skipping Cython modules.')
for mod_dir in self.module_dirs:
if not os.path.isabs(mod_dir):
continue
if not os.path.isdir(mod_dir):
continue
for fn_ in os.listdir(mod_dir):
if fn_.startswith('_'):
continue
if fn_.split('.')[0] in disable:
continue
if (fn_.endswith(('.py', '.pyc', '.pyo', '.so'))
or (cython_enabled and fn_.endswith('.pyx'))
or os.path.isdir(os.path.join(mod_dir, fn_))):
extpos = fn_.rfind('.')
if extpos > 0:
_name = fn_[:extpos]
else:
_name = fn_
names[_name] = os.path.join(mod_dir, fn_)
for name in names:
try:
if names[name].endswith('.pyx'):
# If there's a name which ends in .pyx it means the above
# cython_enabled is True. Continue...
mod = pyximport.load_module(
'{0}.{1}.{2}.{3}'.format(
loaded_base_name,
_mod_type(names[name]),
self.tag,
name
), names[name], tempfile.gettempdir()
)
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(
'{0}.{1}.{2}.{3}'.format(
loaded_base_name, _mod_type(path), self.tag, name
), fn_, path, desc
)
# reload all submodules if necessary
submodules = [
getattr(mod, sname) for sname in dir(mod) if
type(getattr(mod, sname))==type(mod)
]
# reload only custom "sub"modules i.e is a submodule in
# parent module that are still available on disk (i.e. not
# removed during sync_modules)
for submodule in submodules:
try:
smname = '{0}.{1}.{2}'.format(loaded_base_name, self.tag, name)
smfile = os.path.splitext(submodule.__file__)[0] + ".py"
if submodule.__name__.startswith(smname) and os.path.isfile(smfile):
reload(submodule)
except AttributeError:
continue
except ImportError as exc:
log.debug('Failed to import module {0}, this is most likely '
'NOT a problem: {1}'.format(name, exc))
continue
except Exception as exc:
trb = traceback.format_exc()
log.warning('Failed to import module {0}, this is due most '
'likely to a syntax error: {1}'.format(name, trb))
continue
modules.append(mod)
for mod in modules:
virtual = ''
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
mod.__pillar__ = self.pillar
if pack:
if isinstance(pack, list):
for chunk in pack:
setattr(mod, chunk['name'], chunk['value'])
else:
setattr(mod, pack['name'], pack['value'])
# Call a module's initialization method if it exists
if hasattr(mod, '__init__'):
if callable(mod.__init__):
try:
mod.__init__(self.opts)
except TypeError:
pass
if virtual_enable:
if hasattr(mod, '__virtual__'):
if callable(mod.__virtual__):
virtual = mod.__virtual__()
for attr in dir(mod):
if attr.startswith('_'):
continue
if callable(getattr(mod, attr)):
func = getattr(mod, attr)
if isinstance(func, type):
if any([
'Error' in func.__name__,
'Exception' in func.__name__]):
continue
if virtual:
funcs['{0}.{1}'.format(virtual, attr)] = func
self._apply_outputter(func, mod)
elif virtual is False:
pass
else:
funcs[
'{0}.{1}'.format(
mod.__name__[mod.__name__.rindex('.')+1:],
attr
)
] = func
self._apply_outputter(func, mod)
for mod in modules:
if not hasattr(mod, '__salt__'):
mod.__salt__ = funcs
return funcs
def load_modules(self):
'''
Loads all of the modules from module_dirs and returns a list of them
'''
self.modules = []
log.trace('loading {0} in {1}'.format(self.tag, self.module_dirs))
names = {}
disable = set(self.opts.get('disable_{0}s'.format(self.tag), []))
cython_enabled = False
if self.opts.get('cython_enable', True) is True:
try:
import pyximport
pyximport.install()
cython_enabled = True
except ImportError:
log.info('Cython is enabled in the options but not present '
'in the system path. Skipping Cython modules.')
for mod_dir in self.module_dirs:
if not os.path.isabs(mod_dir):
log.trace(
'Skipping {0}, it is not an absolute path'.format(mod_dir))
continue
if not os.path.isdir(mod_dir):
log.trace(
'Skipping {0}, it is not a directory'.format(mod_dir))
continue
for fn_ in os.listdir(mod_dir):
if fn_.startswith('_'):
# skip private modules
# log messages omitted for obviousness
continue
if fn_.split('.')[0] in disable:
log.trace(
'Skipping {0}, it is disabled by configuration'.format(
fn_))
continue
if (fn_.endswith(('.py', '.pyc', '.pyo', '.so'))
or (cython_enabled and fn_.endswith('.pyx'))
or os.path.isdir(os.path.join(mod_dir, fn_))):
extpos = fn_.rfind('.')
if extpos > 0:
_name = fn_[:extpos]
else:
_name = fn_
names[_name] = os.path.join(mod_dir, fn_)
else:
log.trace('Skipping {0}, it does not end with an expected '
'extension'.format(fn_))
for name in names:
try:
if names[name].endswith('.pyx'):
# If there's a name which ends in .pyx it means the above
# cython_enabled is True. Continue...
mod = pyximport.load_module(
'{0}.{1}.{2}.{3}'.format(
self.loaded_base_name,
self.mod_type_check(names[name]), self.tag, name),
names[name], tempfile.gettempdir())
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(
'{0}.{1}.{2}.{3}'.format(self.loaded_base_name,
self.mod_type_check(path),
self.tag, name), fn_, path,
desc)
# reload all submodules if necessary
submodules = [
getattr(mod, sname) for sname in dir(mod)
if isinstance(getattr(mod, sname), mod.__class__)
]
# reload only custom "sub"modules i.e is a submodule in
# parent module that are still available on disk (i.e. not
# removed during sync_modules)
for submodule in submodules:
try:
smname = '{0}.{1}.{2}'.format(
self.loaded_base_name, self.tag, name)
smfile = '{0}.py'.format(
os.path.splitext(submodule.__file__)[0])
if submodule.__name__.startswith(smname) and \
os.path.isfile(smfile):
reload(submodule)
except AttributeError:
continue
except ImportError:
log.debug(
'Failed to import {0} {1}, this is most likely NOT a '
'problem:\n'.format(self.tag, name),
exc_info=True)
continue
except Exception:
log.warning(
'Failed to import {0} {1}, this is due most likely to a '
'syntax error. Traceback raised:\n'.format(self.tag, name),
exc_info=True)
continue
self.modules.append(mod)
def gen_functions(self, pack=None, virtual_enable=True, whitelist=None,
provider_overrides=False):
'''
Return a dict of functions found in the defined module_dirs
'''
log.debug('loading {0} in {1}'.format(self.tag, self.module_dirs))
names = {}
modules = []
funcs = {}
disable = set(self.opts.get('disable_{0}s'.format(self.tag), []))
cython_enabled = False
if self.opts.get('cython_enable', True) is True:
try:
import pyximport
pyximport.install()
cython_enabled = True
except ImportError:
log.info('Cython is enabled in the options but not present '
'in the system path. Skipping Cython modules.')
for mod_dir in self.module_dirs:
if not os.path.isabs(mod_dir):
log.debug(
'Skipping {0}, it is not an absolute path'.format(
mod_dir
)
)
continue
if not os.path.isdir(mod_dir):
log.debug(
'Skipping {0}, it is not a directory'.format(
mod_dir
)
)
continue
for fn_ in os.listdir(mod_dir):
if fn_.startswith('_'):
# skip private modules
# log messages omitted for obviousness
continue
if fn_.split('.')[0] in disable:
log.debug(
'Skipping {0}, it is disabled by configuration'.format(
fn_
)
)
continue
if (fn_.endswith(('.py', '.pyc', '.pyo', '.so'))
or (cython_enabled and fn_.endswith('.pyx'))
or os.path.isdir(os.path.join(mod_dir, fn_))):
extpos = fn_.rfind('.')
if extpos > 0:
_name = fn_[:extpos]
else:
_name = fn_
names[_name] = os.path.join(mod_dir, fn_)
else:
log.debug(
'Skipping {0}, it does not end with an expected '
'extension'.format(
fn_
)
)
for name in names:
try:
if names[name].endswith('.pyx'):
# If there's a name which ends in .pyx it means the above
# cython_enabled is True. Continue...
mod = pyximport.load_module(
'{0}.{1}.{2}.{3}'.format(
self.loaded_base_name,
self.mod_type_check(names[name]),
self.tag,
name
), names[name], tempfile.gettempdir()
)
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(
'{0}.{1}.{2}.{3}'.format(
self.loaded_base_name,
self.mod_type_check(path),
self.tag,
name
), fn_, path, desc
)
# reload all submodules if necessary
submodules = [
getattr(mod, sname) for sname in dir(mod) if
isinstance(getattr(mod, sname), mod.__class__)
]
# reload only custom "sub"modules i.e is a submodule in
# parent module that are still available on disk (i.e. not
# removed during sync_modules)
for submodule in submodules:
try:
smname = '{0}.{1}.{2}'.format(
self.loaded_base_name,
self.tag,
name
)
smfile = '{0}.py'.format(
os.path.splitext(submodule.__file__)[0]
)
if submodule.__name__.startswith(smname) and \
os.path.isfile(smfile):
reload(submodule)
except AttributeError:
continue
except ImportError:
log.debug(
'Failed to import {0} {1}, this is most likely NOT a '
'problem:\n'.format(
self.tag, name
),
exc_info=True
)
continue
except Exception:
log.warning(
'Failed to import {0} {1}, this is due most likely to a '
'syntax error. Traceback raised:\n'.format(
self.tag, name
),
exc_info=True
)
continue
modules.append(mod)
for mod in modules:
virtual = ''
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
mod.__pillar__ = self.pillar
if pack:
if isinstance(pack, list):
for chunk in pack:
if not isinstance(chunk, dict):
continue
try:
setattr(mod, chunk['name'], chunk['value'])
except KeyError:
pass
else:
setattr(mod, pack['name'], pack['value'])
# Call a module's initialization method if it exists
if hasattr(mod, '__init__'):
if callable(mod.__init__):
try:
mod.__init__(self.opts)
except TypeError:
pass
# Trim the full pathname to just the module
# this will be the short name that other salt modules and state
# will refer to it as.
module_name = mod.__name__.rsplit('.', 1)[-1]
if virtual_enable:
# if virtual modules are enabled, we need to look for the
# __virtual__() function inside that module and run it.
# This function will return either a new name for the module,
# an empty string(won't be loaded but you just need to check
# against the same python type, a string) or False.
# This allows us to have things like the pkg module working on
# all platforms under the name 'pkg'. It also allows for
# modules like augeas_cfg to be referred to as 'augeas', which
# would otherwise have namespace collisions. And finally it
# allows modules to return False if they are not intended to
# run on the given platform or are missing dependencies.
try:
if hasattr(mod, '__virtual__'):
if callable(mod.__virtual__):
virtual = mod.__virtual__()
if not virtual:
# if __virtual__() evaluates to false then the
# module wasn't meant for this platform or it's
# not supposed to load for some other reason.
# Some modules might accidentally return None
# and are improperly loaded
if virtual is None:
log.warning(
'{0}.__virtual__() is wrongly '
'returning `None`. It should either '
'return `True`, `False` or a new '
'name. If you\'re the developer '
'of the module {1!r}, please fix '
'this.'.format(
mod.__name__,
module_name
)
)
continue
if virtual is not True and module_name != virtual:
# If __virtual__ returned True the module will
# be loaded with the same name, if it returned
# other value than `True`, it should be a new
# name for the module.
# Update the module name with the new name
log.debug(
'Loaded {0} as virtual {1}'.format(
module_name, virtual
)
)
if not hasattr(mod, '__virtualname__'):
salt.utils.warn_until(
'Hydrogen',
'The {0!r} module is renaming itself '
'in it\'s __virtual__() function ({1} '
'=> {2}). Please set it\'s virtual '
'name as the \'__virtualname__\' '
'module attribute. Example: '
'"__virtualname__ = {2!r}"'.format(
mod.__name__,
module_name,
virtual
)
)
module_name = virtual
elif virtual and hasattr(mod, '__virtualname__'):
module_name = mod.__virtualname__
except KeyError:
# Key errors come out of the virtual function when passing
# in incomplete grains sets, these can be safely ignored
# and logged to debug, still, it includes the traceback to
# help debugging.
log.debug(
'KeyError when loading {0}'.format(module_name),
exc_info=True
)
except Exception:
# If the module throws an exception during __virtual__()
# then log the information and continue to the next.
log.exception(
'Failed to read the virtual function for '
'{0}: {1}'.format(
self.tag, module_name
)
)
continue
if whitelist:
# If a whitelist is defined then only load the module if it is
# in the whitelist
if module_name not in whitelist:
continue
if getattr(mod, '__load__', False) is not False:
log.info(
'The functions from module {0!r} are being loaded from '
'the provided __load__ attribute'.format(
module_name
)
)
for attr in getattr(mod, '__load__', dir(mod)):
if attr.startswith('_'):
# skip private attributes
# log messages omitted for obviousness
continue
if callable(getattr(mod, attr)):
# check to make sure this is callable
func = getattr(mod, attr)
if isinstance(func, type):
# skip callables that might be exceptions
if any(['Error' in func.__name__,
'Exception' in func.__name__]):
continue
# now that callable passes all the checks, add it to the
# library of available functions of this type
# Let's get the function name.
# If the module has the __func_alias__ attribute, it must
# be a dictionary mapping in the form of(key -> value):
# <real-func-name> -> <desired-func-name>
#
# It default's of course to the found callable attribute
# name if no alias is defined.
funcname = getattr(mod, '__func_alias__', {}).get(
attr, attr
)
# functions are namespaced with their module name
module_func_name = '{0}.{1}'.format(module_name, funcname)
funcs[module_func_name] = func
log.trace(
'Added {0} to {1}'.format(module_func_name, self.tag)
)
self._apply_outputter(func, mod)
# Handle provider overrides
if provider_overrides and self.opts.get('providers', False):
if isinstance(self.opts['providers'], dict):
for mod, provider in self.opts['providers'].items():
newfuncs = raw_mod(self.opts, provider, funcs)
if newfuncs:
for newfunc in newfuncs:
f_key = '{0}{1}'.format(
mod, newfunc[newfunc.rindex('.'):]
)
funcs[f_key] = newfuncs[newfunc]
# now that all the functions have been collected, iterate back over
# the available modules and inject the special __salt__ namespace that
# contains these functions.
for mod in modules:
if not hasattr(mod, '__salt__') or (
not in_pack(pack, '__salt__') and
not str(mod.__name__).startswith('salt.loaded.int.grain')
):
mod.__salt__ = funcs
elif not in_pack(pack, '__salt__') and str(mod.__name__).startswith('salt.loaded.int.grain'):
mod.__salt__.update(funcs)
return funcs
def gen_functions(self, pack=None):
'''
Return a dict of functions found in the defined module_dirs
'''
names = {}
modules = []
funcs = {}
cython_enabled = False
if self.opts.get('cython_enable', True) is True:
try:
import pyximport
pyximport.install()
cython_enabled = True
except ImportError:
log.info('Cython is enabled in options put not present '
'on the system path. Skipping Cython modules.')
for mod_dir in self.module_dirs:
if not os.path.isabs(mod_dir):
continue
if not os.path.isdir(mod_dir):
continue
for fn_ in os.listdir(mod_dir):
if fn_.startswith('_'):
continue
if fn_.endswith('.py')\
or fn_.endswith('.pyc')\
or fn_.endswith('.pyo')\
or fn_.endswith('.so')\
or (cython_enabled and fn_.endswith('.pyx')):
names[fn_[:fn_.rindex('.')]] = os.path.join(mod_dir, fn_)
for name in names:
try:
if names[name].endswith('.pyx'):
# If there's a name which ends in .pyx it means the above
# cython_enabled is True. Continue...
mod = pyximport.load_module(name, names[name], '/tmp')
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(name, fn_, path, desc)
except ImportError as exc:
log.debug(('Failed to import module {0}, this is most likely'
' NOT a problem: {1}').format(name, exc))
continue
modules.append(mod)
for mod in modules:
virtual = ''
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
if pack:
if isinstance(pack, list):
for chunk in pack:
setattr(mod, chunk['name'], chunk['value'])
else:
setattr(mod, pack['name'], pack['value'])
# Call a module's initialization method if it exists
if hasattr(mod, '__init__'):
if callable(mod.__init__):
try:
mod.__init__()
except TypeError:
pass
if hasattr(mod, '__virtual__'):
if callable(mod.__virtual__):
virtual = mod.__virtual__()
for attr in dir(mod):
if attr.startswith('_'):
continue
if callable(getattr(mod, attr)):
if virtual:
func = getattr(mod, attr)
funcs[virtual + '.' + attr] = func
self._apply_outputter(func, mod)
elif virtual is False:
pass
else:
func = getattr(mod, attr)
funcs[mod.__name__ + '.' + attr] = func
self._apply_outputter(func, mod)
for mod in modules:
if not hasattr(mod, '__salt__'):
mod.__salt__ = funcs
return funcs
def gen_module(self, name, functions, pack=None):
'''
Load a single module and pack it with the functions passed
'''
full = ''
mod = None
for mod_dir in self.module_dirs:
if not os.path.isabs(mod_dir):
continue
if not os.path.isdir(mod_dir):
continue
fn_ = os.path.join(mod_dir, name)
for ext in ('.py', '.pyo', '.pyc', '.so'):
full_test = '{0}{1}'.format(fn_, ext)
if os.path.isfile(full_test):
full = full_test
if not full:
return None
try:
if full.endswith('.pyx') and self.opts['cython_enable']:
mod = pyximport.load_module(name, full, '/tmp')
else:
fn_, path, desc = imp.find_module(name, self.module_dirs)
mod = imp.load_module(
'{0}_{1}'.format(name, self.tag),
fn_,
path,
desc
)
except ImportError as exc:
log.debug(('Failed to import module {0}: {1}').format(name, exc))
return mod
except Exception as exc:
log.warning(('Failed to import module {0}, this is due most'
' likely to a syntax error: {1}').format(name, exc))
return mod
if hasattr(mod, '__opts__'):
mod.__opts__.update(self.opts)
else:
mod.__opts__ = self.opts
mod.__grains__ = self.grains
if pack:
if isinstance(pack, list):
for chunk in pack:
setattr(mod, chunk['name'], chunk['value'])
else:
setattr(mod, pack['name'], pack['value'])
# Call a module's initialization method if it exists
if hasattr(mod, '__init__'):
if callable(mod.__init__):
try:
mod.__init__()
except TypeError:
pass
funcs = {}
for attr in dir(mod):
if attr.startswith('_'):
continue
if callable(getattr(mod, attr)):
func = getattr(mod, attr)
if hasattr(func, '__bases__'):
if 'BaseException' in func.__bases__:
# the callable object is an exception, don't load it
continue
funcs[
'{0}.{1}'.format(
mod.__name__[:mod.__name__.rindex('_')],
attr)
] = func
self._apply_outputter(func, mod)
if not hasattr(mod, '__salt__'):
mod.__salt__ = functions
return funcs
