def cfg_logging(cfg=None):
r"""Set logging levels from config options.
Args:
cfg (:class:`yggdrasil.config.YggConfigParser`, optional):
Config parser with options that should be used to update the
environment. Defaults to :data:`yggdrasil.config.ygg_cfg`.
"""
is_model = tools.is_subprocess()
if cfg is None:
cfg = ygg_cfg
_LOG_FORMAT = "%(levelname)s:%(module)s.%(funcName)s[%(lineno)d]:%(message)s"
logging.basicConfig(level=logging.INFO, format=_LOG_FORMAT)
logLevelYGG = eval('logging.%s' % cfg.get('debug', 'ygg', 'NOTSET'))
logLevelRMQ = eval('logging.%s' % cfg.get('debug', 'rmq', 'INFO'))
logLevelCLI = eval('logging.%s' % cfg.get('debug', 'client', 'INFO'))
ygg_logger = logging.getLogger("yggdrasil")
rmq_logger = logging.getLogger("pika")
if is_model:
ygg_logger.setLevel(level=logLevelCLI)
else:
ygg_logger.setLevel(level=logLevelYGG)
rmq_logger.setLevel(level=logLevelRMQ)
# For models, route the loggs to stdout so that they are displayed by the
# model driver.
if is_model:
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logLevelCLI)
ygg_logger.addHandler(handler)
rmq_logger.addHandler(handler)
def cfg_logging(cfg=None):
r"""Set logging levels from config options.
Args:
cfg (:class:`yggdrasil.config.YggConfigParser`, optional):
Config parser with options that should be used to update the
environment. Defaults to :data:`yggdrasil.config.ygg_cfg`.
"""
is_model = tools.is_subprocess()
to_stdout = is_model
try: # pragma: no cover
# Direct log messages to stdout in interpreter so that messages
# are not red in notebooks
get_ipython # noqa: F821
in_notebook = True
except BaseException:
in_notebook = False
to_stdout = (is_model or in_notebook)
if cfg is None:
cfg = ygg_cfg
log_format = cfg.get(
'debug', 'format',
"%(levelname)s:%(process)d:%(module)s.%(funcName)s[%(lineno)d]:%(message)s"
)
logLevelYGG = eval(
'logging.%s' %
os.environ.get('YGG_DEBUG', cfg.get('debug', 'ygg', 'NOTSET')))
logLevelRMQ = eval(
'logging.%s' %
os.environ.get('YGG_RMQ_DEBUG', cfg.get('debug', 'rmq', 'INFO')))
logLevelCLI = eval(
'logging.%s' %
os.environ.get('YGG_CLIENT_DEBUG', cfg.get('debug', 'client', 'INFO')))
if is_model:
logLevelYGG = os.environ.get('YGG_MODEL_DEBUG', logLevelCLI)
if not to_stdout:
logging.basicConfig(format=log_format)
ygg_logger = logging.getLogger("yggdrasil")
rmq_logger = logging.getLogger("pika")
ygg_logger.setLevel(level=logLevelYGG)
rmq_logger.setLevel(level=logLevelRMQ)
# For models, route the logs to stdout so that they are
# displayed by the model driver.
if to_stdout:
formatter = logging.Formatter(fmt=log_format)
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(formatter)
handler.setLevel(logLevelYGG)
ygg_logger.handlers = [handler]
rmq_logger.handlers = [handler]
if in_notebook: # pragma: no cover
ygg_logger.propagate = False
rmq_logger.propagate = False
def cfg_logging(cfg=None):
r"""Set logging levels from config options.
Args:
cfg (:class:`yggdrasil.config.YggConfigParser`, optional):
Config parser with options that should be used to update the
environment. Defaults to :data:`yggdrasil.config.ygg_cfg`.
"""
is_model = tools.is_subprocess()
if cfg is None:
cfg = ygg_cfg
_LOG_FORMAT = "%(levelname)s:%(module)s.%(funcName)s[%(lineno)d]:%(message)s"
logging.basicConfig(level=logging.INFO, format=_LOG_FORMAT)
logLevelYGG = eval('logging.%s' % cfg.get('debug', 'ygg', 'NOTSET'))
logLevelRMQ = eval('logging.%s' % cfg.get('debug', 'rmq', 'INFO'))
logLevelCLI = eval('logging.%s' % cfg.get('debug', 'client', 'INFO'))
ygg_logger = logging.getLogger("yggdrasil")
rmq_logger = logging.getLogger("pika")
if is_model:
logLevelMOD = os.environ.get('YGG_MODEL_DEBUG', logLevelCLI)
ygg_logger.setLevel(level=logLevelMOD)
else:
ygg_logger.setLevel(level=logLevelYGG)
rmq_logger.setLevel(level=logLevelRMQ)
to_stdout = False
# For models, route the loggs to stdout so that they are displayed by the
# model driver.
if is_model:
to_stdout = True
try: # pragma: no cover
# Direct log messages to stdout in interpreter so that messages
# are not red in notebooks
get_ipython # noqa: F821
to_stdout = True
ygg_logger.propagate = False
rmq_logger.propagate = False
except BaseException:
pass
if to_stdout:
handler = logging.StreamHandler(sys.stdout)
if is_model:
handler.setLevel(logLevelMOD)
else: # pragma: no cover
# This is only used for notebooks (and interpreters)
handler.setLevel(logLevelYGG)
ygg_logger.handlers = [handler]
rmq_logger.handlers = [handler]
def set_ygg_loglevel(level, cfg=None):
r"""Set the current log level.
Args:
level (str): Level that the log should be set to.
cfg (:class:`yggdrasil.config.YggConfigParser`, optional):
Config parser with options that should be used to update the
environment. Defaults to :data:`yggdrasil.config.ygg_cfg`.
"""
is_model = tools.is_subprocess()
if cfg is None:
cfg = ygg_cfg
if is_model:
opt = 'client'
else:
opt = 'ygg'
cfg.set('debug', opt, level)
logLevelYGG = eval('logging.%s' % level)
ygg_logger = logging.getLogger("yggdrasil")
ygg_logger.setLevel(level=logLevelYGG)
def get_ygg_loglevel(cfg=None, default='DEBUG'):
r"""Get the current log level.
Args:
cfg (:class:`yggdrasil.config.YggConfigParser`, optional):
Config parser with options that should be used to determine the
log level. Defaults to :data:`yggdrasil.config.ygg_cfg`.
default (str, optional): Log level that should be returned if the log
level option is not set in cfg. Defaults to 'DEBUG'.
Returns:
str: Log level string.
"""
is_model = tools.is_subprocess()
if cfg is None:
cfg = ygg_cfg
if is_model:
opt = 'client'
else:
opt = 'ygg'
return cfg.get('debug', opt, default)
def register_example(example_dir):
r"""Register an example based on the contents of the director.
Args:
example_dir (str): Full path to a directory potentially containing an
example.
Returns:
tuple (list, dict, dict): A list of available languages, a dictionary
mapping from language to YAML specification files for the example,
and a dictionary mapping from language to source files for the
example.
"""
# Check that the source directory and test exist
example_base = os.path.basename(example_dir)
srcdir = os.path.join(example_dir, 'src')
testfile = os.path.join(_example_dir, 'tests', 'test_%s.py' % example_base)
if not os.path.isfile(testfile): # pragma: no cover
# TODO: Automate test creation
if (((not tools.is_subprocess())
and (example_base not in ['sbml1', 'sbml2', 'sbml3']))):
logging.error("Missing test file: %s" % testfile)
if not os.path.isdir(srcdir): # pragma: no cover
if not tools.is_subprocess():
logging.error("Missing source directory: %s" % srcdir)
return {}
# Determine which languages are present in the example
lang_base = []
lang_avail = []
lang_search = None
if example_base in ['rpcFib', 'maxMsg']:
lang_search = example_base + 'Cli_%s.yml'
lang_avail += ['all', 'all_nomatlab']
elif example_base.startswith('rpc_'):
lang_search = 'client_%s.yml'
elif example_base == 'root_to_shoot':
lang_avail += ['all', 'all_nomatlab', 'c', 'python']
elif example_base == 'fakeplant':
lang_avail += ['all', 'all_nomatlab', 'c', 'cpp', 'matlab', 'python']
elif example_base in ['types', 'transforms']:
lang_avail += tools.get_supported_lang()
for k in ['cmake', 'make', 'lpy', 'executable']:
lang_avail.remove(k)
elif example_base.startswith('sbml'):
lang_avail = ['sbml']
elif example_base.startswith('osr'):
lang_search = example_base + '_%s.yml'
lang_base += ['osr']
else:
lang_search = example_base + '_%s.yml'
if lang_search is not None:
for match in glob.glob(os.path.join(example_dir, lang_search % '*')):
lang = serialize.process_message(os.path.basename(match),
lang_search)[0]
if lang == 'valgrind':
continue
lang_avail.append(lang.lower())
lang_avail = tuple(sorted(lang_avail))
lang_base = tuple(sorted(lang_base))
# Get YAML and source files for each language
out_yml = {}
out_src = {}
src_is_abs = False
for lang in lang_avail:
yml_names = []
src_names = []
if example_base == 'rpcFib':
if lang == 'all':
lang_set = ('python', 'matlab', 'c')
elif lang == 'all_nomatlab':
lang_set = ('python', 'cpp', 'c')
else:
lang_set = (lang, lang, lang)
yml_names = [
'%sCli_%s.yml' % (example_base, lang_set[0]),
'%sCliPar_%s.yml' % (example_base, lang_set[1]),
'%sSrv_%s.yml' % (example_base, lang_set[2])
]
src_names = [
'%sCli%s' % (example_base, ext_map[lang_set[0]]),
'%sCliPar%s' % (example_base, ext_map[lang_set[1]]),
'%sSrv%s' % (example_base, ext_map[lang_set[2]])
]
elif example_base == 'maxMsg':
if lang == 'all':
lang_set = ('python', 'matlab')
elif lang == 'all_nomatlab':
lang_set = ('python', 'c')
else:
lang_set = (lang, lang)
yml_names = [
'%sCli_%s.yml' % (example_base, lang_set[0]),
'%sSrv_%s.yml' % (example_base, lang_set[1])
]
src_names = [
'%sCli%s' % (example_base, ext_map[lang_set[0]]),
'%sSrv%s' % (example_base, ext_map[lang_set[1]])
]
elif example_base.startswith('rpc_'):
# TODO: Create server examples in other languages
yml_names = ['server_python.yml', 'client_%s.yml' % lang]
src_names = ['server.py', 'client%s' % ext_map[lang]]
elif example_base == 'root_to_shoot':
if lang.startswith('all'):
yml_names = ['root.yml', 'shoot.yml', 'root_to_shoot.yml']
src_names = ['root.c', 'shoot.py']
elif lang == 'python':
yml_names = ['shoot.yml', 'shoot_files.yml']
src_names = ['shoot.py']
elif lang == 'c':
yml_names = ['root.yml', 'root_files.yml']
src_names = ['root.c']
elif example_base == 'fakeplant':
if lang.startswith('all'):
yml_names = [
'canopy.yml', 'light.yml', 'photosynthesis.yml',
'fakeplant.yml'
]
src_names = ['canopy.cpp', 'light.c', 'photosynthesis.py']
if lang == 'all_nomatlab':
yml_names.append('growth_python.yml')
src_names.append('growth.py')
else:
yml_names.append('growth.yml')
src_names.append('growth.m')
elif lang == 'python':
yml_names = ['photosynthesis.yml']
src_names = ['photosynthesis.py']
elif lang == 'c':
yml_names = ['light.yml', 'light_files.yml']
src_names = ['light.c']
elif lang == 'cpp':
yml_names = ['canopy.yml', 'canopy_files.yml']
src_names = ['canopy.cpp']
elif lang == 'matlab':
yml_names = ['growth.yml', 'growth_files.yml']
src_names = ['growth.m']
elif example_base in ['types', 'transforms']:
yml_names = ['%s.yml' % example_base]
src_names = ['src.py', 'dst.py']
elif example_base.startswith('sbml'):
yml_names = ['%s.yml' % example_base]
src_names = ['%s.xml' % example_base]
else:
src_is_abs = True
yml_names = ['%s_%s.yml' % (example_base, lang)]
if lang.startswith('all'):
src_names = []
for lsrc in lang_avail:
if lsrc.startswith('all'):
continue
elif (lang == 'all_nomatlab') and (lsrc == 'matlab'):
continue
src_names += sorted(
glob.glob(os.path.join(srcdir, '*' + ext_map[lsrc])))
else:
src_names = sorted(
glob.glob(os.path.join(srcdir,
'*' + ext_map.get(lang, ''))))
for ilang_base in lang_base:
src_names += sorted(
glob.glob(
os.path.join(srcdir,
'*' + ext_map.get(ilang_base, ''))))
out_yml[lang] = [os.path.join(example_dir, y) for y in yml_names]
if src_is_abs:
out_src[lang] = src_names
else:
out_src[lang] = [os.path.join(srcdir, s) for s in src_names]
if len(out_yml[lang]) == 1:
out_yml[lang] = out_yml[lang][0]
if len(out_src[lang]) == 1:
out_src[lang] = out_src[lang][0]
return lang_base, lang_avail, out_yml, out_src
from datetime import datetime
import os
import psutil
import warnings
import weakref
from yggdrasil import backwards, tools, platform, serialize
from yggdrasil.languages import get_language_dir
from yggdrasil.config import ygg_cfg
from yggdrasil.drivers.InterpretedModelDriver import InterpretedModelDriver
from yggdrasil.tools import TimeOut, sleep
logger = logging.getLogger(__name__)
try: # pragma: matlab
disable_engine = ygg_cfg.get('matlab', 'disable_engine', 'False').lower()
if platform._is_win or (disable_engine == 'true'):
_matlab_engine_installed = False
if not tools.is_subprocess():
logger.debug("matlab.engine disabled")
else:
import matlab.engine
_matlab_engine_installed = True
except ImportError: # pragma: no matlab
logger.debug("Could not import matlab.engine. "
+ "Matlab support for using a sharedEngine will be disabled.")
_matlab_engine_installed = False
_top_lang_dir = get_language_dir('matlab')
_compat_map = {
'R2015b': ['2.7', '3.3', '3.4'],
'R2017a': ['2.7', '3.3', '3.4', '3.5'],
'R2017b': ['2.7', '3.3', '3.4', '3.5', '3.6'],
