def cfg_logging(cfg=None): r"""Set logging levels from config options. Args: cfg (:class:`yggdrasil.config.YggConfigParser`, optional): Config parser with options that should be used to update the environment. Defaults to :data:`yggdrasil.config.ygg_cfg`. """ is_model = tools.is_subprocess() if cfg is None: cfg = ygg_cfg _LOG_FORMAT = "%(levelname)s:%(module)s.%(funcName)s[%(lineno)d]:%(message)s" logging.basicConfig(level=logging.INFO, format=_LOG_FORMAT) logLevelYGG = eval('logging.%s' % cfg.get('debug', 'ygg', 'NOTSET')) logLevelRMQ = eval('logging.%s' % cfg.get('debug', 'rmq', 'INFO')) logLevelCLI = eval('logging.%s' % cfg.get('debug', 'client', 'INFO')) ygg_logger = logging.getLogger("yggdrasil") rmq_logger = logging.getLogger("pika") if is_model: ygg_logger.setLevel(level=logLevelCLI) else: ygg_logger.setLevel(level=logLevelYGG) rmq_logger.setLevel(level=logLevelRMQ) # For models, route the loggs to stdout so that they are displayed by the # model driver. if is_model: handler = logging.StreamHandler(sys.stdout) handler.setLevel(logLevelCLI) ygg_logger.addHandler(handler) rmq_logger.addHandler(handler)
def cfg_logging(cfg=None): r"""Set logging levels from config options. Args: cfg (:class:`yggdrasil.config.YggConfigParser`, optional): Config parser with options that should be used to update the environment. Defaults to :data:`yggdrasil.config.ygg_cfg`. """ is_model = tools.is_subprocess() to_stdout = is_model try: # pragma: no cover # Direct log messages to stdout in interpreter so that messages # are not red in notebooks get_ipython # noqa: F821 in_notebook = True except BaseException: in_notebook = False to_stdout = (is_model or in_notebook) if cfg is None: cfg = ygg_cfg log_format = cfg.get( 'debug', 'format', "%(levelname)s:%(process)d:%(module)s.%(funcName)s[%(lineno)d]:%(message)s" ) logLevelYGG = eval( 'logging.%s' % os.environ.get('YGG_DEBUG', cfg.get('debug', 'ygg', 'NOTSET'))) logLevelRMQ = eval( 'logging.%s' % os.environ.get('YGG_RMQ_DEBUG', cfg.get('debug', 'rmq', 'INFO'))) logLevelCLI = eval( 'logging.%s' % os.environ.get('YGG_CLIENT_DEBUG', cfg.get('debug', 'client', 'INFO'))) if is_model: logLevelYGG = os.environ.get('YGG_MODEL_DEBUG', logLevelCLI) if not to_stdout: logging.basicConfig(format=log_format) ygg_logger = logging.getLogger("yggdrasil") rmq_logger = logging.getLogger("pika") ygg_logger.setLevel(level=logLevelYGG) rmq_logger.setLevel(level=logLevelRMQ) # For models, route the logs to stdout so that they are # displayed by the model driver. if to_stdout: formatter = logging.Formatter(fmt=log_format) handler = logging.StreamHandler(sys.stdout) handler.setFormatter(formatter) handler.setLevel(logLevelYGG) ygg_logger.handlers = [handler] rmq_logger.handlers = [handler] if in_notebook: # pragma: no cover ygg_logger.propagate = False rmq_logger.propagate = False
def cfg_logging(cfg=None): r"""Set logging levels from config options. Args: cfg (:class:`yggdrasil.config.YggConfigParser`, optional): Config parser with options that should be used to update the environment. Defaults to :data:`yggdrasil.config.ygg_cfg`. """ is_model = tools.is_subprocess() if cfg is None: cfg = ygg_cfg _LOG_FORMAT = "%(levelname)s:%(module)s.%(funcName)s[%(lineno)d]:%(message)s" logging.basicConfig(level=logging.INFO, format=_LOG_FORMAT) logLevelYGG = eval('logging.%s' % cfg.get('debug', 'ygg', 'NOTSET')) logLevelRMQ = eval('logging.%s' % cfg.get('debug', 'rmq', 'INFO')) logLevelCLI = eval('logging.%s' % cfg.get('debug', 'client', 'INFO')) ygg_logger = logging.getLogger("yggdrasil") rmq_logger = logging.getLogger("pika") if is_model: logLevelMOD = os.environ.get('YGG_MODEL_DEBUG', logLevelCLI) ygg_logger.setLevel(level=logLevelMOD) else: ygg_logger.setLevel(level=logLevelYGG) rmq_logger.setLevel(level=logLevelRMQ) to_stdout = False # For models, route the loggs to stdout so that they are displayed by the # model driver. if is_model: to_stdout = True try: # pragma: no cover # Direct log messages to stdout in interpreter so that messages # are not red in notebooks get_ipython # noqa: F821 to_stdout = True ygg_logger.propagate = False rmq_logger.propagate = False except BaseException: pass if to_stdout: handler = logging.StreamHandler(sys.stdout) if is_model: handler.setLevel(logLevelMOD) else: # pragma: no cover # This is only used for notebooks (and interpreters) handler.setLevel(logLevelYGG) ygg_logger.handlers = [handler] rmq_logger.handlers = [handler]
def set_ygg_loglevel(level, cfg=None): r"""Set the current log level. Args: level (str): Level that the log should be set to. cfg (:class:`yggdrasil.config.YggConfigParser`, optional): Config parser with options that should be used to update the environment. Defaults to :data:`yggdrasil.config.ygg_cfg`. """ is_model = tools.is_subprocess() if cfg is None: cfg = ygg_cfg if is_model: opt = 'client' else: opt = 'ygg' cfg.set('debug', opt, level) logLevelYGG = eval('logging.%s' % level) ygg_logger = logging.getLogger("yggdrasil") ygg_logger.setLevel(level=logLevelYGG)
def get_ygg_loglevel(cfg=None, default='DEBUG'): r"""Get the current log level. Args: cfg (:class:`yggdrasil.config.YggConfigParser`, optional): Config parser with options that should be used to determine the log level. Defaults to :data:`yggdrasil.config.ygg_cfg`. default (str, optional): Log level that should be returned if the log level option is not set in cfg. Defaults to 'DEBUG'. Returns: str: Log level string. """ is_model = tools.is_subprocess() if cfg is None: cfg = ygg_cfg if is_model: opt = 'client' else: opt = 'ygg' return cfg.get('debug', opt, default)
def register_example(example_dir): r"""Register an example based on the contents of the director. Args: example_dir (str): Full path to a directory potentially containing an example. Returns: tuple (list, dict, dict): A list of available languages, a dictionary mapping from language to YAML specification files for the example, and a dictionary mapping from language to source files for the example. """ # Check that the source directory and test exist example_base = os.path.basename(example_dir) srcdir = os.path.join(example_dir, 'src') testfile = os.path.join(_example_dir, 'tests', 'test_%s.py' % example_base) if not os.path.isfile(testfile): # pragma: no cover # TODO: Automate test creation if (((not tools.is_subprocess()) and (example_base not in ['sbml1', 'sbml2', 'sbml3']))): logging.error("Missing test file: %s" % testfile) if not os.path.isdir(srcdir): # pragma: no cover if not tools.is_subprocess(): logging.error("Missing source directory: %s" % srcdir) return {} # Determine which languages are present in the example lang_base = [] lang_avail = [] lang_search = None if example_base in ['rpcFib', 'maxMsg']: lang_search = example_base + 'Cli_%s.yml' lang_avail += ['all', 'all_nomatlab'] elif example_base.startswith('rpc_'): lang_search = 'client_%s.yml' elif example_base == 'root_to_shoot': lang_avail += ['all', 'all_nomatlab', 'c', 'python'] elif example_base == 'fakeplant': lang_avail += ['all', 'all_nomatlab', 'c', 'cpp', 'matlab', 'python'] elif example_base in ['types', 'transforms']: lang_avail += tools.get_supported_lang() for k in ['cmake', 'make', 'lpy', 'executable']: lang_avail.remove(k) elif example_base.startswith('sbml'): lang_avail = ['sbml'] elif example_base.startswith('osr'): lang_search = example_base + '_%s.yml' lang_base += ['osr'] else: lang_search = example_base + '_%s.yml' if lang_search is not None: for match in glob.glob(os.path.join(example_dir, lang_search % '*')): lang = serialize.process_message(os.path.basename(match), lang_search)[0] if lang == 'valgrind': continue lang_avail.append(lang.lower()) lang_avail = tuple(sorted(lang_avail)) lang_base = tuple(sorted(lang_base)) # Get YAML and source files for each language out_yml = {} out_src = {} src_is_abs = False for lang in lang_avail: yml_names = [] src_names = [] if example_base == 'rpcFib': if lang == 'all': lang_set = ('python', 'matlab', 'c') elif lang == 'all_nomatlab': lang_set = ('python', 'cpp', 'c') else: lang_set = (lang, lang, lang) yml_names = [ '%sCli_%s.yml' % (example_base, lang_set[0]), '%sCliPar_%s.yml' % (example_base, lang_set[1]), '%sSrv_%s.yml' % (example_base, lang_set[2]) ] src_names = [ '%sCli%s' % (example_base, ext_map[lang_set[0]]), '%sCliPar%s' % (example_base, ext_map[lang_set[1]]), '%sSrv%s' % (example_base, ext_map[lang_set[2]]) ] elif example_base == 'maxMsg': if lang == 'all': lang_set = ('python', 'matlab') elif lang == 'all_nomatlab': lang_set = ('python', 'c') else: lang_set = (lang, lang) yml_names = [ '%sCli_%s.yml' % (example_base, lang_set[0]), '%sSrv_%s.yml' % (example_base, lang_set[1]) ] src_names = [ '%sCli%s' % (example_base, ext_map[lang_set[0]]), '%sSrv%s' % (example_base, ext_map[lang_set[1]]) ] elif example_base.startswith('rpc_'): # TODO: Create server examples in other languages yml_names = ['server_python.yml', 'client_%s.yml' % lang] src_names = ['server.py', 'client%s' % ext_map[lang]] elif example_base == 'root_to_shoot': if lang.startswith('all'): yml_names = ['root.yml', 'shoot.yml', 'root_to_shoot.yml'] src_names = ['root.c', 'shoot.py'] elif lang == 'python': yml_names = ['shoot.yml', 'shoot_files.yml'] src_names = ['shoot.py'] elif lang == 'c': yml_names = ['root.yml', 'root_files.yml'] src_names = ['root.c'] elif example_base == 'fakeplant': if lang.startswith('all'): yml_names = [ 'canopy.yml', 'light.yml', 'photosynthesis.yml', 'fakeplant.yml' ] src_names = ['canopy.cpp', 'light.c', 'photosynthesis.py'] if lang == 'all_nomatlab': yml_names.append('growth_python.yml') src_names.append('growth.py') else: yml_names.append('growth.yml') src_names.append('growth.m') elif lang == 'python': yml_names = ['photosynthesis.yml'] src_names = ['photosynthesis.py'] elif lang == 'c': yml_names = ['light.yml', 'light_files.yml'] src_names = ['light.c'] elif lang == 'cpp': yml_names = ['canopy.yml', 'canopy_files.yml'] src_names = ['canopy.cpp'] elif lang == 'matlab': yml_names = ['growth.yml', 'growth_files.yml'] src_names = ['growth.m'] elif example_base in ['types', 'transforms']: yml_names = ['%s.yml' % example_base] src_names = ['src.py', 'dst.py'] elif example_base.startswith('sbml'): yml_names = ['%s.yml' % example_base] src_names = ['%s.xml' % example_base] else: src_is_abs = True yml_names = ['%s_%s.yml' % (example_base, lang)] if lang.startswith('all'): src_names = [] for lsrc in lang_avail: if lsrc.startswith('all'): continue elif (lang == 'all_nomatlab') and (lsrc == 'matlab'): continue src_names += sorted( glob.glob(os.path.join(srcdir, '*' + ext_map[lsrc]))) else: src_names = sorted( glob.glob(os.path.join(srcdir, '*' + ext_map.get(lang, '')))) for ilang_base in lang_base: src_names += sorted( glob.glob( os.path.join(srcdir, '*' + ext_map.get(ilang_base, '')))) out_yml[lang] = [os.path.join(example_dir, y) for y in yml_names] if src_is_abs: out_src[lang] = src_names else: out_src[lang] = [os.path.join(srcdir, s) for s in src_names] if len(out_yml[lang]) == 1: out_yml[lang] = out_yml[lang][0] if len(out_src[lang]) == 1: out_src[lang] = out_src[lang][0] return lang_base, lang_avail, out_yml, out_src
from datetime import datetime import os import psutil import warnings import weakref from yggdrasil import backwards, tools, platform, serialize from yggdrasil.languages import get_language_dir from yggdrasil.config import ygg_cfg from yggdrasil.drivers.InterpretedModelDriver import InterpretedModelDriver from yggdrasil.tools import TimeOut, sleep logger = logging.getLogger(__name__) try: # pragma: matlab disable_engine = ygg_cfg.get('matlab', 'disable_engine', 'False').lower() if platform._is_win or (disable_engine == 'true'): _matlab_engine_installed = False if not tools.is_subprocess(): logger.debug("matlab.engine disabled") else: import matlab.engine _matlab_engine_installed = True except ImportError: # pragma: no matlab logger.debug("Could not import matlab.engine. " + "Matlab support for using a sharedEngine will be disabled.") _matlab_engine_installed = False _top_lang_dir = get_language_dir('matlab') _compat_map = { 'R2015b': ['2.7', '3.3', '3.4'], 'R2017a': ['2.7', '3.3', '3.4', '3.5'], 'R2017b': ['2.7', '3.3', '3.4', '3.5', '3.6'],