import os
import copy
import pprint
import yaml
from yggdrasil import tools
from yggdrasil.components import import_component
from yggdrasil.languages import get_language_ext
from yggdrasil.metaschema.datatypes import get_type_class
from yggdrasil.tests import long_running
from yggdrasil.examples import _example_dir
from yggdrasil.examples.tests import ExampleTstBase
from yggdrasil.metaschema.datatypes import encode_type
_all_lang = tools.get_supported_lang()
_typed_lang = tuple(
[x for x in _all_lang if import_component('model', x).is_typed])
@long_running
class TestExampleTypes(ExampleTstBase):
r"""Test the Types example."""
example_name = 'types'
iter_over = ['language', 'type', 'method']
iter_skip = [(set(_typed_lang), '*', 'run_example_generic'),
(set(_typed_lang), '*', 'run_example_pointers'),
(set(['c']), '*', 'run_example_c_nolengths'),
(set(['c']), '*', 'run_example_c_prefixes'),
('c', set(['string', 'bytes',
'unicode']), 'run_example_c_nolengths'),
('c', set(['1darray', 'ndarray']), 'run_example_c_prefixes'),
def update_language_config(languages=None,
skip_warnings=False,
disable_languages=None,
enable_languages=None,
lang_kwargs=None,
overwrite=False,
verbose=False):
r"""Update configuration options for a language driver.
Args:
languages (list, optional): List of languages to configure.
Defaults to None and all supported languages will be
configured.
skip_warnings (bool, optional): If True, warnings about missing options
will not be raised. Defaults to False.
disable_languages (list, optional): List of languages that should be
disabled. Defaults to an empty list.
enable_languages (list, optional): List of languages that should be
enabled. Defaults to an empty list.
overwrite (bool, optional): If True, the existing file will be overwritten.
Defaults to False.
verbose (bool, optional): If True, information about the config file
will be displayed. Defaults to False.
lang_kwargs (dict, optional): Dictionary containing language
specific keyword arguments. Defaults to {}.
"""
from yggdrasil.components import import_component
if verbose:
logger.info(
"Updating user configuration file for yggdrasil at:\n\t%s" %
usr_config_file)
miss = []
if (languages is None) or overwrite:
all_languages = tools.get_supported_lang()
languages = [
'c', 'c++', 'make', 'cmake', 'python', 'lpy', 'r', 'matlab'
]
for lang in all_languages:
if lang.lower() not in languages:
languages.append(lang)
elif not isinstance(languages, list):
languages = [languages]
if disable_languages is None:
disable_languages = []
if enable_languages is None:
enable_languages = []
if lang_kwargs is None:
lang_kwargs = {}
if overwrite:
shutil.copy(def_config_file, usr_config_file)
ygg_cfg_usr.reload()
drivers = OrderedDict([(lang, import_component('model', lang))
for lang in languages])
drv = list(get_language_order(drivers).values())
for idrv in drv:
if (((idrv.language in disable_languages)
and (idrv.language in enable_languages))):
logger.info(("%s language both enabled and disabled. "
"No action will be taken.") % idrv.language)
elif idrv.language in disable_languages:
ygg_cfg_usr.set(idrv.language, 'disable', 'True')
elif idrv.language in enable_languages:
ygg_cfg_usr.set(idrv.language, 'disable', 'False')
if ygg_cfg_usr.get(idrv.language, 'disable',
'False').lower() == 'true':
continue # pragma: no cover
miss += idrv.configure(ygg_cfg_usr,
**lang_kwargs.get(idrv.language, {}))
ygg_cfg_usr.update_file()
ygg_cfg.reload()
if not skip_warnings:
for sect, opt, desc in miss: # pragma: windows
warnings.warn(("Could not set option %s in section %s. " +
"Please set this in %s to: %s") %
(opt, sect, ygg_cfg_usr.file_to_update, desc),
RuntimeWarning)
def test_get_supported():
r"""Test get_supported_<platforms/lang/comm>."""
tools.get_supported_platforms()
tools.get_supported_lang()
tools.get_supported_comm()
'sphinx.ext.doctest',
'sphinx.ext.intersphinx',
'sphinx.ext.todo',
'sphinx.ext.coverage',
'sphinx.ext.mathjax',
'sphinx.ext.ifconfig',
'sphinx.ext.viewcode',
'sphinx.ext.githubpages',
'sphinx.ext.napoleon',
'sphinxarg.ext',
'breathe',
]
relative_files = []
for lang in tools.get_supported_lang():
if lang == 'python':
continue
try:
lang_dir = languages.get_language_dir(lang)
lang_ext = components.import_component('model', lang).get_language_ext()
except ValueError:
continue
for iext in lang_ext:
relative_files += [os.path.relpath(f, start=srcdir) for f in
glob.glob(os.path.join(lang_dir, '*' + iext))]
relative_files += [os.path.relpath(f, start=srcdir) for f in
glob.glob(os.path.join(lang_dir, '*', '*' + iext))]
breathe_projects = {"yggdrasil": doxydir}
breathe_default_project = "yggdrasil"
def update_config():
r"""Update the user config file for yggdrasil."""
from yggdrasil import config, tools, platform
parser = argparse.ArgumentParser(
description='Update the user config file.')
parser.add_argument(
'--show-file',
action='store_true',
help='Print the path to the config file without updating it.')
parser.add_argument('--remove-file',
action='store_true',
help='Remove the existing config file and return.')
parser.add_argument('--overwrite',
action='store_true',
help='Overwrite the existing file.')
parser.add_argument('--languages',
nargs='+',
default=tools.get_supported_lang(),
help=('One or more languages that should be'
'configured.'))
parser.add_argument('--disable-languages',
nargs='+',
default=[],
dest='disable_languages',
help='One or more languages that should be disabled.')
parser.add_argument('--enable-languages',
nargs='+',
default=[],
help='One or more languages that should be enabled.')
if ('-h' in sys.argv) or ('--help' in sys.argv):
prelang = tools.get_supported_lang()
else:
prelang = parser.parse_known_args()[0].languages
lang_args = {}
lang_args2kwargs = {}
if platform._is_mac:
lang_args.setdefault('c', [])
lang_args['c'].append((('--macos-sdkroot', '--sdkroot'), {
'help': ('The full path to the MacOS SDK '
'that should be used.')
}))
for lang in prelang:
if lang in lang_args:
lang_args2kwargs[lang] = []
for args, kwargs in lang_args.get(lang, []):
parser.add_argument(*args, **kwargs)
lang_args2kwargs[lang].append(parser._actions[-1].dest)
args = parser.parse_args()
lang_kwargs = {
lang: {k: getattr(args, k)
for k in alist}
for lang, alist in lang_args2kwargs.items()
}
if args.show_file:
print('Config file located here: %s' % config.usr_config_file)
if args.remove_file and os.path.isfile(config.usr_config_file):
os.remove(config.usr_config_file)
if args.show_file or args.remove_file:
return
config.update_language_config(args.languages,
overwrite=args.overwrite,
verbose=True,
disable_languages=args.disable_languages,
enable_languages=args.enable_languages,
lang_kwargs=lang_kwargs)
def ygginfo():
r"""Print information about yggdrasil installation."""
from yggdrasil import __version__, tools, config, platform
from yggdrasil.components import import_component
lang_list = tools.get_installed_lang()
prefix = ' '
curr_prefix = ''
vardict = [
('Location', os.path.dirname(__file__)), ('Version', __version__),
('Languages', ', '.join(lang_list)),
('Communication Mechanisms', ', '.join(tools.get_installed_comm())),
('Default Comm Mechanism', tools.get_default_comm()),
('Config File', config.usr_config_file)
]
parser = argparse.ArgumentParser(
description=
'Display information about the current yggdrasil installation.')
parser.add_argument(
'--no-languages',
action='store_true',
dest='no_languages',
help='Don\'t print information about individual languages.')
parser.add_argument(
'--verbose',
action='store_true',
help='Increase the verbosity of the printed information.')
args = parser.parse_args()
try:
# Add language information
if not args.no_languages:
# Install languages
vardict.append(('Installed Languages:', ''))
curr_prefix += prefix
for lang in sorted(lang_list):
drv = import_component('model', lang)
vardict.append((curr_prefix + '%s:' % lang.upper(), ''))
curr_prefix += prefix
if lang == 'executable':
vardict.append((curr_prefix + 'Location', ''))
else:
exec_name = drv.language_executable()
if not os.path.isabs(exec_name):
exec_name = shutil.which(exec_name)
vardict.append((curr_prefix + 'Location', exec_name))
vardict.append(
(curr_prefix + 'Version', drv.language_version()))
curr_prefix = curr_prefix.rsplit(prefix, 1)[0]
curr_prefix = curr_prefix.rsplit(prefix, 1)[0]
# Not installed languages
vardict.append(("Languages Not Installed:", ''))
curr_prefix += prefix
for lang in tools.get_supported_lang():
if lang in lang_list:
continue
drv = import_component('model', lang)
vardict.append((curr_prefix + '%s:' % lang.upper(), ''))
curr_prefix += prefix
vardict.append((curr_prefix + "Language Installed",
drv.is_language_installed()))
vardict.append((curr_prefix + "Base Languages Installed",
drv.are_base_languages_installed()))
if not drv.are_base_languages_installed():
vardict.append(
(curr_prefix + "Base Languages Not Installed", [
b for b in drv.base_languages if
(not import_component('model', b).is_installed())
]))
vardict.append((curr_prefix + "Dependencies Installed",
drv.are_dependencies_installed()))
vardict.append((curr_prefix + "Interface Installed",
drv.is_interface_installed()))
vardict.append(
(curr_prefix + "Comm Installed", drv.is_comm_installed()))
vardict.append(
(curr_prefix + "Configured", drv.is_configured()))
vardict.append((curr_prefix + "Disabled", drv.is_disabled()))
curr_prefix = curr_prefix.rsplit(prefix, 1)[0]
curr_prefix = curr_prefix.rsplit(prefix, 1)[0]
# Add verbose information
if args.verbose:
# Conda info
if os.environ.get('CONDA_PREFIX', ''):
if platform._is_win: # pragma: windows
out = tools.bytes2str(
subprocess.check_output('conda info',
shell=True)).strip()
else:
out = tools.bytes2str(
subprocess.check_output(['conda', 'info'])).strip()
curr_prefix += prefix
vardict.append((curr_prefix + 'Conda Info:',
"\n%s%s" % (curr_prefix + prefix,
("\n" + curr_prefix + prefix).join(
out.splitlines(False)))))
curr_prefix = curr_prefix.rsplit(prefix, 1)[0]
# R and reticulate info
Rdrv = import_component("model", "R")
if Rdrv.is_installed():
env_reticulate = copy.deepcopy(os.environ)
env_reticulate['RETICULATE_PYTHON'] = sys.executable
# Stack size
out = Rdrv.run_executable(["-e", "Cstack_info()"]).strip()
vardict.append((curr_prefix + "R Cstack_info:",
"\n%s%s" % (curr_prefix + prefix,
("\n" + curr_prefix + prefix).join(
out.splitlines(False)))))
# Compilation tools
interp = 'R'.join(Rdrv.get_interpreter().rsplit('Rscript', 1))
vardict.append((curr_prefix + "R C Compiler:", ""))
curr_prefix += prefix
for x in ['CC', 'CFLAGS', 'CXX', 'CXXFLAGS']:
out = tools.bytes2str(
subprocess.check_output([interp, 'CMD', 'config',
x])).strip()
vardict.append((curr_prefix + x,
"%s" % ("\n" + curr_prefix + prefix).join(
out.splitlines(False))))
curr_prefix = curr_prefix.rsplit(prefix, 1)[0]
# Session info
out = Rdrv.run_executable(["-e", "sessionInfo()"]).strip()
vardict.append((curr_prefix + "R sessionInfo:",
"\n%s%s" % (curr_prefix + prefix,
("\n" + curr_prefix + prefix).join(
out.splitlines(False)))))
# Reticulate conda_list
if os.environ.get('CONDA_PREFIX', ''):
out = Rdrv.run_executable([
"-e",
("library(reticulate); "
"reticulate::conda_list()")
],
env=env_reticulate).strip()
vardict.append(
(curr_prefix + "R reticulate::conda_list():",
"\n%s%s" % (curr_prefix + prefix,
("\n" + curr_prefix + prefix).join(
out.splitlines(False)))))
# Windows python versions
if platform._is_win: # pragma: windows
out = Rdrv.run_executable([
"-e",
("library(reticulate); "
"reticulate::py_versions_windows()")
],
env=env_reticulate).strip()
vardict.append(
(curr_prefix + "R reticulate::py_versions_windows():",
"\n%s%s" % (curr_prefix + prefix,
("\n" + curr_prefix + prefix).join(
out.splitlines(False)))))
# conda_binary
if platform._is_win: # pragma: windows
out = Rdrv.run_executable([
"-e",
("library(reticulate); "
"conda <- reticulate:::conda_binary(\"auto\"); "
"system(paste(conda, \"info --json\"))")
],
env=env_reticulate).strip()
vardict.append(
(curr_prefix + "R reticulate::py_versions_windows():",
"\n%s%s" % (curr_prefix + prefix,
("\n" + curr_prefix + prefix).join(
out.splitlines(False)))))
# Reticulate py_config
out = Rdrv.run_executable([
"-e", ("library(reticulate); "
"reticulate::py_config()")
],
env=env_reticulate).strip()
vardict.append((curr_prefix + "R reticulate::py_config():",
"\n%s%s" % (curr_prefix + prefix,
("\n" + curr_prefix + prefix).join(
out.splitlines(False)))))
finally:
# Print things
max_len = max(len(x[0]) for x in vardict)
lines = []
line_format = '%-' + str(max_len) + 's' + prefix + '%s'
for k, v in vardict:
lines.append(line_format % (k, v))
logger.info("yggdrasil info:\n%s" % '\n'.join(lines))
def register_example(example_dir):
r"""Register an example based on the contents of the director.
Args:
example_dir (str): Full path to a directory potentially containing an
example.
Returns:
tuple (list, dict, dict): A list of available languages, a dictionary
mapping from language to YAML specification files for the example,
and a dictionary mapping from language to source files for the
example.
"""
# Check that the source directory and test exist
example_base = os.path.basename(example_dir)
srcdir = os.path.join(example_dir, 'src')
testfile = os.path.join(_example_dir, 'tests', 'test_%s.py' % example_base)
if not os.path.isfile(testfile): # pragma: no cover
# TODO: Automate test creation
if (((not tools.is_subprocess())
and (example_base not in ['sbml1', 'sbml2', 'sbml3']))):
logging.error("Missing test file: %s" % testfile)
if not os.path.isdir(srcdir): # pragma: no cover
if not tools.is_subprocess():
logging.error("Missing source directory: %s" % srcdir)
return {}
# Determine which languages are present in the example
lang_base = []
lang_avail = []
lang_search = None
if example_base in ['rpcFib', 'maxMsg']:
lang_search = example_base + 'Cli_%s.yml'
lang_avail += ['all', 'all_nomatlab']
elif example_base.startswith('rpc_'):
lang_search = 'client_%s.yml'
elif example_base == 'root_to_shoot':
lang_avail += ['all', 'all_nomatlab', 'c', 'python']
elif example_base == 'fakeplant':
lang_avail += ['all', 'all_nomatlab', 'c', 'cpp', 'matlab', 'python']
elif example_base in ['types', 'transforms']:
lang_avail += tools.get_supported_lang()
for k in ['cmake', 'make', 'lpy', 'executable']:
lang_avail.remove(k)
elif example_base.startswith('sbml'):
lang_avail = ['sbml']
elif example_base.startswith('osr'):
lang_search = example_base + '_%s.yml'
lang_base += ['osr']
else:
lang_search = example_base + '_%s.yml'
if lang_search is not None:
for match in glob.glob(os.path.join(example_dir, lang_search % '*')):
lang = serialize.process_message(os.path.basename(match),
lang_search)[0]
if lang == 'valgrind':
continue
lang_avail.append(lang.lower())
lang_avail = tuple(sorted(lang_avail))
lang_base = tuple(sorted(lang_base))
# Get YAML and source files for each language
out_yml = {}
out_src = {}
src_is_abs = False
for lang in lang_avail:
yml_names = []
src_names = []
if example_base == 'rpcFib':
if lang == 'all':
lang_set = ('python', 'matlab', 'c')
elif lang == 'all_nomatlab':
lang_set = ('python', 'cpp', 'c')
else:
lang_set = (lang, lang, lang)
yml_names = [
'%sCli_%s.yml' % (example_base, lang_set[0]),
'%sCliPar_%s.yml' % (example_base, lang_set[1]),
'%sSrv_%s.yml' % (example_base, lang_set[2])
]
src_names = [
'%sCli%s' % (example_base, ext_map[lang_set[0]]),
'%sCliPar%s' % (example_base, ext_map[lang_set[1]]),
'%sSrv%s' % (example_base, ext_map[lang_set[2]])
]
elif example_base == 'maxMsg':
if lang == 'all':
lang_set = ('python', 'matlab')
elif lang == 'all_nomatlab':
lang_set = ('python', 'c')
else:
lang_set = (lang, lang)
yml_names = [
'%sCli_%s.yml' % (example_base, lang_set[0]),
'%sSrv_%s.yml' % (example_base, lang_set[1])
]
src_names = [
'%sCli%s' % (example_base, ext_map[lang_set[0]]),
'%sSrv%s' % (example_base, ext_map[lang_set[1]])
]
elif example_base.startswith('rpc_'):
# TODO: Create server examples in other languages
yml_names = ['server_python.yml', 'client_%s.yml' % lang]
src_names = ['server.py', 'client%s' % ext_map[lang]]
elif example_base == 'root_to_shoot':
if lang.startswith('all'):
yml_names = ['root.yml', 'shoot.yml', 'root_to_shoot.yml']
src_names = ['root.c', 'shoot.py']
elif lang == 'python':
yml_names = ['shoot.yml', 'shoot_files.yml']
src_names = ['shoot.py']
elif lang == 'c':
yml_names = ['root.yml', 'root_files.yml']
src_names = ['root.c']
elif example_base == 'fakeplant':
if lang.startswith('all'):
yml_names = [
'canopy.yml', 'light.yml', 'photosynthesis.yml',
'fakeplant.yml'
]
src_names = ['canopy.cpp', 'light.c', 'photosynthesis.py']
if lang == 'all_nomatlab':
yml_names.append('growth_python.yml')
src_names.append('growth.py')
else:
yml_names.append('growth.yml')
src_names.append('growth.m')
elif lang == 'python':
yml_names = ['photosynthesis.yml']
src_names = ['photosynthesis.py']
elif lang == 'c':
yml_names = ['light.yml', 'light_files.yml']
src_names = ['light.c']
elif lang == 'cpp':
yml_names = ['canopy.yml', 'canopy_files.yml']
src_names = ['canopy.cpp']
elif lang == 'matlab':
yml_names = ['growth.yml', 'growth_files.yml']
src_names = ['growth.m']
elif example_base in ['types', 'transforms']:
yml_names = ['%s.yml' % example_base]
src_names = ['src.py', 'dst.py']
elif example_base.startswith('sbml'):
yml_names = ['%s.yml' % example_base]
src_names = ['%s.xml' % example_base]
else:
src_is_abs = True
yml_names = ['%s_%s.yml' % (example_base, lang)]
if lang.startswith('all'):
src_names = []
for lsrc in lang_avail:
if lsrc.startswith('all'):
continue
elif (lang == 'all_nomatlab') and (lsrc == 'matlab'):
continue
src_names += sorted(
glob.glob(os.path.join(srcdir, '*' + ext_map[lsrc])))
else:
src_names = sorted(
glob.glob(os.path.join(srcdir,
'*' + ext_map.get(lang, ''))))
for ilang_base in lang_base:
src_names += sorted(
glob.glob(
os.path.join(srcdir,
'*' + ext_map.get(ilang_base, ''))))
out_yml[lang] = [os.path.join(example_dir, y) for y in yml_names]
if src_is_abs:
out_src[lang] = src_names
else:
out_src[lang] = [os.path.join(srcdir, s) for s in src_names]
if len(out_yml[lang]) == 1:
out_yml[lang] = out_yml[lang][0]
if len(out_src[lang]) == 1:
out_src[lang] = out_src[lang][0]
return lang_base, lang_avail, out_yml, out_src