import os import copy import pprint import yaml from yggdrasil import tools from yggdrasil.components import import_component from yggdrasil.languages import get_language_ext from yggdrasil.metaschema.datatypes import get_type_class from yggdrasil.tests import long_running from yggdrasil.examples import _example_dir from yggdrasil.examples.tests import ExampleTstBase from yggdrasil.metaschema.datatypes import encode_type _all_lang = tools.get_supported_lang() _typed_lang = tuple( [x for x in _all_lang if import_component('model', x).is_typed]) @long_running class TestExampleTypes(ExampleTstBase): r"""Test the Types example.""" example_name = 'types' iter_over = ['language', 'type', 'method'] iter_skip = [(set(_typed_lang), '*', 'run_example_generic'), (set(_typed_lang), '*', 'run_example_pointers'), (set(['c']), '*', 'run_example_c_nolengths'), (set(['c']), '*', 'run_example_c_prefixes'), ('c', set(['string', 'bytes', 'unicode']), 'run_example_c_nolengths'), ('c', set(['1darray', 'ndarray']), 'run_example_c_prefixes'),
def update_language_config(languages=None, skip_warnings=False, disable_languages=None, enable_languages=None, lang_kwargs=None, overwrite=False, verbose=False): r"""Update configuration options for a language driver. Args: languages (list, optional): List of languages to configure. Defaults to None and all supported languages will be configured. skip_warnings (bool, optional): If True, warnings about missing options will not be raised. Defaults to False. disable_languages (list, optional): List of languages that should be disabled. Defaults to an empty list. enable_languages (list, optional): List of languages that should be enabled. Defaults to an empty list. overwrite (bool, optional): If True, the existing file will be overwritten. Defaults to False. verbose (bool, optional): If True, information about the config file will be displayed. Defaults to False. lang_kwargs (dict, optional): Dictionary containing language specific keyword arguments. Defaults to {}. """ from yggdrasil.components import import_component if verbose: logger.info( "Updating user configuration file for yggdrasil at:\n\t%s" % usr_config_file) miss = [] if (languages is None) or overwrite: all_languages = tools.get_supported_lang() languages = [ 'c', 'c++', 'make', 'cmake', 'python', 'lpy', 'r', 'matlab' ] for lang in all_languages: if lang.lower() not in languages: languages.append(lang) elif not isinstance(languages, list): languages = [languages] if disable_languages is None: disable_languages = [] if enable_languages is None: enable_languages = [] if lang_kwargs is None: lang_kwargs = {} if overwrite: shutil.copy(def_config_file, usr_config_file) ygg_cfg_usr.reload() drivers = OrderedDict([(lang, import_component('model', lang)) for lang in languages]) drv = list(get_language_order(drivers).values()) for idrv in drv: if (((idrv.language in disable_languages) and (idrv.language in enable_languages))): logger.info(("%s language both enabled and disabled. " "No action will be taken.") % idrv.language) elif idrv.language in disable_languages: ygg_cfg_usr.set(idrv.language, 'disable', 'True') elif idrv.language in enable_languages: ygg_cfg_usr.set(idrv.language, 'disable', 'False') if ygg_cfg_usr.get(idrv.language, 'disable', 'False').lower() == 'true': continue # pragma: no cover miss += idrv.configure(ygg_cfg_usr, **lang_kwargs.get(idrv.language, {})) ygg_cfg_usr.update_file() ygg_cfg.reload() if not skip_warnings: for sect, opt, desc in miss: # pragma: windows warnings.warn(("Could not set option %s in section %s. " + "Please set this in %s to: %s") % (opt, sect, ygg_cfg_usr.file_to_update, desc), RuntimeWarning)
def test_get_supported(): r"""Test get_supported_<platforms/lang/comm>.""" tools.get_supported_platforms() tools.get_supported_lang() tools.get_supported_comm()
'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', 'sphinx.ext.githubpages', 'sphinx.ext.napoleon', 'sphinxarg.ext', 'breathe', ] relative_files = [] for lang in tools.get_supported_lang(): if lang == 'python': continue try: lang_dir = languages.get_language_dir(lang) lang_ext = components.import_component('model', lang).get_language_ext() except ValueError: continue for iext in lang_ext: relative_files += [os.path.relpath(f, start=srcdir) for f in glob.glob(os.path.join(lang_dir, '*' + iext))] relative_files += [os.path.relpath(f, start=srcdir) for f in glob.glob(os.path.join(lang_dir, '*', '*' + iext))] breathe_projects = {"yggdrasil": doxydir} breathe_default_project = "yggdrasil"
def update_config(): r"""Update the user config file for yggdrasil.""" from yggdrasil import config, tools, platform parser = argparse.ArgumentParser( description='Update the user config file.') parser.add_argument( '--show-file', action='store_true', help='Print the path to the config file without updating it.') parser.add_argument('--remove-file', action='store_true', help='Remove the existing config file and return.') parser.add_argument('--overwrite', action='store_true', help='Overwrite the existing file.') parser.add_argument('--languages', nargs='+', default=tools.get_supported_lang(), help=('One or more languages that should be' 'configured.')) parser.add_argument('--disable-languages', nargs='+', default=[], dest='disable_languages', help='One or more languages that should be disabled.') parser.add_argument('--enable-languages', nargs='+', default=[], help='One or more languages that should be enabled.') if ('-h' in sys.argv) or ('--help' in sys.argv): prelang = tools.get_supported_lang() else: prelang = parser.parse_known_args()[0].languages lang_args = {} lang_args2kwargs = {} if platform._is_mac: lang_args.setdefault('c', []) lang_args['c'].append((('--macos-sdkroot', '--sdkroot'), { 'help': ('The full path to the MacOS SDK ' 'that should be used.') })) for lang in prelang: if lang in lang_args: lang_args2kwargs[lang] = [] for args, kwargs in lang_args.get(lang, []): parser.add_argument(*args, **kwargs) lang_args2kwargs[lang].append(parser._actions[-1].dest) args = parser.parse_args() lang_kwargs = { lang: {k: getattr(args, k) for k in alist} for lang, alist in lang_args2kwargs.items() } if args.show_file: print('Config file located here: %s' % config.usr_config_file) if args.remove_file and os.path.isfile(config.usr_config_file): os.remove(config.usr_config_file) if args.show_file or args.remove_file: return config.update_language_config(args.languages, overwrite=args.overwrite, verbose=True, disable_languages=args.disable_languages, enable_languages=args.enable_languages, lang_kwargs=lang_kwargs)
def ygginfo(): r"""Print information about yggdrasil installation.""" from yggdrasil import __version__, tools, config, platform from yggdrasil.components import import_component lang_list = tools.get_installed_lang() prefix = ' ' curr_prefix = '' vardict = [ ('Location', os.path.dirname(__file__)), ('Version', __version__), ('Languages', ', '.join(lang_list)), ('Communication Mechanisms', ', '.join(tools.get_installed_comm())), ('Default Comm Mechanism', tools.get_default_comm()), ('Config File', config.usr_config_file) ] parser = argparse.ArgumentParser( description= 'Display information about the current yggdrasil installation.') parser.add_argument( '--no-languages', action='store_true', dest='no_languages', help='Don\'t print information about individual languages.') parser.add_argument( '--verbose', action='store_true', help='Increase the verbosity of the printed information.') args = parser.parse_args() try: # Add language information if not args.no_languages: # Install languages vardict.append(('Installed Languages:', '')) curr_prefix += prefix for lang in sorted(lang_list): drv = import_component('model', lang) vardict.append((curr_prefix + '%s:' % lang.upper(), '')) curr_prefix += prefix if lang == 'executable': vardict.append((curr_prefix + 'Location', '')) else: exec_name = drv.language_executable() if not os.path.isabs(exec_name): exec_name = shutil.which(exec_name) vardict.append((curr_prefix + 'Location', exec_name)) vardict.append( (curr_prefix + 'Version', drv.language_version())) curr_prefix = curr_prefix.rsplit(prefix, 1)[0] curr_prefix = curr_prefix.rsplit(prefix, 1)[0] # Not installed languages vardict.append(("Languages Not Installed:", '')) curr_prefix += prefix for lang in tools.get_supported_lang(): if lang in lang_list: continue drv = import_component('model', lang) vardict.append((curr_prefix + '%s:' % lang.upper(), '')) curr_prefix += prefix vardict.append((curr_prefix + "Language Installed", drv.is_language_installed())) vardict.append((curr_prefix + "Base Languages Installed", drv.are_base_languages_installed())) if not drv.are_base_languages_installed(): vardict.append( (curr_prefix + "Base Languages Not Installed", [ b for b in drv.base_languages if (not import_component('model', b).is_installed()) ])) vardict.append((curr_prefix + "Dependencies Installed", drv.are_dependencies_installed())) vardict.append((curr_prefix + "Interface Installed", drv.is_interface_installed())) vardict.append( (curr_prefix + "Comm Installed", drv.is_comm_installed())) vardict.append( (curr_prefix + "Configured", drv.is_configured())) vardict.append((curr_prefix + "Disabled", drv.is_disabled())) curr_prefix = curr_prefix.rsplit(prefix, 1)[0] curr_prefix = curr_prefix.rsplit(prefix, 1)[0] # Add verbose information if args.verbose: # Conda info if os.environ.get('CONDA_PREFIX', ''): if platform._is_win: # pragma: windows out = tools.bytes2str( subprocess.check_output('conda info', shell=True)).strip() else: out = tools.bytes2str( subprocess.check_output(['conda', 'info'])).strip() curr_prefix += prefix vardict.append((curr_prefix + 'Conda Info:', "\n%s%s" % (curr_prefix + prefix, ("\n" + curr_prefix + prefix).join( out.splitlines(False))))) curr_prefix = curr_prefix.rsplit(prefix, 1)[0] # R and reticulate info Rdrv = import_component("model", "R") if Rdrv.is_installed(): env_reticulate = copy.deepcopy(os.environ) env_reticulate['RETICULATE_PYTHON'] = sys.executable # Stack size out = Rdrv.run_executable(["-e", "Cstack_info()"]).strip() vardict.append((curr_prefix + "R Cstack_info:", "\n%s%s" % (curr_prefix + prefix, ("\n" + curr_prefix + prefix).join( out.splitlines(False))))) # Compilation tools interp = 'R'.join(Rdrv.get_interpreter().rsplit('Rscript', 1)) vardict.append((curr_prefix + "R C Compiler:", "")) curr_prefix += prefix for x in ['CC', 'CFLAGS', 'CXX', 'CXXFLAGS']: out = tools.bytes2str( subprocess.check_output([interp, 'CMD', 'config', x])).strip() vardict.append((curr_prefix + x, "%s" % ("\n" + curr_prefix + prefix).join( out.splitlines(False)))) curr_prefix = curr_prefix.rsplit(prefix, 1)[0] # Session info out = Rdrv.run_executable(["-e", "sessionInfo()"]).strip() vardict.append((curr_prefix + "R sessionInfo:", "\n%s%s" % (curr_prefix + prefix, ("\n" + curr_prefix + prefix).join( out.splitlines(False))))) # Reticulate conda_list if os.environ.get('CONDA_PREFIX', ''): out = Rdrv.run_executable([ "-e", ("library(reticulate); " "reticulate::conda_list()") ], env=env_reticulate).strip() vardict.append( (curr_prefix + "R reticulate::conda_list():", "\n%s%s" % (curr_prefix + prefix, ("\n" + curr_prefix + prefix).join( out.splitlines(False))))) # Windows python versions if platform._is_win: # pragma: windows out = Rdrv.run_executable([ "-e", ("library(reticulate); " "reticulate::py_versions_windows()") ], env=env_reticulate).strip() vardict.append( (curr_prefix + "R reticulate::py_versions_windows():", "\n%s%s" % (curr_prefix + prefix, ("\n" + curr_prefix + prefix).join( out.splitlines(False))))) # conda_binary if platform._is_win: # pragma: windows out = Rdrv.run_executable([ "-e", ("library(reticulate); " "conda <- reticulate:::conda_binary(\"auto\"); " "system(paste(conda, \"info --json\"))") ], env=env_reticulate).strip() vardict.append( (curr_prefix + "R reticulate::py_versions_windows():", "\n%s%s" % (curr_prefix + prefix, ("\n" + curr_prefix + prefix).join( out.splitlines(False))))) # Reticulate py_config out = Rdrv.run_executable([ "-e", ("library(reticulate); " "reticulate::py_config()") ], env=env_reticulate).strip() vardict.append((curr_prefix + "R reticulate::py_config():", "\n%s%s" % (curr_prefix + prefix, ("\n" + curr_prefix + prefix).join( out.splitlines(False))))) finally: # Print things max_len = max(len(x[0]) for x in vardict) lines = [] line_format = '%-' + str(max_len) + 's' + prefix + '%s' for k, v in vardict: lines.append(line_format % (k, v)) logger.info("yggdrasil info:\n%s" % '\n'.join(lines))
def register_example(example_dir): r"""Register an example based on the contents of the director. Args: example_dir (str): Full path to a directory potentially containing an example. Returns: tuple (list, dict, dict): A list of available languages, a dictionary mapping from language to YAML specification files for the example, and a dictionary mapping from language to source files for the example. """ # Check that the source directory and test exist example_base = os.path.basename(example_dir) srcdir = os.path.join(example_dir, 'src') testfile = os.path.join(_example_dir, 'tests', 'test_%s.py' % example_base) if not os.path.isfile(testfile): # pragma: no cover # TODO: Automate test creation if (((not tools.is_subprocess()) and (example_base not in ['sbml1', 'sbml2', 'sbml3']))): logging.error("Missing test file: %s" % testfile) if not os.path.isdir(srcdir): # pragma: no cover if not tools.is_subprocess(): logging.error("Missing source directory: %s" % srcdir) return {} # Determine which languages are present in the example lang_base = [] lang_avail = [] lang_search = None if example_base in ['rpcFib', 'maxMsg']: lang_search = example_base + 'Cli_%s.yml' lang_avail += ['all', 'all_nomatlab'] elif example_base.startswith('rpc_'): lang_search = 'client_%s.yml' elif example_base == 'root_to_shoot': lang_avail += ['all', 'all_nomatlab', 'c', 'python'] elif example_base == 'fakeplant': lang_avail += ['all', 'all_nomatlab', 'c', 'cpp', 'matlab', 'python'] elif example_base in ['types', 'transforms']: lang_avail += tools.get_supported_lang() for k in ['cmake', 'make', 'lpy', 'executable']: lang_avail.remove(k) elif example_base.startswith('sbml'): lang_avail = ['sbml'] elif example_base.startswith('osr'): lang_search = example_base + '_%s.yml' lang_base += ['osr'] else: lang_search = example_base + '_%s.yml' if lang_search is not None: for match in glob.glob(os.path.join(example_dir, lang_search % '*')): lang = serialize.process_message(os.path.basename(match), lang_search)[0] if lang == 'valgrind': continue lang_avail.append(lang.lower()) lang_avail = tuple(sorted(lang_avail)) lang_base = tuple(sorted(lang_base)) # Get YAML and source files for each language out_yml = {} out_src = {} src_is_abs = False for lang in lang_avail: yml_names = [] src_names = [] if example_base == 'rpcFib': if lang == 'all': lang_set = ('python', 'matlab', 'c') elif lang == 'all_nomatlab': lang_set = ('python', 'cpp', 'c') else: lang_set = (lang, lang, lang) yml_names = [ '%sCli_%s.yml' % (example_base, lang_set[0]), '%sCliPar_%s.yml' % (example_base, lang_set[1]), '%sSrv_%s.yml' % (example_base, lang_set[2]) ] src_names = [ '%sCli%s' % (example_base, ext_map[lang_set[0]]), '%sCliPar%s' % (example_base, ext_map[lang_set[1]]), '%sSrv%s' % (example_base, ext_map[lang_set[2]]) ] elif example_base == 'maxMsg': if lang == 'all': lang_set = ('python', 'matlab') elif lang == 'all_nomatlab': lang_set = ('python', 'c') else: lang_set = (lang, lang) yml_names = [ '%sCli_%s.yml' % (example_base, lang_set[0]), '%sSrv_%s.yml' % (example_base, lang_set[1]) ] src_names = [ '%sCli%s' % (example_base, ext_map[lang_set[0]]), '%sSrv%s' % (example_base, ext_map[lang_set[1]]) ] elif example_base.startswith('rpc_'): # TODO: Create server examples in other languages yml_names = ['server_python.yml', 'client_%s.yml' % lang] src_names = ['server.py', 'client%s' % ext_map[lang]] elif example_base == 'root_to_shoot': if lang.startswith('all'): yml_names = ['root.yml', 'shoot.yml', 'root_to_shoot.yml'] src_names = ['root.c', 'shoot.py'] elif lang == 'python': yml_names = ['shoot.yml', 'shoot_files.yml'] src_names = ['shoot.py'] elif lang == 'c': yml_names = ['root.yml', 'root_files.yml'] src_names = ['root.c'] elif example_base == 'fakeplant': if lang.startswith('all'): yml_names = [ 'canopy.yml', 'light.yml', 'photosynthesis.yml', 'fakeplant.yml' ] src_names = ['canopy.cpp', 'light.c', 'photosynthesis.py'] if lang == 'all_nomatlab': yml_names.append('growth_python.yml') src_names.append('growth.py') else: yml_names.append('growth.yml') src_names.append('growth.m') elif lang == 'python': yml_names = ['photosynthesis.yml'] src_names = ['photosynthesis.py'] elif lang == 'c': yml_names = ['light.yml', 'light_files.yml'] src_names = ['light.c'] elif lang == 'cpp': yml_names = ['canopy.yml', 'canopy_files.yml'] src_names = ['canopy.cpp'] elif lang == 'matlab': yml_names = ['growth.yml', 'growth_files.yml'] src_names = ['growth.m'] elif example_base in ['types', 'transforms']: yml_names = ['%s.yml' % example_base] src_names = ['src.py', 'dst.py'] elif example_base.startswith('sbml'): yml_names = ['%s.yml' % example_base] src_names = ['%s.xml' % example_base] else: src_is_abs = True yml_names = ['%s_%s.yml' % (example_base, lang)] if lang.startswith('all'): src_names = [] for lsrc in lang_avail: if lsrc.startswith('all'): continue elif (lang == 'all_nomatlab') and (lsrc == 'matlab'): continue src_names += sorted( glob.glob(os.path.join(srcdir, '*' + ext_map[lsrc]))) else: src_names = sorted( glob.glob(os.path.join(srcdir, '*' + ext_map.get(lang, '')))) for ilang_base in lang_base: src_names += sorted( glob.glob( os.path.join(srcdir, '*' + ext_map.get(ilang_base, '')))) out_yml[lang] = [os.path.join(example_dir, y) for y in yml_names] if src_is_abs: out_src[lang] = src_names else: out_src[lang] = [os.path.join(srcdir, s) for s in src_names] if len(out_yml[lang]) == 1: out_yml[lang] = out_yml[lang][0] if len(out_src[lang]) == 1: out_src[lang] = out_src[lang][0] return lang_base, lang_avail, out_yml, out_src