def test_default_schema(): r"""Test getting default schema.""" s = schema.get_schema() assert (s is not None) schema.clear_schema() assert (schema._schema is None) s = schema.get_schema() assert (s is not None) for k in s.keys(): assert (isinstance(s[k].subtypes, list)) assert (isinstance(s[k].classes, list)) for ksub in s[k].classes: s[k].get_subtype_properties(ksub)
def executeGraph(self, graph): """Execute graph.""" cisgraph = fbpToCis(graph['content']) user = self.getCurrentUser() username = user['login'] cisgraph = as_str(cisgraph, recurse=True, allow_pass=True) # Write to temp file and validate tmpfile = tempfile.NamedTemporaryFile(suffix="yml", prefix="cis", delete=False) yaml.safe_dump(cisgraph, tmpfile, default_flow_style=False) yml_prep = prep_yaml(tmpfile.name) os.remove(tmpfile.name) s = get_schema() yml_norm = s.normalize(yml_prep) try: s.validate(yml_norm) except BaseException as e: print(e) raise RestException('Invalid graph %s', 400, e) self.setRawResponse() yaml_graph = pyaml.dump(cisgraph) #print('Executing graph: ' + str(yaml_graph)) return execGraph(yaml_graph, username)
def generate_component_tests(comptype, base_class, target_globals, directory, class_attr='_cls'): r"""Function that generates tests for all component subtypes based on the registered classes. Args: comptype (str): Component type. base_class (class): Base python test class. target_globals (dict): Globals dictionary that class should be added to. directory (str): Directory to check for tests or file in directory that should be checked for tests. class_attr (str, optional): Attribute that should be set to the name of the class being tested. """ from yggdrasil.schema import get_schema _schema = get_schema() if os.path.isfile(directory): directory = os.path.dirname(directory) if comptype not in _schema.keys(): # pragma: debug raise NotImplementedError("%s is not a component type." % comptype) for subtype in _schema[comptype].subtypes: subtype_cls = _schema[comptype].subtype2class[subtype] new_cls_name = 'Test%s' % subtype_cls new_cls_file = os.path.join(directory, 'test_%s' % subtype_cls) + '.py' if os.path.isfile(new_cls_file): continue cls_attr = {class_attr: subtype_cls} new_cls = type(new_cls_name, (base_class, ), cls_attr) target_globals[new_cls.__name__] = new_cls del new_cls
def create_component(comptype, subtype=None, **kwargs): r"""Dynamically create an instance of a component with the specified options as outlined in the component schema. This function requires loading the component schemas and so should not be used at the module or class level to prevent circular dependencies. Args: comptype (str): Component type. subtype (str, optional): Component subtype. If subtype is not one of the registered subtypes for the specified comptype, subtype is treated as the name of the class. Defaults to None if not provided and the default subtype defined in the schema for the specified component will be used. If the subtype is specified by the component subtype key in the remaining kwargs, that subtype will be used instead. **kwargs: Additional keyword arguments are treated as options for the component as outlined in the component schema. Returns: ComponentBase: Instance of the specified component type/subtype and options. Raises: ValueError: If comptype is not a registered component type. """ from yggdrasil.schema import get_schema s = get_schema().get(comptype, None) if s is None: # pragma: debug raise ValueError("Unrecognized component type: %s" % comptype) if s.subtype_key in kwargs: subtype = kwargs[s.subtype_key] if subtype is None: subtype = s.identify_subtype(kwargs) cls = import_component(comptype, subtype=subtype, **kwargs) return cls(**kwargs)
def test_get_schema_subtype(): r"""Test get_schema_subtype for allow_instance=True.""" component = 'serializer' subtype = 'direct' doc = {'seritype': subtype} valid = components.create_component(component, seritype=subtype) invalid = components.create_component(component, seritype='json') s = schema.get_schema() kwargs = {'subtype': subtype, 'allow_instance': True} s.validate_component(component, doc, **kwargs) s.validate_component(component, valid, **kwargs) assert_raises(ValidationError, s.validate_component, component, invalid, **kwargs) s.validate_component(component, doc, subtype=subtype) assert_raises(ValidationError, s.validate_component, component, valid, subtype=subtype) # Test for base s.validate_component(component, valid, subtype='base', allow_instance=True) s.validate_component(component, invalid, subtype='base', allow_instance=True)
def write_component_table(comp='all', table_type='all', **kwargs): r"""Write a component table to a file. Args: comp (str): Name of a component type to create a table for. Defaults to 'all' and tables are created for each of the registered components. table_type (str, optional): Type of table that should be created for the component. Defaults to 'all'. Supported values include: * 'all': Create each type of table for the specified component. * 'general': Create a table of properties common to all components of the specified type. * 'specific': Create a table of properties specific to only some components of the specified type. * 'subtype': Create a table describing the subtypes available for the specified component type. **kwargs: Additional keyword arguments are passed to component2table and write_table. Returns: str, list: Name of file or files created. """ from yggdrasil.schema import get_schema s = get_schema() table_type_list = ['subtype', 'general', 'specific'] # Loop if comp == 'all': fname = kwargs.get("fname", None) fname_base = kwargs.get("fname_base", None) assert ((fname is None) and (fname_base is None)) out = [] for k in s.keys(): new_out = write_component_table(comp=k, table_type=table_type, **kwargs) if isinstance(new_out, list): out += new_out else: out.append(new_out) return out if table_type == 'all': fname = kwargs.get("fname", None) fname_base = kwargs.get("fname_base", None) assert ((fname is None) and (fname_base is None)) out = [] for k in table_type_list: out.append(write_component_table(comp=comp, table_type=k, **kwargs)) return out # Construct file name fname_format = 'schema_table_%s_%s.rst' kwargs.setdefault('subtype_ref', (fname_format % (comp, 'subtype')).replace( '.rst', '_rst')) kwargs.setdefault('fname_base', fname_format % (comp, table_type)) lines = component2table(comp, table_type, **kwargs) return write_table(lines, **kwargs)
def _init_single_comm(self, name, io, comm_kws, **kwargs): r"""Parse keyword arguments for input/output comm.""" self.debug("Creating %s comm", io) s = get_schema() if comm_kws is None: comm_kws = dict() if io == 'input': direction = 'recv' comm_type = self._icomm_type touches_model = self._is_output attr_comm = 'icomm' comm_kws['close_on_eof_recv'] = False else: direction = 'send' comm_type = self._ocomm_type touches_model = self._is_input attr_comm = 'ocomm' comm_kws['direction'] = direction comm_kws['dont_open'] = True comm_kws['reverse_names'] = True comm_kws.setdefault('comm', {'comm': comm_type}) assert (name == self.name) comm_kws.setdefault('name', name) if not isinstance(comm_kws['comm'], list): comm_kws['comm'] = [comm_kws['comm']] for i, x in enumerate(comm_kws['comm']): if x is None: comm_kws['comm'][i] = dict() elif not isinstance(x, dict): comm_kws['comm'][i] = dict(comm=x) comm_kws['comm'][i].setdefault('comm', comm_type) any_files = False all_files = True if not touches_model: comm_kws['no_suffix'] = True ikws = [] for x in comm_kws['comm']: if get_comm_class(x['comm']).is_file: any_files = True ikws += s['file'].get_subtype_properties(x['comm']) else: all_files = False ikws += s['comm'].get_subtype_properties(x['comm']) ikws = list(set(ikws)) for k in ikws: if (k not in comm_kws) and (k in kwargs): comm_kws[k] = kwargs.pop(k) if ('comm_env' in kwargs) and ('comm_env' not in comm_kws): comm_kws['env'] = kwargs.pop('comm_env') if any_files and (io == 'input'): kwargs.setdefault('timeout_send_1st', 60) self.debug('%s comm_kws:\n%s', attr_comm, self.pprint(comm_kws, 1)) setattr(self, attr_comm, new_comm(comm_kws.pop('name'), **comm_kws)) setattr(self, '%s_kws' % attr_comm, comm_kws) if touches_model: self.env.update(getattr(self, attr_comm).opp_comms) elif not all_files: self.comm_env.update(getattr(self, attr_comm).opp_comms) return kwargs
def test_default_schema(): r"""Test getting default schema.""" s = schema.get_schema() assert (s is not None) schema.clear_schema() assert (schema._schema is None) s = schema.get_schema() assert (s is not None) for k in s.keys(): assert (isinstance(s[k].subtypes, list)) assert (isinstance(s[k].classes, list)) for ksub in s[k].classes: s[k].get_subtype_properties(ksub) s[k].default_subtype s.get_schema(relaxed=True) s.get_schema(allow_instance=True) s.definitions s.form_schema
def parse_component(yml, ctype, existing=None): r"""Parse a yaml entry for a component, adding it to the list of existing components. Args: yml (dict): YAML dictionary for a component. ctype (str): Component type. This can be 'input', 'output', 'model', or 'connection'. existing (dict, optional): Dictionary of existing components. Defaults to empty dict. Raises: TypeError: If yml is not a dictionary. ValueError: If dtype is not 'input', 'output', 'model', or 'connection'. ValueError: If the component already exists. Returns: dict: All components identified. """ s = get_schema() if not isinstance(yml, dict): raise YAMLSpecificationError( "Component entry in yml must be a dictionary.") ctype_list = [ 'input', 'output', 'model', 'connection', 'model_input', 'model_output' ] if existing is None: existing = {k: {} for k in ctype_list} if ctype not in ctype_list: raise YAMLSpecificationError("'%s' is not a recognized component.") # Parse based on type if ctype == 'model': existing = parse_model(yml, existing) elif ctype == 'connection': existing = parse_connection(yml, existing) elif ctype in ['input', 'output']: for k in ['icomm_kws', 'ocomm_kws']: if k not in yml: continue for x in yml[k]['comm']: if 'comm' not in x: if 'filetype' in x: x['comm'] = s['file'].subtype2class[x['filetype']] elif 'commtype' in x: x['comm'] = s['comm'].subtype2class[x['commtype']] # Ensure component dosn't already exist if yml['name'] in existing[ctype]: pprint.pprint(existing) pprint.pprint(yml) raise YAMLSpecificationError( "%s is already a registered '%s' component." % (yml['name'], ctype)) existing[ctype][yml['name']] = yml return existing
def __init__(self, *args, **kwargs): super(FilterTransform, self).__init__(*args, **kwargs) if isinstance(self.filter, dict): from yggdrasil.schema import get_schema from yggdrasil.components import create_component filter_schema = get_schema().get('filter') filter_kws = dict(self.filter, subtype=filter_schema.identify_subtype( self.filter)) self.filter = create_component('filter', **filter_kws)
def test_normalize(): r"""Test normalization of legacy formats.""" s = schema.get_schema() for x, y in _normalize_objects: a = s.normalize(x, backwards_compat=True) try: assert_equal(a, y) except BaseException: # pragma: debug pprint.pprint(a) pprint.pprint(y) raise
def parse_model(yml, existing): r"""Parse a yaml entry for a model. Args: yml (dict): YAML dictionary for a model. existing (dict): Dictionary of existing components. Returns: dict: Updated log of all entries. """ _lang2driver = get_schema()['model'].subtype2class language = yml.pop('language') yml['driver'] = _lang2driver[language] # Add server driver if yml.get('is_server', False): srv = { 'name': '%s:%s' % (yml['name'], yml['name']), 'commtype': 'default', 'datatype': { 'type': 'bytes' }, 'driver': 'ServerDriver', 'args': yml['name'] + '_SERVER', 'working_dir': yml['working_dir'] } yml['inputs'].append(srv) yml['clients'] = [] # Add client driver if yml.get('client_of', []): srv_names = yml['client_of'] yml['client_of'] = srv_names for srv in srv_names: cli = { 'name': '%s:%s_%s' % (yml['name'], srv, yml['name']), 'commtype': 'default', 'datatype': { 'type': 'bytes' }, 'driver': 'ClientDriver', 'args': srv + '_SERVER', 'working_dir': yml['working_dir'] } yml['outputs'].append(cli) # Model index and I/O channels yml['model_index'] = len(existing['model']) for io in ['inputs', 'outputs']: for x in yml[io]: x['model_driver'] = [yml['name']] x['partner_language'] = language existing = parse_component(x, io[:-1], existing=existing) return existing
def test_normalize(): r"""Test normalization of legacy formats.""" s = schema.get_schema() for x, y in _normalize_objects: a = s.normalize(x, backwards_compat=True) # , show_errors=True) try: assert_equal(a, y) except BaseException: # pragma: debug print("Unexpected Normalization:\n\nA:") pprint.pprint(a) print('\nB:') pprint.pprint(y) raise
def get_supported_comm(): r"""Get a list of the communication mechanisms supported by yggdrasil. Returns: list: The names of communication mechanisms supported by yggdrasil. """ from yggdrasil import schema s = schema.get_schema() out = s['comm'].classes for k in ['CommBase', 'DefaultComm']: if k in out: out.remove(k) return list(set(out))
def get_supported_lang(): r"""Get a list of the model programming languages that are supported by yggdrasil. Returns: list: The names of programming languages supported by yggdrasil. """ from yggdrasil import schema s = schema.get_schema() out = s['model'].subtypes if 'c++' in out: out[out.index('c++')] = 'cpp' return list(set(out))
def generate_component_subtests(comptype, suffix, target_globals, parent_module_name, new_attr=None, module_name_format='test_%s', class_name_format='Test%s', skip_subtypes=[]): r"""Function that generates tests for all component subtypes that subclass the original test class for the subtype. Args: comptype (str): Component type. suffix (str): Suffix to be added to the end of the original test class name to get the name for the new test. target_globals (dict): Globals dictionary that class should be added to. parent_module_name (str): Parent module that contains test classes for the specified component type. new_attr (dict, optional): Attributes to add to the test classes. Defaults to None and will be an empty dict. module_name_format (str, optional): Format string that should be used to locate the original test modules. Defaults to 'test_%s'. class_name_format (str, optional): Format string that should be used to both name generated test classes and locate the original test classes. Defaults to 'Test%s'. skip_subtypes (list, optional): List of subtypes to skip. Defaults to empty list. """ from yggdrasil.schema import get_schema _schema = get_schema() if new_attr is None: # pragma: debug new_attr = {} if comptype not in _schema.keys(): # pragma: debug raise NotImplementedError("%s is not a component type." % comptype) for subtype in _schema[comptype].subtypes: if subtype in skip_subtypes: continue subtype_cls = _schema[comptype].subtype2class[subtype] old_mod_name = (parent_module_name + '.' + (module_name_format % subtype_cls)) old_cls_name = class_name_format % subtype_cls new_cls_name = class_name_format % (subtype_cls + suffix.title()) base_class = getattr(importlib.import_module(old_mod_name), old_cls_name) new_cls = type(new_cls_name, (base_class, ), new_attr) target_globals[new_cls.__name__] = new_cls del new_cls
def generate_component_tests(comptype, base_class, target_globals, directory, class_attr='_cls', new_attr=None, class_name_format='Test%s', class_file_format='test_%s.py'): r"""Function that generates tests for all component subtypes based on the registered classes. Args: comptype (str): Component type. base_class (class): Base python test class. target_globals (dict): Globals dictionary that class should be added to. directory (str): Directory to check for tests or file in directory that should be checked for tests. class_attr (str, optional): Attribute that should be set to the name of the class being tested. new_attr (dict, optional): Attributes to add to the test classes. Defaults to None and will be an empty dict. class_name_format (str, optional): Format string that should be used to name generated test class's. Defaults to 'Test%s'. class_file_format (str, optional): Format string that should be used to name test files that are checked for. Defaults to 'test_%s.py'. """ from yggdrasil.schema import get_schema _schema = get_schema() if new_attr is None: new_attr = {} if os.path.isfile(directory): directory = os.path.dirname(directory) if comptype not in _schema.keys(): # pragma: debug raise NotImplementedError("%s is not a component type." % comptype) for subtype in _schema[comptype].subtypes: subtype_cls = _schema[comptype].subtype2class[subtype] new_cls_name = class_name_format % subtype_cls new_cls_file = os.path.join(directory, class_file_format % subtype_cls) if os.path.isfile(new_cls_file): continue inew_attr = copy.deepcopy(new_attr) inew_attr.setdefault(class_attr, subtype_cls) new_cls = type(new_cls_name, (base_class, ), inew_attr) target_globals[new_cls.__name__] = new_cls del new_cls
def is_lang_installed(lang): r"""Check to see if yggdrasil can run models written in a programming language on the current machine. Args: lang (str): Programming language to check. Returns: bool: True if models in the provided language can be run on the current machine, False otherwise. """ from yggdrasil import schema, drivers s = schema.get_schema() drv = drivers.import_driver(s['model'].subtype2class[lang]) return drv.is_installed()
def test_create_schema(): r"""Test creating new schema.""" fname = 'test_schema.yml' if os.path.isfile(fname): # pragma: debug os.remove(fname) # Test saving/loading schema s0 = schema.create_schema() s0.save(fname) assert (s0 is not None) assert (os.path.isfile(fname)) s1 = schema.get_schema(fname) assert_equal(s1.schema, s0.schema) # assert_equal(s1, s0) os.remove(fname) # Test getting schema s2 = schema.load_schema(fname) assert (os.path.isfile(fname)) assert_equal(s2, s0) os.remove(fname)
def test_save_load_schema(): r"""Test saving & loading schema.""" fname = 'test_schema.yml' if os.path.isfile(fname): # pragma: debug os.remove(fname) # Test saving/loading schema s0 = schema.load_schema() s0.save(fname) assert (s0 is not None) assert (os.path.isfile(fname)) s1 = schema.get_schema(fname) assert (s1.schema == s0.schema) # assert(s1 == s0) os.remove(fname) # Test getting schema s2 = schema.load_schema(fname) assert (os.path.isfile(fname)) assert (s2.schema == s0.schema) assert (s2 == s0) os.remove(fname)
def parse_yaml(files): r"""Parse list of yaml files. Args: files (str, list): Either the path to a single yaml file or a list of yaml files. Raises: ValueError: If the yml dictionary is missing a required keyword or has an invalid value. RuntimeError: If one of the I/O channels is not initialized with driver information. Returns: dict: Dictionary of information parsed from the yamls. """ s = get_schema() # Parse files using schema yml_prep = prep_yaml(files) # print('prepped') # pprint.pprint(yml_prep) yml_norm = s.validate(yml_prep, normalize=True) # print('normalized') # pprint.pprint(yml_norm) # Parse models, then connections to ensure connections can be processed existing = None for k in ['models', 'connections']: for yml in yml_norm[k]: existing = parse_component(yml, k[:-1], existing=existing) # Make sure that I/O channels initialized for io in ['input', 'output']: for k, v in existing[io].items(): if 'driver' not in v: raise RuntimeError("No driver established for %s channel %s" % (io, k)) # Link io drivers back to models existing = link_model_io(existing) # print('drivers') # pprint.pprint(existing) return existing
def convertGraph(self, graph): """Convert graph.""" cisgraph = fbpToCis(graph['content']) cisgraph = as_str(cisgraph, recurse=True, allow_pass=True) # Write to temp file and validate tmpfile = tempfile.NamedTemporaryFile(suffix="yml", prefix="cis", delete=False) yaml.safe_dump(cisgraph, tmpfile, default_flow_style=False) yml_prep = prep_yaml(tmpfile.name) os.remove(tmpfile.name) s = get_schema() yml_norm = s.normalize(yml_prep) try: s.validate(yml_norm) except BaseException as e: print(e) raise RestException('Invalid graph %s', 400, e) self.setRawResponse() return pyaml.dump(cisgraph)
def parse_model(yml, existing): r"""Parse a yaml entry for a model. Args: yml (dict): YAML dictionary for a model. existing (dict): Dictionary of existing components. Returns: dict: Updated log of all entries. """ _lang2driver = get_schema()['model'].subtype2class language = yml.pop('language') yml['driver'] = _lang2driver[language] # Add server input if yml.get('is_server', False): srv = { 'name': '%s:%s' % (yml['name'], yml['name']), 'commtype': 'default', 'datatype': { 'type': 'bytes' }, 'args': yml['name'] + '_SERVER', 'working_dir': yml['working_dir'] } if yml.get('function', False) and isinstance(yml['is_server'], bool): if (len(yml['inputs']) == 1) and (len(yml['outputs']) == 1): yml['is_server'] = { 'input': yml['inputs'][0]['name'], 'output': yml['outputs'][0]['name'] } else: raise YAMLSpecificationError( "The 'is_server' parameter is boolean for the model '%s' " "and the 'function' parameter is also set. " "If the 'function' and 'is_server' parameters are used " "together, the 'is_server' parameter must be a mapping " "with 'input' and 'output' entries specifying which of " "the function's input/output variables should be received" "/sent from/to clients. e.g. \n" "\t-input: input_variable\n" "\t-output: output_variables\n" % yml['name']) replaces = None if isinstance(yml['is_server'], dict): replaces = {} for io in ['input', 'output']: replaces[io] = None if not yml['is_server'][io].startswith('%s:' % yml['name']): yml['is_server'][io] = '%s:%s' % (yml['name'], yml['is_server'][io]) for i, x in enumerate(yml[io + 's']): if x['name'] == yml['is_server'][io]: replaces[io] = x replaces[io + '_index'] = i yml[io + 's'].pop(i) break else: raise YAMLSpecificationError( "Failed to locate an existing %s channel " "with the name %s." % (io, yml['is_server'][io])) srv['server_replaces'] = replaces yml['inputs'].insert(replaces['input_index'], srv) else: yml['inputs'].append(srv) yml['clients'] = [] existing['server'].setdefault(srv['name'], { 'clients': [], 'model_name': yml['name'] }) if replaces: existing['server'][srv['name']]['replaces'] = replaces # Mark timesync clients timesync = yml.pop('timesync_client_of', []) if timesync: yml.setdefault('client_of', []) yml['client_of'] += timesync # Add client output if yml.get('client_of', []): for srv in yml['client_of']: srv_name = '%s:%s' % (srv, srv) if srv in timesync: cli_name = '%s:%s' % (yml['name'], srv) else: cli_name = '%s:%s_%s' % (yml['name'], srv, yml['name']) cli = {'name': cli_name, 'working_dir': yml['working_dir']} yml['outputs'].append(cli) existing['server'].setdefault(srv_name, { 'clients': [], 'model_name': srv }) existing['server'][srv_name]['clients'].append(cli_name) # Model index and I/O channels yml['model_index'] = len(existing['model']) for io in ['inputs', 'outputs']: for x in yml[io]: if ((yml.get('function', False) and (not x.get('outside_loop', False)) and yml.get('is_server', False))): x.setdefault('dont_copy', True) if yml.get('allow_threading', False) or ((yml.get('copies', 1) > 1) and (not x.get('dont_copy', False))): x['allow_multiple_comms'] = True # TODO: Replace model_driver with partner_model? x['model_driver'] = [yml['name']] x['partner_model'] = yml['name'] if yml.get('copies', 1) > 1: x['partner_copies'] = yml['copies'] x['partner_language'] = language existing = parse_component(x, io[:-1], existing=existing) for k in yml.get('env', {}).keys(): if not isinstance(yml['env'][k], str): yml['env'][k] = json.dumps(yml['env'][k]) return existing
def parse_yaml(files, complete_partial=False, partial_commtype=None, model_only=False, model_submission=False, yaml_param=None, directory_for_clones=None): r"""Parse list of yaml files. Args: files (str, list): Either the path to a single yaml file or a list of yaml files. complete_partial (bool, optional): If True, unpaired input/output channels are allowed and reserved for use (e.g. for calling the model as a function). Defaults to False. partial_commtype (dict, optional): Communicator kwargs that should be be used for the connections to the unpaired channels when complete_partial is True. Defaults to None and will be ignored. model_only (bool, optional): If True, the YAML will not be evaluated as a complete integration and only the individual components will be parsed. Defaults to False. model_submission (bool, optional): If True, the YAML will be evaluated as a submission to the yggdrasil model repository and model_only will be set to True. Defaults to False. yaml_param (dict, optional): Parameters that should be used in mustache formatting of YAML files. Defaults to None and is ignored. directory_for_clones (str, optional): Directory that git repositories should be cloned into. Defaults to None and the current working directory will be used. Raises: ValueError: If the yml dictionary is missing a required keyword or has an invalid value. RuntimeError: If one of the I/O channels is not initialized with driver information. Returns: dict: Dictionary of information parsed from the yamls. """ s = get_schema() # Parse files using schema yml_prep = prep_yaml(files, yaml_param=yaml_param, directory_for_clones=directory_for_clones) # print('prepped') # pprint.pprint(yml_prep) if model_submission: models = [] for yml in yml_prep['models']: wd = yml.pop('working_dir', None) x = s.validate_model_submission(yml) if wd: x['working_dir'] = wd models.append(x) yml_prep['models'] = models model_only = True yml_norm = s.validate(yml_prep, normalize=True, no_defaults=True, required_defaults=True) # print('normalized') # pprint.pprint(yml_norm) # Determine if any of the models require synchronization timesync_names = [] for yml in yml_norm['models']: if yml.get('timesync', False): if yml['timesync'] is True: yml['timesync'] = 'timesync' if not isinstance(yml['timesync'], list): yml['timesync'] = [yml['timesync']] for i, tsync in enumerate(yml['timesync']): if isinstance(tsync, str): tsync = {'name': tsync} yml['timesync'][i] = tsync timesync_names.append(tsync['name']) yml.setdefault('timesync_client_of', []) yml['timesync_client_of'].append(tsync['name']) for tsync in set(timesync_names): for m in yml_norm['models']: if m['name'] == tsync: assert (m['language'] == 'timesync') m.update(is_server=True, inputs=[], outputs=[]) break else: yml_norm['models'].append({ 'name': tsync, 'args': [], 'language': 'timesync', 'is_server': True, 'working_dir': os.getcwd(), 'inputs': [], 'outputs': [] }) # Parse models, then connections to ensure connections can be processed existing = None for k in ['models', 'connections']: for yml in yml_norm[k]: existing = parse_component(yml, k[:-1], existing=existing) # Exit early if model_only: return yml_norm # Add stand-in model that uses unpaired channels if complete_partial: existing = complete_partial_integration( existing, complete_partial, partial_commtype=partial_commtype) # Create server/client connections for srv, srv_info in existing['server'].items(): clients = srv_info['clients'] if srv not in existing['input']: continue yml = { 'inputs': [{ 'name': x } for x in clients], 'outputs': [{ 'name': srv }], 'driver': 'RPCRequestDriver', 'name': existing['input'][srv]['model_driver'][0] } if srv_info.get('replaces', None): yml['outputs'][0].update({ k: v for k, v in srv_info['replaces']['input'].items() if k not in ['name'] }) yml['response_kwargs'] = { k: v for k, v in srv_info['replaces']['output'].items() if k not in ['name'] } existing = parse_component(yml, 'connection', existing=existing) existing['model'][yml['dst_models'][0]]['clients'] = yml['src_models'] existing.pop('server') # Make sure that servers have clients and clients have servers for k, v in existing['model'].items(): if v.get('is_server', False): for x in existing['model'].values(): if v['name'] in x.get('client_of', []): break else: raise YAMLSpecificationError( "Server '%s' does not have any clients.", k) elif v.get('client_of', False): for s in v['client_of']: missing_servers = [] if s not in existing['model']: missing_servers.append(s) if missing_servers: print(list(existing['model'].keys())) raise YAMLSpecificationError( "Servers %s do not exist, but '%s' is a client of them." % (missing_servers, v['name'])) # Make sure that I/O channels initialized opp_map = {'input': 'output', 'output': 'input'} for io in ['input', 'output']: remove = [] for k in list(existing[io].keys()): v = existing[io][k] if 'driver' not in v: if v.get('is_default', False): remove.append(k) elif 'default_file' in v: new_conn = { io + 's': [v['default_file']], opp_map[io] + 's': [v] } existing = parse_component(new_conn, 'connection', existing=existing) elif (io == 'input') and ('default_value' in v): # TODO: The keys moved should be automated based on schema # if the ValueComm has anymore parameters added vdef = { 'name': v['name'], 'default_value': v.pop('default_value'), 'count': v.pop('count', 1), 'commtype': 'value' } new_conn = {'inputs': [vdef], 'outputs': [v]} existing = parse_component(new_conn, 'connection', existing=existing) else: raise YAMLSpecificationError( "No driver established for %s channel %s" % (io, k)) # Remove unused default channels for k in remove: for m in existing[io][k]['model_driver']: for i, x in enumerate(existing['model'][m][io + 's']): if x['name'] == k: existing['model'][m][io + 's'].pop(i) break existing[io].pop(k) # Link io drivers back to models existing = link_model_io(existing) # print('drivers') # pprint.pprint(existing) return existing
def test_ConnectionDriverOnexit_errors(): r"""Test that errors are raised for invalid onexit.""" assert_raises(ValueError, ConnectionDriver, 'test', onexit='invalid') def test_ConnectionDriverTranslate_errors(): r"""Test that errors are raised for invalid translators.""" assert(not hasattr(invalid_translate, '__call__')) assert_raises(ValueError, ConnectionDriver, 'test', translator=invalid_translate) # Dynamically create tests based on registered file classes s = get_schema() comm_types = list(s['comm'].schema_subtypes.keys()) for k in comm_types: if k == _default_comm: # pragma: debug continue # Output ocls = type('Test%sOutputDriver' % k, (TestConnectionDriver, ), {'ocomm_name': k, 'driver': 'OutputDriver', 'args': 'test'}) # Input icls = type('Test%sInputDriver' % k, (TestConnectionDriver, ), {'icomm_name': k, 'driver': 'InputDriver', 'args': 'test'}) # Flags
def parse_connection(yml, existing): r"""Parse a yaml entry for a connection between I/O channels. Args: yml (dict): YAML dictionary for a connection. existing (dict): Dictionary of existing components. Raises: RuntimeError: If the 'inputs' entry is not a model output or file. RuntimeError: If neither the 'inputs' or 'outputs' entries correspond to model I/O channels. Returns: dict: Updated log of all entries. """ schema = get_schema() # File input is_file = {'inputs': [], 'outputs': []} iname_list = [] for x in yml['inputs']: is_file['inputs'].append(schema.is_valid_component('file', x)) if is_file['inputs'][-1]: fname = os.path.expanduser(x['name']) if not os.path.isabs(fname): fname = os.path.join(x['working_dir'], fname) fname = os.path.normpath(fname) if (not os.path.isfile(fname)) and (not x.get( 'wait_for_creation', False)): raise YAMLSpecificationError( ("Input '%s' not found in any of the registered " + "model outputs and is not a file.") % x['name']) x['address'] = fname elif 'default_value' in x: x['address'] = x['default_value'] else: iname_list.append(x['name']) # File output oname_list = [] for x in yml['outputs']: is_file['outputs'].append(schema.is_valid_component('file', x)) if is_file['outputs'][-1]: fname = os.path.expanduser(x['name']) if not x.get('in_temp', False): if not os.path.isabs(fname): fname = os.path.join(x['working_dir'], fname) fname = os.path.normpath(fname) x['address'] = fname else: oname_list.append(x['name']) iname = ','.join(iname_list) oname = ','.join(oname_list) if not iname: args = oname elif not oname: args = iname else: args = '%s_to_%s' % (iname, oname) name = args # Connection xx = {'src_models': [], 'dst_models': [], 'inputs': [], 'outputs': []} for i, y in enumerate(yml['inputs']): if is_file['inputs'][i] or ('default_value' in y): xx['inputs'].append(y) else: new = existing['output'][y['name']] new.update(y) xx['inputs'].append(new) xx['src_models'] += existing['output'][y['name']]['model_driver'] del existing['output'][y['name']] for i, y in enumerate(yml['outputs']): if is_file['outputs'][i]: xx['outputs'].append(y) else: new = existing['input'][y['name']] new.update(y) xx['outputs'].append(new) xx['dst_models'] += existing['input'][y['name']]['model_driver'] del existing['input'][y['name']] # TODO: Split comms if models are not co-located and the main # process needs access to the message passed yml.update(xx) yml.setdefault('driver', 'ConnectionDriver') yml.setdefault('name', name) return existing
def component2table(comp, table_type, include_required=None, subtype_ref=None, **kwargs): r"""Create a table describing a component. Args: comp (str): Name of a component type to create a table for. table_type (str): Type of table that should be created for the component. Supported values include: * 'general': Create a table of properties common to all components of the specified type. * 'specific': Create a table of properties specific to only some components of the specified type. * 'subtype': Create a table describing the subtypes available for the specified component type. include_required (bool, optional): If True, a required column is included. Defaults to True if table_type is 'general' and False otherwise. subtype_ref (str, optional): Reference for the subtype table for the specified component that should be used in the description for the subtype property. Defaults to None and is ignored. **kwargs: Additional keyword arguments are passed to dict2table. Returns: list: Lines comprising the table. """ from yggdrasil.schema import get_schema if include_required is None: if table_type == 'general': include_required = True else: include_required = False if table_type in ['subtype', 'specific']: kwargs.setdefault('prune_empty_columns', True) args = {} s = get_schema() subtype_key = s[comp].subtype_key if table_type in ['general', 'specific']: # Set defaults kwargs.setdefault('key_column_name', 'option') kwargs.setdefault('val_column_name', 'description') kwargs.setdefault('column_order', [kwargs['key_column_name'], 'type', 'required', kwargs['val_column_name']]) if (not include_required) and ('required' in kwargs['column_order']): kwargs['column_order'].remove('required') # Get list of component subtypes if table_type == 'general': s_comp_list = [s[comp].get_subtype_schema('base', unique=True)] else: s_comp_list = [s[comp].get_subtype_schema(x, unique=True) for x in s[comp].classes] # Loop over subtyeps out_apply = {} for s_comp in s_comp_list: for k, v in s_comp['properties'].items(): if (k == subtype_key) and (table_type == 'specific'): continue if k not in args: args[k] = {'type': v.get('type', ''), 'description': v.get('description', '')} if include_required: if k in s_comp.get('required', []): args[k]['required'] = 'X' else: args[k]['required'] = '' if (table_type == 'specific'): if k not in out_apply: out_apply[k] = [] out_apply[k] += s_comp['properties'][subtype_key]['enum'] elif (subtype_ref is not None) and (k == subtype_key): args[k]['description'] += ( ' (Options described :ref:`here <%s>`)' % subtype_ref) if table_type == 'specific': for k, v in args.items(): v['Valid for \'%s\' of' % subtype_key] = list(set(out_apply[k])) elif table_type == 'subtype': kwargs.setdefault('key_column_name', subtype_key) for x, subtypes in s[comp].schema_subtypes.items(): s_comp = s[comp].get_subtype_schema(x, unique=True) subt = subtypes[0] args[subt] = { 'description': s_comp['properties'][subtype_key].get( 'description', '')} if len(subtypes) > 1: args[subt]['aliases'] = subtypes[1:] if s_comp['properties'][subtype_key].get('default', None) in subtypes: args[subt]['description'] = ('[DEFAULT] ' + args[subt]['description']) else: raise ValueError("Unsupported table_type: '%s'" % table_type) return dict2table(args, **kwargs)
def __init__(self, skip_component_schema_normalization=None, **kwargs): if skip_component_schema_normalization is None: skip_component_schema_normalization = (not (os.environ.get( 'YGG_VALIDATE_COMPONENTS', 'None').lower() in ['true', '1'])) comptype = self._schema_type if (comptype is None) and (not self._schema_properties): self.extra_kwargs = kwargs return subtype = None if self._schema_subtype_key is not None: subtype = getattr( self, self._schema_subtype_key, getattr(self, '_%s' % self._schema_subtype_key, None)) # Fall back to some simple parsing/normalization to save time on # full jsonschema normalization for k, v in self._schema_properties.items(): if k in self._schema_excluded_from_class: continue default = v.get('default', None) if (k == self._schema_subtype_key) and (subtype is not None): default = subtype if default is not None: kwargs.setdefault(k, copy.deepcopy(default)) if v.get('type', None) == 'array': if isinstance(kwargs.get(k, None), (bytes, str)): kwargs[k] = kwargs[k].split() # Parse keyword arguments using schema if (((comptype is not None) and (subtype is not None) and (not skip_component_schema_normalization))): from yggdrasil.schema import get_schema s = get_schema().get_component_schema( comptype, subtype, relaxed=True, allow_instance_definitions=True) props = list(s['properties'].keys()) if not skip_component_schema_normalization: from yggdrasil import metaschema kwargs.setdefault(self._schema_subtype_key, subtype) # Remove properties that shouldn't ve validated in class for k in self._schema_excluded_from_class_validation: if k in s['properties']: del s['properties'][k] # Validate and normalize metaschema.validate_instance(kwargs, s, normalize=False) # TODO: Normalization performance needs improvement # import pprint # print('before') # pprint.pprint(kwargs_comp) # kwargs_comp = metaschema.validate_instance(kwargs_comp, s, # normalize=True) # kwargs.update(kwargs_comp) # print('normalized') # pprint.pprint(kwargs_comp) else: props = self._schema_properties.keys() # Set attributes based on properties for k in props: if k in self._schema_excluded_from_class: continue v = kwargs.pop(k, None) if getattr(self, k, None) is None: setattr(self, k, v) # elif (getattr(self, k) != v) and (v is not None): # warnings.warn(("The schema property '%s' is provided as a " # "keyword with a value of %s, but the class " # "already has an attribute of the same name " # "with the value %s.") # % (k, v, getattr(self, k))) self.extra_kwargs = kwargs
import os import shutil from yggdrasil import schema, metaschema schema_dir = os.path.join(os.path.dirname(__file__), 'schema') if not os.path.isdir(schema_dir): os.mkdir(schema_dir) indent = ' ' s = schema.get_schema() shutil.copy(metaschema._metaschema_fname, os.path.join(schema_dir, 'metaschema.json')) with open(os.path.join(schema_dir, 'integration.json'), 'w') as fd: metaschema.encoder.encode_json(s.schema, fd=fd, indent=indent, sort_keys=False) schema.get_json_schema(os.path.join(schema_dir, 'integration_strict.json'), indent=indent) schema.get_model_form_schema(os.path.join(schema_dir, 'model_form.json'), indent=indent)
def parse_yaml(files, as_function=False): r"""Parse list of yaml files. Args: files (str, list): Either the path to a single yaml file or a list of yaml files. as_function (bool, optional): If True, the missing input/output channels will be created for using model(s) as a function. Defaults to False. Raises: ValueError: If the yml dictionary is missing a required keyword or has an invalid value. RuntimeError: If one of the I/O channels is not initialized with driver information. Returns: dict: Dictionary of information parsed from the yamls. """ s = get_schema() # Parse files using schema yml_prep = prep_yaml(files) # print('prepped') # pprint.pprint(yml_prep) yml_norm = s.validate(yml_prep, normalize=True) # print('normalized') # pprint.pprint(yml_norm) # Determine if any of the models require synchronization timesync_names = [] for yml in yml_norm['models']: if yml.get('timesync', False): if yml['timesync'] is True: yml['timesync'] = 'timesync' if not isinstance(yml['timesync'], list): yml['timesync'] = [yml['timesync']] for i, tsync in enumerate(yml['timesync']): if isinstance(tsync, str): tsync = {'name': tsync} yml['timesync'][i] = tsync timesync_names.append(tsync['name']) yml.setdefault('timesync_client_of', []) yml['timesync_client_of'].append(tsync['name']) for tsync in set(timesync_names): for m in yml_norm['models']: if m['name'] == tsync: assert (m['language'] == 'timesync') m.update(is_server=True, inputs=[], outputs=[]) break else: yml_norm['models'].append({ 'name': tsync, 'args': [], 'language': 'timesync', 'is_server': True, 'working_dir': os.getcwd(), 'inputs': [], 'outputs': [] }) # Parse models, then connections to ensure connections can be processed existing = None for k in ['models', 'connections']: for yml in yml_norm[k]: existing = parse_component(yml, k[:-1], existing=existing) # Create server/client connections for srv, srv_info in existing['server'].items(): clients = srv_info['clients'] if srv not in existing['input']: continue yml = { 'inputs': [{ 'name': x } for x in clients], 'outputs': [{ 'name': srv }], 'driver': 'RPCRequestDriver', 'name': existing['input'][srv]['model_driver'][0] } if srv_info.get('replaces', None): yml['outputs'][0].update({ k: v for k, v in srv_info['replaces']['input'].items() if k not in ['name'] }) yml['response_kwargs'] = { k: v for k, v in srv_info['replaces']['output'].items() if k not in ['name'] } existing = parse_component(yml, 'connection', existing=existing) existing['model'][yml['dst_models'][0]]['clients'] = yml['src_models'] existing.pop('server') if as_function: existing = add_model_function(existing) # Make sure that servers have clients and clients have servers for k, v in existing['model'].items(): if v.get('is_server', False): for x in existing['model'].values(): if v['name'] in x.get('client_of', []): break else: raise YAMLSpecificationError( "Server '%s' does not have any clients.", k) elif v.get('client_of', False): for s in v['client_of']: missing_servers = [] if s not in existing['model']: missing_servers.append(s) if missing_servers: print(list(existing['model'].keys())) raise YAMLSpecificationError( "Servers %s do not exist, but '%s' is a client of them." % (missing_servers, v['name'])) # Make sure that I/O channels initialized opp_map = {'input': 'output', 'output': 'input'} for io in ['input', 'output']: remove = [] for k in list(existing[io].keys()): v = existing[io][k] if 'driver' not in v: if v.get('is_default', False): remove.append(k) elif 'default_file' in v: new_conn = { io + 's': [v['default_file']], opp_map[io] + 's': [v] } existing = parse_component(new_conn, 'connection', existing=existing) elif (io == 'input') and ('default_value' in v): # TODO: The keys moved should be automated based on schema # if the ValueComm has anymore parameters added vdef = { 'name': v['name'], 'default_value': v.pop('default_value'), 'count': v.pop('count', 1), 'commtype': 'value' } new_conn = {'inputs': [vdef], 'outputs': [v]} existing = parse_component(new_conn, 'connection', existing=existing) else: raise YAMLSpecificationError( "No driver established for %s channel %s" % (io, k)) # Remove unused default channels for k in remove: for m in existing[io][k]['model_driver']: for i, x in enumerate(existing['model'][m][io + 's']): if x['name'] == k: existing['model'][m][io + 's'].pop(i) break existing[io].pop(k) # Link io drivers back to models existing = link_model_io(existing) # print('drivers') # pprint.pprint(existing) return existing