def test_default_schema():
r"""Test getting default schema."""
s = schema.get_schema()
assert (s is not None)
schema.clear_schema()
assert (schema._schema is None)
s = schema.get_schema()
assert (s is not None)
for k in s.keys():
assert (isinstance(s[k].subtypes, list))
assert (isinstance(s[k].classes, list))
for ksub in s[k].classes:
s[k].get_subtype_properties(ksub)
def executeGraph(self, graph):
"""Execute graph."""
cisgraph = fbpToCis(graph['content'])
user = self.getCurrentUser()
username = user['login']
cisgraph = as_str(cisgraph, recurse=True, allow_pass=True)
# Write to temp file and validate
tmpfile = tempfile.NamedTemporaryFile(suffix="yml", prefix="cis",
delete=False)
yaml.safe_dump(cisgraph, tmpfile, default_flow_style=False)
yml_prep = prep_yaml(tmpfile.name)
os.remove(tmpfile.name)
s = get_schema()
yml_norm = s.normalize(yml_prep)
try:
s.validate(yml_norm)
except BaseException as e:
print(e)
raise RestException('Invalid graph %s', 400, e)
self.setRawResponse()
yaml_graph = pyaml.dump(cisgraph)
#print('Executing graph: ' + str(yaml_graph))
return execGraph(yaml_graph, username)
def generate_component_tests(comptype,
base_class,
target_globals,
directory,
class_attr='_cls'):
r"""Function that generates tests for all component subtypes
based on the registered classes.
Args:
comptype (str): Component type.
base_class (class): Base python test class.
target_globals (dict): Globals dictionary that class should be
added to.
directory (str): Directory to check for tests or file in directory
that should be checked for tests.
class_attr (str, optional): Attribute that should be set to the
name of the class being tested.
"""
from yggdrasil.schema import get_schema
_schema = get_schema()
if os.path.isfile(directory):
directory = os.path.dirname(directory)
if comptype not in _schema.keys(): # pragma: debug
raise NotImplementedError("%s is not a component type." % comptype)
for subtype in _schema[comptype].subtypes:
subtype_cls = _schema[comptype].subtype2class[subtype]
new_cls_name = 'Test%s' % subtype_cls
new_cls_file = os.path.join(directory, 'test_%s' % subtype_cls) + '.py'
if os.path.isfile(new_cls_file):
continue
cls_attr = {class_attr: subtype_cls}
new_cls = type(new_cls_name, (base_class, ), cls_attr)
target_globals[new_cls.__name__] = new_cls
del new_cls
def create_component(comptype, subtype=None, **kwargs):
r"""Dynamically create an instance of a component with the specified options
as outlined in the component schema. This function requires loading the
component schemas and so should not be used at the module or class level to
prevent circular dependencies.
Args:
comptype (str): Component type.
subtype (str, optional): Component subtype. If subtype is not one of the
registered subtypes for the specified comptype, subtype is treated
as the name of the class. Defaults to None if not provided and the
default subtype defined in the schema for the specified component
will be used. If the subtype is specified by the component subtype
key in the remaining kwargs, that subtype will be used instead.
**kwargs: Additional keyword arguments are treated as options for the
component as outlined in the component schema.
Returns:
ComponentBase: Instance of the specified component type/subtype and
options.
Raises:
ValueError: If comptype is not a registered component type.
"""
from yggdrasil.schema import get_schema
s = get_schema().get(comptype, None)
if s is None: # pragma: debug
raise ValueError("Unrecognized component type: %s" % comptype)
if s.subtype_key in kwargs:
subtype = kwargs[s.subtype_key]
if subtype is None:
subtype = s.identify_subtype(kwargs)
cls = import_component(comptype, subtype=subtype, **kwargs)
return cls(**kwargs)
def test_get_schema_subtype():
r"""Test get_schema_subtype for allow_instance=True."""
component = 'serializer'
subtype = 'direct'
doc = {'seritype': subtype}
valid = components.create_component(component, seritype=subtype)
invalid = components.create_component(component, seritype='json')
s = schema.get_schema()
kwargs = {'subtype': subtype, 'allow_instance': True}
s.validate_component(component, doc, **kwargs)
s.validate_component(component, valid, **kwargs)
assert_raises(ValidationError, s.validate_component, component, invalid,
**kwargs)
s.validate_component(component, doc, subtype=subtype)
assert_raises(ValidationError,
s.validate_component,
component,
valid,
subtype=subtype)
# Test for base
s.validate_component(component, valid, subtype='base', allow_instance=True)
s.validate_component(component,
invalid,
subtype='base',
allow_instance=True)
def write_component_table(comp='all', table_type='all', **kwargs):
r"""Write a component table to a file.
Args:
comp (str): Name of a component type to create a table for. Defaults to
'all' and tables are created for each of the registered components.
table_type (str, optional): Type of table that should be created for the
component. Defaults to 'all'. Supported values include:
* 'all': Create each type of table for the specified component.
* 'general': Create a table of properties common to all components
of the specified type.
* 'specific': Create a table of properties specific to only some
components of the specified type.
* 'subtype': Create a table describing the subtypes available for
the specified component type.
**kwargs: Additional keyword arguments are passed to component2table
and write_table.
Returns:
str, list: Name of file or files created.
"""
from yggdrasil.schema import get_schema
s = get_schema()
table_type_list = ['subtype', 'general', 'specific']
# Loop
if comp == 'all':
fname = kwargs.get("fname", None)
fname_base = kwargs.get("fname_base", None)
assert ((fname is None) and (fname_base is None))
out = []
for k in s.keys():
new_out = write_component_table(comp=k,
table_type=table_type,
**kwargs)
if isinstance(new_out, list):
out += new_out
else:
out.append(new_out)
return out
if table_type == 'all':
fname = kwargs.get("fname", None)
fname_base = kwargs.get("fname_base", None)
assert ((fname is None) and (fname_base is None))
out = []
for k in table_type_list:
out.append(write_component_table(comp=comp, table_type=k,
**kwargs))
return out
# Construct file name
fname_format = 'schema_table_%s_%s.rst'
kwargs.setdefault('subtype_ref',
(fname_format % (comp, 'subtype')).replace(
'.rst', '_rst'))
kwargs.setdefault('fname_base', fname_format % (comp, table_type))
lines = component2table(comp, table_type, **kwargs)
return write_table(lines, **kwargs)
def _init_single_comm(self, name, io, comm_kws, **kwargs):
r"""Parse keyword arguments for input/output comm."""
self.debug("Creating %s comm", io)
s = get_schema()
if comm_kws is None:
comm_kws = dict()
if io == 'input':
direction = 'recv'
comm_type = self._icomm_type
touches_model = self._is_output
attr_comm = 'icomm'
comm_kws['close_on_eof_recv'] = False
else:
direction = 'send'
comm_type = self._ocomm_type
touches_model = self._is_input
attr_comm = 'ocomm'
comm_kws['direction'] = direction
comm_kws['dont_open'] = True
comm_kws['reverse_names'] = True
comm_kws.setdefault('comm', {'comm': comm_type})
assert (name == self.name)
comm_kws.setdefault('name', name)
if not isinstance(comm_kws['comm'], list):
comm_kws['comm'] = [comm_kws['comm']]
for i, x in enumerate(comm_kws['comm']):
if x is None:
comm_kws['comm'][i] = dict()
elif not isinstance(x, dict):
comm_kws['comm'][i] = dict(comm=x)
comm_kws['comm'][i].setdefault('comm', comm_type)
any_files = False
all_files = True
if not touches_model:
comm_kws['no_suffix'] = True
ikws = []
for x in comm_kws['comm']:
if get_comm_class(x['comm']).is_file:
any_files = True
ikws += s['file'].get_subtype_properties(x['comm'])
else:
all_files = False
ikws += s['comm'].get_subtype_properties(x['comm'])
ikws = list(set(ikws))
for k in ikws:
if (k not in comm_kws) and (k in kwargs):
comm_kws[k] = kwargs.pop(k)
if ('comm_env' in kwargs) and ('comm_env' not in comm_kws):
comm_kws['env'] = kwargs.pop('comm_env')
if any_files and (io == 'input'):
kwargs.setdefault('timeout_send_1st', 60)
self.debug('%s comm_kws:\n%s', attr_comm, self.pprint(comm_kws, 1))
setattr(self, attr_comm, new_comm(comm_kws.pop('name'), **comm_kws))
setattr(self, '%s_kws' % attr_comm, comm_kws)
if touches_model:
self.env.update(getattr(self, attr_comm).opp_comms)
elif not all_files:
self.comm_env.update(getattr(self, attr_comm).opp_comms)
return kwargs
def test_default_schema():
r"""Test getting default schema."""
s = schema.get_schema()
assert (s is not None)
schema.clear_schema()
assert (schema._schema is None)
s = schema.get_schema()
assert (s is not None)
for k in s.keys():
assert (isinstance(s[k].subtypes, list))
assert (isinstance(s[k].classes, list))
for ksub in s[k].classes:
s[k].get_subtype_properties(ksub)
s[k].default_subtype
s.get_schema(relaxed=True)
s.get_schema(allow_instance=True)
s.definitions
s.form_schema
def parse_component(yml, ctype, existing=None):
r"""Parse a yaml entry for a component, adding it to the list of
existing components.
Args:
yml (dict): YAML dictionary for a component.
ctype (str): Component type. This can be 'input', 'output',
'model', or 'connection'.
existing (dict, optional): Dictionary of existing components.
Defaults to empty dict.
Raises:
TypeError: If yml is not a dictionary.
ValueError: If dtype is not 'input', 'output', 'model', or
'connection'.
ValueError: If the component already exists.
Returns:
dict: All components identified.
"""
s = get_schema()
if not isinstance(yml, dict):
raise YAMLSpecificationError(
"Component entry in yml must be a dictionary.")
ctype_list = [
'input', 'output', 'model', 'connection', 'model_input', 'model_output'
]
if existing is None:
existing = {k: {} for k in ctype_list}
if ctype not in ctype_list:
raise YAMLSpecificationError("'%s' is not a recognized component.")
# Parse based on type
if ctype == 'model':
existing = parse_model(yml, existing)
elif ctype == 'connection':
existing = parse_connection(yml, existing)
elif ctype in ['input', 'output']:
for k in ['icomm_kws', 'ocomm_kws']:
if k not in yml:
continue
for x in yml[k]['comm']:
if 'comm' not in x:
if 'filetype' in x:
x['comm'] = s['file'].subtype2class[x['filetype']]
elif 'commtype' in x:
x['comm'] = s['comm'].subtype2class[x['commtype']]
# Ensure component dosn't already exist
if yml['name'] in existing[ctype]:
pprint.pprint(existing)
pprint.pprint(yml)
raise YAMLSpecificationError(
"%s is already a registered '%s' component." %
(yml['name'], ctype))
existing[ctype][yml['name']] = yml
return existing
def __init__(self, *args, **kwargs):
super(FilterTransform, self).__init__(*args, **kwargs)
if isinstance(self.filter, dict):
from yggdrasil.schema import get_schema
from yggdrasil.components import create_component
filter_schema = get_schema().get('filter')
filter_kws = dict(self.filter,
subtype=filter_schema.identify_subtype(
self.filter))
self.filter = create_component('filter', **filter_kws)
def test_normalize():
r"""Test normalization of legacy formats."""
s = schema.get_schema()
for x, y in _normalize_objects:
a = s.normalize(x, backwards_compat=True)
try:
assert_equal(a, y)
except BaseException: # pragma: debug
pprint.pprint(a)
pprint.pprint(y)
raise
def parse_model(yml, existing):
r"""Parse a yaml entry for a model.
Args:
yml (dict): YAML dictionary for a model.
existing (dict): Dictionary of existing components.
Returns:
dict: Updated log of all entries.
"""
_lang2driver = get_schema()['model'].subtype2class
language = yml.pop('language')
yml['driver'] = _lang2driver[language]
# Add server driver
if yml.get('is_server', False):
srv = {
'name': '%s:%s' % (yml['name'], yml['name']),
'commtype': 'default',
'datatype': {
'type': 'bytes'
},
'driver': 'ServerDriver',
'args': yml['name'] + '_SERVER',
'working_dir': yml['working_dir']
}
yml['inputs'].append(srv)
yml['clients'] = []
# Add client driver
if yml.get('client_of', []):
srv_names = yml['client_of']
yml['client_of'] = srv_names
for srv in srv_names:
cli = {
'name': '%s:%s_%s' % (yml['name'], srv, yml['name']),
'commtype': 'default',
'datatype': {
'type': 'bytes'
},
'driver': 'ClientDriver',
'args': srv + '_SERVER',
'working_dir': yml['working_dir']
}
yml['outputs'].append(cli)
# Model index and I/O channels
yml['model_index'] = len(existing['model'])
for io in ['inputs', 'outputs']:
for x in yml[io]:
x['model_driver'] = [yml['name']]
x['partner_language'] = language
existing = parse_component(x, io[:-1], existing=existing)
return existing
def test_normalize():
r"""Test normalization of legacy formats."""
s = schema.get_schema()
for x, y in _normalize_objects:
a = s.normalize(x, backwards_compat=True) # , show_errors=True)
try:
assert_equal(a, y)
except BaseException: # pragma: debug
print("Unexpected Normalization:\n\nA:")
pprint.pprint(a)
print('\nB:')
pprint.pprint(y)
raise
def get_supported_comm():
r"""Get a list of the communication mechanisms supported by yggdrasil.
Returns:
list: The names of communication mechanisms supported by yggdrasil.
"""
from yggdrasil import schema
s = schema.get_schema()
out = s['comm'].classes
for k in ['CommBase', 'DefaultComm']:
if k in out:
out.remove(k)
return list(set(out))
def get_supported_lang():
r"""Get a list of the model programming languages that are supported
by yggdrasil.
Returns:
list: The names of programming languages supported by yggdrasil.
"""
from yggdrasil import schema
s = schema.get_schema()
out = s['model'].subtypes
if 'c++' in out:
out[out.index('c++')] = 'cpp'
return list(set(out))
def generate_component_subtests(comptype,
suffix,
target_globals,
parent_module_name,
new_attr=None,
module_name_format='test_%s',
class_name_format='Test%s',
skip_subtypes=[]):
r"""Function that generates tests for all component subtypes that
subclass the original test class for the subtype.
Args:
comptype (str): Component type.
suffix (str): Suffix to be added to the end of the original
test class name to get the name for the new test.
target_globals (dict): Globals dictionary that class should be
added to.
parent_module_name (str): Parent module that contains test
classes for the specified component type.
new_attr (dict, optional): Attributes to add to the test classes.
Defaults to None and will be an empty dict.
module_name_format (str, optional): Format string that should
be used to locate the original test modules. Defaults to
'test_%s'.
class_name_format (str, optional): Format string that should
be used to both name generated test classes and locate the
original test classes. Defaults to 'Test%s'.
skip_subtypes (list, optional): List of subtypes to skip.
Defaults to empty list.
"""
from yggdrasil.schema import get_schema
_schema = get_schema()
if new_attr is None: # pragma: debug
new_attr = {}
if comptype not in _schema.keys(): # pragma: debug
raise NotImplementedError("%s is not a component type." % comptype)
for subtype in _schema[comptype].subtypes:
if subtype in skip_subtypes:
continue
subtype_cls = _schema[comptype].subtype2class[subtype]
old_mod_name = (parent_module_name + '.' +
(module_name_format % subtype_cls))
old_cls_name = class_name_format % subtype_cls
new_cls_name = class_name_format % (subtype_cls + suffix.title())
base_class = getattr(importlib.import_module(old_mod_name),
old_cls_name)
new_cls = type(new_cls_name, (base_class, ), new_attr)
target_globals[new_cls.__name__] = new_cls
del new_cls
def generate_component_tests(comptype,
base_class,
target_globals,
directory,
class_attr='_cls',
new_attr=None,
class_name_format='Test%s',
class_file_format='test_%s.py'):
r"""Function that generates tests for all component subtypes
based on the registered classes.
Args:
comptype (str): Component type.
base_class (class): Base python test class.
target_globals (dict): Globals dictionary that class should be
added to.
directory (str): Directory to check for tests or file in directory
that should be checked for tests.
class_attr (str, optional): Attribute that should be set to the
name of the class being tested.
new_attr (dict, optional): Attributes to add to the test classes.
Defaults to None and will be an empty dict.
class_name_format (str, optional): Format string that should
be used to name generated test class's. Defaults to
'Test%s'.
class_file_format (str, optional): Format string that should
be used to name test files that are checked for. Defaults
to 'test_%s.py'.
"""
from yggdrasil.schema import get_schema
_schema = get_schema()
if new_attr is None:
new_attr = {}
if os.path.isfile(directory):
directory = os.path.dirname(directory)
if comptype not in _schema.keys(): # pragma: debug
raise NotImplementedError("%s is not a component type." % comptype)
for subtype in _schema[comptype].subtypes:
subtype_cls = _schema[comptype].subtype2class[subtype]
new_cls_name = class_name_format % subtype_cls
new_cls_file = os.path.join(directory, class_file_format % subtype_cls)
if os.path.isfile(new_cls_file):
continue
inew_attr = copy.deepcopy(new_attr)
inew_attr.setdefault(class_attr, subtype_cls)
new_cls = type(new_cls_name, (base_class, ), inew_attr)
target_globals[new_cls.__name__] = new_cls
del new_cls
def is_lang_installed(lang):
r"""Check to see if yggdrasil can run models written in a programming
language on the current machine.
Args:
lang (str): Programming language to check.
Returns:
bool: True if models in the provided language can be run on the current
machine, False otherwise.
"""
from yggdrasil import schema, drivers
s = schema.get_schema()
drv = drivers.import_driver(s['model'].subtype2class[lang])
return drv.is_installed()
def test_create_schema():
r"""Test creating new schema."""
fname = 'test_schema.yml'
if os.path.isfile(fname): # pragma: debug
os.remove(fname)
# Test saving/loading schema
s0 = schema.create_schema()
s0.save(fname)
assert (s0 is not None)
assert (os.path.isfile(fname))
s1 = schema.get_schema(fname)
assert_equal(s1.schema, s0.schema)
# assert_equal(s1, s0)
os.remove(fname)
# Test getting schema
s2 = schema.load_schema(fname)
assert (os.path.isfile(fname))
assert_equal(s2, s0)
os.remove(fname)
def test_save_load_schema():
r"""Test saving & loading schema."""
fname = 'test_schema.yml'
if os.path.isfile(fname): # pragma: debug
os.remove(fname)
# Test saving/loading schema
s0 = schema.load_schema()
s0.save(fname)
assert (s0 is not None)
assert (os.path.isfile(fname))
s1 = schema.get_schema(fname)
assert (s1.schema == s0.schema)
# assert(s1 == s0)
os.remove(fname)
# Test getting schema
s2 = schema.load_schema(fname)
assert (os.path.isfile(fname))
assert (s2.schema == s0.schema)
assert (s2 == s0)
os.remove(fname)
def parse_yaml(files):
r"""Parse list of yaml files.
Args:
files (str, list): Either the path to a single yaml file or a list of
yaml files.
Raises:
ValueError: If the yml dictionary is missing a required keyword or has
an invalid value.
RuntimeError: If one of the I/O channels is not initialized with driver
information.
Returns:
dict: Dictionary of information parsed from the yamls.
"""
s = get_schema()
# Parse files using schema
yml_prep = prep_yaml(files)
# print('prepped')
# pprint.pprint(yml_prep)
yml_norm = s.validate(yml_prep, normalize=True)
# print('normalized')
# pprint.pprint(yml_norm)
# Parse models, then connections to ensure connections can be processed
existing = None
for k in ['models', 'connections']:
for yml in yml_norm[k]:
existing = parse_component(yml, k[:-1], existing=existing)
# Make sure that I/O channels initialized
for io in ['input', 'output']:
for k, v in existing[io].items():
if 'driver' not in v:
raise RuntimeError("No driver established for %s channel %s" %
(io, k))
# Link io drivers back to models
existing = link_model_io(existing)
# print('drivers')
# pprint.pprint(existing)
return existing
def convertGraph(self, graph):
"""Convert graph."""
cisgraph = fbpToCis(graph['content'])
cisgraph = as_str(cisgraph, recurse=True, allow_pass=True)
# Write to temp file and validate
tmpfile = tempfile.NamedTemporaryFile(suffix="yml", prefix="cis",
delete=False)
yaml.safe_dump(cisgraph, tmpfile, default_flow_style=False)
yml_prep = prep_yaml(tmpfile.name)
os.remove(tmpfile.name)
s = get_schema()
yml_norm = s.normalize(yml_prep)
try:
s.validate(yml_norm)
except BaseException as e:
print(e)
raise RestException('Invalid graph %s', 400, e)
self.setRawResponse()
return pyaml.dump(cisgraph)
def parse_model(yml, existing):
r"""Parse a yaml entry for a model.
Args:
yml (dict): YAML dictionary for a model.
existing (dict): Dictionary of existing components.
Returns:
dict: Updated log of all entries.
"""
_lang2driver = get_schema()['model'].subtype2class
language = yml.pop('language')
yml['driver'] = _lang2driver[language]
# Add server input
if yml.get('is_server', False):
srv = {
'name': '%s:%s' % (yml['name'], yml['name']),
'commtype': 'default',
'datatype': {
'type': 'bytes'
},
'args': yml['name'] + '_SERVER',
'working_dir': yml['working_dir']
}
if yml.get('function', False) and isinstance(yml['is_server'], bool):
if (len(yml['inputs']) == 1) and (len(yml['outputs']) == 1):
yml['is_server'] = {
'input': yml['inputs'][0]['name'],
'output': yml['outputs'][0]['name']
}
else:
raise YAMLSpecificationError(
"The 'is_server' parameter is boolean for the model '%s' "
"and the 'function' parameter is also set. "
"If the 'function' and 'is_server' parameters are used "
"together, the 'is_server' parameter must be a mapping "
"with 'input' and 'output' entries specifying which of "
"the function's input/output variables should be received"
"/sent from/to clients. e.g. \n"
"\t-input: input_variable\n"
"\t-output: output_variables\n" % yml['name'])
replaces = None
if isinstance(yml['is_server'], dict):
replaces = {}
for io in ['input', 'output']:
replaces[io] = None
if not yml['is_server'][io].startswith('%s:' % yml['name']):
yml['is_server'][io] = '%s:%s' % (yml['name'],
yml['is_server'][io])
for i, x in enumerate(yml[io + 's']):
if x['name'] == yml['is_server'][io]:
replaces[io] = x
replaces[io + '_index'] = i
yml[io + 's'].pop(i)
break
else:
raise YAMLSpecificationError(
"Failed to locate an existing %s channel "
"with the name %s." % (io, yml['is_server'][io]))
srv['server_replaces'] = replaces
yml['inputs'].insert(replaces['input_index'], srv)
else:
yml['inputs'].append(srv)
yml['clients'] = []
existing['server'].setdefault(srv['name'], {
'clients': [],
'model_name': yml['name']
})
if replaces:
existing['server'][srv['name']]['replaces'] = replaces
# Mark timesync clients
timesync = yml.pop('timesync_client_of', [])
if timesync:
yml.setdefault('client_of', [])
yml['client_of'] += timesync
# Add client output
if yml.get('client_of', []):
for srv in yml['client_of']:
srv_name = '%s:%s' % (srv, srv)
if srv in timesync:
cli_name = '%s:%s' % (yml['name'], srv)
else:
cli_name = '%s:%s_%s' % (yml['name'], srv, yml['name'])
cli = {'name': cli_name, 'working_dir': yml['working_dir']}
yml['outputs'].append(cli)
existing['server'].setdefault(srv_name, {
'clients': [],
'model_name': srv
})
existing['server'][srv_name]['clients'].append(cli_name)
# Model index and I/O channels
yml['model_index'] = len(existing['model'])
for io in ['inputs', 'outputs']:
for x in yml[io]:
if ((yml.get('function', False)
and (not x.get('outside_loop', False))
and yml.get('is_server', False))):
x.setdefault('dont_copy', True)
if yml.get('allow_threading',
False) or ((yml.get('copies', 1) > 1) and
(not x.get('dont_copy', False))):
x['allow_multiple_comms'] = True
# TODO: Replace model_driver with partner_model?
x['model_driver'] = [yml['name']]
x['partner_model'] = yml['name']
if yml.get('copies', 1) > 1:
x['partner_copies'] = yml['copies']
x['partner_language'] = language
existing = parse_component(x, io[:-1], existing=existing)
for k in yml.get('env', {}).keys():
if not isinstance(yml['env'][k], str):
yml['env'][k] = json.dumps(yml['env'][k])
return existing
def parse_yaml(files,
complete_partial=False,
partial_commtype=None,
model_only=False,
model_submission=False,
yaml_param=None,
directory_for_clones=None):
r"""Parse list of yaml files.
Args:
files (str, list): Either the path to a single yaml file or a list of
yaml files.
complete_partial (bool, optional): If True, unpaired input/output
channels are allowed and reserved for use (e.g. for calling the
model as a function). Defaults to False.
partial_commtype (dict, optional): Communicator kwargs that should be
be used for the connections to the unpaired channels when
complete_partial is True. Defaults to None and will be ignored.
model_only (bool, optional): If True, the YAML will not be evaluated
as a complete integration and only the individual components will
be parsed. Defaults to False.
model_submission (bool, optional): If True, the YAML will be evaluated
as a submission to the yggdrasil model repository and model_only
will be set to True. Defaults to False.
yaml_param (dict, optional): Parameters that should be used in
mustache formatting of YAML files. Defaults to None and is
ignored.
directory_for_clones (str, optional): Directory that git repositories
should be cloned into. Defaults to None and the current working
directory will be used.
Raises:
ValueError: If the yml dictionary is missing a required keyword or
has an invalid value.
RuntimeError: If one of the I/O channels is not initialized with
driver information.
Returns:
dict: Dictionary of information parsed from the yamls.
"""
s = get_schema()
# Parse files using schema
yml_prep = prep_yaml(files,
yaml_param=yaml_param,
directory_for_clones=directory_for_clones)
# print('prepped')
# pprint.pprint(yml_prep)
if model_submission:
models = []
for yml in yml_prep['models']:
wd = yml.pop('working_dir', None)
x = s.validate_model_submission(yml)
if wd:
x['working_dir'] = wd
models.append(x)
yml_prep['models'] = models
model_only = True
yml_norm = s.validate(yml_prep,
normalize=True,
no_defaults=True,
required_defaults=True)
# print('normalized')
# pprint.pprint(yml_norm)
# Determine if any of the models require synchronization
timesync_names = []
for yml in yml_norm['models']:
if yml.get('timesync', False):
if yml['timesync'] is True:
yml['timesync'] = 'timesync'
if not isinstance(yml['timesync'], list):
yml['timesync'] = [yml['timesync']]
for i, tsync in enumerate(yml['timesync']):
if isinstance(tsync, str):
tsync = {'name': tsync}
yml['timesync'][i] = tsync
timesync_names.append(tsync['name'])
yml.setdefault('timesync_client_of', [])
yml['timesync_client_of'].append(tsync['name'])
for tsync in set(timesync_names):
for m in yml_norm['models']:
if m['name'] == tsync:
assert (m['language'] == 'timesync')
m.update(is_server=True, inputs=[], outputs=[])
break
else:
yml_norm['models'].append({
'name': tsync,
'args': [],
'language': 'timesync',
'is_server': True,
'working_dir': os.getcwd(),
'inputs': [],
'outputs': []
})
# Parse models, then connections to ensure connections can be processed
existing = None
for k in ['models', 'connections']:
for yml in yml_norm[k]:
existing = parse_component(yml, k[:-1], existing=existing)
# Exit early
if model_only:
return yml_norm
# Add stand-in model that uses unpaired channels
if complete_partial:
existing = complete_partial_integration(
existing, complete_partial, partial_commtype=partial_commtype)
# Create server/client connections
for srv, srv_info in existing['server'].items():
clients = srv_info['clients']
if srv not in existing['input']:
continue
yml = {
'inputs': [{
'name': x
} for x in clients],
'outputs': [{
'name': srv
}],
'driver': 'RPCRequestDriver',
'name': existing['input'][srv]['model_driver'][0]
}
if srv_info.get('replaces', None):
yml['outputs'][0].update({
k: v
for k, v in srv_info['replaces']['input'].items()
if k not in ['name']
})
yml['response_kwargs'] = {
k: v
for k, v in srv_info['replaces']['output'].items()
if k not in ['name']
}
existing = parse_component(yml, 'connection', existing=existing)
existing['model'][yml['dst_models'][0]]['clients'] = yml['src_models']
existing.pop('server')
# Make sure that servers have clients and clients have servers
for k, v in existing['model'].items():
if v.get('is_server', False):
for x in existing['model'].values():
if v['name'] in x.get('client_of', []):
break
else:
raise YAMLSpecificationError(
"Server '%s' does not have any clients.", k)
elif v.get('client_of', False):
for s in v['client_of']:
missing_servers = []
if s not in existing['model']:
missing_servers.append(s)
if missing_servers:
print(list(existing['model'].keys()))
raise YAMLSpecificationError(
"Servers %s do not exist, but '%s' is a client of them."
% (missing_servers, v['name']))
# Make sure that I/O channels initialized
opp_map = {'input': 'output', 'output': 'input'}
for io in ['input', 'output']:
remove = []
for k in list(existing[io].keys()):
v = existing[io][k]
if 'driver' not in v:
if v.get('is_default', False):
remove.append(k)
elif 'default_file' in v:
new_conn = {
io + 's': [v['default_file']],
opp_map[io] + 's': [v]
}
existing = parse_component(new_conn,
'connection',
existing=existing)
elif (io == 'input') and ('default_value' in v):
# TODO: The keys moved should be automated based on schema
# if the ValueComm has anymore parameters added
vdef = {
'name': v['name'],
'default_value': v.pop('default_value'),
'count': v.pop('count', 1),
'commtype': 'value'
}
new_conn = {'inputs': [vdef], 'outputs': [v]}
existing = parse_component(new_conn,
'connection',
existing=existing)
else:
raise YAMLSpecificationError(
"No driver established for %s channel %s" % (io, k))
# Remove unused default channels
for k in remove:
for m in existing[io][k]['model_driver']:
for i, x in enumerate(existing['model'][m][io + 's']):
if x['name'] == k:
existing['model'][m][io + 's'].pop(i)
break
existing[io].pop(k)
# Link io drivers back to models
existing = link_model_io(existing)
# print('drivers')
# pprint.pprint(existing)
return existing
def test_ConnectionDriverOnexit_errors():
r"""Test that errors are raised for invalid onexit."""
assert_raises(ValueError, ConnectionDriver, 'test',
onexit='invalid')
def test_ConnectionDriverTranslate_errors():
r"""Test that errors are raised for invalid translators."""
assert(not hasattr(invalid_translate, '__call__'))
assert_raises(ValueError, ConnectionDriver, 'test',
translator=invalid_translate)
# Dynamically create tests based on registered file classes
s = get_schema()
comm_types = list(s['comm'].schema_subtypes.keys())
for k in comm_types:
if k == _default_comm: # pragma: debug
continue
# Output
ocls = type('Test%sOutputDriver' % k,
(TestConnectionDriver, ), {'ocomm_name': k,
'driver': 'OutputDriver',
'args': 'test'})
# Input
icls = type('Test%sInputDriver' % k,
(TestConnectionDriver, ), {'icomm_name': k,
'driver': 'InputDriver',
'args': 'test'})
# Flags
def parse_connection(yml, existing):
r"""Parse a yaml entry for a connection between I/O channels.
Args:
yml (dict): YAML dictionary for a connection.
existing (dict): Dictionary of existing components.
Raises:
RuntimeError: If the 'inputs' entry is not a model output or file.
RuntimeError: If neither the 'inputs' or 'outputs' entries correspond
to model I/O channels.
Returns:
dict: Updated log of all entries.
"""
schema = get_schema()
# File input
is_file = {'inputs': [], 'outputs': []}
iname_list = []
for x in yml['inputs']:
is_file['inputs'].append(schema.is_valid_component('file', x))
if is_file['inputs'][-1]:
fname = os.path.expanduser(x['name'])
if not os.path.isabs(fname):
fname = os.path.join(x['working_dir'], fname)
fname = os.path.normpath(fname)
if (not os.path.isfile(fname)) and (not x.get(
'wait_for_creation', False)):
raise YAMLSpecificationError(
("Input '%s' not found in any of the registered " +
"model outputs and is not a file.") % x['name'])
x['address'] = fname
elif 'default_value' in x:
x['address'] = x['default_value']
else:
iname_list.append(x['name'])
# File output
oname_list = []
for x in yml['outputs']:
is_file['outputs'].append(schema.is_valid_component('file', x))
if is_file['outputs'][-1]:
fname = os.path.expanduser(x['name'])
if not x.get('in_temp', False):
if not os.path.isabs(fname):
fname = os.path.join(x['working_dir'], fname)
fname = os.path.normpath(fname)
x['address'] = fname
else:
oname_list.append(x['name'])
iname = ','.join(iname_list)
oname = ','.join(oname_list)
if not iname:
args = oname
elif not oname:
args = iname
else:
args = '%s_to_%s' % (iname, oname)
name = args
# Connection
xx = {'src_models': [], 'dst_models': [], 'inputs': [], 'outputs': []}
for i, y in enumerate(yml['inputs']):
if is_file['inputs'][i] or ('default_value' in y):
xx['inputs'].append(y)
else:
new = existing['output'][y['name']]
new.update(y)
xx['inputs'].append(new)
xx['src_models'] += existing['output'][y['name']]['model_driver']
del existing['output'][y['name']]
for i, y in enumerate(yml['outputs']):
if is_file['outputs'][i]:
xx['outputs'].append(y)
else:
new = existing['input'][y['name']]
new.update(y)
xx['outputs'].append(new)
xx['dst_models'] += existing['input'][y['name']]['model_driver']
del existing['input'][y['name']]
# TODO: Split comms if models are not co-located and the main
# process needs access to the message passed
yml.update(xx)
yml.setdefault('driver', 'ConnectionDriver')
yml.setdefault('name', name)
return existing
def component2table(comp, table_type, include_required=None,
subtype_ref=None, **kwargs):
r"""Create a table describing a component.
Args:
comp (str): Name of a component type to create a table for.
table_type (str): Type of table that should be created for the component.
Supported values include:
* 'general': Create a table of properties common to all components
of the specified type.
* 'specific': Create a table of properties specific to only some
components of the specified type.
* 'subtype': Create a table describing the subtypes available for
the specified component type.
include_required (bool, optional): If True, a required column is
included. Defaults to True if table_type is 'general' and False
otherwise.
subtype_ref (str, optional): Reference for the subtype table for the
specified component that should be used in the description for the
subtype property. Defaults to None and is ignored.
**kwargs: Additional keyword arguments are passed to dict2table.
Returns:
list: Lines comprising the table.
"""
from yggdrasil.schema import get_schema
if include_required is None:
if table_type == 'general':
include_required = True
else:
include_required = False
if table_type in ['subtype', 'specific']:
kwargs.setdefault('prune_empty_columns', True)
args = {}
s = get_schema()
subtype_key = s[comp].subtype_key
if table_type in ['general', 'specific']:
# Set defaults
kwargs.setdefault('key_column_name', 'option')
kwargs.setdefault('val_column_name', 'description')
kwargs.setdefault('column_order', [kwargs['key_column_name'],
'type', 'required',
kwargs['val_column_name']])
if (not include_required) and ('required' in kwargs['column_order']):
kwargs['column_order'].remove('required')
# Get list of component subtypes
if table_type == 'general':
s_comp_list = [s[comp].get_subtype_schema('base', unique=True)]
else:
s_comp_list = [s[comp].get_subtype_schema(x, unique=True)
for x in s[comp].classes]
# Loop over subtyeps
out_apply = {}
for s_comp in s_comp_list:
for k, v in s_comp['properties'].items():
if (k == subtype_key) and (table_type == 'specific'):
continue
if k not in args:
args[k] = {'type': v.get('type', ''),
'description': v.get('description', '')}
if include_required:
if k in s_comp.get('required', []):
args[k]['required'] = 'X'
else:
args[k]['required'] = ''
if (table_type == 'specific'):
if k not in out_apply:
out_apply[k] = []
out_apply[k] += s_comp['properties'][subtype_key]['enum']
elif (subtype_ref is not None) and (k == subtype_key):
args[k]['description'] += (
' (Options described :ref:`here <%s>`)' % subtype_ref)
if table_type == 'specific':
for k, v in args.items():
v['Valid for \'%s\' of' % subtype_key] = list(set(out_apply[k]))
elif table_type == 'subtype':
kwargs.setdefault('key_column_name', subtype_key)
for x, subtypes in s[comp].schema_subtypes.items():
s_comp = s[comp].get_subtype_schema(x, unique=True)
subt = subtypes[0]
args[subt] = {
'description': s_comp['properties'][subtype_key].get(
'description', '')}
if len(subtypes) > 1:
args[subt]['aliases'] = subtypes[1:]
if s_comp['properties'][subtype_key].get('default', None) in subtypes:
args[subt]['description'] = ('[DEFAULT] '
+ args[subt]['description'])
else:
raise ValueError("Unsupported table_type: '%s'" % table_type)
return dict2table(args, **kwargs)
def __init__(self, skip_component_schema_normalization=None, **kwargs):
if skip_component_schema_normalization is None:
skip_component_schema_normalization = (not (os.environ.get(
'YGG_VALIDATE_COMPONENTS', 'None').lower() in ['true', '1']))
comptype = self._schema_type
if (comptype is None) and (not self._schema_properties):
self.extra_kwargs = kwargs
return
subtype = None
if self._schema_subtype_key is not None:
subtype = getattr(
self, self._schema_subtype_key,
getattr(self, '_%s' % self._schema_subtype_key, None))
# Fall back to some simple parsing/normalization to save time on
# full jsonschema normalization
for k, v in self._schema_properties.items():
if k in self._schema_excluded_from_class:
continue
default = v.get('default', None)
if (k == self._schema_subtype_key) and (subtype is not None):
default = subtype
if default is not None:
kwargs.setdefault(k, copy.deepcopy(default))
if v.get('type', None) == 'array':
if isinstance(kwargs.get(k, None), (bytes, str)):
kwargs[k] = kwargs[k].split()
# Parse keyword arguments using schema
if (((comptype is not None) and (subtype is not None)
and (not skip_component_schema_normalization))):
from yggdrasil.schema import get_schema
s = get_schema().get_component_schema(
comptype,
subtype,
relaxed=True,
allow_instance_definitions=True)
props = list(s['properties'].keys())
if not skip_component_schema_normalization:
from yggdrasil import metaschema
kwargs.setdefault(self._schema_subtype_key, subtype)
# Remove properties that shouldn't ve validated in class
for k in self._schema_excluded_from_class_validation:
if k in s['properties']:
del s['properties'][k]
# Validate and normalize
metaschema.validate_instance(kwargs, s, normalize=False)
# TODO: Normalization performance needs improvement
# import pprint
# print('before')
# pprint.pprint(kwargs_comp)
# kwargs_comp = metaschema.validate_instance(kwargs_comp, s,
# normalize=True)
# kwargs.update(kwargs_comp)
# print('normalized')
# pprint.pprint(kwargs_comp)
else:
props = self._schema_properties.keys()
# Set attributes based on properties
for k in props:
if k in self._schema_excluded_from_class:
continue
v = kwargs.pop(k, None)
if getattr(self, k, None) is None:
setattr(self, k, v)
# elif (getattr(self, k) != v) and (v is not None):
# warnings.warn(("The schema property '%s' is provided as a "
# "keyword with a value of %s, but the class "
# "already has an attribute of the same name "
# "with the value %s.")
# % (k, v, getattr(self, k)))
self.extra_kwargs = kwargs
import os
import shutil
from yggdrasil import schema, metaschema
schema_dir = os.path.join(os.path.dirname(__file__), 'schema')
if not os.path.isdir(schema_dir):
os.mkdir(schema_dir)
indent = ' '
s = schema.get_schema()
shutil.copy(metaschema._metaschema_fname,
os.path.join(schema_dir, 'metaschema.json'))
with open(os.path.join(schema_dir, 'integration.json'), 'w') as fd:
metaschema.encoder.encode_json(s.schema,
fd=fd,
indent=indent,
sort_keys=False)
schema.get_json_schema(os.path.join(schema_dir, 'integration_strict.json'),
indent=indent)
schema.get_model_form_schema(os.path.join(schema_dir, 'model_form.json'),
indent=indent)
def parse_yaml(files, as_function=False):
r"""Parse list of yaml files.
Args:
files (str, list): Either the path to a single yaml file or a list of
yaml files.
as_function (bool, optional): If True, the missing input/output channels
will be created for using model(s) as a function. Defaults to False.
Raises:
ValueError: If the yml dictionary is missing a required keyword or has
an invalid value.
RuntimeError: If one of the I/O channels is not initialized with driver
information.
Returns:
dict: Dictionary of information parsed from the yamls.
"""
s = get_schema()
# Parse files using schema
yml_prep = prep_yaml(files)
# print('prepped')
# pprint.pprint(yml_prep)
yml_norm = s.validate(yml_prep, normalize=True)
# print('normalized')
# pprint.pprint(yml_norm)
# Determine if any of the models require synchronization
timesync_names = []
for yml in yml_norm['models']:
if yml.get('timesync', False):
if yml['timesync'] is True:
yml['timesync'] = 'timesync'
if not isinstance(yml['timesync'], list):
yml['timesync'] = [yml['timesync']]
for i, tsync in enumerate(yml['timesync']):
if isinstance(tsync, str):
tsync = {'name': tsync}
yml['timesync'][i] = tsync
timesync_names.append(tsync['name'])
yml.setdefault('timesync_client_of', [])
yml['timesync_client_of'].append(tsync['name'])
for tsync in set(timesync_names):
for m in yml_norm['models']:
if m['name'] == tsync:
assert (m['language'] == 'timesync')
m.update(is_server=True, inputs=[], outputs=[])
break
else:
yml_norm['models'].append({
'name': tsync,
'args': [],
'language': 'timesync',
'is_server': True,
'working_dir': os.getcwd(),
'inputs': [],
'outputs': []
})
# Parse models, then connections to ensure connections can be processed
existing = None
for k in ['models', 'connections']:
for yml in yml_norm[k]:
existing = parse_component(yml, k[:-1], existing=existing)
# Create server/client connections
for srv, srv_info in existing['server'].items():
clients = srv_info['clients']
if srv not in existing['input']:
continue
yml = {
'inputs': [{
'name': x
} for x in clients],
'outputs': [{
'name': srv
}],
'driver': 'RPCRequestDriver',
'name': existing['input'][srv]['model_driver'][0]
}
if srv_info.get('replaces', None):
yml['outputs'][0].update({
k: v
for k, v in srv_info['replaces']['input'].items()
if k not in ['name']
})
yml['response_kwargs'] = {
k: v
for k, v in srv_info['replaces']['output'].items()
if k not in ['name']
}
existing = parse_component(yml, 'connection', existing=existing)
existing['model'][yml['dst_models'][0]]['clients'] = yml['src_models']
existing.pop('server')
if as_function:
existing = add_model_function(existing)
# Make sure that servers have clients and clients have servers
for k, v in existing['model'].items():
if v.get('is_server', False):
for x in existing['model'].values():
if v['name'] in x.get('client_of', []):
break
else:
raise YAMLSpecificationError(
"Server '%s' does not have any clients.", k)
elif v.get('client_of', False):
for s in v['client_of']:
missing_servers = []
if s not in existing['model']:
missing_servers.append(s)
if missing_servers:
print(list(existing['model'].keys()))
raise YAMLSpecificationError(
"Servers %s do not exist, but '%s' is a client of them."
% (missing_servers, v['name']))
# Make sure that I/O channels initialized
opp_map = {'input': 'output', 'output': 'input'}
for io in ['input', 'output']:
remove = []
for k in list(existing[io].keys()):
v = existing[io][k]
if 'driver' not in v:
if v.get('is_default', False):
remove.append(k)
elif 'default_file' in v:
new_conn = {
io + 's': [v['default_file']],
opp_map[io] + 's': [v]
}
existing = parse_component(new_conn,
'connection',
existing=existing)
elif (io == 'input') and ('default_value' in v):
# TODO: The keys moved should be automated based on schema
# if the ValueComm has anymore parameters added
vdef = {
'name': v['name'],
'default_value': v.pop('default_value'),
'count': v.pop('count', 1),
'commtype': 'value'
}
new_conn = {'inputs': [vdef], 'outputs': [v]}
existing = parse_component(new_conn,
'connection',
existing=existing)
else:
raise YAMLSpecificationError(
"No driver established for %s channel %s" % (io, k))
# Remove unused default channels
for k in remove:
for m in existing[io][k]['model_driver']:
for i, x in enumerate(existing['model'][m][io + 's']):
if x['name'] == k:
existing['model'][m][io + 's'].pop(i)
break
existing[io].pop(k)
# Link io drivers back to models
existing = link_model_io(existing)
# print('drivers')
# pprint.pprint(existing)
return existing
