def _read_nodeinfo(self, name, base_uri, seen, nodename=None):
path = os.path.join(base_uri, name + FILE_EXTENSION)
try:
entity = YamlFile(path).entity
seen[name] = True
merge_base = Entity()
for klass in entity.classes.as_list():
if klass not in seen:
ret = self._read_nodeinfo(klass, self.classes_uri, seen,
name if nodename is None else nodename)[0]
# on every iteration, we merge the result of the
# recursive descend into what we have so far…
merge_base.merge(ret)
# … and finally, we merge what we have at this level into the
# result of the iteration, so that elements at the current level
# overwrite stuff defined by parents
merge_base.merge(entity)
return merge_base, path
except IOError:
if base_uri == self.classes_uri:
raise reclass.errors.ClassNotFound('yaml_fs', name, base_uri, nodename)
else:
raise reclass.errors.NodeNotFound('yaml_fs', name, base_uri)
def _recurse_entity(self,
entity,
merge_base=None,
seen=None,
nodename=None,
environment=None):
if seen is None:
seen = {}
if environment is None:
environment = self._default_environment
if merge_base is None:
merge_base = Entity(name='empty (@{0})'.format(nodename))
for klass in entity.classes.as_list():
if klass not in seen:
try:
class_entity = self._storage.get_class(klass, environment)
except ClassNotFound, e:
e.set_nodename(nodename)
raise e
descent = self._recurse_entity(class_entity,
seen=seen,
nodename=nodename,
environment=environment)
# on every iteration, we merge the result of the recursive
# descent into what we have so far…
merge_base.merge(descent)
seen[klass] = True
def test_merge_newuri(self, **types):
instances = self._make_instances(**types)
newuri = 'test://uri2'
e1 = Entity(*instances, uri='test://uri1')
e2 = Entity(*instances, uri=newuri)
e1.merge(e2)
self.assertEqual(e1.uri, newuri)
def test_merge_newname(self, **types):
instances = self._make_instances(**types)
newname = 'newname'
e1 = Entity(*instances, name='oldname')
e2 = Entity(*instances, name=newname)
e1.merge(e2)
self.assertEqual(e1.name, newname)
def test_merge_newenv(self, **types):
instances = self._make_instances(**types)
newenv = 'new env'
e1 = Entity(*instances, environment='env')
e2 = Entity(*instances, environment=newenv)
e1.merge(e2)
self.assertEqual(e1.environment, newenv)
def test_merge_newname(self, **types):
instances = self._make_instances(**types)
newname = 'newname'
e1 = Entity(*instances, name='oldname')
e2 = Entity(*instances, name=newname)
e1.merge(e2)
self.assertEqual(e1.name, newname)
def test_merge_newenv(self, **types):
instances = self._make_instances(**types)
newenv = 'new env'
e1 = Entity(*instances, environment='env')
e2 = Entity(*instances, environment=newenv)
e1.merge(e2)
self.assertEqual(e1.environment, newenv)
def test_equal_empty_named(self, **types):
instances = self._make_instances(**types)
self.assertEqual(Entity(SETTINGS, *instances),
Entity(SETTINGS, *instances))
name = 'empty'
self.assertEqual(Entity(SETTINGS, *instances, name=name),
Entity(SETTINGS, *instances, name=name))
def test_exports_with_refs(self):
inventory = {'node1': {'a': 1, 'b': 2}, 'node2': {'a': 3, 'b': 4}}
node3_exports = Exports({'a': '${a}', 'b': '${b}'}, SETTINGS, '')
node3_parameters = Parameters({
'name': 'node3',
'a': '${c}',
'b': 5
}, SETTINGS, '')
node3_parameters.merge({'c': 3})
node3_entity = Entity(SETTINGS,
classes=None,
applications=None,
parameters=node3_parameters,
exports=node3_exports)
node3_entity.interpolate_exports()
inventory['node3'] = node3_entity.exports.as_dict()
r = {
'node1': {
'a': 1,
'b': 2
},
'node2': {
'a': 3,
'b': 4
},
'node3': {
'a': 3,
'b': 5
}
}
self.assertDictEqual(inventory, r)
def _get_class_mappings_entity(self, entity):
if not self._class_mappings:
return Entity(self._settings, name='empty (class mappings)')
c = Classes()
if self._settings.class_mappings_match_path:
matchname = entity.pathname
else:
matchname = entity.name
for mapping in self._class_mappings:
matched = False
key, klasses = Core._shlex_split(mapping)
if key[0] == ('/'):
matched = Core._match_regexp(key[1:-1], matchname)
if matched:
for klass in klasses:
c.append_if_new(matched.expand(klass))
else:
if Core._match_glob(key, matchname):
for klass in klasses:
c.append_if_new(klass)
return Entity(self._settings,
classes=c,
name='class mappings for node {0}'.format(entity.name))
def test_unequal_empty_uri(self, **types):
instances = self._make_instances(**types)
uri = 'test://uri'
self.assertNotEqual(Entity(*instances, uri=uri),
Entity(*instances, uri=uri[::-1]))
for i in instances:
i.__eq__.assert_called_once_with(i)
def test_unequal_empty_named(self, **types):
instances = self._make_instances(**types)
name = 'empty'
self.assertNotEqual(Entity(*instances, name=name),
Entity(*instances, name=name[::-1]))
for i in instances:
i.__eq__.assert_called_once_with(i)
def test_merge_newuri(self, **types):
instances = self._make_instances(**types)
newuri = 'test://uri2'
e1 = Entity(*instances, uri='test://uri1')
e2 = Entity(*instances, uri=newuri)
e1.merge(e2)
self.assertEqual(e1.uri, newuri)
def test_as_dict(self, **types):
instances = self._make_instances(**types)
entity = Entity(*instances, name='test')
comp = {}
comp['classes'] = instances[0].as_list()
comp['applications'] = instances[1].as_list()
comp['parameters'] = instances[2].as_dict()
d = entity.as_dict()
self.assertDictEqual(d, comp)
def test_as_dict(self, **types):
instances = self._make_instances(**types)
entity = Entity(*instances, name='test', environment='test')
comp = {}
comp['classes'] = instances[0].as_list()
comp['applications'] = instances[1].as_list()
comp['parameters'] = instances[2].as_dict()
comp['environment'] = 'test'
d = entity.as_dict()
self.assertDictEqual(d, comp)
def _nodeinfo(self, nodename):
node_entity = self._storage.get_node(nodename)
base_entity = Entity(name='base')
base_entity.merge(self._get_class_mappings_entity(node_entity.name))
base_entity.merge(self._get_input_data_entity())
seen = {}
merge_base = self._recurse_entity(base_entity, seen=seen,
nodename=base_entity.name)
ret = self._recurse_entity(node_entity, merge_base, seen=seen,
nodename=node_entity.name)
ret.interpolate()
return ret
def _nodeinfo(self, nodename):
node_entity = self._storage.get_node(nodename)
base_entity = Entity(name='base')
base_entity.merge(self._get_class_mappings_entity(node_entity.name))
base_entity.merge(self._get_input_data_entity())
seen = {}
merge_base = self._recurse_entity(base_entity, seen=seen,
nodename=base_entity.name)
ret = self._recurse_entity(node_entity, merge_base, seen=seen,
nodename=node_entity.name)
try:
ret.interpolate()
return ret
except UndefinedVariableError as e:
e.set_context('%s @ %s' % (e.context, nodename))
raise e
def test_constructor_default(self, **mocks):
# Actually test the real objects by calling the default constructor,
# all other tests shall pass instances to the constructor
e = Entity()
self.assertEqual(e.name, '')
self.assertIsInstance(e.classes, Classes)
self.assertIsInstance(e.applications, Applications)
self.assertIsInstance(e.parameters, Parameters)
def test_interpolate_list_types(self):
node1_exports = Exports({'exps': ['${one}']}, SETTINGS, 'first')
node1_parameters = Parameters(
{
'alpha': ['${two}', '${three}'],
'one': 1,
'two': 2,
'three': 3
}, SETTINGS, 'first')
node1_entity = Entity(SETTINGS,
classes=None,
applications=None,
parameters=node1_parameters,
exports=node1_exports)
node2_exports = Exports({'exps': '${alpha}'}, SETTINGS, 'second')
node2_parameters = Parameters({}, SETTINGS, 'second')
node2_entity = Entity(SETTINGS,
classes=None,
applications=None,
parameters=node2_parameters,
exports=node2_exports)
result = {'exps': [1, 2, 3]}
node1_entity.merge(node2_entity)
node1_entity.interpolate(None)
self.assertIs(type(node1_entity.exports.as_dict()['exps']), list)
self.assertDictEqual(node1_entity.exports.as_dict(), result)
def _populate_with_class_mappings(self, nodename):
if not self._class_mappings:
return Entity(name='empty')
c = Classes()
for mapping in self._class_mappings:
matched = False
key, klasses = Core._shlex_split(mapping)
if key[0] == ('/'):
matched = Core._match_regexp(key[1:-1], nodename)
if matched:
for klass in klasses:
c.append_if_new(matched.expand(klass))
else:
if Core._match_glob(key, nodename):
for klass in klasses:
c.append_if_new(klass)
return Entity(classes=c,
name='class mappings for node {0}'.format(nodename))
def test_constructor_empty(self, **types):
instances = self._make_instances(**types)
e = Entity(*instances)
self.assertEqual(e.name, '')
cl, al, pl = [getattr(i, '__len__') for i in instances]
self.assertEqual(len(e.classes), cl.return_value)
cl.assert_called_once_with()
self.assertEqual(len(e.applications), al.return_value)
al.assert_called_once_with()
self.assertEqual(len(e.parameters), pl.return_value)
pl.assert_called_once_with()
def _node_entity(self, nodename):
node_entity = self._storage.get_node(nodename, self._settings)
if node_entity.environment == None:
node_entity.environment = self._settings.default_environment
base_entity = Entity(self._settings, name='base')
base_entity.merge(self._get_class_mappings_entity(node_entity.name))
base_entity.merge(self._get_input_data_entity())
base_entity.merge_parameters(self._get_automatic_parameters(nodename, node_entity.environment))
seen = {}
merge_base = self._recurse_entity(base_entity, seen=seen, nodename=nodename,
environment=node_entity.environment)
return self._recurse_entity(node_entity, merge_base=merge_base, context=merge_base, seen=seen,
nodename=nodename, environment=node_entity.environment)
def _recurse_entity(self, entity, merge_base=None, seen=None, nodename=None):
if seen is None:
seen = {}
if merge_base is None:
merge_base = Entity(name="empty (@{0})".format(nodename))
for klass in entity.classes.as_list():
if klass not in seen:
try:
class_entity = self._storage.get_class(klass)
except ClassNotFound, e:
e.set_nodename(nodename)
raise e
descent = self._recurse_entity(class_entity, seen=seen, nodename=nodename)
# on every iteration, we merge the result of the recursive
# descent into what we have so far…
merge_base.merge(descent)
seen[klass] = True
def _recurse_entity(self, entity, merge_base=None, seen=None, nodename=None):
if seen is None:
seen = {}
if merge_base is None:
merge_base = Entity(name='empty (@{0})'.format(nodename))
for klass in entity.classes.as_list():
if klass not in seen:
try:
class_entity = self._classes_cache[klass]
except KeyError, e:
class_entity = self._get_class(klass, nodename)
self._classes_cache[klass] = class_entity
descent = self._recurse_entity(class_entity, seen=seen,
nodename=nodename)
# on every iteration, we merge the result of the recursive
# descent into what we have so far…
merge_base.merge(descent)
seen[klass] = True
def _read_nodeinfo(self, name, base_uri, seen, nodename=None):
path = os.path.join(base_uri, name + FILE_EXTENSION)
try:
entity = YamlFile(path).entity
seen[name] = True
merge_base = Entity()
for klass in entity.classes.as_list():
if klass not in seen:
ret = self._read_nodeinfo(
klass, self.classes_uri, seen,
name if nodename is None else nodename)[0]
# on every iteration, we merge the result of the
# recursive descend into what we have so far…
merge_base.merge(ret)
# … and finally, we merge what we have at this level into the
# result of the iteration, so that elements at the current level
# overwrite stuff defined by parents
merge_base.merge(entity)
return merge_base, path
except IOError:
if base_uri == self.classes_uri:
raise reclass.errors.ClassNotFound('yaml_fs', name, base_uri,
nodename)
else:
raise reclass.errors.NodeNotFound('yaml_fs', name, base_uri)
def _recurse_entity(self, entity, merge_base=None, context=None, seen=None, nodename=None, environment=None):
if seen is None:
seen = {}
if environment is None:
environment = self._settings.default_environment
if merge_base is None:
merge_base = Entity(self._settings, name='empty (@{0})'.format(nodename))
if context is None:
context = Entity(self._settings, name='empty (@{0})'.format(nodename))
for klass in entity.classes.as_list():
# class name contain reference
num_references = klass.count(self._settings.reference_sentinels[0]) +\
klass.count(self._settings.export_sentinels[0])
if num_references > 0:
try:
klass = str(self._parser.parse(klass, self._settings).render(merge_base.parameters.as_dict(), {}))
except ResolveError as e:
try:
klass = str(self._parser.parse(klass, self._settings).render(context.parameters.as_dict(), {}))
except ResolveError as e:
raise ClassNameResolveError(klass, nodename, entity.uri)
if klass not in seen:
try:
class_entity = self._storage.get_class(klass, environment, self._settings)
except ClassNotFound as e:
if self._settings.ignore_class_notfound:
if self._cnf_r.match(klass):
if self._settings.ignore_class_notfound_warning:
# TODO, add logging handler
print("[WARNING] Reclass class not found: '%s'. Skipped!" % klass, file=sys.stderr)
continue
e.nodename = nodename
e.uri = entity.uri
raise
descent = self._recurse_entity(class_entity, context=context, seen=seen,
nodename=nodename, environment=environment)
# on every iteration, we merge the result of the recursive
# descent into what we have so far…
merge_base.merge(descent)
seen[klass] = True
# … and finally, we merge what we have at this level into the
# result of the iteration, so that elements at the current level
# overwrite stuff defined by parents
merge_base.merge(entity)
return merge_base
def test_reference_to_an_export(self):
inventory = {'node1': {'a': 1, 'b': 2}, 'node2': {'a': 3, 'b': 4}}
node3_exports = Exports({'a': '${a}', 'b': '${b}'}, SETTINGS, '')
node3_parameters = Parameters(
{
'name': 'node3',
'ref': '${exp}',
'a': '${c}',
'b': 5
}, SETTINGS, '')
node3_parameters.merge({'c': 3, 'exp': '$[ exports:a ]'})
node3_entity = Entity(SETTINGS,
classes=None,
applications=None,
parameters=node3_parameters,
exports=node3_exports)
node3_entity.interpolate_exports()
inventory['node3'] = node3_entity.exports.as_dict()
node3_entity.interpolate(inventory)
res_inv = {
'node1': {
'a': 1,
'b': 2
},
'node2': {
'a': 3,
'b': 4
},
'node3': {
'a': 3,
'b': 5
}
}
res_params = {
'a': 3,
'c': 3,
'b': 5,
'name': 'node3',
'exp': {
'node1': 1,
'node3': 3,
'node2': 3
},
'ref': {
'node1': 1,
'node3': 3,
'node2': 3
}
}
self.assertDictEqual(node3_parameters.as_dict(), res_params)
self.assertDictEqual(inventory, res_inv)
def _nodeinfo(self, nodename):
node_entity = self._storage.get_node(nodename)
base_entity = Entity(name='base')
base_entity.merge(self._get_class_mappings_entity(node_entity.name))
base_entity.merge(self._get_input_data_entity())
seen = {}
merge_base = self._recurse_entity(base_entity,
seen=seen,
nodename=base_entity.name)
ret = self._recurse_entity(node_entity,
merge_base,
seen=seen,
nodename=node_entity.name)
ret.interpolate()
return ret
def _recurse_entity(self,
entity,
merge_base=None,
seen=None,
nodename=None,
environment=None):
if seen is None:
seen = {}
if environment is None:
environment = self._settings.default_environment
if merge_base is None:
merge_base = Entity(self._settings,
name='empty (@{0})'.format(nodename))
for klass in entity.classes.as_list():
if klass not in seen:
try:
class_entity = self._storage.get_class(
klass, environment, self._settings)
except ClassNotFound as e:
if self._settings.ignore_class_notfound:
if self._cnf_r.match(klass):
if self._settings.ignore_class_notfound_warning:
# TODO, add logging handler
print >> sys.stderr, "[WARNING] Reclass class not found: '%s'. Skipped!" % klass
continue
e.nodename = nodename
e.uri = entity.uri
raise
descent = self._recurse_entity(class_entity,
seen=seen,
nodename=nodename,
environment=environment)
# on every iteration, we merge the result of the recursive
# descent into what we have so far…
merge_base.merge(descent)
seen[klass] = True
# … and finally, we merge what we have at this level into the
# result of the iteration, so that elements at the current level
# overwrite stuff defined by parents
merge_base.merge(entity)
return merge_base
def _nodeinfo(self, nodename):
node_entity = self._storage.get_node(nodename)
base_entity = Entity(name='base')
base_entity.merge(self._get_class_mappings_entity(node_entity.name))
base_entity.merge(self._get_input_data_entity())
seen = {}
merge_base = self._recurse_entity(base_entity,
seen=seen,
nodename=base_entity.name)
ret = self._recurse_entity(node_entity,
merge_base,
seen=seen,
nodename=node_entity.name)
try:
ret.interpolate()
return ret
except UndefinedVariableError as e:
e.set_context('%s @ %s' % (e.context, nodename))
raise e
def test_unequal_types(self, **types):
instances = self._make_instances(**types)
self.assertNotEqual(Entity(*instances, name='empty'), None)
for i in instances:
self.assertEqual(i.__eq__.call_count, 0)
def test_equal_empty(self, **types):
instances = self._make_instances(**types)
self.assertEqual(Entity(*instances), Entity(*instances))
for i in instances:
i.__eq__.assert_called_once_with(i)
def test_merge(self, **types):
instances = self._make_instances(**types)
e = Entity(*instances)
e.merge(e)
for i, fn in zip(instances, ('merge_unique', 'merge_unique', 'merge')):
getattr(i, fn).assert_called_once_with(i)
def test_constructor_empty_uri(self, **types):
uri = 'test://uri'
e = Entity(*self._make_instances(**types), uri=uri)
self.assertEqual(e.uri, uri)
def test_constructor_empty_named(self, **types):
name = 'empty'
e = Entity(*self._make_instances(**types), name=name)
self.assertEqual(e.name, name)
def test_exports_with_nested_references(self):
inventory = {
'node1': {
'alpha': {
'a': 1,
'b': 2
}
},
'node2': {
'alpha': {
'a': 3,
'b': 4
}
}
}
node3_exports = Exports({'alpha': '${alpha}'})
node3_parameters = Parameters({
'name': 'node3',
'alpha': {
'a': '${one}',
'b': '${two}'
},
'beta': '$[ exports:alpha ]',
'one': '111',
'two': '${three}',
'three': '123'
})
node3_entity = Entity(None, None, node3_parameters, node3_exports)
res_params = {
'beta': {
'node1': {
'a': 1,
'b': 2
},
'node3': {
'a': '111',
'b': '123'
},
'node2': {
'a': 3,
'b': 4
}
},
'name': 'node3',
'alpha': {
'a': '111',
'b': '123'
},
'three': '123',
'two': '123',
'one': '111'
}
res_inv = {
'node1': {
'alpha': {
'a': 1,
'b': 2
}
},
'node2': {
'alpha': {
'a': 3,
'b': 4
}
},
'node3': {
'alpha': {
'a': '111',
'b': '123'
}
}
}
node3_entity.interpolate_exports()
inventory['node3'] = node3_entity.exports.as_dict()
node3_entity.interpolate('node3', inventory)
self.assertDictEqual(node3_parameters.as_dict(), res_params)
self.assertDictEqual(inventory, res_inv)
def _get_input_data_entity(self):
if not self._input_data:
return Entity(name='empty (input data)')
p = Parameters(self._input_data)
return Entity(parameters=p, name='input data')
def test_constructor_empty_env(self, **types):
env = 'not base'
e = Entity(*self._make_instances(**types), environment=env)
self.assertEqual(e.environment, env)
def test_merge(self, **types):
instances = self._make_instances(**types)
e = Entity(*instances)
e.merge(e)
for i, fn in zip(instances, ('merge_unique', 'merge_unique', 'merge')):
getattr(i, fn).assert_called_once_with(i)
def _test_constructor_wrong_types(self, which_replace, **types):
instances = self._make_instances(**types)
instances[which_replace] = 'Invalid type'
e = Entity(*instances)
