def testConfigListAllNDWithoutAll(self):
"""test all groups listing without all upcall (nD)"""
# Even in nD, ensure that $GROUP is a simple group that has been previously expanded
f = make_temp_file("""
# A comment
[Main]
default: local
[local]
map: if [[ $GROUP == "x1y[3-4]" ]]; then exit 1; elif [[ $GROUP == "x1y1" ]]; then echo rack[1-5]z[1-42]; else echo rack[6-10]z[1-42]; fi
#all:
list: echo x1y1 x1y2 x1y[3-4]
#reverse:
""")
res = GroupResolverConfig(f.name, illegal_chars=ILLEGAL_GROUP_CHARS)
nodeset = NodeSet("rack3z40", resolver=res)
self.assertEqual(str(NodeSet.fromall(resolver=res)),
"rack[1-10]z[1-42]")
self.assertEqual(res.grouplist(), ['x1y1', 'x1y2', 'x1y[3-4]']) # raw
self.assertEqual(grouplist(resolver=res),
['x1y1', 'x1y2', 'x1y3', 'x1y4']) # cleaned
# test "@*" magic group listing
nodeset = NodeSet("@*", resolver=res)
self.assertEqual(str(nodeset), "rack[1-10]z[1-42]")
# with group source
nodeset = NodeSet("@local:*", resolver=res)
self.assertEqual(str(nodeset), "rack[1-10]z[1-42]")
nodeset = NodeSet("rack11z1,@local:*,rack11z[2-42]", resolver=res)
self.assertEqual(str(nodeset), "rack[1-11]z[1-42]")
def testConfigGroupsDirExistsNoOther(self):
"""test groups with groupsdir defined (real, no other)"""
dname1 = make_temp_dir()
dname2 = make_temp_dir()
f = make_temp_file("""
[Main]
default: new_local
groupsdir: %s %s
""" % (dname1, dname2))
f2 = make_temp_file("""
[new_local]
map: echo example[1-100]
#all:
list: echo bar
#reverse:
""",
suffix=".conf",
dir=dname2)
try:
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
self.assertEqual(nodeset.regroup(), "@bar")
self.assertEqual(str(NodeSet("@bar", resolver=res)),
"example[1-100]")
finally:
f2.close()
f.close()
shutil.rmtree(dname1, ignore_errors=True)
shutil.rmtree(dname2, ignore_errors=True)
def test_fromall_grouplist(self):
"""test NodeSet.fromall() without all upcall"""
# Group Source that has no all upcall and that can handle special char
test_groups2 = makeTestG2()
source = UpcallGroupSource("simple",
"sed -n 's/^$GROUP:\(.*\)/\\1/p' %s" % test_groups2.name,
None,
"sed -n 's/^\([0-9A-Za-z_-\%%]*\):.*/\\1/p' %s"
% test_groups2.name,
None)
res = GroupResolver(source)
# fromall will trigger ParserEngine.grouplist() that we want to test here
nsall = NodeSet.fromall(resolver=res)
# if working, group resolution worked with % char
self.assertEqual(str(NodeSet.fromall(resolver=res)), "montana[32-55,87-90]")
self.assertEqual(len(nsall), 28)
# btw explicitly check escaped char
nsesc = NodeSet('@escape%test', resolver=res)
self.assertEqual(str(nsesc), 'montana[87-90]')
self.assertEqual(len(nsesc), 4)
nsesc2 = NodeSet('@esc%test2', resolver=res)
self.assertEqual(nsesc, nsesc2)
ns = NodeSet('montana[87-90]', resolver=res)
# could also result in escape%test?
self.assertEqual(ns.regroup(), '@esc%test2')
def testConfigGroupsDirExistsNoOther(self):
"""test groups with groupsdir defined (real, no other)"""
dname1 = make_temp_dir()
dname2 = make_temp_dir()
f = make_temp_file("""
[Main]
default: new_local
groupsdir: %s %s
""" % (dname1, dname2))
f2 = make_temp_file("""
[new_local]
map: echo example[1-100]
#all:
list: echo bar
#reverse:
""", suffix=".conf", dir=dname2)
try:
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
self.assertEqual(nodeset.regroup(), "@bar")
self.assertEqual(str(NodeSet("@bar", resolver=res)), "example[1-100]")
finally:
f2.close()
f.close()
shutil.rmtree(dname1, ignore_errors=True)
shutil.rmtree(dname2, ignore_errors=True)
def testConfigListAllWithAll(self):
"""test all groups listing with all upcall"""
f = make_temp_file("""
# A comment
[Main]
default: local
[local]
map: echo example[1-100]
all: echo foo bar
list: echo foo
#reverse:
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-50]", resolver=res)
self.assertEqual(str(nodeset), "example[1-50]")
self.assertEqual(str(NodeSet.fromall(resolver=res)), "bar,foo")
# test "@*" magic group listing
nodeset = NodeSet("@*", resolver=res)
self.assertEqual(str(nodeset), "bar,foo")
nodeset = NodeSet("rab,@*,oof", resolver=res)
self.assertEqual(str(nodeset), "bar,foo,oof,rab")
# with group source
nodeset = NodeSet("@local:*", resolver=res)
self.assertEqual(str(nodeset), "bar,foo")
nodeset = NodeSet("rab,@local:*,oof", resolver=res)
self.assertEqual(str(nodeset), "bar,foo,oof,rab")
def testConfigCrossRefs(self):
"""test groups config with cross references"""
f = make_temp_file("""
# A comment
[Main]
default: other
[local]
map: echo example[1-100]
[other]
map: echo "foo: @local:foo" | sed -n 's/^$GROUP:\(.*\)/\\1/p'
[third]
map: echo -e "bar: @ref-rel\\nref-rel: @other:foo\\nref-all: @*" | sed -n 's/^$GROUP:\(.*\)/\\1/p'
list: echo bar
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("@other:foo", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
# @third:bar -> @ref-rel (third) -> @other:foo -> @local:foo -> nodes
nodeset = NodeSet("@third:bar", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
nodeset = NodeSet("@third:ref-all", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
def testGroupSyntaxes(self):
"""test NodeSet group operation syntaxes"""
nodeset = NodeSet("@gpu")
self.assertEqual(str(nodeset), "montana[38-41]")
nodeset = NodeSet("@chassis[1-3,5]&@chassis[2-3]")
self.assertEqual(str(nodeset), "montana[34-37]")
nodeset1 = NodeSet("@io!@mds")
nodeset2 = NodeSet("@oss")
self.assertEqual(str(nodeset1), str(nodeset2))
self.assertEqual(str(nodeset1), "montana[4-5]")
def testGroupGroups(self):
"""test NodeSet.groups()"""
nodeset = NodeSet("montana[32-37,42-55]")
self.assertEqual(sorted(nodeset.groups().keys()), ['@all', '@chassis1', '@chassis10', '@chassis11', '@chassis12', '@chassis2', '@chassis3', '@chassis6', '@chassis7', '@chassis8', '@chassis9', '@compute'])
testns = NodeSet()
for gnodes, inodes in nodeset.groups().itervalues():
testns.update(inodes)
self.assertEqual(testns, nodeset)
def test_clear_cache(self):
"""test GroupSource.clear_cache()"""
source = StaticGroupSource('cache', {'map': {'a': 'foo1', 'b': 'foo2'} })
# create custom resolver with default source
res = GroupResolver(source)
# Populate map cache
self.assertEqual("foo1", str(NodeSet("@a", resolver=res)))
self.assertEqual("foo2", str(NodeSet("@b", resolver=res)))
self.assertEqual(len(source._cache['map']), 2)
# Clear cache
source.clear_cache()
self.assertEqual(len(source._cache['map']), 0)
def testConfigBasicNoMain(self):
"""test groups with a local config without main section"""
f = make_temp_file("""
# A comment
[local]
map: echo example[1-100]
#all:
list: echo foo
#reverse:
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
self.assertEqual(nodeset.regroup(), "@foo")
self.assertEqual(str(NodeSet("@foo", resolver=res)), "example[1-100]")
def testConfigBasicNoMain(self):
"""test groups with a local config without main section"""
f = make_temp_file("""
# A comment
[local]
map: echo example[1-100]
#all:
list: echo foo
#reverse:
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
self.assertEqual(nodeset.regroup(), "@foo")
self.assertEqual(str(NodeSet("@foo", resolver=res)), "example[1-100]")
def testGroupListDefault(self):
"""test NodeSet group listing GroupResolver.grouplist()"""
groups = ClusterShell.NodeSet.RESOLVER_STD_GROUP.grouplist()
self.assertEqual(len(groups), 20)
helper_groups = grouplist()
self.assertEqual(len(helper_groups), 20)
total = 0
nodes = NodeSet()
for group in groups:
ns = NodeSet("@%s" % group)
total += len(ns)
nodes.update(ns)
self.assertEqual(total, 310)
all_nodes = NodeSet.fromall()
self.assertEqual(len(all_nodes), len(nodes))
self.assertEqual(all_nodes, nodes)
def testGroupResolverMinimal(self):
"""test NodeSet with minimal GroupResolver"""
test_groups1 = makeTestG1()
source = GroupSource("minimal",
"sed -n 's/^$GROUP:\(.*\)/\\1/p' %s" % test_groups1.name,
None, None, None)
# create custom resolver with default source
res = GroupResolver(source)
nodeset = NodeSet("@gpu", resolver=res)
self.assertEqual(nodeset, NodeSet("montana[38-41]"))
self.assertEqual(str(nodeset), "montana[38-41]")
self.assertRaises(NodeSetExternalError, NodeSet.fromall, resolver=res)
def testGroupListSource2(self):
"""test NodeSet group listing GroupResolver.grouplist(source)"""
groups = std_group_resolver().grouplist("source2")
self.assertEqual(len(groups), 2)
total = 0
for group in groups:
total += len(NodeSet("@source2:%s" % group))
self.assertEqual(total, 24)
def test_expired_cache(self):
"""test GroupSource cache entries expired according to config"""
# create custom resolver with default source
source = StaticGroupSource('cache', {'map': {'a': 'foo1', 'b': 'foo2'} })
source.cache_delay = 0.2
res = GroupResolver(source)
# Populate map cache
self.assertEqual("foo1", str(NodeSet("@a", resolver=res)))
self.assertEqual("foo2", str(NodeSet("@b", resolver=res)))
self.assertEqual(len(source._cache['map']), 2)
# Be sure 0.2 cache delay is expired (especially for old Python version)
time.sleep(0.25)
source._data['map']['a'] = 'something_else'
self.assertEqual('something_else', str(NodeSet("@a", resolver=res)))
def testGroupListDefault(self):
"""test NodeSet group listing GroupResolver.grouplist()"""
groups = std_group_resolver().grouplist()
self.assertEqual(len(groups), 20)
helper_groups = grouplist()
self.assertEqual(len(helper_groups), 20)
total = 0
nodes = NodeSet()
for group in groups:
ns = NodeSet("@%s" % group)
total += len(ns)
nodes.update(ns)
self.assertEqual(total, 310)
all_nodes = NodeSet.fromall()
self.assertEqual(len(all_nodes), len(nodes))
self.assertEqual(all_nodes, nodes)
def testGroupListSource2(self):
"""test NodeSet group listing GroupResolver.grouplist(source)"""
groups = ClusterShell.NodeSet.RESOLVER_STD_GROUP.grouplist("source2")
self.assertEqual(len(groups), 2)
total = 0
for group in groups:
total += len(NodeSet("@source2:%s" % group))
self.assertEqual(total, 24)
def testConfigWithEmptyList(self):
"""test groups with list upcall returning nothing"""
f = make_temp_file("""
# A comment
[Main]
default: local
[local]
map: echo example[1-100]
#all:
list: :
reverse: echo foo
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
self.assertEqual(nodeset.regroup(), "@foo")
def testConfigWithEmptyList(self):
"""test groups with list upcall returning nothing"""
f = make_temp_file("""
# A comment
[Main]
default: local
[local]
map: echo example[1-100]
#all:
list: :
reverse: echo foo
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
self.assertEqual(nodeset.regroup(), "@foo")
def testConfigNoListButReverseQuery(self):
"""test groups with no list but reverse upcall"""
f = make_temp_file("""
# A comment
[Main]
default: local
[local]
map: echo example[1-100]
#all:
#list: echo foo
reverse: echo foo
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
self.assertEqual(nodeset.regroup(), "@foo")
def testConfigNoListButReverseQuery(self):
"""test groups with no list but reverse upcall"""
f = make_temp_file("""
# A comment
[Main]
default: local
[local]
map: echo example[1-100]
#all:
#list: echo foo
reverse: echo foo
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
self.assertEqual(nodeset.regroup(), "@foo")
def testConfigEmpty(self):
"""test groups with an empty configuration file"""
f = make_temp_file("")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
self.assertRaises(GroupResolverSourceError, nodeset.regroup)
# non existant group
self.assertRaises(GroupResolverSourceError, NodeSet, "@bar", resolver=res)
def testConfigBasicLocalAlternative(self):
"""test groups with a basic local config file (= alternative)"""
f = make_temp_file("""
# A comment
[Main]
default=local
[local]
map=echo example[1-100]
#all=
list=echo foo
#reverse=
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
self.assertEqual(nodeset.regroup(), "@foo")
self.assertEqual(str(NodeSet("@foo", resolver=res)), "example[1-100]")
def testConfigBasicEmptyDefault(self):
"""test groups with a empty default namespace"""
f = make_temp_file("""
# A comment
[Main]
default:
[local]
map: echo example[1-100]
#all:
list: echo foo
#reverse:
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
self.assertEqual(nodeset.regroup(), "@foo")
self.assertEqual(str(NodeSet("@foo", resolver=res)), "example[1-100]")
def testConfigBasicLocalAlternative(self):
"""test groups with a basic local config file (= alternative)"""
f = make_temp_file("""
# A comment
[Main]
default=local
[local]
map=echo example[1-100]
#all=
list=echo foo
#reverse=
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
self.assertEqual(nodeset.regroup(), "@foo")
self.assertEqual(str(NodeSet("@foo", resolver=res)), "example[1-100]")
def testConfigGroupsDirDummy(self):
"""test groups with groupsdir defined (dummy)"""
f = make_temp_file("""
[Main]
default: local
groupsdir: /path/to/nowhere
[local]
map: echo example[1-100]
#all:
list: echo foo
#reverse:
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
self.assertEqual(nodeset.regroup(), "@foo")
self.assertEqual(str(NodeSet("@foo", resolver=res)), "example[1-100]")
def testConfigNoListNoReverse(self):
"""test groups with no list and not reverse upcall"""
f = make_temp_file("""
# A comment
[Main]
default: local
[local]
map: echo example[1-100]
#all:
#list:
#reverse:
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
# not able to regroup, should still return valid nodeset
self.assertEqual(nodeset.regroup(), "example[1-100]")
def testConfigNoListNoReverse(self):
"""test groups with no list and not reverse upcall"""
f = make_temp_file("""
# A comment
[Main]
default: local
[local]
map: echo example[1-100]
#all:
#list:
#reverse:
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
# not able to regroup, should still return valid nodeset
self.assertEqual(nodeset.regroup(), "example[1-100]")
def testConfigCFGDIR(self):
"""test groups with $CFGDIR use in upcalls"""
f = make_temp_file("""
[Main]
default: local
[local]
map: echo example[1-100]
list: basename $CFGDIR
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
# just a trick to check $CFGDIR resolution...
tmpgroup = os.path.basename(os.path.dirname(f.name))
self.assertEqual(nodeset.groups().keys(), ['@%s' % tmpgroup])
self.assertEqual(str(nodeset), "example[1-100]")
self.assertEqual(nodeset.regroup(), "@%s" % tmpgroup)
self.assertEqual(str(NodeSet("@%s" % tmpgroup, resolver=res)),
"example[1-100]")
def testConfigBasicEmptyDefault(self):
"""test groups with a empty default namespace"""
f = make_temp_file("""
# A comment
[Main]
default:
[local]
map: echo example[1-100]
#all:
list: echo foo
#reverse:
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
self.assertEqual(nodeset.regroup(), "@foo")
self.assertEqual(str(NodeSet("@foo", resolver=res)), "example[1-100]")
def testConfigGroupsDirDummy(self):
"""test groups with groupsdir defined (dummy)"""
f = make_temp_file("""
[Main]
default: local
groupsdir: /path/to/nowhere
[local]
map: echo example[1-100]
#all:
list: echo foo
#reverse:
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
self.assertEqual(nodeset.regroup(), "@foo")
self.assertEqual(str(NodeSet("@foo", resolver=res)), "example[1-100]")
def test_config_cache_delay(self):
"""test group config cache_delay options"""
f = make_temp_file("""
[local]
cache_delay: 0.2
map: echo foo1
""")
res = GroupResolverConfig(f.name)
self.assertEqual(res._sources['local'].cache_delay, 0.2)
self.assertEqual("foo1", str(NodeSet("@local:foo", resolver=res)))
def testConfigBasicLocalVerbose(self):
"""test groups with a basic local config file (verbose)"""
f = make_temp_file("""
# A comment
[Main]
default: local
[local]
map: echo example[1-100]
#all:
list: echo foo
#reverse:
""")
res = GroupResolverConfig(f.name)
res.set_verbosity(1)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
self.assertEqual(nodeset.regroup(), "@foo")
self.assertEqual(str(NodeSet("@foo", resolver=res)), "example[1-100]")
def testConfigBasicLocalVerbose(self):
"""test groups with a basic local config file (verbose)"""
f = make_temp_file("""
# A comment
[Main]
default: local
[local]
map: echo example[1-100]
#all:
list: echo foo
#reverse:
""")
res = GroupResolverConfig(f.name)
res.set_verbosity(1)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
self.assertEqual(nodeset.regroup(), "@foo")
self.assertEqual(str(NodeSet("@foo", resolver=res)), "example[1-100]")
def testConfigListAllWithoutAll(self):
"""test all groups listing without all upcall"""
f = make_temp_file("""
# A comment
[Main]
default: local
[local]
map: echo example[1-100]
#all:
list: echo foo bar
#reverse:
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-50]", resolver=res)
self.assertEqual(str(nodeset), "example[1-50]")
self.assertEqual(str(NodeSet.fromall(resolver=res)), "example[1-100]")
# test "@*" magic group listing
nodeset = NodeSet("@*", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
nodeset = NodeSet("@*,example[101-104]", resolver=res)
self.assertEqual(str(nodeset), "example[1-104]")
nodeset = NodeSet("example[105-149],@*,example[101-104]", resolver=res)
self.assertEqual(str(nodeset), "example[1-149]")
# with group source
nodeset = NodeSet("@local:*", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
nodeset = NodeSet("example0,@local:*,example[101-110]", resolver=res)
self.assertEqual(str(nodeset), "example[0-110]")
def testGroupGroups(self):
"""test NodeSet.groups()"""
nodeset = NodeSet("montana[32-37,42-55]")
self.assertEqual(sorted(nodeset.groups().keys()), ['@all', '@chassis1', '@chassis10', '@chassis11', '@chassis12', '@chassis2', '@chassis3', '@chassis6', '@chassis7', '@chassis8', '@chassis9', '@compute'])
testns = NodeSet()
for gnodes, inodes in nodeset.groups().itervalues():
testns.update(inodes)
self.assertEqual(testns, nodeset)
def testConfigIllegalCharsND(self):
"""test group list containing illegal characters"""
f = make_temp_file("""
# A comment
[Main]
default: local
[local]
map: echo rack[6-10]z[1-42]
#all:
list: echo x1y1 x1y2 @illegal x1y[3-4]
#reverse:
""")
res = GroupResolverConfig(f.name, illegal_chars=ILLEGAL_GROUP_CHARS)
nodeset = NodeSet("rack3z40", resolver=res)
self.assertRaises(GroupResolverIllegalCharError, res.grouplist)
def testConfigRegroupWrongNamespace(self):
"""test groups by calling regroup(wrong_namespace)"""
f = make_temp_file("""
# A comment
[Main]
default: local
[local]
map: echo example[1-100]
#all:
list: echo foo
#reverse:
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertRaises(GroupResolverSourceError, nodeset.regroup, "unknown")
def testConfigCrossRefs(self):
"""test groups config with cross references"""
f = make_temp_file("""
# A comment
[Main]
default: other
[local]
map: echo example[1-100]
[other]
map: echo "foo: @local:foo" | sed -n 's/^$GROUP:\(.*\)/\\1/p'
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("@other:foo", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
def testConfigQueryFailed(self):
"""test groups with config and failed query"""
f = make_temp_file("""
# A comment
[Main]
default: local
[local]
map: false
#all:
list: echo foo
#reverse:
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
self.assertRaises(NodeSetExternalError, nodeset.regroup)
def testConfigIllegalChars(self):
"""test groups with illegal characters"""
f = make_temp_file("""
# A comment
[Main]
default: local
[local]
map: echo example[1-100]
#all:
list: echo 'foo *'
reverse: echo f^oo
""")
res = GroupResolverConfig(f.name, illegal_chars=set("@,&!&^*"))
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertRaises(GroupResolverIllegalCharError, nodeset.groups)
self.assertRaises(GroupResolverIllegalCharError, nodeset.regroup)
def testConfigBasicLocal(self):
"""test groups with a basic local config file"""
f = make_temp_file("""
# A comment
[Main]
default: local
[local]
map: echo example[1-100]
#all:
list: echo foo
#reverse:
""")
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
self.assertEqual(nodeset.regroup(), "@foo")
self.assertEqual(nodeset.groups().keys(), ["@foo"])
self.assertEqual(str(NodeSet("@foo", resolver=res)), "example[1-100]")
# No 'all' defined: all_nodes() should raise an error
self.assertRaises(GroupSourceNoUpcall, res.all_nodes)
# No 'reverse' defined: node_groups() should raise an error
self.assertRaises(GroupSourceNoUpcall, res.node_groups, "example1")
# regroup with rest
nodeset = NodeSet("example[1-101]", resolver=res)
self.assertEqual(nodeset.regroup(), "@foo,example101")
# regroup incomplete
nodeset = NodeSet("example[50-200]", resolver=res)
self.assertEqual(nodeset.regroup(), "example[50-200]")
# regroup no matching
nodeset = NodeSet("example[102-200]", resolver=res)
self.assertEqual(nodeset.regroup(), "example[102-200]")
def testConfigGroupsMultipleDirs(self):
"""test groups with multiple confdir defined"""
dname1 = make_temp_dir()
dname2 = make_temp_dir()
# Notes:
# - use dname1 two times to check dup checking code
# - use quotes on one of the directory path
f = make_temp_file("""
[Main]
default: local2
confdir: "%s" %s %s
[local]
map: echo example[1-100]
list: echo foo
""" % (dname1, dname2, dname1))
fs1 = make_temp_file("""
[local1]
map: echo loc1node[1-100]
list: echo bar
""", suffix=".conf", dir=dname1)
fs2 = make_temp_file("""
[local2]
map: echo loc2node[02-50]
list: echo toto
""", suffix=".conf", dir=dname2)
try:
res = GroupResolverConfig(f.name)
nodeset = NodeSet("example[1-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
# local
self.assertEqual(nodeset.regroup("local"), "@local:foo")
self.assertEqual(str(NodeSet("@local:foo", resolver=res)), "example[1-100]")
# local1
nodeset = NodeSet("loc1node[1-100]", resolver=res)
self.assertEqual(nodeset.regroup("local1"), "@local1:bar")
self.assertEqual(str(NodeSet("@local1:bar", resolver=res)), "loc1node[1-100]")
# local2
nodeset = NodeSet("loc2node[02-50]", resolver=res)
self.assertEqual(nodeset.regroup(), "@toto") # default group source
self.assertEqual(str(NodeSet("@toto", resolver=res)), "loc2node[02-50]")
finally:
fs2.close()
fs1.close()
f.close()
shutil.rmtree(dname2, ignore_errors=True)
shutil.rmtree(dname1, ignore_errors=True)
def testGroupResolverReverse(self):
"""test NodeSet GroupResolver with reverse upcall"""
test_groups3 = makeTestG3()
test_reverse3 = makeTestR3()
source = GroupSource("test",
"sed -n 's/^$GROUP:\(.*\)/\\1/p' %s" % test_groups3.name,
"sed -n 's/^all:\(.*\)/\\1/p' %s" % test_groups3.name,
"sed -n 's/^\([0-9A-Za-z_-]*\):.*/\\1/p' %s" % test_groups3.name,
"awk -F: '/^$NODE:/ { gsub(\",\",\"\\n\",$2); print $2 }' %s" % test_reverse3.name)
# create custom resolver with default source
res = GroupResolver(source)
nodeset = NodeSet("@all", resolver=res)
self.assertEqual(nodeset, NodeSet("montana[32-55]"))
self.assertEqual(str(nodeset), "montana[32-55]")
self.assertEqual(nodeset.regroup(), "@all")
self.assertEqual(nodeset.regroup(), "@all")
nodeset = NodeSet("@overclock", resolver=res)
self.assertEqual(nodeset, NodeSet("montana[41-42]"))
self.assertEqual(str(nodeset), "montana[41-42]")
self.assertEqual(nodeset.regroup(), "@overclock")
self.assertEqual(nodeset.regroup(), "@overclock")
nodeset = NodeSet("@gpu,@overclock", resolver=res)
self.assertEqual(nodeset, NodeSet("montana[38-42]"))
self.assertEqual(str(nodeset), "montana[38-42]")
# un-overlap :)
self.assertEqual(nodeset.regroup(), "@gpu,montana42")
self.assertEqual(nodeset.regroup(), "@gpu,montana42")
self.assertEqual(nodeset.regroup(overlap=True), "@gpu,@overclock")
nodeset = NodeSet("montana41", resolver=res)
self.assertEqual(nodeset.regroup(), "montana41")
self.assertEqual(nodeset.regroup(), "montana41")
# test regroup code when using unindexed node
nodeset = NodeSet("idaho", resolver=res)
self.assertEqual(nodeset.regroup(), "@single")
self.assertEqual(nodeset.regroup(), "@single")
nodeset = NodeSet("@single", resolver=res)
self.assertEqual(str(nodeset), "idaho")
# unresolved unindexed:
nodeset = NodeSet("utah", resolver=res)
self.assertEqual(nodeset.regroup(), "utah")
self.assertEqual(nodeset.regroup(), "utah")
nodeset = NodeSet("@all!montana38", resolver=res)
self.assertEqual(nodeset, NodeSet("montana[32-37,39-55]"))
self.assertEqual(str(nodeset), "montana[32-37,39-55]")
self.assertEqual(nodeset.regroup(), "@para,montana[39-41]")
self.assertEqual(nodeset.regroup(), "@para,montana[39-41]")
self.assertEqual(nodeset.regroup(overlap=True),
"@chassis[1-3],@login,@overclock,@para,montana[39-40]")
self.assertEqual(nodeset.regroup(overlap=True),
"@chassis[1-3],@login,@overclock,@para,montana[39-40]")
nodeset = NodeSet("montana[32-37]", resolver=res)
self.assertEqual(nodeset.regroup(), "@chassis[1-3]")
self.assertEqual(nodeset.regroup(), "@chassis[1-3]")
def testGroupNoPrefix(self):
"""test NodeSet group noprefix option"""
nodeset = NodeSet("montana[32-37,42-55]")
self.assertEqual(nodeset.regroup("source2"), "@source2:para")
self.assertEqual(nodeset.regroup("source2", noprefix=True), "@para")
def test_yaml_basic(self):
"""test groups with a basic YAML config file"""
dname = make_temp_dir()
f = make_temp_file("""
# A comment
[Main]
default: yaml
autodir: %s
""" % dname)
yamlfile = make_temp_file("""
yaml:
foo: example[1-4,91-100],example90
bar: example[5-89]
""", suffix=".yaml", dir=dname)
res = GroupResolverConfig(f.name)
# Group resolution
nodeset = NodeSet("@foo", resolver=res)
self.assertEqual(str(nodeset), "example[1-4,90-100]")
nodeset = NodeSet("@bar", resolver=res)
self.assertEqual(str(nodeset), "example[5-89]")
nodeset = NodeSet("@foo,@bar", resolver=res)
self.assertEqual(str(nodeset), "example[1-100]")
nodeset = NodeSet("@unknown", resolver=res)
self.assertEqual(len(nodeset), 0)
# Regroup
nodeset = NodeSet("example[1-4,90-100]", resolver=res)
self.assertEqual(str(nodeset), "example[1-4,90-100]")
self.assertEqual(nodeset.regroup(), "@foo")
self.assertEqual(nodeset.groups().keys(), ["@foo"])
self.assertEqual(str(NodeSet("@foo", resolver=res)), "example[1-4,90-100]")
# No 'all' defined: all_nodes() should raise an error
self.assertRaises(GroupSourceError, res.all_nodes)
# No 'reverse' defined: node_groups() should raise an error
self.assertRaises(GroupSourceError, res.node_groups, "example1")
# regroup with rest
nodeset = NodeSet("example[1-101]", resolver=res)
self.assertEqual(nodeset.regroup(), "@bar,@foo,example101")
# regroup incomplete
nodeset = NodeSet("example[50-200]", resolver=res)
self.assertEqual(nodeset.regroup(), "example[50-200]")
# regroup no matching
nodeset = NodeSet("example[102-200]", resolver=res)
self.assertEqual(nodeset.regroup(), "example[102-200]")
def testGroupResolverND(self):
"""test NodeSet with simple custom GroupResolver (nD)"""
test_groups4 = makeTestG4()
source = UpcallGroupSource("simple",
"sed -n 's/^$GROUP:\(.*\)/\\1/p' %s" % test_groups4.name,
"sed -n 's/^all:\(.*\)/\\1/p' %s" % test_groups4.name,
"sed -n 's/^\([0-9A-Za-z_-]*\):.*/\\1/p' %s" % test_groups4.name,
None)
# create custom resolver with default source
res = GroupResolver(source)
self.assertFalse(res.has_node_groups())
self.assertFalse(res.has_node_groups("dummy_namespace"))
nodeset = NodeSet("@rack-x1y2", resolver=res)
self.assertEqual(nodeset, NodeSet("idaho[2-3]z1"))
self.assertEqual(str(nodeset), "idaho[2-3]z1")
nodeset = NodeSet("@rack-y1", resolver=res)
self.assertEqual(str(nodeset), "idaho[1-2,4-5]z1")
nodeset = NodeSet("@rack-all", resolver=res)
self.assertEqual(str(nodeset), "idaho[1-7]z1")
# test NESTED nD groups()
self.assertEqual(sorted(nodeset.groups().keys()),
['@rack-all', '@rack-x1', '@rack-x1y1', '@rack-x1y2',
'@rack-x2', '@rack-x2y1', '@rack-x2y2', '@rack-y1',
'@rack-y2'])
self.assertEqual(sorted(nodeset.groups(groupsource="simple").keys()),
['@simple:rack-all', '@simple:rack-x1',
'@simple:rack-x1y1', '@simple:rack-x1y2',
'@simple:rack-x2', '@simple:rack-x2y1',
'@simple:rack-x2y2', '@simple:rack-y1',
'@simple:rack-y2'])
self.assertEqual(sorted(nodeset.groups(groupsource="simple",
noprefix=True).keys()),
['@rack-all', '@rack-x1', '@rack-x1y1', '@rack-x1y2',
'@rack-x2', '@rack-x2y1', '@rack-x2y2', '@rack-y1',
'@rack-y2'])
testns = NodeSet()
for gnodes, inodes in nodeset.groups().itervalues():
testns.update(inodes)
self.assertEqual(testns, nodeset)
# more tests with nested groups
nodeset = NodeSet("idaho5z1", resolver=res)
self.assertEqual(sorted(nodeset.groups().keys()),
['@rack-all', '@rack-x2', '@rack-x2y1', '@rack-y1'])
nodeset = NodeSet("idaho5z1,idaho4z1", resolver=res)
self.assertEqual(sorted(nodeset.groups().keys()),
['@rack-all', '@rack-x2', '@rack-x2y1', '@rack-y1'])
nodeset = NodeSet("idaho5z1,idaho7z1", resolver=res)
self.assertEqual(sorted(nodeset.groups().keys()),
['@rack-all', '@rack-x2', '@rack-x2y1', '@rack-x2y2',
'@rack-y1', '@rack-y2'])
def testGroupNoPrefix(self):
"""test NodeSet group noprefix option"""
nodeset = NodeSet("montana[32-37,42-55]")
self.assertEqual(nodeset.regroup("source2"), "@source2:para")
self.assertEqual(nodeset.regroup("source2", noprefix=True), "@para")
def testGroupResolverReverse(self):
"""test NodeSet GroupResolver with reverse upcall"""
test_groups3 = makeTestG3()
test_reverse3 = makeTestR3()
source = GroupSource(
"test", "sed -n 's/^$GROUP:\(.*\)/\\1/p' %s" % test_groups3.name,
"sed -n 's/^all:\(.*\)/\\1/p' %s" % test_groups3.name,
"sed -n 's/^\([0-9A-Za-z_-]*\):.*/\\1/p' %s" % test_groups3.name,
"awk -F: '/^$NODE:/ { gsub(\",\",\"\\n\",$2); print $2 }' %s" %
test_reverse3.name)
# create custom resolver with default source
res = GroupResolver(source)
nodeset = NodeSet("@all", resolver=res)
self.assertEqual(nodeset, NodeSet("montana[32-55]"))
self.assertEqual(str(nodeset), "montana[32-55]")
self.assertEqual(nodeset.regroup(), "@all")
self.assertEqual(nodeset.regroup(), "@all")
nodeset = NodeSet("@overclock", resolver=res)
self.assertEqual(nodeset, NodeSet("montana[41-42]"))
self.assertEqual(str(nodeset), "montana[41-42]")
self.assertEqual(nodeset.regroup(), "@overclock")
self.assertEqual(nodeset.regroup(), "@overclock")
nodeset = NodeSet("@gpu,@overclock", resolver=res)
self.assertEqual(nodeset, NodeSet("montana[38-42]"))
self.assertEqual(str(nodeset), "montana[38-42]")
# un-overlap :)
self.assertEqual(nodeset.regroup(), "@gpu,montana42")
self.assertEqual(nodeset.regroup(), "@gpu,montana42")
self.assertEqual(nodeset.regroup(overlap=True), "@gpu,@overclock")
nodeset = NodeSet("montana41", resolver=res)
self.assertEqual(nodeset.regroup(), "montana41")
self.assertEqual(nodeset.regroup(), "montana41")
# test regroup code when using unindexed node
nodeset = NodeSet("idaho", resolver=res)
self.assertEqual(nodeset.regroup(), "@single")
self.assertEqual(nodeset.regroup(), "@single")
nodeset = NodeSet("@single", resolver=res)
self.assertEqual(str(nodeset), "idaho")
# unresolved unindexed:
nodeset = NodeSet("utah", resolver=res)
self.assertEqual(nodeset.regroup(), "utah")
self.assertEqual(nodeset.regroup(), "utah")
nodeset = NodeSet("@all!montana38", resolver=res)
self.assertEqual(nodeset, NodeSet("montana[32-37,39-55]"))
self.assertEqual(str(nodeset), "montana[32-37,39-55]")
self.assertEqual(nodeset.regroup(), "@para,montana[39-41]")
self.assertEqual(nodeset.regroup(), "@para,montana[39-41]")
self.assertEqual(
nodeset.regroup(overlap=True),
"@chassis[1-3],@login,@overclock,@para,montana[39-40]")
self.assertEqual(
nodeset.regroup(overlap=True),
"@chassis[1-3],@login,@overclock,@para,montana[39-40]")
nodeset = NodeSet("montana[32-37]", resolver=res)
self.assertEqual(nodeset.regroup(), "@chassis[1-3]")
self.assertEqual(nodeset.regroup(), "@chassis[1-3]")
