def test_check_node_state_lookup_00(self):
"""
Test: check for the existence and values of the
pbs.REVERSE_STATE_CHANGES dictionary
run a hook that converts a state change hex into a string, then search
for it in the server log.
"""
self.add_pbs_python_path_to_sys_path()
import pbs
self.logger.debug("---- %s TEST STARTED ----" % get_method_name(self))
self.server.manager(MGR_CMD_SET, SERVER, {'log_events': 4095})
attrs = {'event': 'modifyvnode', 'enabled': 'True', 'debug': 'True'}
hook_name_00 = 'x1234'
hook_body_00 = get_hook_body_reverse_node_state()
ret = self.server.create_hook(hook_name_00, attrs)
self.assertTrue(ret, "Could not create hook %s" % hook_name_00)
ret = self.server.import_hook(hook_name_00, hook_body_00)
for mom in self.server.moms.values():
start_time = time.time()
mom.restart()
self.server.log_match("Node;%s;node up" % mom.fqdn,
starttime=start_time)
self.server.log_match("Node;%s;node down" % mom.fqdn,
starttime=start_time)
for value, key in pbs.REVERSE_NODE_STATE.items():
self.server.log_match("key:%s value:%s" % (key, value),
starttime=start_time)
self.logger.debug("---- %s TEST ENDED ----" % get_method_name(self))
def test_hook_state_changes_02(self):
"""
Test: stop and start the pbs server; look for proper log messages
"""
self.logger.debug("---- %s TEST STARTED ----" % get_method_name(self))
self.server.manager(MGR_CMD_SET, SERVER, {'log_events': 4095})
attrs = {'event': 'modifyvnode', 'enabled': 'True', 'debug': 'True'}
hook_name_00 = 's1234'
hook_body_00 = get_hook_body_modifyvnode_param_rpt()
ret = self.server.create_hook(hook_name_00, attrs)
self.assertTrue(ret, "Could not create hook %s" % hook_name_00)
ret = self.server.import_hook(hook_name_00, hook_body_00)
# stop the server and then start it
start_time = time.time()
state_chain_start_time = start_time
self.server.stop()
self.server.start()
# look for messages indicating all the vnodes came up
for mom in self.server.moms.values():
self.checkLog(start_time,
mom.fqdn,
check_up=True,
check_down=False)
self.checkNodeFree(start_time)
# Verify each preceeding state matches the current previous state
state_chain_end_time = time.time()
self.checkpreviousStateChain(state_chain_start_time,
state_chain_end_time, mom.shortname)
self.logger.debug("---- %s TEST ENDED ----" % get_method_name(self))
def test_check_node_state_constants_00(self):
"""
Test: verify expected node state constants and associated reverse map
are defined in the pbs module and contain the expected values.
"""
self.logger.debug("---- %s TEST STARTED ----" % get_method_name(self))
self.add_pbs_python_path_to_sys_path()
import pbs
self.assertEqual(
len(pbs.REVERSE_NODE_STATE), len(node_states),
"node state count mismatch: actual=%s, expected:%s" %
(len(pbs.REVERSE_NODE_STATE), len(node_states)))
for attr, value in node_states.items():
self.logger.debug("checking attribute '%s' in pbs module", attr)
self.assertTrue(hasattr(pbs, attr),
"pbs.%s does not exist." % attr)
self.assertEqual(
getattr(pbs, attr), value,
"pbs.%s is incorrect: actual=%s, expected=%s." %
(attr, getattr(pbs, attr), value))
self.assertIn(value, pbs.REVERSE_NODE_STATE)
self.assertEqual(
pbs.REVERSE_NODE_STATE[value], attr,
("pbs.REVERSE_NODE_STATE[%s] is incorrect: actual=%s, " +
"expected=%s.") %
(value, pbs.REVERSE_NODE_STATE[value], attr))
self.logger.debug("---- %s TEST ENDED ----" % get_method_name(self))
def test_hook_state_changes_01(self):
"""
Test each mom for state changes:
1. sigkill the mom; check server log for expected messages
2. start the mom; check server log for expected messages
3. verify the chain of reported current/previous states is unbroken
"""
self.logger.debug("---- %s TEST STARTED ----" % get_method_name(self))
self.server.manager(MGR_CMD_SET, SERVER, {'log_events': 4095})
attrs = {'event': 'modifyvnode', 'enabled': 'True', 'debug': 'True'}
hook_name_00 = 'p1234'
hook_body_00 = get_hook_body_modifyvnode_param_rpt()
ret = self.server.create_hook(hook_name_00, attrs)
self.assertTrue(ret, "Could not create hook %s" % hook_name_00)
ret = self.server.import_hook(hook_name_00, hook_body_00)
for mom in self.server.moms.values():
self.logger.debug(" ***sigkilling mom:%s", mom.fqdn)
start_time = time.time()
state_chain_start_time = start_time
mom.signal('-KILL')
self.checkLog(start_time,
mom.fqdn,
check_up=False,
check_down=True)
self.checkNodeDown(start_time)
start_time = time.time()
mom.start()
self.checkLog(start_time,
mom.fqdn,
check_up=True,
check_down=False)
self.checkNodeFree(start_time)
# Verify each preceeding state matches the current previous state
state_chain_end_time = time.time()
self.checkpreviousStateChain(state_chain_start_time,
state_chain_end_time, mom.shortname)
self.logger.debug("---- %s TEST ENDED ----" % get_method_name(self))
def test_hook_state_changes_01(self):
"""
Test: this will test four things:
1. It will sigkill pbs_mom, look for the proper log messages.
2. It will check the log for the proper hook messages
3. It will bring up pbs_mom, look for the proper log messages.
4. It will check the log for the proper hook messages
"""
self.logger.debug("---- %s TEST STARTED ----" % get_method_name(self))
self.server.manager(MGR_CMD_SET, SERVER, {'log_events': 4095})
attrs = {'event': 'modifyvnode', 'enabled': 'True', 'debug': 'True'}
hook_name_00 = 'p1234'
hook_body_00 = get_hook_body_modifyvnode_param_rpt()
ret = self.server.create_hook(hook_name_00, attrs)
self.assertTrue(ret, "Could not create hook %s" % hook_name_00)
ret = self.server.import_hook(hook_name_00, hook_body_00)
for mom in self.server.moms.values():
self.logger.debug(" ***sigkilling mom:%s", mom.fqdn)
start_time = int(time.time())
mom.signal('-KILL')
self.checkLog(start_time,
mom.fqdn,
check_up=False,
check_down=True)
self.server.log_match("v.state_hex=0x2 v_o.state_hex=0x0",
starttime=start_time)
start_time = int(time.time())
mom.start()
self.checkLog(start_time,
mom.fqdn,
check_up=True,
check_down=False)
self.server.log_match("v.state_hex=0x0 v_o.state_hex=0x400",
starttime=start_time)
self.logger.debug("---- %s TEST ENDED ----" % get_method_name(self))
def test_hook_state_changes_00(self):
"""
Test: induce a variety of vnode state changes with debug turned on
and inspect the pbs log for expected entries
"""
if os.getuid() != 0 or sys.platform in ('cygwin', 'win32'):
self.skipTest("Test need to run as root")
self.logger.debug("---- %s TEST STARTED ----" % get_method_name(self))
self.server.manager(MGR_CMD_SET, SERVER, {'log_events': 4095})
attrs = {'event': 'modifyvnode', 'enabled': 'True', 'debug': 'True'}
hook_name_00 = 'm1234'
hook_body_00 = get_hook_body_modifyvnode_param_rpt()
ret = self.server.create_hook(hook_name_00, attrs)
self.assertTrue(ret, "Could not create hook %s" % hook_name_00)
ret = self.server.import_hook(hook_name_00, hook_body_00)
# print info about the test deployment
self.logger.debug("socket.gethostname():%s" % socket.gethostname())
self.logger.debug("***self.server.name:%s" % str(self.server.name))
self.logger.debug("self.server.moms:%s" % str(self.server.moms))
self.logger.debug("self.server.hostname=%s" % self.server.hostname)
nodeinfo = self.server.status(NODE)
# test effects of various state changes on each mom
for mom in self.server.moms.values():
# State change test: mom stop
start_time = time.time()
state_chain_start_time = start_time
mom.stop()
self.checkLog(start_time,
mom.fqdn,
check_up=False,
check_down=True)
self.checkNodeDown(start_time)
# State change test: mom start
start_time = time.time()
mom.start()
self.checkLog(start_time,
mom.fqdn,
check_up=True,
check_down=False)
self.checkNodeFree(start_time)
# State change test: mom restart
start_time = time.time()
mom.restart()
self.checkLog(start_time, mom.fqdn, check_up=True, check_down=True)
self.checkNodeDown(start_time)
self.checkNodeFree(start_time)
# State change test: take mom offline then online
# take offline
start_time = time.time()
self.logger.debug(" ***offline mom:%s" % mom)
self.server.manager(MGR_CMD_SET,
NODE, {'state': (INCR, 'offline')},
id=mom.shortname)
self.checkLog(start_time,
mom.fqdn,
check_up=False,
check_down=False)
self.checkNodeOffline(start_time)
# back online
start_time = time.time()
self.logger.debug(" ***online mom:%s" % mom)
self.server.manager(MGR_CMD_SET,
NODE, {'state': (DECR, 'offline')},
id=mom.shortname)
self.checkLog(start_time,
mom.fqdn,
check_up=False,
check_down=False)
self.checkNodeFree(start_time)
# State change test: create and release maintenance reservation
start_time = time.time()
res_start_time = start_time + 15
res_end_time = res_start_time + 1
attrs = {
'reserve_start': res_start_time,
'reserve_end': res_end_time
}
self.logger.debug(" ***reserve & release mom:%s" % mom)
rid = self.server.submit(
Reservation(ROOT_USER, attrs, hosts=[mom.shortname]))
self.logger.debug("rid=%s" % rid)
self.checkLog(start_time,
mom.fqdn,
check_up=False,
check_down=False)
self.checkNodeResvExclusive(start_time)
self.checkNodeFree(start_time)
# Verify each preceeding state matches the current previous state
state_chain_end_time = time.time()
self.checkpreviousStateChain(state_chain_start_time,
state_chain_end_time, mom.shortname)
self.logger.debug("---- %s TEST ENDED ----" % get_method_name(self))
def test_hook_state_changes_00(self):
"""
Test: induce a variety of vnode state changes with debug turned on
and inspect the pbs log for expected entries
"""
self.logger.debug("---- %s TEST STARTED ----" % get_method_name(self))
self.server.manager(MGR_CMD_SET, SERVER, {'log_events': 4095})
attrs = {'event': 'modifyvnode', 'enabled': 'True', 'debug': 'True'}
hook_name_00 = 'm1234'
hook_body_00 = get_hook_body_modifyvnode_param_rpt()
ret = self.server.create_hook(hook_name_00, attrs)
self.assertTrue(ret, "Could not create hook %s" % hook_name_00)
ret = self.server.import_hook(hook_name_00, hook_body_00)
# print info about the test deployment
self.logger.debug("socket.gethostname():%s" % socket.gethostname())
self.logger.debug("***self.server.name:%s" % str(self.server.name))
self.logger.debug("self.server.moms:%s" % str(self.server.moms))
self.logger.debug("self.server.hostname=%s" % self.server.hostname)
nodeinfo = self.server.status(NODE)
# test effects of various state changes on each mom
for mom in self.server.moms.values():
# State change test: mom stop
start_time = int(time.time())
mom.stop()
self.checkLog(start_time,
mom.fqdn,
check_up=False,
check_down=True)
self.server.log_match("v.state_hex=0x2 v_o.state_hex=0x0",
starttime=start_time)
# State change test: mom start
start_time = int(time.time())
mom.start()
self.checkLog(start_time,
mom.fqdn,
check_up=True,
check_down=False)
self.server.log_match("v.state_hex=0x0 v_o.state_hex=0x400",
starttime=start_time)
# State change test: mom restart
start_time = int(time.time())
mom.restart()
self.checkLog(start_time, mom.fqdn, check_up=True, check_down=True)
self.server.log_match("v.state_hex=0x2 v_o.state_hex=0x0",
starttime=start_time)
self.server.log_match("v.state_hex=0x0 v_o.state_hex=0x400",
starttime=start_time)
# State change test: take mom offline then online
# take offline
start_time = int(time.time())
self.logger.debug(" ***offline mom:%s" % mom)
self.server.manager(MGR_CMD_SET,
NODE, {'state': (INCR, 'offline')},
id=mom.shortname)
self.checkLog(start_time,
mom.fqdn,
check_up=False,
check_down=False)
self.server.log_match("v.state_hex=0x1 v_o.state_hex=0x0",
starttime=start_time)
# back online
start_time = int(time.time())
self.logger.debug(" ***online mom:%s" % mom)
self.server.manager(MGR_CMD_SET,
NODE, {'state': (DECR, 'offline')},
id=mom.shortname)
self.checkLog(start_time,
mom.fqdn,
check_up=False,
check_down=False)
self.server.log_match("v.state_hex=0x0 v_o.state_hex=0x1",
starttime=start_time)
# State change test: create and release maintenance reservation
start_time = int(time.time())
res_start_time = start_time + 15
res_end_time = res_start_time + 1
attrs = {
'reserve_start': res_start_time,
'reserve_end': res_end_time
}
self.logger.debug(" ***reserve & release mom:%s" % mom)
rid = self.server.submit(
Reservation(ROOT_USER, attrs, hosts=[mom.shortname]))
self.logger.debug("rid=%s" % rid)
self.checkLog(start_time,
mom.fqdn,
check_up=False,
check_down=False)
self.server.log_match("v.state_hex=0x2000 v_o.state_hex=0x0",
starttime=start_time)
self.server.log_match("v.state_hex=0x0 v_o.state_hex=0x2000",
starttime=start_time)
# State change test: create and delete vnode
# TODO: add impl
# State change test: induce ND_STATE_MAINTENANCE state
# TODO: add impl
self.logger.debug("---- %s TEST ENDED ----" % get_method_name(self))