def test_check_node_state_lookup_00(self): """ Test: check for the existence and values of the pbs.REVERSE_STATE_CHANGES dictionary run a hook that converts a state change hex into a string, then search for it in the server log. """ self.add_pbs_python_path_to_sys_path() import pbs self.logger.debug("---- %s TEST STARTED ----" % get_method_name(self)) self.server.manager(MGR_CMD_SET, SERVER, {'log_events': 4095}) attrs = {'event': 'modifyvnode', 'enabled': 'True', 'debug': 'True'} hook_name_00 = 'x1234' hook_body_00 = get_hook_body_reverse_node_state() ret = self.server.create_hook(hook_name_00, attrs) self.assertTrue(ret, "Could not create hook %s" % hook_name_00) ret = self.server.import_hook(hook_name_00, hook_body_00) for mom in self.server.moms.values(): start_time = time.time() mom.restart() self.server.log_match("Node;%s;node up" % mom.fqdn, starttime=start_time) self.server.log_match("Node;%s;node down" % mom.fqdn, starttime=start_time) for value, key in pbs.REVERSE_NODE_STATE.items(): self.server.log_match("key:%s value:%s" % (key, value), starttime=start_time) self.logger.debug("---- %s TEST ENDED ----" % get_method_name(self))
def test_hook_state_changes_02(self): """ Test: stop and start the pbs server; look for proper log messages """ self.logger.debug("---- %s TEST STARTED ----" % get_method_name(self)) self.server.manager(MGR_CMD_SET, SERVER, {'log_events': 4095}) attrs = {'event': 'modifyvnode', 'enabled': 'True', 'debug': 'True'} hook_name_00 = 's1234' hook_body_00 = get_hook_body_modifyvnode_param_rpt() ret = self.server.create_hook(hook_name_00, attrs) self.assertTrue(ret, "Could not create hook %s" % hook_name_00) ret = self.server.import_hook(hook_name_00, hook_body_00) # stop the server and then start it start_time = time.time() state_chain_start_time = start_time self.server.stop() self.server.start() # look for messages indicating all the vnodes came up for mom in self.server.moms.values(): self.checkLog(start_time, mom.fqdn, check_up=True, check_down=False) self.checkNodeFree(start_time) # Verify each preceeding state matches the current previous state state_chain_end_time = time.time() self.checkpreviousStateChain(state_chain_start_time, state_chain_end_time, mom.shortname) self.logger.debug("---- %s TEST ENDED ----" % get_method_name(self))
def test_check_node_state_constants_00(self): """ Test: verify expected node state constants and associated reverse map are defined in the pbs module and contain the expected values. """ self.logger.debug("---- %s TEST STARTED ----" % get_method_name(self)) self.add_pbs_python_path_to_sys_path() import pbs self.assertEqual( len(pbs.REVERSE_NODE_STATE), len(node_states), "node state count mismatch: actual=%s, expected:%s" % (len(pbs.REVERSE_NODE_STATE), len(node_states))) for attr, value in node_states.items(): self.logger.debug("checking attribute '%s' in pbs module", attr) self.assertTrue(hasattr(pbs, attr), "pbs.%s does not exist." % attr) self.assertEqual( getattr(pbs, attr), value, "pbs.%s is incorrect: actual=%s, expected=%s." % (attr, getattr(pbs, attr), value)) self.assertIn(value, pbs.REVERSE_NODE_STATE) self.assertEqual( pbs.REVERSE_NODE_STATE[value], attr, ("pbs.REVERSE_NODE_STATE[%s] is incorrect: actual=%s, " + "expected=%s.") % (value, pbs.REVERSE_NODE_STATE[value], attr)) self.logger.debug("---- %s TEST ENDED ----" % get_method_name(self))
def test_hook_state_changes_01(self): """ Test each mom for state changes: 1. sigkill the mom; check server log for expected messages 2. start the mom; check server log for expected messages 3. verify the chain of reported current/previous states is unbroken """ self.logger.debug("---- %s TEST STARTED ----" % get_method_name(self)) self.server.manager(MGR_CMD_SET, SERVER, {'log_events': 4095}) attrs = {'event': 'modifyvnode', 'enabled': 'True', 'debug': 'True'} hook_name_00 = 'p1234' hook_body_00 = get_hook_body_modifyvnode_param_rpt() ret = self.server.create_hook(hook_name_00, attrs) self.assertTrue(ret, "Could not create hook %s" % hook_name_00) ret = self.server.import_hook(hook_name_00, hook_body_00) for mom in self.server.moms.values(): self.logger.debug(" ***sigkilling mom:%s", mom.fqdn) start_time = time.time() state_chain_start_time = start_time mom.signal('-KILL') self.checkLog(start_time, mom.fqdn, check_up=False, check_down=True) self.checkNodeDown(start_time) start_time = time.time() mom.start() self.checkLog(start_time, mom.fqdn, check_up=True, check_down=False) self.checkNodeFree(start_time) # Verify each preceeding state matches the current previous state state_chain_end_time = time.time() self.checkpreviousStateChain(state_chain_start_time, state_chain_end_time, mom.shortname) self.logger.debug("---- %s TEST ENDED ----" % get_method_name(self))
def test_hook_state_changes_01(self): """ Test: this will test four things: 1. It will sigkill pbs_mom, look for the proper log messages. 2. It will check the log for the proper hook messages 3. It will bring up pbs_mom, look for the proper log messages. 4. It will check the log for the proper hook messages """ self.logger.debug("---- %s TEST STARTED ----" % get_method_name(self)) self.server.manager(MGR_CMD_SET, SERVER, {'log_events': 4095}) attrs = {'event': 'modifyvnode', 'enabled': 'True', 'debug': 'True'} hook_name_00 = 'p1234' hook_body_00 = get_hook_body_modifyvnode_param_rpt() ret = self.server.create_hook(hook_name_00, attrs) self.assertTrue(ret, "Could not create hook %s" % hook_name_00) ret = self.server.import_hook(hook_name_00, hook_body_00) for mom in self.server.moms.values(): self.logger.debug(" ***sigkilling mom:%s", mom.fqdn) start_time = int(time.time()) mom.signal('-KILL') self.checkLog(start_time, mom.fqdn, check_up=False, check_down=True) self.server.log_match("v.state_hex=0x2 v_o.state_hex=0x0", starttime=start_time) start_time = int(time.time()) mom.start() self.checkLog(start_time, mom.fqdn, check_up=True, check_down=False) self.server.log_match("v.state_hex=0x0 v_o.state_hex=0x400", starttime=start_time) self.logger.debug("---- %s TEST ENDED ----" % get_method_name(self))
def test_hook_state_changes_00(self): """ Test: induce a variety of vnode state changes with debug turned on and inspect the pbs log for expected entries """ if os.getuid() != 0 or sys.platform in ('cygwin', 'win32'): self.skipTest("Test need to run as root") self.logger.debug("---- %s TEST STARTED ----" % get_method_name(self)) self.server.manager(MGR_CMD_SET, SERVER, {'log_events': 4095}) attrs = {'event': 'modifyvnode', 'enabled': 'True', 'debug': 'True'} hook_name_00 = 'm1234' hook_body_00 = get_hook_body_modifyvnode_param_rpt() ret = self.server.create_hook(hook_name_00, attrs) self.assertTrue(ret, "Could not create hook %s" % hook_name_00) ret = self.server.import_hook(hook_name_00, hook_body_00) # print info about the test deployment self.logger.debug("socket.gethostname():%s" % socket.gethostname()) self.logger.debug("***self.server.name:%s" % str(self.server.name)) self.logger.debug("self.server.moms:%s" % str(self.server.moms)) self.logger.debug("self.server.hostname=%s" % self.server.hostname) nodeinfo = self.server.status(NODE) # test effects of various state changes on each mom for mom in self.server.moms.values(): # State change test: mom stop start_time = time.time() state_chain_start_time = start_time mom.stop() self.checkLog(start_time, mom.fqdn, check_up=False, check_down=True) self.checkNodeDown(start_time) # State change test: mom start start_time = time.time() mom.start() self.checkLog(start_time, mom.fqdn, check_up=True, check_down=False) self.checkNodeFree(start_time) # State change test: mom restart start_time = time.time() mom.restart() self.checkLog(start_time, mom.fqdn, check_up=True, check_down=True) self.checkNodeDown(start_time) self.checkNodeFree(start_time) # State change test: take mom offline then online # take offline start_time = time.time() self.logger.debug(" ***offline mom:%s" % mom) self.server.manager(MGR_CMD_SET, NODE, {'state': (INCR, 'offline')}, id=mom.shortname) self.checkLog(start_time, mom.fqdn, check_up=False, check_down=False) self.checkNodeOffline(start_time) # back online start_time = time.time() self.logger.debug(" ***online mom:%s" % mom) self.server.manager(MGR_CMD_SET, NODE, {'state': (DECR, 'offline')}, id=mom.shortname) self.checkLog(start_time, mom.fqdn, check_up=False, check_down=False) self.checkNodeFree(start_time) # State change test: create and release maintenance reservation start_time = time.time() res_start_time = start_time + 15 res_end_time = res_start_time + 1 attrs = { 'reserve_start': res_start_time, 'reserve_end': res_end_time } self.logger.debug(" ***reserve & release mom:%s" % mom) rid = self.server.submit( Reservation(ROOT_USER, attrs, hosts=[mom.shortname])) self.logger.debug("rid=%s" % rid) self.checkLog(start_time, mom.fqdn, check_up=False, check_down=False) self.checkNodeResvExclusive(start_time) self.checkNodeFree(start_time) # Verify each preceeding state matches the current previous state state_chain_end_time = time.time() self.checkpreviousStateChain(state_chain_start_time, state_chain_end_time, mom.shortname) self.logger.debug("---- %s TEST ENDED ----" % get_method_name(self))
def test_hook_state_changes_00(self): """ Test: induce a variety of vnode state changes with debug turned on and inspect the pbs log for expected entries """ self.logger.debug("---- %s TEST STARTED ----" % get_method_name(self)) self.server.manager(MGR_CMD_SET, SERVER, {'log_events': 4095}) attrs = {'event': 'modifyvnode', 'enabled': 'True', 'debug': 'True'} hook_name_00 = 'm1234' hook_body_00 = get_hook_body_modifyvnode_param_rpt() ret = self.server.create_hook(hook_name_00, attrs) self.assertTrue(ret, "Could not create hook %s" % hook_name_00) ret = self.server.import_hook(hook_name_00, hook_body_00) # print info about the test deployment self.logger.debug("socket.gethostname():%s" % socket.gethostname()) self.logger.debug("***self.server.name:%s" % str(self.server.name)) self.logger.debug("self.server.moms:%s" % str(self.server.moms)) self.logger.debug("self.server.hostname=%s" % self.server.hostname) nodeinfo = self.server.status(NODE) # test effects of various state changes on each mom for mom in self.server.moms.values(): # State change test: mom stop start_time = int(time.time()) mom.stop() self.checkLog(start_time, mom.fqdn, check_up=False, check_down=True) self.server.log_match("v.state_hex=0x2 v_o.state_hex=0x0", starttime=start_time) # State change test: mom start start_time = int(time.time()) mom.start() self.checkLog(start_time, mom.fqdn, check_up=True, check_down=False) self.server.log_match("v.state_hex=0x0 v_o.state_hex=0x400", starttime=start_time) # State change test: mom restart start_time = int(time.time()) mom.restart() self.checkLog(start_time, mom.fqdn, check_up=True, check_down=True) self.server.log_match("v.state_hex=0x2 v_o.state_hex=0x0", starttime=start_time) self.server.log_match("v.state_hex=0x0 v_o.state_hex=0x400", starttime=start_time) # State change test: take mom offline then online # take offline start_time = int(time.time()) self.logger.debug(" ***offline mom:%s" % mom) self.server.manager(MGR_CMD_SET, NODE, {'state': (INCR, 'offline')}, id=mom.shortname) self.checkLog(start_time, mom.fqdn, check_up=False, check_down=False) self.server.log_match("v.state_hex=0x1 v_o.state_hex=0x0", starttime=start_time) # back online start_time = int(time.time()) self.logger.debug(" ***online mom:%s" % mom) self.server.manager(MGR_CMD_SET, NODE, {'state': (DECR, 'offline')}, id=mom.shortname) self.checkLog(start_time, mom.fqdn, check_up=False, check_down=False) self.server.log_match("v.state_hex=0x0 v_o.state_hex=0x1", starttime=start_time) # State change test: create and release maintenance reservation start_time = int(time.time()) res_start_time = start_time + 15 res_end_time = res_start_time + 1 attrs = { 'reserve_start': res_start_time, 'reserve_end': res_end_time } self.logger.debug(" ***reserve & release mom:%s" % mom) rid = self.server.submit( Reservation(ROOT_USER, attrs, hosts=[mom.shortname])) self.logger.debug("rid=%s" % rid) self.checkLog(start_time, mom.fqdn, check_up=False, check_down=False) self.server.log_match("v.state_hex=0x2000 v_o.state_hex=0x0", starttime=start_time) self.server.log_match("v.state_hex=0x0 v_o.state_hex=0x2000", starttime=start_time) # State change test: create and delete vnode # TODO: add impl # State change test: induce ND_STATE_MAINTENANCE state # TODO: add impl self.logger.debug("---- %s TEST ENDED ----" % get_method_name(self))