def test_oobm_background_powerstate_sync(self): """ Tests out-of-band management background powerstate sync """ self.debug("Testing oobm background sync") interval = list_configurations( self.apiclient, name='outofbandmanagement.sync.interval' )[0].value self.configureAndEnableOobm() self.startIpmiServer() bmc = IpmiServerContext().bmc bmc.powerstate = 'on' self.checkSyncToState('On', interval) bmc.powerstate = 'off' self.checkSyncToState('Off', interval) self.server.shutdown() self.server.server_close() # Check for unknown state (ipmi server not reachable) self.checkSyncToState('Unknown', interval)
def test_oobm_background_powerstate_sync(self): """ Tests out-of-band management background powerstate sync """ self.debug("Testing oobm background sync") interval = list_configurations( self.apiclient, name='outofbandmanagement.sync.interval' )[0].value self.configureAndEnableOobm() self.startIpmiServer() bmc = IpmiServerContext().bmc bmc.powerstate = 'on' self.checkSyncToState('On', interval) bmc.powerstate = 'off' self.checkSyncToState('Off', interval) self.server.shutdown() self.server.server_close() # Check for unknown state (ipmi server not reachable) self.checkSyncToState('Unknown', interval)
def configureAndStartIpmiServer(self, power_state=None): """ Setup ipmisim and enable out-of-band management for host """ self.configureAndEnableOobm() self.startIpmiServer() if power_state: bmc = IpmiServerContext().bmc bmc.powerstate = power_state
def configureAndStartIpmiServer(self, power_state=None): """ Setup ipmisim and enable out-of-band management for host """ self.configureAndEnableOobm() self.startIpmiServer() if power_state: bmc = IpmiServerContext().bmc bmc.powerstate = power_state
def configureAndEnableOobm(self, power_state=None): """ Setup ipmisim and enable out-of-band management for host """ self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd()) response = self.apiclient.enableOutOfBandManagementForHost(self.getOobmEnableCmd()) self.assertEqual(response.enabled, True) if power_state: bmc = IpmiServerContext().bmc bmc.powerstate = power_state
def configureAndEnableOobm(self, power_state=None): """ Setup ipmisim and enable out-of-band management for host """ self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd()) response = self.apiclient.enableOutOfBandManagementForHost(self.getOobmEnableCmd()) self.assertEqual(response.enabled, True) if power_state: bmc = IpmiServerContext().bmc bmc.powerstate = power_state
def test_oobm_background_powerstate_sync(self): """ Tests out-of-band management background powerstate sync """ self.debug("Testing oobm background sync") self.configureAndEnableOobm() bmc = IpmiServerContext().bmc bmc.powerstate = 'on' self.checkSyncToState('On', 2) bmc.powerstate = 'off' self.checkSyncToState('Off', 2) # Check for unknown state (ipmi server not reachable) cmd = self.getOobmConfigCmd() cmd.port = 1 response = self.apiclient.configureOutOfBandManagement(cmd) self.checkSyncToState('Unknown', 2)
def test_oobm_background_powerstate_sync(self): """ Tests out-of-band management background powerstate sync """ self.debug("Testing oobm background sync") interval = list_configurations( self.apiclient, name='outofbandmanagement.sync.interval' )[0].value self.configureAndEnableOobm() bmc = IpmiServerContext().bmc bmc.powerstate = 'on' self.checkSyncToState('On', interval) bmc.powerstate = 'off' self.checkSyncToState('Off', interval) # Check for unknown state (ipmi server not reachable) cmd = self.getOobmConfigCmd() cmd.port = 1 response = self.apiclient.configureOutOfBandManagement(cmd) self.checkSyncToState('Unknown', interval)
def test_oobm_zchange_password(self): """ Tests out-of-band management change password feature """ self.configureAndEnableOobm() self.startIpmiServer() self.debug("Testing oobm change password") cmd = changeOutOfBandManagementPassword.changeOutOfBandManagementPasswordCmd( ) cmd.hostid = self.getHost().id cmd.password = "******" try: response = self.apiclient.changeOutOfBandManagementPassword(cmd) self.assertEqual(response.status, True) except Exception as e: if "packet session id 0x0 does not match active session" in str(e): raise self.skipTest("Known ipmitool issue hit, skipping test") raise e bmc = IpmiServerContext().bmc bmc.powerstate = 'on' response = self.issuePowerActionCmd('STATUS') self.assertEqual(response.status, True) self.assertEqual(response.powerstate, 'On') # Reset configuration, resets password self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd()) self.assertEqual(response.status, True) alerts = Alert.list(self.apiclient, keyword="auth-error", listall=True) alertCount = 0 if alerts: alertCount = len(alerts) try: response = self.issuePowerActionCmd('STATUS') self.fail("Expected an exception to be thrown, failing") except Exception: pass alerts = Alert.list(self.apiclient, keyword="auth-error", listall=True) # At least one alert was sent self.assertTrue((len(alerts) - alertCount) > 0)
def test_oobm_zchange_password(self): """ Tests out-of-band management change password feature """ self.configureAndEnableOobm() self.startIpmiServer() self.debug("Testing oobm change password") alerts = Alert.list(self.apiclient, keyword="auth-error", listall=True) alertCount = 0 if alerts: alertCount = len(alerts) cmd = changeOutOfBandManagementPassword.changeOutOfBandManagementPasswordCmd() cmd.hostid = self.getHost().id cmd.password = "******" try: response = self.apiclient.changeOutOfBandManagementPassword(cmd) self.assertEqual(response.status, True) except Exception as e: if "packet session id 0x0 does not match active session" in str(e): raise self.skipTest("Known ipmitool issue hit, skipping test") raise e bmc = IpmiServerContext().bmc bmc.powerstate = 'on' response = self.issuePowerActionCmd('STATUS') self.assertEqual(response.status, True) self.assertEqual(response.powerstate, 'On') # Reset configuration, resets password self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd()) self.assertEqual(response.status, True) try: response = self.issuePowerActionCmd('STATUS') self.fail("Expected an exception to be thrown, failing") except Exception: pass alerts = Alert.list(self.apiclient, keyword="auth-error", listall=True) self.assertTrue((len(alerts) - alertCount) >= 0)
def test_oobm_zchange_password(self): """ Tests out-of-band management change password feature """ self.configureAndEnableOobm() self.startIpmiServer() self.debug("Testing oobm change password") cmd = changeOutOfBandManagementPassword.changeOutOfBandManagementPasswordCmd() cmd.hostid = self.getHost().id cmd.password = "******" response = self.apiclient.changeOutOfBandManagementPassword(cmd) self.assertEqual(response.status, True) bmc = IpmiServerContext().bmc bmc.powerstate = 'on' response = self.issuePowerActionCmd('STATUS') self.assertEqual(response.status, True) self.assertEqual(response.powerstate, 'On') # Reset configuration, resets password self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd()) self.assertEqual(response.status, True) alerts = Alert.list(self.apiclient, keyword="auth-error", listall=True) alertCount = 0 if alerts: alertCount = len(alerts) try: response = self.issuePowerActionCmd('STATUS') self.fail("Expected an exception to be thrown, failing") except Exception: pass alerts = Alert.list(self.apiclient, keyword="auth-error", listall=True) # At least one alert was sent self.assertTrue((len(alerts) - alertCount) > 0)
def test_oobm_multiple_mgmt_server_ownership(self): """ Tests out-of-band management ownership expiry across multi-mgmt server """ self.configureAndEnableOobm() cloudstackVersion = Configurations.listCapabilities( self.apiclient).cloudstackversion currentMsHosts = [] mshosts = self.dbclient.execute( "select msid from mshost where version='%s' and removed is NULL and state='Up'" % (cloudstackVersion)) if len(mshosts) > 0: currentMsHosts = map(lambda row: row[0], mshosts) # Inject fake ms host self.dbclient.execute( "insert into mshost (msid,runid,name,state,version,service_ip,service_port,last_update) values (%s,%s,'oobm-marvin-fakebox', 'Down', '%s', '127.0.0.1', '22', NOW())" % (self.getFakeMsId(), self.getFakeMsRunId(), cloudstackVersion)) # Pass ownership to the fake ms id self.dbclient.execute( "update oobm set mgmt_server_id=%d where port=%d" % (self.getFakeMsId(), self.getIpmiServerPort())) self.debug("Testing oobm background sync") pingInterval = float( list_configurations(self.apiclient, name='ping.interval')[0].value) pingTimeout = float( list_configurations(self.apiclient, name='ping.timeout')[0].value) def removeFakeMgmtServer(fakeMsRunId): rows = self.dbclient.execute( "select * from mshost_peer where peer_runid=%s" % fakeMsRunId) if len(rows) > 0: self.debug( "Mgmt server is now trying to contact the fake mgmt server" ) self.dbclient.execute( "update mshost set removed=now() where runid=%s" % fakeMsRunId) self.dbclient.execute( "update mshost_peer set peer_state='Down' where peer_runid=%s" % fakeMsRunId) return True, None return False, None def checkOobmOwnershipExpiry(serverPort, fakeMsId): rows = self.dbclient.execute( "select mgmt_server_id from oobm where port=%d" % (serverPort)) if len(rows) > 0 and rows[0][0] != fakeMsId: self.debug( "Out-of-band management ownership expired as node was detected to be gone" ) return True, None return False, None retry_interval = 1 + (pingInterval * pingTimeout / 10) res, _ = wait_until(retry_interval, 10, removeFakeMgmtServer, self.getFakeMsRunId()) if not res: self.fail( "Management server failed to turn down or remove fake mgmt server" ) res, _ = wait_until(retry_interval, 100, checkOobmOwnershipExpiry, self.getIpmiServerPort(), self.getFakeMsId()) if not res: self.fail( "Management server failed to expire ownership of fenced peer") self.debug("Testing oobm background sync should claim new ownership") interval = list_configurations( self.apiclient, name='outofbandmanagement.sync.interval')[0].value self.startIpmiServer() bmc = IpmiServerContext().bmc bmc.powerstate = 'on' self.checkSyncToState('On', interval) result = self.dbclient.execute( "select mgmt_server_id from oobm where port=%d" % (self.getIpmiServerPort())) newOwnerId = result[0][0] self.assertTrue(newOwnerId in currentMsHosts)
def test_oobm_enabledisable_across_clusterzones(self): """ Tests out-of-band management enable/disable feature at cluster and zone level sequentially Zone > Cluster > Host """ self.configureAndEnableOobm() self.startIpmiServer() bmc = IpmiServerContext().bmc bmc.powerstate = 'off' host = self.getHost() # Disable at zone level cmd = disableOutOfBandManagementForZone.disableOutOfBandManagementForZoneCmd( ) cmd.zoneid = host.zoneid response = self.apiclient.disableOutOfBandManagementForZone(cmd) # Disable at cluster level cmd = disableOutOfBandManagementForCluster.disableOutOfBandManagementForClusterCmd( ) cmd.clusterid = host.clusterid response = self.apiclient.disableOutOfBandManagementForCluster(cmd) # Disable at host level cmd = disableOutOfBandManagementForHost.disableOutOfBandManagementForHostCmd( ) cmd.hostid = host.id response = self.apiclient.disableOutOfBandManagementForHost(cmd) try: self.issuePowerActionCmd('STATUS') self.fail("Exception was expected, oobm is disabled at zone level") except Exception: pass # Enable at zone level cmd = enableOutOfBandManagementForZone.enableOutOfBandManagementForZoneCmd( ) cmd.zoneid = host.zoneid response = self.apiclient.enableOutOfBandManagementForZone(cmd) try: self.issuePowerActionCmd('STATUS') self.fail( "Exception was expected, oobm is disabled at cluster level") except Exception: pass # Check background thread syncs state to Disabled response = self.getHost(hostId=host.id).outofbandmanagement self.assertEqual(response.powerstate, 'Disabled') self.dbclient.execute( "update oobm set power_state='On' where port=%d" % self.getIpmiServerPort()) interval = list_configurations( self.apiclient, name='outofbandmanagement.sync.interval')[0].value self.checkSyncToState('Disabled', interval) # Enable at cluster level cmd = enableOutOfBandManagementForCluster.enableOutOfBandManagementForClusterCmd( ) cmd.clusterid = host.clusterid response = self.apiclient.enableOutOfBandManagementForCluster(cmd) try: self.issuePowerActionCmd('STATUS') self.fail("Exception was expected, oobm is disabled at host level") except Exception: pass # Enable at host level cmd = enableOutOfBandManagementForHost.enableOutOfBandManagementForHostCmd( ) cmd.hostid = host.id response = self.apiclient.enableOutOfBandManagementForHost(cmd) response = self.issuePowerActionCmd('STATUS') self.assertEqual(response.powerstate, 'Off')
def test_oobm_multiple_mgmt_server_ownership(self): """ Tests out-of-band management ownership expiry across multi-mgmt server """ self.configureAndEnableOobm() cloudstackVersion = Configurations.listCapabilities(self.apiclient).cloudstackversion currentMsHosts = [] mshosts = self.dbclient.execute("select msid from mshost where version='%s' and removed is NULL and state='Up'" % (cloudstackVersion)) if len(mshosts) > 0: currentMsHosts = map(lambda row: row[0], mshosts) # Inject fake ms host self.dbclient.execute("insert into mshost (msid,runid,name,state,version,service_ip,service_port,last_update) values (%s,%s,'oobm-marvin-fakebox', 'Down', '%s', '127.0.0.1', '22', NOW())" % (self.getFakeMsId(), self.getFakeMsRunId(), cloudstackVersion)) # Pass ownership to the fake ms id self.dbclient.execute("update oobm set mgmt_server_id=%d where port=%d" % (self.getFakeMsId(), self.getIpmiServerPort())) self.debug("Testing oobm background sync") pingInterval = float(list_configurations( self.apiclient, name='ping.interval' )[0].value) pingTimeout = float(list_configurations( self.apiclient, name='ping.timeout' )[0].value) def removeFakeMgmtServer(fakeMsRunId): rows = self.dbclient.execute("select * from mshost_peer where peer_runid=%s" % fakeMsRunId) if len(rows) > 0: self.debug("Mgmt server is now trying to contact the fake mgmt server") self.dbclient.execute("update mshost set removed=now() where runid=%s" % fakeMsRunId) self.dbclient.execute("update mshost_peer set peer_state='Down' where peer_runid=%s" % fakeMsRunId) return True, None return False, None def checkOobmOwnershipExpiry(serverPort, fakeMsId): rows = self.dbclient.execute("select mgmt_server_id from oobm where port=%d" % (serverPort)) if len(rows) > 0 and rows[0][0] != fakeMsId: self.debug("Out-of-band management ownership expired as node was detected to be gone") return True, None return False, None retry_interval = 1 + (pingInterval * pingTimeout / 10) res, _ = wait_until(retry_interval, 10, removeFakeMgmtServer, self.getFakeMsRunId()) if not res: self.fail("Management server failed to turn down or remove fake mgmt server") res, _ = wait_until(retry_interval, 100, checkOobmOwnershipExpiry, self.getIpmiServerPort(), self.getFakeMsId()) if not res: self.fail("Management server failed to expire ownership of fenced peer") self.debug("Testing oobm background sync should claim new ownership") interval = list_configurations( self.apiclient, name='outofbandmanagement.sync.interval' )[0].value self.startIpmiServer() bmc = IpmiServerContext().bmc bmc.powerstate = 'on' self.checkSyncToState('On', interval) result = self.dbclient.execute("select mgmt_server_id from oobm where port=%d" % (self.getIpmiServerPort())) newOwnerId = result[0][0] self.assertTrue(newOwnerId in currentMsHosts)
def test_oobm_enabledisable_across_clusterzones(self): """ Tests out-of-band management enable/disable feature at cluster and zone level sequentially Zone > Cluster > Host """ self.configureAndEnableOobm() self.startIpmiServer() bmc = IpmiServerContext().bmc bmc.powerstate = 'off' host = self.getHost() # Disable at zone level cmd = disableOutOfBandManagementForZone.disableOutOfBandManagementForZoneCmd() cmd.zoneid = host.zoneid response = self.apiclient.disableOutOfBandManagementForZone(cmd) # Disable at cluster level cmd = disableOutOfBandManagementForCluster.disableOutOfBandManagementForClusterCmd() cmd.clusterid = host.clusterid response = self.apiclient.disableOutOfBandManagementForCluster(cmd) # Disable at host level cmd = disableOutOfBandManagementForHost.disableOutOfBandManagementForHostCmd() cmd.hostid = host.id response = self.apiclient.disableOutOfBandManagementForHost(cmd) try: self.issuePowerActionCmd('STATUS') self.fail("Exception was expected, oobm is disabled at zone level") except Exception: pass # Enable at zone level cmd = enableOutOfBandManagementForZone.enableOutOfBandManagementForZoneCmd() cmd.zoneid = host.zoneid response = self.apiclient.enableOutOfBandManagementForZone(cmd) try: self.issuePowerActionCmd('STATUS') self.fail("Exception was expected, oobm is disabled at cluster level") except Exception: pass # Check background thread syncs state to Disabled response = self.getHost(hostId=host.id).outofbandmanagement self.assertEqual(response.powerstate, 'Disabled') self.dbclient.execute("update oobm set power_state='On' where port=%d" % self.getIpmiServerPort()) interval = list_configurations( self.apiclient, name='outofbandmanagement.sync.interval' )[0].value self.checkSyncToState('Disabled', interval) # Enable at cluster level cmd = enableOutOfBandManagementForCluster.enableOutOfBandManagementForClusterCmd() cmd.clusterid = host.clusterid response = self.apiclient.enableOutOfBandManagementForCluster(cmd) try: self.issuePowerActionCmd('STATUS') self.fail("Exception was expected, oobm is disabled at host level") except Exception: pass # Enable at host level cmd = enableOutOfBandManagementForHost.enableOutOfBandManagementForHostCmd() cmd.hostid = host.id response = self.apiclient.enableOutOfBandManagementForHost(cmd) response = self.issuePowerActionCmd('STATUS') self.assertEqual(response.powerstate, 'Off')