def test_oobm_background_powerstate_sync(self):
"""
Tests out-of-band management background powerstate sync
"""
self.debug("Testing oobm background sync")
interval = list_configurations(
self.apiclient,
name='outofbandmanagement.sync.interval'
)[0].value
self.configureAndEnableOobm()
self.startIpmiServer()
bmc = IpmiServerContext().bmc
bmc.powerstate = 'on'
self.checkSyncToState('On', interval)
bmc.powerstate = 'off'
self.checkSyncToState('Off', interval)
self.server.shutdown()
self.server.server_close()
# Check for unknown state (ipmi server not reachable)
self.checkSyncToState('Unknown', interval)
def test_oobm_background_powerstate_sync(self):
"""
Tests out-of-band management background powerstate sync
"""
self.debug("Testing oobm background sync")
interval = list_configurations(
self.apiclient,
name='outofbandmanagement.sync.interval'
)[0].value
self.configureAndEnableOobm()
self.startIpmiServer()
bmc = IpmiServerContext().bmc
bmc.powerstate = 'on'
self.checkSyncToState('On', interval)
bmc.powerstate = 'off'
self.checkSyncToState('Off', interval)
self.server.shutdown()
self.server.server_close()
# Check for unknown state (ipmi server not reachable)
self.checkSyncToState('Unknown', interval)
def configureAndStartIpmiServer(self, power_state=None):
"""
Setup ipmisim and enable out-of-band management for host
"""
self.configureAndEnableOobm()
self.startIpmiServer()
if power_state:
bmc = IpmiServerContext().bmc
bmc.powerstate = power_state
def configureAndStartIpmiServer(self, power_state=None):
"""
Setup ipmisim and enable out-of-band management for host
"""
self.configureAndEnableOobm()
self.startIpmiServer()
if power_state:
bmc = IpmiServerContext().bmc
bmc.powerstate = power_state
def configureAndEnableOobm(self, power_state=None):
"""
Setup ipmisim and enable out-of-band management for host
"""
self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd())
response = self.apiclient.enableOutOfBandManagementForHost(self.getOobmEnableCmd())
self.assertEqual(response.enabled, True)
if power_state:
bmc = IpmiServerContext().bmc
bmc.powerstate = power_state
def configureAndEnableOobm(self, power_state=None):
"""
Setup ipmisim and enable out-of-band management for host
"""
self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd())
response = self.apiclient.enableOutOfBandManagementForHost(self.getOobmEnableCmd())
self.assertEqual(response.enabled, True)
if power_state:
bmc = IpmiServerContext().bmc
bmc.powerstate = power_state
def setUpClass(cls):
testClient = super(TestOutOfBandManagement, cls).getClsTestClient()
cls.apiclient = testClient.getApiClient()
cls.services = testClient.getParsedTestDataConfig()
cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests())
cls.host = None
cls.cleanup = []
# use random port for ipmisim
s = socket.socket()
s.bind(('', 0))
cls.serverPort = s.getsockname()[1]
s.close()
def startIpmiServer(tname, server):
try:
server.serve_forever()
except Exception:
pass
IpmiServerContext('reset')
ThreadedIpmiServer.allow_reuse_address = True
server = ThreadedIpmiServer(('0.0.0.0', cls.serverPort), IpmiServer)
_thread.start_new_thread(startIpmiServer, (
"ipmi-server",
server,
))
cls.server = server
def setUpClass(cls):
testClient = super(TestOutOfBandManagement, cls).getClsTestClient()
cls.apiclient = testClient.getApiClient()
cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__
cls.services = testClient.getParsedTestDataConfig()
cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests())
cls.host = None
cls.cleanup = []
cls.hypervisor = cls.testClient.getHypervisorInfo()
if (cls.hypervisor.lower() != "simulator"):
cls.skipIfMSIsUnsupported(cls)
# use random port for ipmisim
s = socket.socket()
s.bind(('', 0))
cls.serverPort = s.getsockname()[1]
s.close()
def startIpmiServer(tname, server):
try:
server.serve_forever()
except Exception: pass
IpmiServerContext('reset')
ThreadedIpmiServer.allow_reuse_address = True
server = ThreadedIpmiServer(('0.0.0.0', cls.serverPort), IpmiServer)
_thread.start_new_thread(startIpmiServer, ("ipmi-server", server,))
cls.server = server
def test_oobm_zchange_password(self):
"""
Tests out-of-band management change password feature
"""
self.configureAndEnableOobm()
self.startIpmiServer()
self.debug("Testing oobm change password")
cmd = changeOutOfBandManagementPassword.changeOutOfBandManagementPasswordCmd(
)
cmd.hostid = self.getHost().id
cmd.password = "******"
try:
response = self.apiclient.changeOutOfBandManagementPassword(cmd)
self.assertEqual(response.status, True)
except Exception as e:
if "packet session id 0x0 does not match active session" in str(e):
raise self.skipTest("Known ipmitool issue hit, skipping test")
raise e
bmc = IpmiServerContext().bmc
bmc.powerstate = 'on'
response = self.issuePowerActionCmd('STATUS')
self.assertEqual(response.status, True)
self.assertEqual(response.powerstate, 'On')
# Reset configuration, resets password
self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd())
self.assertEqual(response.status, True)
alerts = Alert.list(self.apiclient, keyword="auth-error", listall=True)
alertCount = 0
if alerts:
alertCount = len(alerts)
try:
response = self.issuePowerActionCmd('STATUS')
self.fail("Expected an exception to be thrown, failing")
except Exception:
pass
alerts = Alert.list(self.apiclient, keyword="auth-error", listall=True)
# At least one alert was sent
self.assertTrue((len(alerts) - alertCount) > 0)
def ipmiserver():
context = IpmiServerContext()
server = socketserver.UDPServer(('0.0.0.0', 9999), IpmiServer)
t = Thread(target=server.serve_forever)
t.daemon = True
t.start()
yield server.server_address, context.bmc
server.shutdown()
def tearDownClass(cls):
try:
cleanup_resources(cls.apiclient, cls.cleanup)
cls.server.shutdown()
cls.server.server_close()
IpmiServerContext('reset')
except Exception as e:
raise Exception("Warning: Exception during cleanup : %s" % e)
def test_oobm_zchange_password(self):
"""
Tests out-of-band management change password feature
"""
self.configureAndEnableOobm()
self.startIpmiServer()
self.debug("Testing oobm change password")
alerts = Alert.list(self.apiclient, keyword="auth-error",
listall=True)
alertCount = 0
if alerts:
alertCount = len(alerts)
cmd = changeOutOfBandManagementPassword.changeOutOfBandManagementPasswordCmd()
cmd.hostid = self.getHost().id
cmd.password = "******"
try:
response = self.apiclient.changeOutOfBandManagementPassword(cmd)
self.assertEqual(response.status, True)
except Exception as e:
if "packet session id 0x0 does not match active session" in str(e):
raise self.skipTest("Known ipmitool issue hit, skipping test")
raise e
bmc = IpmiServerContext().bmc
bmc.powerstate = 'on'
response = self.issuePowerActionCmd('STATUS')
self.assertEqual(response.status, True)
self.assertEqual(response.powerstate, 'On')
# Reset configuration, resets password
self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd())
self.assertEqual(response.status, True)
try:
response = self.issuePowerActionCmd('STATUS')
self.fail("Expected an exception to be thrown, failing")
except Exception: pass
alerts = Alert.list(self.apiclient, keyword="auth-error",
listall=True)
self.assertTrue((len(alerts) - alertCount) >= 0)
def startIpmiServer(self):
def startIpmiServer(tname, server):
self.debug("Starting ipmisim server")
try:
server.serve_forever()
except Exception: pass
IpmiServerContext('reset')
ThreadedIpmiServer.allow_reuse_address = False
server = ThreadedIpmiServer(('0.0.0.0', self.getIpmiServerPort()), IpmiServer)
_thread.start_new_thread(startIpmiServer, ("ipmi-server", server,))
self.server = server
def test_oobm_background_powerstate_sync(self):
"""
Tests out-of-band management background powerstate sync
"""
self.debug("Testing oobm background sync")
self.configureAndEnableOobm()
bmc = IpmiServerContext().bmc
bmc.powerstate = 'on'
self.checkSyncToState('On', 2)
bmc.powerstate = 'off'
self.checkSyncToState('Off', 2)
# Check for unknown state (ipmi server not reachable)
cmd = self.getOobmConfigCmd()
cmd.port = 1
response = self.apiclient.configureOutOfBandManagement(cmd)
self.checkSyncToState('Unknown', 2)
def test_oobm_zchange_password(self):
"""
Tests out-of-band management change password feature
"""
self.configureAndEnableOobm()
self.startIpmiServer()
self.debug("Testing oobm change password")
cmd = changeOutOfBandManagementPassword.changeOutOfBandManagementPasswordCmd()
cmd.hostid = self.getHost().id
cmd.password = "******"
response = self.apiclient.changeOutOfBandManagementPassword(cmd)
self.assertEqual(response.status, True)
bmc = IpmiServerContext().bmc
bmc.powerstate = 'on'
response = self.issuePowerActionCmd('STATUS')
self.assertEqual(response.status, True)
self.assertEqual(response.powerstate, 'On')
# Reset configuration, resets password
self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd())
self.assertEqual(response.status, True)
alerts = Alert.list(self.apiclient, keyword="auth-error",
listall=True)
alertCount = 0
if alerts:
alertCount = len(alerts)
try:
response = self.issuePowerActionCmd('STATUS')
self.fail("Expected an exception to be thrown, failing")
except Exception: pass
alerts = Alert.list(self.apiclient, keyword="auth-error",
listall=True)
# At least one alert was sent
self.assertTrue((len(alerts) - alertCount) > 0)
def tearDown(self):
try:
self.dbclient.execute("delete from oobm where port=%d" % self.getIpmiServerPort())
self.dbclient.execute("delete from mshost_peer where peer_runid=%s" % self.getFakeMsRunId())
self.dbclient.execute("delete from mshost where runid=%s" % self.getFakeMsRunId())
self.dbclient.execute("delete from cluster_details where name='outOfBandManagementEnabled'")
self.dbclient.execute("delete from data_center_details where name='outOfBandManagementEnabled'")
cleanup_resources(self.apiclient, self.cleanup)
if self.server:
self.server.shutdown()
self.server.server_close()
IpmiServerContext('reset')
except Exception as e:
raise Exception("Warning: Exception during cleanup : %s" % e)
def test_oobm_background_powerstate_sync(self):
"""
Tests out-of-band management background powerstate sync
"""
self.debug("Testing oobm background sync")
interval = list_configurations(
self.apiclient,
name='outofbandmanagement.sync.interval'
)[0].value
self.configureAndEnableOobm()
bmc = IpmiServerContext().bmc
bmc.powerstate = 'on'
self.checkSyncToState('On', interval)
bmc.powerstate = 'off'
self.checkSyncToState('Off', interval)
# Check for unknown state (ipmi server not reachable)
cmd = self.getOobmConfigCmd()
cmd.port = 1
response = self.apiclient.configureOutOfBandManagement(cmd)
self.checkSyncToState('Unknown', interval)
def test_oobm_multiple_mgmt_server_ownership(self):
"""
Tests out-of-band management ownership expiry across multi-mgmt server
"""
self.configureAndEnableOobm()
cloudstackVersion = Configurations.listCapabilities(
self.apiclient).cloudstackversion
currentMsHosts = []
mshosts = self.dbclient.execute(
"select msid from mshost where version='%s' and removed is NULL and state='Up'"
% (cloudstackVersion))
if len(mshosts) > 0:
currentMsHosts = map(lambda row: row[0], mshosts)
# Inject fake ms host
self.dbclient.execute(
"insert into mshost (msid,runid,name,state,version,service_ip,service_port,last_update) values (%s,%s,'oobm-marvin-fakebox', 'Down', '%s', '127.0.0.1', '22', NOW())"
% (self.getFakeMsId(), self.getFakeMsRunId(), cloudstackVersion))
# Pass ownership to the fake ms id
self.dbclient.execute(
"update oobm set mgmt_server_id=%d where port=%d" %
(self.getFakeMsId(), self.getIpmiServerPort()))
self.debug("Testing oobm background sync")
pingInterval = float(
list_configurations(self.apiclient, name='ping.interval')[0].value)
pingTimeout = float(
list_configurations(self.apiclient, name='ping.timeout')[0].value)
def removeFakeMgmtServer(fakeMsRunId):
rows = self.dbclient.execute(
"select * from mshost_peer where peer_runid=%s" % fakeMsRunId)
if len(rows) > 0:
self.debug(
"Mgmt server is now trying to contact the fake mgmt server"
)
self.dbclient.execute(
"update mshost set removed=now() where runid=%s" %
fakeMsRunId)
self.dbclient.execute(
"update mshost_peer set peer_state='Down' where peer_runid=%s"
% fakeMsRunId)
return True, None
return False, None
def checkOobmOwnershipExpiry(serverPort, fakeMsId):
rows = self.dbclient.execute(
"select mgmt_server_id from oobm where port=%d" % (serverPort))
if len(rows) > 0 and rows[0][0] != fakeMsId:
self.debug(
"Out-of-band management ownership expired as node was detected to be gone"
)
return True, None
return False, None
retry_interval = 1 + (pingInterval * pingTimeout / 10)
res, _ = wait_until(retry_interval, 10, removeFakeMgmtServer,
self.getFakeMsRunId())
if not res:
self.fail(
"Management server failed to turn down or remove fake mgmt server"
)
res, _ = wait_until(retry_interval, 100, checkOobmOwnershipExpiry,
self.getIpmiServerPort(), self.getFakeMsId())
if not res:
self.fail(
"Management server failed to expire ownership of fenced peer")
self.debug("Testing oobm background sync should claim new ownership")
interval = list_configurations(
self.apiclient, name='outofbandmanagement.sync.interval')[0].value
self.startIpmiServer()
bmc = IpmiServerContext().bmc
bmc.powerstate = 'on'
self.checkSyncToState('On', interval)
result = self.dbclient.execute(
"select mgmt_server_id from oobm where port=%d" %
(self.getIpmiServerPort()))
newOwnerId = result[0][0]
self.assertTrue(newOwnerId in currentMsHosts)
def test_oobm_enabledisable_across_clusterzones(self):
"""
Tests out-of-band management enable/disable feature at cluster
and zone level sequentially Zone > Cluster > Host
"""
self.configureAndEnableOobm()
self.startIpmiServer()
bmc = IpmiServerContext().bmc
bmc.powerstate = 'off'
host = self.getHost()
# Disable at zone level
cmd = disableOutOfBandManagementForZone.disableOutOfBandManagementForZoneCmd(
)
cmd.zoneid = host.zoneid
response = self.apiclient.disableOutOfBandManagementForZone(cmd)
# Disable at cluster level
cmd = disableOutOfBandManagementForCluster.disableOutOfBandManagementForClusterCmd(
)
cmd.clusterid = host.clusterid
response = self.apiclient.disableOutOfBandManagementForCluster(cmd)
# Disable at host level
cmd = disableOutOfBandManagementForHost.disableOutOfBandManagementForHostCmd(
)
cmd.hostid = host.id
response = self.apiclient.disableOutOfBandManagementForHost(cmd)
try:
self.issuePowerActionCmd('STATUS')
self.fail("Exception was expected, oobm is disabled at zone level")
except Exception:
pass
# Enable at zone level
cmd = enableOutOfBandManagementForZone.enableOutOfBandManagementForZoneCmd(
)
cmd.zoneid = host.zoneid
response = self.apiclient.enableOutOfBandManagementForZone(cmd)
try:
self.issuePowerActionCmd('STATUS')
self.fail(
"Exception was expected, oobm is disabled at cluster level")
except Exception:
pass
# Check background thread syncs state to Disabled
response = self.getHost(hostId=host.id).outofbandmanagement
self.assertEqual(response.powerstate, 'Disabled')
self.dbclient.execute(
"update oobm set power_state='On' where port=%d" %
self.getIpmiServerPort())
interval = list_configurations(
self.apiclient, name='outofbandmanagement.sync.interval')[0].value
self.checkSyncToState('Disabled', interval)
# Enable at cluster level
cmd = enableOutOfBandManagementForCluster.enableOutOfBandManagementForClusterCmd(
)
cmd.clusterid = host.clusterid
response = self.apiclient.enableOutOfBandManagementForCluster(cmd)
try:
self.issuePowerActionCmd('STATUS')
self.fail("Exception was expected, oobm is disabled at host level")
except Exception:
pass
# Enable at host level
cmd = enableOutOfBandManagementForHost.enableOutOfBandManagementForHostCmd(
)
cmd.hostid = host.id
response = self.apiclient.enableOutOfBandManagementForHost(cmd)
response = self.issuePowerActionCmd('STATUS')
self.assertEqual(response.powerstate, 'Off')
def test_oobm_multiple_mgmt_server_ownership(self):
"""
Tests out-of-band management ownership expiry across multi-mgmt server
"""
self.configureAndEnableOobm()
cloudstackVersion = Configurations.listCapabilities(self.apiclient).cloudstackversion
currentMsHosts = []
mshosts = self.dbclient.execute("select msid from mshost where version='%s' and removed is NULL and state='Up'" % (cloudstackVersion))
if len(mshosts) > 0:
currentMsHosts = map(lambda row: row[0], mshosts)
# Inject fake ms host
self.dbclient.execute("insert into mshost (msid,runid,name,state,version,service_ip,service_port,last_update) values (%s,%s,'oobm-marvin-fakebox', 'Down', '%s', '127.0.0.1', '22', NOW())" % (self.getFakeMsId(), self.getFakeMsRunId(), cloudstackVersion))
# Pass ownership to the fake ms id
self.dbclient.execute("update oobm set mgmt_server_id=%d where port=%d" % (self.getFakeMsId(), self.getIpmiServerPort()))
self.debug("Testing oobm background sync")
pingInterval = float(list_configurations(
self.apiclient,
name='ping.interval'
)[0].value)
pingTimeout = float(list_configurations(
self.apiclient,
name='ping.timeout'
)[0].value)
def removeFakeMgmtServer(fakeMsRunId):
rows = self.dbclient.execute("select * from mshost_peer where peer_runid=%s" % fakeMsRunId)
if len(rows) > 0:
self.debug("Mgmt server is now trying to contact the fake mgmt server")
self.dbclient.execute("update mshost set removed=now() where runid=%s" % fakeMsRunId)
self.dbclient.execute("update mshost_peer set peer_state='Down' where peer_runid=%s" % fakeMsRunId)
return True, None
return False, None
def checkOobmOwnershipExpiry(serverPort, fakeMsId):
rows = self.dbclient.execute("select mgmt_server_id from oobm where port=%d" % (serverPort))
if len(rows) > 0 and rows[0][0] != fakeMsId:
self.debug("Out-of-band management ownership expired as node was detected to be gone")
return True, None
return False, None
retry_interval = 1 + (pingInterval * pingTimeout / 10)
res, _ = wait_until(retry_interval, 10, removeFakeMgmtServer, self.getFakeMsRunId())
if not res:
self.fail("Management server failed to turn down or remove fake mgmt server")
res, _ = wait_until(retry_interval, 100, checkOobmOwnershipExpiry, self.getIpmiServerPort(), self.getFakeMsId())
if not res:
self.fail("Management server failed to expire ownership of fenced peer")
self.debug("Testing oobm background sync should claim new ownership")
interval = list_configurations(
self.apiclient,
name='outofbandmanagement.sync.interval'
)[0].value
self.startIpmiServer()
bmc = IpmiServerContext().bmc
bmc.powerstate = 'on'
self.checkSyncToState('On', interval)
result = self.dbclient.execute("select mgmt_server_id from oobm where port=%d" % (self.getIpmiServerPort()))
newOwnerId = result[0][0]
self.assertTrue(newOwnerId in currentMsHosts)
def test_oobm_enabledisable_across_clusterzones(self):
"""
Tests out-of-band management enable/disable feature at cluster
and zone level sequentially Zone > Cluster > Host
"""
self.configureAndEnableOobm()
self.startIpmiServer()
bmc = IpmiServerContext().bmc
bmc.powerstate = 'off'
host = self.getHost()
# Disable at zone level
cmd = disableOutOfBandManagementForZone.disableOutOfBandManagementForZoneCmd()
cmd.zoneid = host.zoneid
response = self.apiclient.disableOutOfBandManagementForZone(cmd)
# Disable at cluster level
cmd = disableOutOfBandManagementForCluster.disableOutOfBandManagementForClusterCmd()
cmd.clusterid = host.clusterid
response = self.apiclient.disableOutOfBandManagementForCluster(cmd)
# Disable at host level
cmd = disableOutOfBandManagementForHost.disableOutOfBandManagementForHostCmd()
cmd.hostid = host.id
response = self.apiclient.disableOutOfBandManagementForHost(cmd)
try:
self.issuePowerActionCmd('STATUS')
self.fail("Exception was expected, oobm is disabled at zone level")
except Exception: pass
# Enable at zone level
cmd = enableOutOfBandManagementForZone.enableOutOfBandManagementForZoneCmd()
cmd.zoneid = host.zoneid
response = self.apiclient.enableOutOfBandManagementForZone(cmd)
try:
self.issuePowerActionCmd('STATUS')
self.fail("Exception was expected, oobm is disabled at cluster level")
except Exception: pass
# Check background thread syncs state to Disabled
response = self.getHost(hostId=host.id).outofbandmanagement
self.assertEqual(response.powerstate, 'Disabled')
self.dbclient.execute("update oobm set power_state='On' where port=%d" % self.getIpmiServerPort())
interval = list_configurations(
self.apiclient,
name='outofbandmanagement.sync.interval'
)[0].value
self.checkSyncToState('Disabled', interval)
# Enable at cluster level
cmd = enableOutOfBandManagementForCluster.enableOutOfBandManagementForClusterCmd()
cmd.clusterid = host.clusterid
response = self.apiclient.enableOutOfBandManagementForCluster(cmd)
try:
self.issuePowerActionCmd('STATUS')
self.fail("Exception was expected, oobm is disabled at host level")
except Exception: pass
# Enable at host level
cmd = enableOutOfBandManagementForHost.enableOutOfBandManagementForHostCmd()
cmd.hostid = host.id
response = self.apiclient.enableOutOfBandManagementForHost(cmd)
response = self.issuePowerActionCmd('STATUS')
self.assertEqual(response.powerstate, 'Off')
