def validate(self):
"""
Validate that the storage domain is accessible.
"""
self.log.info("sdUUID=%s", self.sdUUID)
self.invalidateMetadata()
if not len(self.getMetadata()):
raise se.StorageDomainAccessError(self.sdUUID)
def chkVG(vgName):
cmd = ["vgck", vgName]
rc, out, err = _lvminfo.cmd(cmd, _lvminfo._getVGDevs((vgName, )))
if rc != 0:
_lvminfo._invalidatevgs(vgName)
_lvminfo._invalidatelvs(vgName)
raise se.StorageDomainAccessError("%s: %s" % (vgName, err))
return True
def getStats(self):
"""
Get storage domain statistics
"""
# self.log.info("sdUUID=%s", self.sdUUID)
stats = {'disktotal': '',
'diskfree': '',
'mdavalid': True,
'mdathreshold': True,
'mdasize': 0,
'mdafree': 0}
try:
st = self.oop.os.statvfs(self.domaindir)
stats['disktotal'] = str(st.f_frsize * st.f_blocks)
stats['diskfree'] = str(st.f_frsize * st.f_bavail)
except OSError as e:
self.log.info("sdUUID=%s %s", self.sdUUID, str(e))
if e.errno == errno.ESTALE:
raise se.FileStorageDomainStaleNFSHandle
raise se.StorageDomainAccessError(self.sdUUID)
return stats
class TestMonitorThreadMonitoring(VdsmTestCase):
# In this state we do:
# 1. If refresh timeout has expired, remove the domain from the cache
# 2. call domain.selftest()
# 3. call domain.getStats()
# 4. call domain.validateMaster()
# 5. call domain.hasHostId()
# 6. call domain.getVersion()
#
# - When path check completes, we get a callback from the checker thread
# and update monitor status.
# - On failure, we abort the process, and set status.error.
# - If this is the first failure, we emit a domain state change event with
# valid=False.
# - On success, if the previous status.error was set, we emit a domain
# state change event with valid=True.
# - If everything was successful, and host id not acquired yet, acquire it
# - We repeat this forever until the monitor is stopped.
def test_unknown_to_valid(self):
with monitor_env() as env:
domain = FakeDomain("uuid")
monitor.sdCache.domains["uuid"] = domain
env.thread.start()
# First cycle suceeds, but path status is not avialale yet
env.wait_for_cycle()
status = env.thread.getStatus()
self.assertFalse(status.actual)
self.assertEqual(env.event.received, [])
# When path succeeds, emit VALID event
env.checker.complete(domain.getMonitoringPath(), FakeCheckResult())
status = env.thread.getStatus()
self.assertTrue(status.actual)
self.assertTrue(status.valid)
self.assertEqual(env.event.received, [(('uuid', True), {})])
@permutations([
("selftest", OSError),
("selftest", UnexpectedError),
("getStats", se.FileStorageDomainStaleNFSHandle),
# TODO: Uncomment when test is fixed to not check a type.
# ("getStats", se.StorageDomainAccessError("fake-sd-id")),
("getStats", UnexpectedError),
("validateMaster", OSError),
("validateMaster", UnexpectedError),
("getVersion", OSError),
("getVersion", UnexpectedError),
])
def test_from_unknown_to_invalid_domain(self, method, exception):
with monitor_env() as env:
domain = FakeDomain("uuid")
domain.errors[method] = exception
monitor.sdCache.domains["uuid"] = domain
env.thread.start()
# First cycle fail, emit event without waiting for path status
env.wait_for_cycle()
status = env.thread.getStatus()
self.assertTrue(status.actual)
self.assertFalse(status.valid)
# TODO: Should assert we got an exception set by the test instead
# of checking a type.
self.assertIsInstance(status.error, exception)
self.assertEqual(env.event.received, [(('uuid', False), {})])
@permutations([[se.MiscFileReadException], [UnexpectedError]])
def test_from_unknown_to_invalid_path(self, exception):
with monitor_env() as env:
domain = FakeDomain("uuid")
monitor.sdCache.domains["uuid"] = domain
env.thread.start()
# First cycle succeed, but path status is not available yet
env.wait_for_cycle()
status = env.thread.getStatus()
self.assertFalse(status.actual)
self.assertEqual(env.event.received, [])
# When path fail, emit INVALID event
env.checker.complete(domain.getMonitoringPath(),
FakeCheckResult(exception))
status = env.thread.getStatus()
self.assertTrue(status.actual)
self.assertFalse(status.valid)
self.assertIsInstance(status.error, exception)
self.assertEqual(env.event.received, [(('uuid', False), {})])
@permutations([
("selftest", OSError),
("selftest", UnexpectedError),
("getStats", se.FileStorageDomainStaleNFSHandle),
("getStats", se.StorageDomainAccessError("fake-sd-id")),
("getStats", UnexpectedError),
("validateMaster", OSError),
("validateMaster", UnexpectedError),
("getVersion", OSError),
("getVersion", UnexpectedError),
])
def test_from_invalid_to_valid_domain(self, method, exception):
with monitor_env() as env:
domain = FakeDomain("uuid")
domain.errors[method] = exception
monitor.sdCache.domains["uuid"] = domain
env.thread.start()
# First cycle fail, and emit INVALID event
env.wait_for_cycle()
del env.event.received[0]
# Path status succeeds, but domain status is not valid, so no event
# is emitted.
env.wait_for_cycle()
env.checker.complete(domain.getMonitoringPath(), FakeCheckResult())
status = env.thread.getStatus()
self.assertTrue(status.actual)
self.assertFalse(status.valid)
self.assertEqual(env.event.received, [])
# When next cycle succeeds, emit VALID event
del domain.errors[method]
env.wait_for_cycle()
status = env.thread.getStatus()
self.assertTrue(status.valid)
self.assertEqual(env.event.received, [(('uuid', True), {})])
@permutations([[se.MiscFileReadException], [UnexpectedError]])
def test_from_invalid_to_valid_path(self, exception):
with monitor_env() as env:
domain = FakeDomain("uuid")
monitor.sdCache.domains["uuid"] = domain
env.thread.start()
# First cycle succeed, but path status fail, emit INVALID event
env.wait_for_cycle()
env.checker.complete(domain.getMonitoringPath(),
FakeCheckResult(exception))
del env.event.received[0]
# Both domain status and pass status succeed, emit VALID event
env.wait_for_cycle()
env.checker.complete(domain.getMonitoringPath(), FakeCheckResult())
status = env.thread.getStatus()
self.assertTrue(status.valid)
self.assertEqual(env.event.received, [(('uuid', True), {})])
def test_keeps_valid(self):
with monitor_env() as env:
domain = FakeDomain("uuid")
monitor.sdCache.domains["uuid"] = domain
env.thread.start()
# Both domain status and path status succeed and emit VALID event
env.wait_for_cycle()
env.checker.complete(domain.getMonitoringPath(), FakeCheckResult())
del env.event.received[0]
# Both succeed again, no event emitted - domain monitor state did
# not change (valid -> valid)
env.wait_for_cycle()
env.checker.complete(domain.getMonitoringPath(), FakeCheckResult())
status = env.thread.getStatus()
self.assertTrue(status.valid)
self.assertEqual(env.event.received, [])
@permutations([
("selftest", OSError),
("selftest", UnexpectedError),
("getStats", se.FileStorageDomainStaleNFSHandle),
# TODO: Uncomment when test is fixed to not check a type.
# ("getStats", se.StorageDomainAccessError("fake-sd-id")),
("getStats", UnexpectedError),
("validateMaster", OSError),
("validateMaster", UnexpectedError),
("getVersion", OSError),
("getVersion", UnexpectedError),
])
def test_from_valid_to_invalid_domain(self, method, exception):
with monitor_env() as env:
domain = FakeDomain("uuid")
monitor.sdCache.domains["uuid"] = domain
env.thread.start()
# Both domain status and path status succeed and emit VALID event
env.wait_for_cycle()
env.checker.complete(domain.getMonitoringPath(), FakeCheckResult())
del env.event.received[0]
# Domain status fail, emit INVALID event
domain.errors[method] = exception
env.wait_for_cycle()
status = env.thread.getStatus()
self.assertFalse(status.valid)
# TODO: Should assert we got an exception set by the test instead
# of checking a type.
self.assertIsInstance(status.error, exception)
self.assertEqual(env.event.received, [(('uuid', False), {})])
@permutations([[se.MiscFileReadException], [UnexpectedError]])
def test_from_valid_to_invalid_path(self, exception):
with monitor_env() as env:
domain = FakeDomain("uuid")
monitor.sdCache.domains["uuid"] = domain
env.thread.start()
# Both domain status and path status succeed and emit VALID event
env.wait_for_cycle()
env.checker.complete(domain.getMonitoringPath(), FakeCheckResult())
del env.event.received[0]
env.wait_for_cycle()
env.checker.complete(domain.getMonitoringPath(),
FakeCheckResult(exception))
status = env.thread.getStatus()
self.assertFalse(status.valid)
self.assertIsInstance(status.error, exception)
self.assertEqual(env.event.received, [(('uuid', False), {})])
def test_acquire_host_id(self):
with monitor_env() as env:
domain = FakeDomain("uuid")
monitor.sdCache.domains["uuid"] = domain
env.thread.start()
# Both domain status and path status succeed
env.wait_for_cycle()
env.checker.complete(domain.getMonitoringPath(), FakeCheckResult())
self.assertFalse(domain.acquired)
# Acquire host id on the next cycle
env.wait_for_cycle()
self.assertTrue(domain.acquired)
def test_acquire_host_id_after_error(self):
with monitor_env() as env:
domain = FakeDomain("uuid")
domain.errors["selftest"] = OSError
monitor.sdCache.domains["uuid"] = domain
env.thread.start()
# Domain status fail, emit INVALID event
env.wait_for_cycle()
del domain.errors["selftest"]
# Both domain status and path status succeed
env.wait_for_cycle()
env.checker.complete(domain.getMonitoringPath(), FakeCheckResult())
self.assertFalse(domain.acquired)
# Acquire host id on the next cycle
env.wait_for_cycle()
self.assertTrue(domain.acquired)
def test_acquire_host_id_if_lost(self):
with monitor_env() as env:
domain = FakeDomain("uuid")
monitor.sdCache.domains["uuid"] = domain
env.thread.start()
# Both domain status and path status succeed
env.wait_for_cycle()
env.checker.complete(domain.getMonitoringPath(), FakeCheckResult())
self.assertFalse(domain.acquired)
# Acquire host id on the next cycle
env.wait_for_cycle()
self.assertTrue(domain.acquired)
# Simulate loosing host id - acquire again because status is valid
domain.acquired = False
env.wait_for_cycle()
self.assertTrue(domain.acquired)
def test_dont_acquire_host_id_on_iso_domain(self):
with monitor_env() as env:
domain = FakeDomain("uuid", iso_dir="/path")
monitor.sdCache.domains["uuid"] = domain
env.thread.start()
env.wait_for_cycle()
env.checker.complete(domain.getMonitoringPath(), FakeCheckResult())
self.assertFalse(domain.acquired)
def test_dont_acquire_host_id_on_error(self):
with monitor_env() as env:
domain = FakeDomain("uuid")
domain.errors["selftest"] = OSError
monitor.sdCache.domains["uuid"] = domain
env.thread.start()
env.wait_for_cycle()
env.checker.complete(domain.getMonitoringPath(), FakeCheckResult())
self.assertFalse(domain.acquired)
@permutations([[se.AcquireHostIdFailure], [UnexpectedError]])
def test_acquire_host_id_retry_after_error(self, exception):
with monitor_env() as env:
domain = FakeDomain("uuid")
domain.errors['acquireHostId'] = exception
monitor.sdCache.domains["uuid"] = domain
env.thread.start()
env.wait_for_cycle()
self.assertFalse(domain.acquired)
del domain.errors["acquireHostId"]
env.checker.complete(domain.getMonitoringPath(), FakeCheckResult())
self.assertFalse(domain.acquired)
# Acquire on next cycle
env.wait_for_cycle()
self.assertTrue(domain.acquired)
def test_refresh(self):
with monitor_env(refresh=MONITOR_INTERVAL * 1.5) as env:
domain = FakeDomain("uuid")
monitor.sdCache.domains["uuid"] = domain
env.thread.start()
# Domain will be removed after the refresh timeout
env.wait_for_cycle()
self.assertIn(domain.sdUUID, monitor.sdCache.domains)
env.wait_for_cycle()
# Refresh timeout will expires during next cycle
env.wait_for_cycle()
self.assertNotIn(domain.sdUUID, monitor.sdCache.domains)
