def construct(self):
self.hostImplementation = FakeHost()
self.fakeInaugurate = Empty()
self.fakeInaugurate.provideLabel = self.provideLabelForInauguration
self.fakeInaugurate.register = self.registerForInauguration
self.fakeInaugurate.unregister = self.unregisterForInauguration
self.fakeTFTPBoot = FakeTFTPBoot(self)
self.fakeDnsmasq = Empty()
self.fakeDnsmasq.addIfNotAlready = self.dnsmasqAddIfNotAlready
self.fakeReclaimHost = Empty()
self.patchWithSpecValidation(fakeObject=self.fakeReclaimHost,
realMethod=reclaimhostspooler.ReclaimHostSpooler.cold,
fakeMethod=self.reclaimHostCold)
self.patchWithSpecValidation(fakeObject=self.fakeReclaimHost,
realMethod=reclaimhostspooler.ReclaimHostSpooler.soft,
fakeMethod=self.reclaimHostSoft)
self.fakeTFTPBoot.expectToBeConfiguredForInaugurator()
self.expectedDnsmasqAddIfNotAlready = True
self.expectedClearDisk = False
hoststatemachine.HostStateMachine.ALLOW_CLEARING_OF_DISK = True
self.tested = hoststatemachine.HostStateMachine(
hostImplementation=self.hostImplementation,
inaugurate=self.fakeInaugurate, tftpboot=self.fakeTFTPBoot, dnsmasq=self.fakeDnsmasq,
reclaimHost=self.fakeReclaimHost)
self.tested.setDestroyCallback(self.destroyHost)
self.assertIs(self.tested.hostImplementation(), self.hostImplementation)
self.fakeTFTPBoot.validateConfiguredOnceForInaugurator()
assert self.checkInCallback is not None
assert self.doneCallback is not None
def construct(self):
self.hostImplementation = FakeHost()
self.fakeInaugurate = Empty()
self.fakeInaugurate.provideLabel = self.provideLabelForInauguration
self.fakeInaugurate.register = self.registerForInauguration
self.fakeInaugurate.unregister = self.unregisterForInauguration
self.fakeTFTPBoot = Empty()
self.fakeTFTPBoot.inauguratorCommandLine = self.inauguratorCommandLine
self.fakeTFTPBoot.configureForInaugurator = self.tftpbootConfigureForInaugurator
self.fakeTFTPBoot.configureForLocalBoot = self.tftpbootConfigureForLocalBoot
self.fakeDnsmasq = Empty()
self.fakeDnsmasq.addIfNotAlready = self.dnsmasqAddIfNotAlready
self.fakeReclaimHost = Empty()
self.fakeReclaimHost.cold = self.reclaimHostCold
self.fakeReclaimHost.soft = self.reclaimHostSoft
self.expectedTFTPBootToBeConfiguredForInaugurator = True
self.expectedDnsmasqAddIfNotAlready = True
self.expectedClearDisk = False
hoststatemachine.HostStateMachine.ALLOW_CLEARING_OF_DISK = True
self.tested = hoststatemachine.HostStateMachine(
hostImplementation=self.hostImplementation,
inaugurate=self.fakeInaugurate, tftpboot=self.fakeTFTPBoot, dnsmasq=self.fakeDnsmasq,
reclaimHost=self.fakeReclaimHost)
self.tested.setDestroyCallback(self.destroyHost)
self.assertIs(self.tested.hostImplementation(), self.hostImplementation)
self.assertFalse(self.expectedTFTPBootToBeConfiguredForInaugurator)
assert self.checkInCallback is not None
assert self.doneCallback is not None
def setUp(self):
super(Test, self).setUp(moduleInWhichToSetupMocks=reclaimhostspooler)
self._host = FakeHost()
self._hostStateMachine = FakeHostStateMachine(self._host)
self._hosts = Hosts()
self._hosts.add(self._hostStateMachine)
self._tftpboot = FakeTFTPBoot()
self._wasThereAnAttemptToCreateFifo = False
self._fakeSoftReclaimRequestFifoPath = "/fakeNotifyFifoPath"
self._fakeSoftReclaimFailedFifoPath = "/fakeSoftReclaimFailedFifoPath"
self._origKillSelf = reclaimhostspooler.suicide.killSelf
reclaimhostspooler.suicide.killSelf = mock.Mock()
self._actualColdReclamationRequests = []
self._tested = self._generateTestedInstanceWithMockedThreading()
self._expectedRequests = []
self._coldReclaimCallbackRaisesException = False
class Test(unittest.TestCase):
def setUp(self):
globallock._lock.acquire()
self.addCleanup(self.releaseGlobalLock)
self.checkInCallback = None
self.doneCallback = None
self.failureCallback = None
self.expectedProvidedLabel = None
self.provideLabelRaises = False
self.expectedReportedState = None
timer.scheduleIn = self.scheduleTimerIn
timer.cancelAllByTag = self.cancelAllTimersByTag
self.currentTimer = None
self.currentTimerTag = None
self.expectedColdReclaim = False
self.expectReconfigureBIOS = False
self.expectedHardReset = True
self.expectedSoftReclaim = False
self.expectedSelfDestruct = False
self.softReclaimFailedCallback = None
self.construct()
@classmethod
def setUpClass(cls):
cls.configureLogging()
@classmethod
def configureLogging(self):
logger = logging.getLogger()
verbosity = int(os.getenv("VERBOSITY", 0))
logLevels = {0: logging.CRITICAL + 1,
1: logging.ERROR,
2: logging.INFO,
3: logging.DEBUG}
maxVerbosity = max(logLevels.keys())
if verbosity > maxVerbosity:
verbosity = maxVerbosity
elif verbosity < 0:
verbosity = 0
logLevel = logLevels[verbosity]
logger.setLevel(logLevel)
def releaseGlobalLock(self):
globallock._lock.release()
def construct(self):
self.hostImplementation = FakeHost()
self.fakeInaugurate = Empty()
self.fakeInaugurate.provideLabel = self.provideLabelForInauguration
self.fakeInaugurate.register = self.registerForInauguration
self.fakeInaugurate.unregister = self.unregisterForInauguration
self.fakeTFTPBoot = FakeTFTPBoot(self)
self.fakeDnsmasq = Empty()
self.fakeDnsmasq.addIfNotAlready = self.dnsmasqAddIfNotAlready
self.fakeReclaimHost = Empty()
self.patchWithSpecValidation(fakeObject=self.fakeReclaimHost,
realMethod=reclaimhostspooler.ReclaimHostSpooler.cold,
fakeMethod=self.reclaimHostCold)
self.patchWithSpecValidation(fakeObject=self.fakeReclaimHost,
realMethod=reclaimhostspooler.ReclaimHostSpooler.soft,
fakeMethod=self.reclaimHostSoft)
self.fakeTFTPBoot.expectToBeConfiguredForInaugurator()
self.expectedDnsmasqAddIfNotAlready = True
self.expectedClearDisk = False
hoststatemachine.HostStateMachine.ALLOW_CLEARING_OF_DISK = True
self.tested = hoststatemachine.HostStateMachine(
hostImplementation=self.hostImplementation,
inaugurate=self.fakeInaugurate, tftpboot=self.fakeTFTPBoot, dnsmasq=self.fakeDnsmasq,
reclaimHost=self.fakeReclaimHost)
self.tested.setDestroyCallback(self.destroyHost)
self.assertIs(self.tested.hostImplementation(), self.hostImplementation)
self.fakeTFTPBoot.validateConfiguredOnceForInaugurator()
assert self.checkInCallback is not None
assert self.doneCallback is not None
def patchWithSpecValidation(self, fakeObject, realMethod, fakeMethod):
specValidator = mock.create_autospec(realMethod)
methodName = realMethod.__name__
def useFakeMethodWithSpecValidation(*args, **kwargs):
fakeSelf = None
try:
specValidator(fakeSelf, *args, **kwargs)
except TypeError as ex:
msg = "It seems that method '%(methodName)s' was used with the wrong argument " \
"specification; args:%(args)s, kwargs: %(kwargs)s " \
% dict(methodName=methodName, args=args, kwargs=kwargs)
ex.message = "%(msg)s. Original message: '%(origMessage)s'." % dict(msg=msg,
origMessage=ex.message)
ex.args = (ex.message,)
raise ex
fakeMethod(*args, **kwargs)
setattr(fakeObject, methodName, useFakeMethodWithSpecValidation)
def destroyHost(self, stateMachine):
self.assertIs(stateMachine, self.tested)
self.assertTrue(self.expectedSelfDestruct)
self.expectedSelfDestruct = False
def scheduleTimerIn(self, timeout, callback, tag):
self.assertIs(self.currentTimer, None)
self.assertIs(self.currentTimerTag, None)
self.currentTimer = callback
self.currentTimerTag = tag
def cancelAllTimersByTag(self, tag):
if self.currentTimerTag is not None:
self.assertIsNot(self.currentTimer, None)
self.assertIs(self.currentTimerTag, tag)
self.currentTimer = None
self.currentTimerTag = None
def triggerTimeout(self):
self.assertIsNot(self.currentTimer, None)
self.assertIsNot(self.currentTimerTag, None)
self.currentTimer()
self.currentTimer = None
self.currentTimerTag = None
def registerForInauguration(self, id, checkInCallback, doneCallback, progressCallback, failureCallback):
self.assertEquals(id, self.hostImplementation.id())
self.assertIs(self.checkInCallback, None)
self.assertIs(self.doneCallback, None)
self.checkInCallback = checkInCallback
self.doneCallback = doneCallback
self.progressCallback = progressCallback
self.failureCallback = failureCallback
def unregisterForInauguration(self, id):
self.assertIsNot(self.checkInCallback, None)
self.assertIsNot(self.doneCallback, None)
self.assertIsNot(self.progressCallback, None)
self.checkInCallback = None
self.doneCallback = None
self.progressCallback = None
def assertRegisteredForInauguration(self, id):
self.assertEquals(id, self.hostImplementation.id())
self.assertIsNot(self.checkInCallback, None)
self.assertIsNot(self.doneCallback, None)
self.assertIsNot(self.progressCallback, None)
def assertUnegisteredForInauguration(self, id):
self.assertIs(self.checkInCallback, None)
self.assertIs(self.doneCallback, None)
self.assertIs(self.progressCallback, None)
def provideLabelForInauguration(self, id, label):
self.assertEquals(id, self.hostImplementation.id())
if self.provideLabelRaises:
raise Exception("Provide label raises on purpose, as part of test")
self.actualProvidedLabel = label
def isObjectInitialized(self):
return hasattr(self, 'tested')
def dnsmasqAddIfNotAlready(self, mac, ip):
self.assertEquals(mac, self.hostImplementation.primaryMACAddress())
self.assertEquals(ip, self.hostImplementation.ipAddress())
self.assertTrue(self.expectedDnsmasqAddIfNotAlready)
self.expectedDnsmasqAddIfNotAlready = False
def reclaimHostCold(self, hostImplementation, reconfigureBIOS=False, hardReset=False):
self.assertIs(hostImplementation, self.hostImplementation)
self.assertTrue(self.expectedColdReclaim)
self.expectedColdReclaim = False
self.assertEqual(self.expectReconfigureBIOS, reconfigureBIOS)
self.assertEqual(self.expectedHardReset, hardReset)
def reclaimHostSoft(self, hostImplementation, serialPort, isInauguratorActive=False,
maxUptime=9999):
self.assertIs(hostImplementation, self.hostImplementation)
self.assertTrue(self.expectedSoftReclaim)
self.expectedSoftReclaim = False
self.softReclaimFailedCallback = self.tested.softReclaimFailed
def validateProvidedLabel(self, expected):
self.assertEquals(self.actualProvidedLabel, expected)
self.expectedProvidedLabel = None
def validateCheckInCallbackProvidesLabelImmediately(self, label):
self.assertIn(self.tested.state(), [
hoststatemachine.STATE_SOFT_RECLAMATION,
hoststatemachine.STATE_COLD_RECLAMATION])
self.assertIs(self.expectedProvidedLabel, None)
self.expectedProvidedLabel = label
self.assertIs(self.expectedReportedState, None)
self.expectedReportedState = hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED
self.checkInCallback()
self.validateProvidedLabel(expected=label)
self.assertIs(self.expectedReportedState, None)
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED)
def stateChangedCallback(self, tested):
self.assertIs(tested, self.tested)
self.assertIsNot(self.expectedReportedState, None)
self.assertEquals(tested.state(), self.expectedReportedState)
self.expectedReportedState = None
def inaugurationDone(self):
self.assertIn(self.tested.state(), [hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED])
self.assertIs(self.expectedProvidedLabel, None)
self.assertIs(self.expectedReportedState, None)
self.expectedReportedState = hoststatemachine.STATE_INAUGURATION_DONE
self.fakeTFTPBoot.expectToBeConfiguredForLocalBoot()
self.doneCallback()
self.assertIs(self.expectedProvidedLabel, None)
self.assertIs(self.expectedReportedState, None)
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_DONE)
self.fakeTFTPBoot.validateConfiguredOnceForLocalBoot()
self.assertIs(self.currentTimer, None)
def inaugurationFailed(self, isLastAttemptBeforeRevertingToColdReclamation=False):
self.assertIn(self.tested.state(), [hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED])
self.assertIs(self.expectedProvidedLabel, None)
self.assertIs(self.expectedReportedState, None)
if isLastAttemptBeforeRevertingToColdReclamation:
expectedReportedState = hoststatemachine.STATE_COLD_RECLAMATION
else:
expectedReportedState = hoststatemachine.STATE_SOFT_RECLAMATION
self.expectedReportedState = expectedReportedState
self.fakeTFTPBoot.expectToBeConfiguredForInaugurator()
self.expectedDnsmasqAddIfNotAlready = True
self.expectedSoftReclaim = True
self.failureCallback(message="Some osmosis failure")
self.assertIs(self.expectedProvidedLabel, None)
self.assertIs(self.expectedReportedState, None)
self.assertEquals(self.tested.state(), expectedReportedState)
self.fakeTFTPBoot.validateConfiguredOnceForInaugurator()
def assign(self, label, hint):
self.tested.assign(self.stateChangedCallback, label, hint)
self.assertEquals(self.tested.imageLabel(), label)
self.assertEquals(self.tested.imageHint(), hint)
def test_vmLifeCycle_Normal(self):
self.assertRegisteredForInauguration(self.hostImplementation.id())
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SOFT_RECLAMATION)
self.assign("fake image label", "fake image hint")
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SOFT_RECLAMATION)
self.validateCheckInCallbackProvidesLabelImmediately("fake image label")
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED)
self.inaugurationDone()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_DONE)
self.assertRegisteredForInauguration(self.hostImplementation.id())
def unassignCausesSoftReclaim(self):
self.assertFalse(self.expectedSoftReclaim)
self.expectedSoftReclaim = True
self.fakeTFTPBoot.expectToBeConfiguredForInaugurator()
self.expectedDnsmasqAddIfNotAlready = True
self.tested.unassign()
self.assertFalse(self.expectedSoftReclaim)
self.assertFalse(self.expectedColdReclaim)
self.fakeTFTPBoot.validateConfiguredOnceForInaugurator()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SOFT_RECLAMATION)
self.assertRegisteredForInauguration(self.hostImplementation.id())
def validateCallCausesColdReclamation(self, call):
self.assertFalse(self.expectedColdReclaim)
self.expectedColdReclaim = True
self.fakeTFTPBoot.expectToBeConfiguredForInaugurator()
self.expectedDnsmasqAddIfNotAlready = True
call()
self.assertFalse(self.expectedColdReclaim)
self.fakeTFTPBoot.validateConfiguredOnceForInaugurator()
self.assertFalse(self.expectedDnsmasqAddIfNotAlready)
self.assertEquals(self.tested.state(), hoststatemachine.STATE_COLD_RECLAMATION)
self.assertRegisteredForInauguration(self.hostImplementation.id())
def validateCallCausesSoftReclamation(self, call):
self.assertFalse(self.expectedColdReclaim)
self.expectedSoftReclaim = True
self.fakeTFTPBoot.expectToBeConfiguredForInaugurator()
self.expectedDnsmasqAddIfNotAlready = True
call()
self.assertFalse(self.expectedSoftReclaim)
self.fakeTFTPBoot.validateConfiguredOnceForInaugurator()
self.assertFalse(self.expectedDnsmasqAddIfNotAlready)
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SOFT_RECLAMATION)
self.assertRegisteredForInauguration(self.hostImplementation.id())
def validateCallCausesReclamationAndStateReport(self, call, state):
self.assertIs(self.expectedReportedState, None)
self.expectedReportedState = state
if state == hoststatemachine.STATE_COLD_RECLAMATION:
self.validateCallCausesColdReclamation(call)
elif state == hoststatemachine.STATE_SOFT_RECLAMATION:
self.validateCallCausesSoftReclamation(call)
else:
self.assertFalse(True, state)
self.assertIs(self.expectedReportedState, None)
def validateCallCausesSoftReclamationAndStateReport(self, call):
self.validateCallCausesReclamationAndStateReport(call, hoststatemachine.STATE_SOFT_RECLAMATION)
def validateCallCausesColdReclamationAndStateReport(self, call):
self.validateCallCausesReclamationAndStateReport(call, hoststatemachine.STATE_COLD_RECLAMATION)
def test_vmLifeCycle_OrderlyRelease(self):
self.assign("fake image label", "fake image hint")
self.validateCheckInCallbackProvidesLabelImmediately("fake image label")
self.inaugurationDone()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_DONE)
self.unassignCausesSoftReclaim()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SOFT_RECLAMATION)
self.assertRegisteredForInauguration(self.hostImplementation.id())
def test_vmLifeCycle_OrderlyRelease_QuickReclamationDidNotWork(self):
self.assign("fake image label", "fake image hint")
self.validateCheckInCallbackProvidesLabelImmediately("fake image label")
self.inaugurationDone()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_DONE)
self.unassignCausesSoftReclaim()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SOFT_RECLAMATION)
self.validateCallCausesColdReclamation(self.softReclaimFailedCallback)
self.assertEquals(self.tested.state(), hoststatemachine.STATE_COLD_RECLAMATION)
self.assertRegisteredForInauguration(self.hostImplementation.id())
def validateAssignCallbackProvidesLabelImmediately(self, label, hint):
self.assertEquals(self.tested.state(), hoststatemachine.STATE_CHECKED_IN)
self.assertIs(self.expectedProvidedLabel, None)
self.expectedProvidedLabel = label
self.assertIs(self.expectedReportedState, None)
self.expectedReportedState = hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED
self.assign(label, hint)
self.validateProvidedLabel(expected=label)
self.assertIs(self.expectedReportedState, None)
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED)
def test_vmLifeCycle_Reuse_ReachedCheckeInBeforeReuse(self):
self.assign("fake image label", "fake image hint")
self.validateCheckInCallbackProvidesLabelImmediately("fake image label")
self.inaugurationDone()
self.unassignCausesSoftReclaim()
self.checkInCallbackLingers()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_CHECKED_IN)
self.validateAssignCallbackProvidesLabelImmediately("fake image label 2", "fake image hint 2")
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED)
self.inaugurationDone()
self.unassignCausesSoftReclaim()
self.assertRegisteredForInauguration(self.hostImplementation.id())
def test_vmLifeCycle_Reuse_ReassignedBeforeReachingCheckeIn(self):
self.assign("fake image label", "fake image hint")
self.validateCheckInCallbackProvidesLabelImmediately("fake image label")
self.inaugurationDone()
self.unassignCausesSoftReclaim()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SOFT_RECLAMATION)
self.assign("fake image label 2", "fake image hint 2")
self.validateCheckInCallbackProvidesLabelImmediately("fake image label 2")
self.inaugurationDone()
self.unassignCausesSoftReclaim()
def checkInCallbackLingers(self):
self.assertIn(self.tested.state(), [
hoststatemachine.STATE_SOFT_RECLAMATION,
hoststatemachine.STATE_COLD_RECLAMATION])
self.checkInCallback()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_CHECKED_IN)
self.assertIs(self.currentTimer, None)
def test_vmLifeCycle_QuickReclamationFailedWhenAssigned_UserDecidesToUnassign(self):
self.assign("fake image label", "fake image hint")
self.validateCheckInCallbackProvidesLabelImmediately("fake image label")
self.inaugurationDone()
self.unassignCausesSoftReclaim()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SOFT_RECLAMATION)
self.assign("fake image label", "fake image hint")
self.assertIsNot(self.softReclaimFailedCallback, None)
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SOFT_RECLAMATION)
self.validateCallCausesColdReclamationAndStateReport(self.softReclaimFailedCallback)
self.tested.unassign()
self.checkInCallbackLingers()
self.assertRegisteredForInauguration(self.hostImplementation.id())
def test_vmLifeCycle_QuickReclamationFailedWithTimeoutWhenAssigned_UserDecidesToUnassign(self):
self.assign("fake image label", "fake image hint")
self.validateCallCausesColdReclamationAndStateReport(self.currentTimer)
self.tested.unassign()
self.checkInCallbackLingers()
self.assertRegisteredForInauguration(self.hostImplementation.id())
def test_coldReclamationSavesTheDay(self):
self.validateCallCausesColdReclamation(self.currentTimer)
self.checkInCallbackLingers()
def validateTimerCausesSelfDestruct(self):
self.assertFalse(self.expectedSelfDestruct)
self.expectedSelfDestruct = True
self.hostImplementation.expectedDestroy = True
self.currentTimer()
self.assertFalse(self.expectedSelfDestruct)
self.assertFalse(self.hostImplementation.expectedDestroy)
self.assertUnegisteredForInauguration(self.hostImplementation.id())
def validateTimerCausesSelfDestructionAndStateReport(self):
self.assertIs(self.expectedReportedState, None)
self.expectedReportedState = hoststatemachine.STATE_DESTROYED
self.validateTimerCausesSelfDestruct()
self.assertIs(self.expectedReportedState, None)
def validateDestructionOfHost(self, isAssigned):
if isAssigned:
validationMethod = self.validateCallCausesColdReclamationAndStateReport
else:
validationMethod = self.validateCallCausesColdReclamation
klass = hoststatemachine.HostStateMachine
for retryNr in range(1, klass.NR_CONSECUTIVE_ERRORS_BEFORE_DESTRUCTION + 1):
if retryNr > klass.NR_CONSECUTIVE_ERRORS_BEFORE_HARD_RESET or retryNr == 1:
self.expectedHardReset = True
else:
self.expectedHardReset = False
if retryNr == klass.NR_CONSECUTIVE_ERRORS_BEFORE_CLEARING_DISK + 1:
self.expectedClearDisk = True
if retryNr == klass.NR_CONSECUTIVE_ERRORS_BEFORE_RECONFIGURING_BIOS + 1:
self.expectReconfigureBIOS = True
validationMethod(self.currentTimer)
def validateTimeoutOnSoftReclamation(self, isLastAttemptBeforeRevertingToColdReclamation=False):
self.expectedDnsmasqAddIfNotAlready = True
self.expectedSoftReclaim = True
if isLastAttemptBeforeRevertingToColdReclamation:
expectedReportedState = hoststatemachine.STATE_COLD_RECLAMATION
else:
expectedReportedState = hoststatemachine.STATE_SOFT_RECLAMATION
self.expectedReportedState = expectedReportedState
self.fakeTFTPBoot.expectToBeConfiguredForInaugurator()
self.currentTimer()
self.assertIs(self.expectedProvidedLabel, None)
self.assertIs(self.expectedReportedState, None)
self.assertEquals(self.tested.state(), expectedReportedState)
self.fakeTFTPBoot.validateConfiguredOnceForInaugurator()
def _reachMaxInaugurationFailureCountByFailureReports(self, imageLabel):
failureCallback = self.inaugurationFailed
self._reachMaxInaugurationFailureCount(imageLabel, failureCallback)
def _reachMaxInaugurationFailureCountBySoftReclamationTimeouts(self, imageLabel):
failureCallback = self.validateTimeoutOnSoftReclamation
self._reachMaxInaugurationFailureCount(imageLabel, failureCallback)
def _reachMaxInaugurationFailureCount(self, imageLabel, failureCallback):
nrRetries = hoststatemachine.HostStateMachine.MAX_NR_CONSECUTIVE_INAUGURATION_FAILURES
for _ in xrange(nrRetries):
self.validateCheckInCallbackProvidesLabelImmediately(imageLabel)
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED)
failureCallback()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SOFT_RECLAMATION)
def validateInaugurationDoneMessageReloadsSoftReclamationRetries(self, failureCallback):
self.assign("fake image label", "fake image hint")
self._reachMaxInaugurationFailureCountByFailureReports("fake image label")
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SOFT_RECLAMATION)
self.validateCheckInCallbackProvidesLabelImmediately("fake image label")
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED)
self.inaugurationDone()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_DONE)
self.unassignCausesSoftReclaim()
self.assign("fake image label", "fake image hint")
self._reachMaxInaugurationFailureCountByFailureReports("fake image label")
def test_vmLifeCycle_AllReclamationRetriesFail_NoUser(self):
self.validateDestructionOfHost(isAssigned=False)
self.validateTimerCausesSelfDestruct()
def test_vmLifeCycle_AllReclamationRetriesFail_WithUser(self):
self.assign("fake image label", "fake image hint")
self.validateDestructionOfHost(isAssigned=True)
self.validateTimerCausesSelfDestructionAndStateReport()
self.assertUnegisteredForInauguration(self.hostImplementation.id())
def test_lateInaugurationDoneMessageDoesNotChangeState(self):
self.assign("fake image label", "fake image hint")
self.validateCheckInCallbackProvidesLabelImmediately("fake image label")
self.inaugurationDone()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_DONE)
self.doneCallback()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_DONE)
def test_checkInWhileNotReclaiming(self):
label = "fake image label"
self.assign(label, "fake image hint")
self.validateCheckInCallbackProvidesLabelImmediately(label)
self.assertIs(self.expectedReportedState, None)
self.expectedReportedState = hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED
self.assertIs(self.expectedProvidedLabel, None)
self.checkInCallback()
self.assertIs(self.expectedProvidedLabel, None)
self.assertIs(self.expectedReportedState, None)
def test_softReclamationFailureWhileDestroyedDoesNotChangeState(self):
self.validateDestructionOfHost(isAssigned=False)
self.validateTimerCausesSelfDestruct()
self.assertUnegisteredForInauguration(self.hostImplementation.id())
self.assertEquals(self.tested.state(), hoststatemachine.STATE_DESTROYED)
self.tested.softReclaimFailed()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_DESTROYED)
def test_vmLifeCycle_notAFreshVM(self):
self.currentTimer = None
self.currentTimerTag = None
self.checkInCallback = None
self.doneCallback = None
self.expectedDnsmasqAddIfNotAlready = True
self.fakeTFTPBoot.expectToBeConfiguredForInaugurator()
self.fakeDnsmasq.addIfNotAlready = mock.Mock()
self.fakeTFTPBoot.configureForInaugurator = mock.Mock()
self.expectedColdReclaim = True
self.tested = hoststatemachine.HostStateMachine(
hostImplementation=self.hostImplementation, inaugurate=self.fakeInaugurate,
tftpboot=self.fakeTFTPBoot, dnsmasq=self.fakeDnsmasq, freshVMJustStarted=False,
reclaimHost=self.fakeReclaimHost)
self.tested.setDestroyCallback(self.destroyHost)
self.assertIs(self.tested.hostImplementation(), self.hostImplementation)
assert self.checkInCallback is not None
assert self.doneCallback is not None
self.assertEquals(self.tested.state(), hoststatemachine.STATE_COLD_RECLAMATION)
def test_vmLifeCycle_inauguratorProgress(self):
self.assign("fake image label", "fake image hint")
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SOFT_RECLAMATION)
self.cancelAllTimersByTag(self.tested)
self.progressCallback(dict(percent=100))
self.assertIs(self.currentTimerTag, None)
self.validateCheckInCallbackProvidesLabelImmediately("fake image label")
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED)
self.cancelAllTimersByTag(self.tested)
self.progressCallback(dict(percent=100))
self.assertIs(self.currentTimerTag, None)
self.progressCallback(dict(state='fetching', percent=100))
self.assertIsNot(self.currentTimerTag, None)
self.progressCallback(dict(state='whatisthisstate', percent=100))
self.assertIsNot(self.currentTimerTag, None)
self.inaugurationDone()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_DONE)
self.cancelAllTimersByTag(self.tested)
self.progressCallback(dict(percent=100))
self.assertIs(self.currentTimerTag, None)
self.unassignCausesSoftReclaim()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SOFT_RECLAMATION)
self.validateCallCausesColdReclamation(self.softReclaimFailedCallback)
self.assertEquals(self.tested.state(), hoststatemachine.STATE_COLD_RECLAMATION)
self.cancelAllTimersByTag(self.tested)
self.progressCallback(dict(percent=100))
self.assertIs(self.currentTimerTag, None)
self.checkInCallback()
self.cancelAllTimersByTag(self.tested)
self.progressCallback(dict(percent=100))
self.assertIs(self.currentTimerTag, None)
def test_clearingOfDiskNotAllowed(self):
hoststatemachine.HostStateMachine.ALLOW_CLEARING_OF_DISK = False
self.expectedClearDisk = False
self.validateCallCausesColdReclamation(self.currentTimer)
self.expectedHardReset = False
self.validateCallCausesColdReclamation(self.currentTimer)
self.validateCallCausesColdReclamation(self.currentTimer)
self.expectedHardReset = True
self.validateCallCausesColdReclamation(self.currentTimer)
def test_timeoutWhenWaitingForInaugurationCausesSoftReclamation(self):
self.assign("fake image label", "fake image hint")
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SOFT_RECLAMATION)
self.validateCheckInCallbackProvidesLabelImmediately("fake image label")
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED)
self.validateCallCausesSoftReclamationAndStateReport(self.currentTimer)
def test_failureDuringInaugurationCausesSoftReclamation(self):
self.checkInCallback()
self.expectedReportedState = hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED
self.assign("fake image label", "fake image hint")
self.inaugurationFailed()
def test_revertToColdReclamationByExhaustingSoftReclamationsDueToInaugurationFailures(self):
self.assign("fake image label", "fake image hint")
self._reachMaxInaugurationFailureCountByFailureReports("fake image label")
self.validateCheckInCallbackProvidesLabelImmediately("fake image label")
self.expectedColdReclaim = True
self.inaugurationFailed(isLastAttemptBeforeRevertingToColdReclamation=True)
self.assertEquals(self.tested.state(), hoststatemachine.STATE_COLD_RECLAMATION)
def test_stayInSoftReclamationOnTimeoutAfterProvidingLabel(self):
self.assign("fake image label", "fake image hint")
self._reachMaxInaugurationFailureCountBySoftReclamationTimeouts("fake image label")
self.validateCheckInCallbackProvidesLabelImmediately("fake image label")
self.expectedColdReclaim = False
self.inaugurationFailed()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SOFT_RECLAMATION)
def test_coldReclamationBehavesNormallyAfterExhaustionOfSoftReclamationsDueToInaugurationFailures(self):
self.assign("fake image label", "fake image hint")
self._reachMaxInaugurationFailureCountByFailureReports("fake image label")
self.validateDestructionOfHost(isAssigned=True)
def test_coldReclamationBehavesNormallyAfterExhaustionOfSoftReclamationsDueTimeouts(self):
self.assign("fake image label", "fake image hint")
self._reachMaxInaugurationFailureCountBySoftReclamationTimeouts("fake image label")
self.validateDestructionOfHost(isAssigned=True)
def test_unassigningDoesNotCauseDestruction(self):
"""This was created specifically to validate that allocations that are being stopped before the
inauguration is either done, failed or timed out (could be a user manually stopping the
allocation, or a timeout value smaller in the test client than in Rackattack), do not cause the
state machine to be destroyed. To be destroyed, it has to either notify failure or timeout."""
nrRetries = hoststatemachine.HostStateMachine.MAX_NR_CONSECUTIVE_INAUGURATION_FAILURES + \
hoststatemachine.HostStateMachine.NR_CONSECUTIVE_ERRORS_BEFORE_DESTRUCTION + 20
for _ in xrange(nrRetries):
self.assign("fake image label", "fake image hint")
self.validateCheckInCallbackProvidesLabelImmediately("fake image label")
self.unassignCausesSoftReclaim()
def test_InaugurationDoneMessageReloadsSoftReclamationRetriesAfterInaugurationFailureReports(self):
def failureCallback():
self._reachMaxInaugurationFailureCountByFailureReports("fake image label")
self.validateInaugurationDoneMessageReloadsSoftReclamationRetries(failureCallback)
def test_InaugurationDoneMessageReloadsSoftReclamationRetriesAfterSoftReclamationTimeouts(self):
def failureCallback():
self._reachMaxInaugurationFailureCountBySoftReclamationTimeouts("fake image label")
self.validateInaugurationDoneMessageReloadsSoftReclamationRetries(failureCallback)
class Test(unittest.TestCase):
def setUp(self):
globallock._lock.acquire()
self.checkInCallback = None
self.doneCallback = None
self.expectedProvidedLabel = None
self.provideLabelRaises = False
self.expectedReportedState = None
timer.scheduleIn = self.scheduleTimerIn
timer.cancelAllByTag = self.cancelAllTimersByTag
self.currentTimer = None
self.currentTimerTag = None
self.expectedTFTPBootToBeConfiguredForInaugurator = False
self.expectedTFTPBootToBeConfiguredForLocalHost = False
self.expectedColdReclaim = False
self.expectReconfigureBIOS = False
self.expectedSoftReclaim = False
self.expectedSelfDestruct = False
self.softReclaimFailedCallback = None
self.construct()
def tearDown(self):
globallock._lock.release()
def construct(self):
self.hostImplementation = FakeHost()
self.fakeInaugurate = Empty()
self.fakeInaugurate.provideLabel = self.provideLabelForInauguration
self.fakeInaugurate.register = self.registerForInauguration
self.fakeInaugurate.unregister = self.unregisterForInauguration
self.fakeTFTPBoot = Empty()
self.fakeTFTPBoot.inauguratorCommandLine = self.inauguratorCommandLine
self.fakeTFTPBoot.configureForInaugurator = self.tftpbootConfigureForInaugurator
self.fakeTFTPBoot.configureForLocalBoot = self.tftpbootConfigureForLocalBoot
self.fakeDnsmasq = Empty()
self.fakeDnsmasq.addIfNotAlready = self.dnsmasqAddIfNotAlready
self.fakeReclaimHost = Empty()
self.fakeReclaimHost.cold = self.reclaimHostCold
self.fakeReclaimHost.soft = self.reclaimHostSoft
self.expectedTFTPBootToBeConfiguredForInaugurator = True
self.expectedDnsmasqAddIfNotAlready = True
self.expectedClearDisk = False
hoststatemachine.HostStateMachine.ALLOW_CLEARING_OF_DISK = True
self.tested = hoststatemachine.HostStateMachine(
hostImplementation=self.hostImplementation,
inaugurate=self.fakeInaugurate, tftpboot=self.fakeTFTPBoot, dnsmasq=self.fakeDnsmasq,
reclaimHost=self.fakeReclaimHost)
self.tested.setDestroyCallback(self.destroyHost)
self.assertIs(self.tested.hostImplementation(), self.hostImplementation)
self.assertFalse(self.expectedTFTPBootToBeConfiguredForInaugurator)
assert self.checkInCallback is not None
assert self.doneCallback is not None
def destroyHost(self, stateMachine):
self.assertIs(stateMachine, self.tested)
self.assertTrue(self.expectedSelfDestruct)
self.expectedSelfDestruct = False
def scheduleTimerIn(self, timeout, callback, tag):
self.assertIs(self.currentTimer, None)
self.assertIs(self.currentTimerTag, None)
self.currentTimer = callback
self.currentTimerTag = tag
def cancelAllTimersByTag(self, tag):
if self.currentTimerTag is not None:
self.assertIsNot(self.currentTimer, None)
self.assertIs(self.currentTimerTag, tag)
self.currentTimer = None
self.currentTimerTag = None
def triggerTimeout(self):
self.assertIsNot(self.currentTimer, None)
self.assertIsNot(self.currentTimerTag, None)
self.currentTimer()
self.currentTimer = None
self.currentTimerTag = None
def inauguratorCommandLine(self, id, mac, ip):
self.assertEquals(id, self.hostImplementation.id())
self.assertEquals(mac, self.hostImplementation.primaryMACAddress())
self.assertEquals(ip, self.hostImplementation.ipAddress())
return "fake inaugurator command line"
def registerForInauguration(self, id, checkInCallback, doneCallback, progressCallback):
self.assertEquals(id, self.hostImplementation.id())
self.assertIs(self.checkInCallback, None)
self.assertIs(self.doneCallback, None)
self.checkInCallback = checkInCallback
self.doneCallback = doneCallback
self.progressCallback = progressCallback
def unregisterForInauguration(self, id):
self.assertIsNot(self.checkInCallback, None)
self.assertIsNot(self.doneCallback, None)
self.assertIsNot(self.progressCallback, None)
self.checkInCallback = None
self.doneCallback = None
self.progressCallback = None
def assertRegisteredForInauguration(self, id):
self.assertEquals(id, self.hostImplementation.id())
self.assertIsNot(self.checkInCallback, None)
self.assertIsNot(self.doneCallback, None)
self.assertIsNot(self.progressCallback, None)
def assertUnegisteredForInauguration(self, id):
self.assertIs(self.checkInCallback, None)
self.assertIs(self.doneCallback, None)
self.assertIs(self.progressCallback, None)
def provideLabelForInauguration(self, id, label):
self.assertEquals(id, self.hostImplementation.id())
if self.provideLabelRaises:
raise Exception("Provide label raises on purpose, as part of test")
self.assertEquals(label, self.expectedProvidedLabel)
self.expectedProvidedLabel = None
def isObjectInitialized(self):
return hasattr(self, 'tested')
def tftpbootConfigureForInaugurator(self, id, mac, ip, clearDisk=False):
self.assertEquals(id, self.hostImplementation.id())
self.assertEquals(mac, self.hostImplementation.primaryMACAddress())
self.assertEquals(ip, self.hostImplementation.ipAddress())
self.assertTrue(self.expectedTFTPBootToBeConfiguredForInaugurator)
self.assertEquals(clearDisk, self.expectedClearDisk)
self.expectedTFTPBootToBeConfiguredForInaugurator = False
def dnsmasqAddIfNotAlready(self, mac, ip):
self.assertEquals(mac, self.hostImplementation.primaryMACAddress())
self.assertEquals(ip, self.hostImplementation.ipAddress())
self.assertTrue(self.expectedDnsmasqAddIfNotAlready)
self.expectedDnsmasqAddIfNotAlready = False
def tftpbootConfigureForLocalBoot(self, mac):
self.assertEquals(mac, self.hostImplementation.primaryMACAddress())
self.assertTrue(self.expectedTFTPBootToBeConfiguredForLocalHost)
self.expectedTFTPBootToBeConfiguredForLocalHost = False
def reclaimHostCold(self, hostImplementation, reconfigureBIOS=False):
self.assertIs(hostImplementation, self.hostImplementation)
self.assertTrue(self.expectedColdReclaim)
self.expectedColdReclaim = False
self.assertEqual(self.expectReconfigureBIOS, reconfigureBIOS)
def reclaimHostSoft(self, hostImplementation):
self.assertIs(hostImplementation, self.hostImplementation)
self.assertTrue(self.expectedSoftReclaim)
self.expectedSoftReclaim = False
self.softReclaimFailedCallback = self.tested.softReclaimFailed
def checkInCallbackProvidedLabelImmidiately(self, label):
self.assertIn(self.tested.state(), [
hoststatemachine.STATE_QUICK_RECLAIMATION_IN_PROGRESS,
hoststatemachine.STATE_SLOW_RECLAIMATION_IN_PROGRESS])
self.assertIs(self.expectedProvidedLabel, None)
self.expectedProvidedLabel = label
self.assertIs(self.expectedReportedState, None)
self.expectedReportedState = hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED
self.checkInCallback()
self.assertIs(self.expectedProvidedLabel, None)
self.assertIs(self.expectedReportedState, None)
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED)
def stateChangedCallback(self, tested):
self.assertIs(tested, self.tested)
self.assertIsNot(self.expectedReportedState, None)
self.assertEquals(tested.state(), self.expectedReportedState)
self.expectedReportedState = None
def inaugurationDone(self):
self.assertIn(self.tested.state(), [hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED])
self.assertIs(self.expectedProvidedLabel, None)
self.assertIs(self.expectedReportedState, None)
self.expectedReportedState = hoststatemachine.STATE_INAUGURATION_DONE
self.assertFalse(self.expectedTFTPBootToBeConfiguredForLocalHost)
self.expectedTFTPBootToBeConfiguredForLocalHost = True
self.doneCallback()
self.assertIs(self.expectedProvidedLabel, None)
self.assertIs(self.expectedReportedState, None)
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_DONE)
self.assertFalse(self.expectedTFTPBootToBeConfiguredForLocalHost)
self.assertIs(self.currentTimer, None)
def assign(self, label, hint):
self.tested.assign(self.stateChangedCallback, label, hint)
self.assertEquals(self.tested.imageLabel(), label)
self.assertEquals(self.tested.imageHint(), hint)
def test_vmLifeCycle_Normal(self):
self.assertRegisteredForInauguration(self.hostImplementation.id())
self.assertEquals(self.tested.state(), hoststatemachine.STATE_QUICK_RECLAIMATION_IN_PROGRESS)
self.assign("fake image label", "fake image hint")
self.assertEquals(self.tested.state(), hoststatemachine.STATE_QUICK_RECLAIMATION_IN_PROGRESS)
self.checkInCallbackProvidedLabelImmidiately("fake image label")
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED)
self.inaugurationDone()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_DONE)
self.assertRegisteredForInauguration(self.hostImplementation.id())
def unassignCausesSoftReclaim(self):
self.assertFalse(self.expectedSoftReclaim)
self.expectedSoftReclaim = True
self.assertFalse(self.expectedTFTPBootToBeConfiguredForInaugurator)
self.expectedTFTPBootToBeConfiguredForInaugurator = True
self.expectedDnsmasqAddIfNotAlready = True
self.tested.unassign()
self.assertFalse(self.expectedSoftReclaim)
self.assertFalse(self.expectedTFTPBootToBeConfiguredForInaugurator)
self.assertEquals(self.tested.state(), hoststatemachine.STATE_QUICK_RECLAIMATION_IN_PROGRESS)
self.assertRegisteredForInauguration(self.hostImplementation.id())
def callCausesColdReclaim(self, call):
self.assertFalse(self.expectedColdReclaim)
self.expectedColdReclaim = True
self.expectedTFTPBootToBeConfiguredForInaugurator = True
self.expectedDnsmasqAddIfNotAlready = True
call()
self.assertFalse(self.expectedColdReclaim)
self.assertFalse(self.expectedTFTPBootToBeConfiguredForInaugurator)
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SLOW_RECLAIMATION_IN_PROGRESS)
self.assertRegisteredForInauguration(self.hostImplementation.id())
def callCausesColdReclaimAndStateChange(self, call, state):
self.assertIs(self.expectedReportedState, None)
self.expectedReportedState = state
self.callCausesColdReclaim(call)
self.assertIs(self.expectedReportedState, None)
def test_vmLifeCycle_OrderlyRelease(self):
self.assign("fake image label", "fake image hint")
self.checkInCallbackProvidedLabelImmidiately("fake image label")
self.inaugurationDone()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_DONE)
self.unassignCausesSoftReclaim()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_QUICK_RECLAIMATION_IN_PROGRESS)
self.assertRegisteredForInauguration(self.hostImplementation.id())
def test_vmLifeCycle_OrderlyRelease_QuickReclaimationDidNotWork(self):
self.assign("fake image label", "fake image hint")
self.checkInCallbackProvidedLabelImmidiately("fake image label")
self.inaugurationDone()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_DONE)
self.unassignCausesSoftReclaim()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_QUICK_RECLAIMATION_IN_PROGRESS)
self.callCausesColdReclaim(self.softReclaimFailedCallback)
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SLOW_RECLAIMATION_IN_PROGRESS)
self.assertRegisteredForInauguration(self.hostImplementation.id())
def assignCallbackProvidedLabelImmidiately(self, label, hint):
self.assertEquals(self.tested.state(), hoststatemachine.STATE_CHECKED_IN)
self.assertIs(self.expectedProvidedLabel, None)
self.expectedProvidedLabel = label
self.assertIs(self.expectedReportedState, None)
self.expectedReportedState = hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED
self.assign(label, hint)
self.assertIs(self.expectedProvidedLabel, None)
self.assertIs(self.expectedReportedState, None)
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED)
def test_vmLifeCycle_Reuse_ReachedCheckeInBeforeReuse(self):
self.assign("fake image label", "fake image hint")
self.checkInCallbackProvidedLabelImmidiately("fake image label")
self.inaugurationDone()
self.unassignCausesSoftReclaim()
self.checkInCallbackLingers()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_CHECKED_IN)
self.assignCallbackProvidedLabelImmidiately("fake image label 2", "fake image hint 2")
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED)
self.inaugurationDone()
self.unassignCausesSoftReclaim()
self.assertRegisteredForInauguration(self.hostImplementation.id())
def test_vmLifeCycle_Reuse_ReassignedBeforeReachingCheckeIn(self):
self.assign("fake image label", "fake image hint")
self.checkInCallbackProvidedLabelImmidiately("fake image label")
self.inaugurationDone()
self.unassignCausesSoftReclaim()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_QUICK_RECLAIMATION_IN_PROGRESS)
self.assign("fake image label 2", "fake image hint 2")
self.checkInCallbackProvidedLabelImmidiately("fake image label 2")
self.inaugurationDone()
self.unassignCausesSoftReclaim()
def checkInCallbackLingers(self):
self.assertIn(self.tested.state(), [
hoststatemachine.STATE_QUICK_RECLAIMATION_IN_PROGRESS,
hoststatemachine.STATE_SLOW_RECLAIMATION_IN_PROGRESS])
self.checkInCallback()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_CHECKED_IN)
self.assertIs(self.currentTimer, None)
def test_vmLifeCycle_QuickReclaimationFailedWhenAssigned_UserDecidesToUnassign(self):
self.assign("fake image label", "fake image hint")
self.checkInCallbackProvidedLabelImmidiately("fake image label")
self.inaugurationDone()
self.unassignCausesSoftReclaim()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_QUICK_RECLAIMATION_IN_PROGRESS)
self.assign("fake image label", "fake image hint")
self.assertIsNot(self.softReclaimFailedCallback, None)
self.assertEquals(self.tested.state(), hoststatemachine.STATE_QUICK_RECLAIMATION_IN_PROGRESS)
self.callCausesColdReclaimAndStateChange(
self.softReclaimFailedCallback, hoststatemachine.STATE_SLOW_RECLAIMATION_IN_PROGRESS)
self.tested.unassign()
self.checkInCallbackLingers()
self.assertRegisteredForInauguration(self.hostImplementation.id())
def test_vmLifeCycle_QuickReclaimationFailedWithTimeoutWhenAssigned_UserDecidesToUnassign(self):
self.assign("fake image label", "fake image hint")
self.callCausesColdReclaimAndStateChange(
self.currentTimer, hoststatemachine.STATE_SLOW_RECLAIMATION_IN_PROGRESS)
self.tested.unassign()
self.checkInCallbackLingers()
self.assertRegisteredForInauguration(self.hostImplementation.id())
def test_coldReclaimationSavesTheDay(self):
self.callCausesColdReclaim(self.currentTimer)
self.checkInCallbackLingers()
def timerCausesSelfDestruct(self):
self.assertFalse(self.expectedSelfDestruct)
self.expectedSelfDestruct = True
self.hostImplementation.expectedDestroy = True
self.currentTimer()
self.assertFalse(self.expectedSelfDestruct)
self.assertFalse(self.hostImplementation.expectedDestroy)
self.assertUnegisteredForInauguration(self.hostImplementation.id())
def timerCausesSelfDestructAndStateChange(self):
self.assertIs(self.expectedReportedState, None)
self.expectedReportedState = hoststatemachine.STATE_DESTROYED
self.timerCausesSelfDestruct()
self.assertIs(self.expectedReportedState, None)
def test_vmLifeCycle_AllReclaimationRetriesFail_NoUser(self):
self.callCausesColdReclaim(self.currentTimer)
self.callCausesColdReclaim(self.currentTimer)
self.expectedClearDisk = True
self.callCausesColdReclaim(self.currentTimer)
self.callCausesColdReclaim(self.currentTimer)
self.expectReconfigureBIOS = True
self.callCausesColdReclaim(self.currentTimer)
self.timerCausesSelfDestruct()
self.assertUnegisteredForInauguration(self.hostImplementation.id())
def test_vmLifeCycle_AllReclaimationRetriesFail_WithUser(self):
self.assign("fake image label", "fake image hint")
self.callCausesColdReclaimAndStateChange(
self.currentTimer, hoststatemachine.STATE_SLOW_RECLAIMATION_IN_PROGRESS)
self.callCausesColdReclaimAndStateChange(
self.currentTimer, hoststatemachine.STATE_SLOW_RECLAIMATION_IN_PROGRESS)
self.expectedClearDisk = True
self.callCausesColdReclaimAndStateChange(
self.currentTimer, hoststatemachine.STATE_SLOW_RECLAIMATION_IN_PROGRESS)
self.callCausesColdReclaimAndStateChange(
self.currentTimer, hoststatemachine.STATE_SLOW_RECLAIMATION_IN_PROGRESS)
self.expectReconfigureBIOS = True
self.callCausesColdReclaimAndStateChange(
self.currentTimer, hoststatemachine.STATE_SLOW_RECLAIMATION_IN_PROGRESS)
self.timerCausesSelfDestructAndStateChange()
self.assertUnegisteredForInauguration(self.hostImplementation.id())
def test_vmLifeCycle_UnableToProvideLabel_ColdReclaim(self):
self.assign("fake image label", "fake image hint")
self.checkInCallbackProvidedLabelImmidiately("fake image label")
self.inaugurationDone()
self.unassignCausesSoftReclaim()
self.checkInCallbackLingers()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_CHECKED_IN)
self.provideLabelRaises = True
self.expectedReportedState = hoststatemachine.STATE_SLOW_RECLAIMATION_IN_PROGRESS
self.expectedTFTPBootToBeConfiguredForInaugurator = True
self.expectedDnsmasqAddIfNotAlready = True
self.expectedColdReclaim = True
self.assign("fake image label", "fake image hint")
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SLOW_RECLAIMATION_IN_PROGRESS)
self.assertRegisteredForInauguration(self.hostImplementation.id())
def test_lateInaugurationDoneMessageDoesNotChangeState(self):
self.assign("fake image label", "fake image hint")
self.checkInCallbackProvidedLabelImmidiately("fake image label")
self.inaugurationDone()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_DONE)
self.doneCallback()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_DONE)
def test_checkInWhileNotReclaiming(self):
label = "fake image label"
self.assign(label, "fake image hint")
self.checkInCallbackProvidedLabelImmidiately(label)
self.assertIs(self.expectedReportedState, None)
self.expectedReportedState = hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED
self.expectedProvidedLabel = label
self.checkInCallback()
self.assertIs(self.expectedProvidedLabel, None)
self.assertIs(self.expectedReportedState, None)
def test_softReclaimFailedWhileDestroyed(self):
self.callCausesColdReclaim(self.currentTimer)
self.callCausesColdReclaim(self.currentTimer)
self.expectedClearDisk = True
self.callCausesColdReclaim(self.currentTimer)
self.callCausesColdReclaim(self.currentTimer)
self.expectReconfigureBIOS = True
self.callCausesColdReclaim(self.currentTimer)
self.timerCausesSelfDestruct()
self.assertUnegisteredForInauguration(self.hostImplementation.id())
self.assertEquals(self.tested.state(), hoststatemachine.STATE_DESTROYED)
self.tested.softReclaimFailed()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_DESTROYED)
def test_vmLifeCycle_notAFreshVM(self):
self.currentTimer = None
self.currentTimerTag = None
self.checkInCallback = None
self.doneCallback = None
self.expectedDnsmasqAddIfNotAlready = True
self.expectedTFTPBootToBeConfiguredForInaugurator = True
self.fakeDnsmasq.addIfNotAlready = mock.Mock()
self.fakeTFTPBoot.configureForInaugurator = mock.Mock()
self.expectedColdReclaim = True
self.tested = hoststatemachine.HostStateMachine(
hostImplementation=self.hostImplementation, inaugurate=self.fakeInaugurate,
tftpboot=self.fakeTFTPBoot, dnsmasq=self.fakeDnsmasq, freshVMJustStarted=False,
reclaimHost=self.fakeReclaimHost)
self.tested.setDestroyCallback(self.destroyHost)
self.assertIs(self.tested.hostImplementation(), self.hostImplementation)
assert self.checkInCallback is not None
assert self.doneCallback is not None
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SLOW_RECLAIMATION_IN_PROGRESS)
def test_vmLifeCycle_inauguratorProgress(self):
self.assign("fake image label", "fake image hint")
self.assertEquals(self.tested.state(), hoststatemachine.STATE_QUICK_RECLAIMATION_IN_PROGRESS)
self.cancelAllTimersByTag(self.tested)
self.progressCallback(dict(percent=100))
self.assertIs(self.currentTimerTag, None)
self.checkInCallbackProvidedLabelImmidiately("fake image label")
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_LABEL_PROVIDED)
self.cancelAllTimersByTag(self.tested)
self.progressCallback(dict(percent=100))
self.assertIs(self.currentTimerTag, None)
self.progressCallback(dict(state='fetching', percent=100))
self.assertIsNot(self.currentTimerTag, None)
self.progressCallback(dict(state='whatisthisstate', percent=100))
self.assertIsNot(self.currentTimerTag, None)
self.inaugurationDone()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_INAUGURATION_DONE)
self.cancelAllTimersByTag(self.tested)
self.progressCallback(dict(percent=100))
self.assertIs(self.currentTimerTag, None)
self.unassignCausesSoftReclaim()
self.assertEquals(self.tested.state(), hoststatemachine.STATE_QUICK_RECLAIMATION_IN_PROGRESS)
self.callCausesColdReclaim(self.softReclaimFailedCallback)
self.assertEquals(self.tested.state(), hoststatemachine.STATE_SLOW_RECLAIMATION_IN_PROGRESS)
self.cancelAllTimersByTag(self.tested)
self.progressCallback(dict(percent=100))
self.assertIs(self.currentTimerTag, None)
self.checkInCallback()
self.cancelAllTimersByTag(self.tested)
self.progressCallback(dict(percent=100))
self.assertIs(self.currentTimerTag, None)
def test_ClearingOfDiskNotAllowed(self):
hoststatemachine.HostStateMachine.ALLOW_CLEARING_OF_DISK = False
self.expectedClearDisk = False
self.callCausesColdReclaim(self.currentTimer)
self.callCausesColdReclaim(self.currentTimer)
self.callCausesColdReclaim(self.currentTimer)
self.callCausesColdReclaim(self.currentTimer)
class Test(EpollEventLoopTestCase):
def setUp(self):
super(Test, self).setUp(moduleInWhichToSetupMocks=reclaimhostspooler)
self._host = FakeHost()
self._hostStateMachine = FakeHostStateMachine(self._host)
self._hosts = Hosts()
self._hosts.add(self._hostStateMachine)
self._tftpboot = FakeTFTPBoot()
self._wasThereAnAttemptToCreateFifo = False
self._fakeSoftReclaimRequestFifoPath = "/fakeNotifyFifoPath"
self._fakeSoftReclaimFailedFifoPath = "/fakeSoftReclaimFailedFifoPath"
self._origKillSelf = reclaimhostspooler.suicide.killSelf
reclaimhostspooler.suicide.killSelf = mock.Mock()
self._actualColdReclamationRequests = []
self._tested = self._generateTestedInstanceWithMockedThreading()
self._expectedRequests = []
self._coldReclaimCallbackRaisesException = False
def tearDown(self):
reclaimhostspooler.suicide.killSelf = self._origKillSelf
super(Test, self).tearDown()
def test_FifosCreatedIfDoNotExist(self):
self.assertFalse(self._fakeFilesystem.Exists(self._fakeSoftReclaimRequestFifoPath))
self.assertFalse(self._fakeFilesystem.Exists(self._fakeSoftReclaimFailedFifoPath))
self._continueWithEventLoop()
self.assertTrue(self._fakeFilesystem.Exists(self._fakeSoftReclaimRequestFifoPath))
self.assertTrue(self._fakeFilesystem.Exists(self._fakeSoftReclaimFailedFifoPath))
def test_FifosStayIfAlreadyExist(self):
self._pipeMethodsMock.osMkfifo(self._fakeSoftReclaimRequestFifoPath)
self._pipeMethodsMock.osMkfifo(self._fakeSoftReclaimFailedFifoPath)
self.assertTrue(self._fakeFilesystem.Exists(self._fakeSoftReclaimRequestFifoPath))
self.assertTrue(self._fakeFilesystem.Exists(self._fakeSoftReclaimFailedFifoPath))
self._continueWithEventLoop()
self.assertTrue(self._fakeFilesystem.Exists(self._fakeSoftReclaimRequestFifoPath))
self.assertTrue(self._fakeFilesystem.Exists(self._fakeSoftReclaimFailedFifoPath))
def test_SoftReclaimRequest(self):
self._validateSoftReclaimFlow()
def test_SoftReclaimFailed(self):
self._continueWithEventLoop()
softReclaimFailedFifoWriteFd = self._pipeMethodsMock.osOpen(self._fakeSoftReclaimFailedFifoPath, os.O_WRONLY)
self._pipeMethodsMock.osWrite(softReclaimFailedFifoWriteFd, self._host.id())
self._continueWithEventLoop()
self._hostStateMachine.softReclaimFailed.assert_called_once_with()
def test_NoCrashOnSoftReclaimFailedCallbackFaliure(self):
self._hostStateMachine.softReclaimFailed.side_effect = Exception("don't crash because of me")
self._continueWithEventLoop()
softReclaimFailedFifoWriteFd = self._pipeMethodsMock.osOpen(self._fakeSoftReclaimFailedFifoPath, os.O_WRONLY)
self._pipeMethodsMock.osWrite(softReclaimFailedFifoWriteFd, self._host.id())
self._continueWithEventLoop()
self._hostStateMachine.softReclaimFailed.assert_called_once_with()
self._hostStateMachine.softReclaimFailed.side_effect = None
self._validateSoftReclaimFlow()
def test_NoCrashOnSoftReclaimFailedMsgForNonExistentHost(self):
self._continueWithEventLoop()
softReclaimFailedFifoWriteFd = self._pipeMethodsMock.osOpen(self._fakeSoftReclaimFailedFifoPath, os.O_WRONLY)
self._pipeMethodsMock.osWrite(softReclaimFailedFifoWriteFd, "non-existent")
self._continueWithEventLoop()
self._validateSoftReclaimFlow()
def test_NoCrashOnSoftReclaimFailedForEmptyHost(self):
self._continueWithEventLoop()
softReclaimFailedFifoWriteFd = self._pipeMethodsMock.osOpen(self._fakeSoftReclaimFailedFifoPath, os.O_WRONLY)
self._pipeMethodsMock.osWrite(softReclaimFailedFifoWriteFd, "fake id,,")
self._continueWithEventLoop()
self._validateSoftReclaimFlow()
def test_SuicideOnFailure(self):
self._continueWithEventLoop()
self._addColdReclamationRequest(self._host)
self._coldReclaimCallbackRaisesException = True
self.assertRaises(ValueError, self._continueWithEventLoop)
reclaimhostspooler.suicide.killSelf.assert_called_once_with()
def test_NoCrashOnSoftReclaimRequestForHostWithEmptyName(self):
self._host._id = ""
self._validateSoftReclaimFlow()
def test_ColdReclaimRequest(self):
self._continueWithEventLoop()
self._addColdReclamationRequest(self._host)
self._continueWithEventLoop()
self._validateExpectedRequests()
def test_ColdReclaimRequestWithHardReset(self):
self._continueWithEventLoop()
self._addColdReclamationRequest(self._host, hardReset=True)
self._continueWithEventLoop()
self._validateExpectedRequests()
def _addSoftReclamationRequest(self, host):
self._expectedRequests.append(["soft", self._host])
request = greenlet.greenlet(lambda: self._tested.soft(self._host))
request.switch()
def _addColdReclamationRequest(self, host, hardReset=False):
self._expectedRequests.append(["cold", self._host, hardReset])
request = greenlet.greenlet(lambda: self._tested.cold(self._host, hardReset=hardReset))
request.switch()
def _validateExpectedRequests(self):
self._continueWithEventLoop()
for request in self._expectedRequests:
requestType = request[0]
if requestType == "cold":
actual = self._actualColdReclamationRequests.pop(0)
expected = request[1:]
elif requestType == "soft":
host = request[1]
expected = self._getEncodedSoftRequest(host)
actual = self._pipeMethodsMock.getFifoContent(self._fakeSoftReclaimRequestFifoPath)
actual = base64.decodestring(actual)
else:
self.assertFalse(True)
self.assertEquals(actual, expected)
def _validateSoftReclaimFlow(self):
self._continueWithEventLoop()
self._addSoftReclamationRequest(self._host)
self._validateExpectedRequests()
def _handleColdReclamationRequest(self, host, hardReset):
if self._coldReclaimCallbackRaisesException:
raise ValueError("Ignore me")
self._actualColdReclamationRequests.append([host, hardReset])
def _getEncodedColdRequest(self, host):
requestArgs = [host.id(), host.primaryMACAddress()]
decodedRequest = ",".join(requestArgs)
encodedRequest = base64.encodestring(decodedRequest)
return encodedRequest
def _getEncodedSoftRequest(self, host):
credentials = host.rootSSHCredentials()
requestArgs = [
"soft",
host.id(),
credentials["hostname"],
credentials["username"],
credentials["password"],
host.primaryMACAddress(),
host.targetDevice(),
"False",
]
return ",".join(requestArgs)
def _generateTestedInstance(self):
ReclaimHostSpoolerWithColdReclamation._handleColdReclamationRequest = self._handleColdReclamationRequest
instance = ReclaimHostSpoolerWithColdReclamation(
self._hosts, self._fakeSoftReclaimRequestFifoPath, self._fakeSoftReclaimFailedFifoPath
)
return instance
