def testCreateSwapControllerAndDeleteIt(self, **_kwargs):
# Instantiates, then deletes SwapController
sc = SwapController(concurrency=3)
self.assertEqual(len(sc._slotAgents), 3)
del sc
def testProgramAbortOnInputReaderThreadCrash(
self, osExitMock, _slotAgentClassMock, modelSwapperInterfaceClassMock):
# Verify that a crash in notification reader thread results in a call
# to os._exit()
# Configure ModelSwapperInterface mock
swapperMock = modelSwapperInterfaceClassMock.return_value
swapperMock.initSchedulerNotification.side_effect = Exception(
"testProgramAbortOnInputReaderThreadCrash")
osExitArgQ = Queue.Queue()
osExitMock.side_effect = osExitArgQ.put
sc = SwapController(concurrency=1)
self.assertEqual(len(sc._slotAgents), 1)
# Request stop, so that the main loop will exit ASAP
sc.requestStopTS()
# Run SwapController in a thread
def runSwapControllerThread(sc, runResultQ):
try:
g_logger.info("Swap Controller run-thread is running")
try:
r = sc.run()
except Exception as e:
runResultQ.put(e)
else:
runResultQ.put(r)
finally:
g_logger.info("Swap Controller run-thread is exiting")
runResultQ = Queue.Queue()
scThread = threading.Thread(
target=runSwapControllerThread,
name="runSwapControllerThread",
args=(sc, runResultQ))
scThread.setDaemon(True)
scThread.start()
# Wait for os._exit to be called
self.assertEqual(
osExitArgQ.get(timeout=5),
SwapController._EXIT_CODE_ON_FAILURE_OF_NOTIFICATION_READER_THREAD)
# Wait for the run-thread to stop
g_logger.info("Waiting for SwapController run-thread to stop")
scThread.join(timeout=5)
self.assertFalse(scThread.isAlive())
g_logger.info("SwapController run-thread stopped")
runResult = runResultQ.get_nowait()
self.assertIsNone(runResult)
def testDetectNotificationReaderThreadTargetCallFailed(
self, _slotAgentClassMock, _modelSwapperInterfaceClassMock):
# Reproduce failure to invoke the SwapController's input-reader thread
# target and verify that SwapController's event loop raises the expected
# exception
sc = SwapController(concurrency=1)
self.assertEqual(len(sc._slotAgents), 1)
# Patch SwapController's input-thread object with one that will exhibit a
# failure while trying to call the thread target
def expectTwoArgs(_a, _b):
pass
t = threading.Thread(target=expectTwoArgs)
t.setDaemon(True)
patch.multiple(sc, _notificationReaderThread=t).start()
# Attempt to run it in a thread
def runSwapControllerThread(sc, runResultQ):
try:
g_logger.info("Swap Controller run-thread is running")
try:
r = sc.run()
except Exception as e:
runResultQ.put(e)
else:
runResultQ.put(r)
finally:
g_logger.info("Swap Controller run-thread is exiting")
runResultQ = Queue.Queue()
scThread = threading.Thread(
target=runSwapControllerThread,
name="runSwapControllerThread",
args=(sc, runResultQ))
scThread.setDaemon(True)
scThread.start()
# Wait for the run-thread to stop
g_logger.info("Waiting for SwapController run-thread to stop")
scThread.join(timeout=5)
self.assertFalse(scThread.isAlive())
g_logger.info("SwapController run-thread stopped")
# Confirm the expected exception
runResult = runResultQ.get_nowait()
self.assertIsInstance(runResult, AssertionError)
self.assertIn("Notification-reader thread failed to start in time",
runResult.args[0])
def __init__(self, concurrency):
self._logger = _getLogger()
self._concurrency = concurrency
self._signalPipeReadFD, self._signalPipeWriteFD = os.pipe()
# Make the write end non-blocking to prevent accidental deadlocking of the
# signal dispatcher
fcntl.fcntl(
self._signalPipeWriteFD, fcntl.F_SETFL,
fcntl.fcntl(self._signalPipeWriteFD, fcntl.F_GETFL)
| os.O_NONBLOCK)
# Register for signals of interest
self._signalsOfInterest = [
signal.SIGHUP, signal.SIGTERM, signal.SIGINT
]
for sig in self._signalsOfInterest:
signal.signal(sig, self._handleSignal)
# Create the slot agents and swap controller.
self._swapController = SwapController(concurrency=concurrency)
def testRunSwapControllerAndStopIt(
self, _slotAgentClassMock, modelSwapperInterfaceClassMock):
# Instantiate SwapController instance, run it in a separate thread,
# then stop SwapController.
# Configure ModelSwapperInterface instance mock
swapperMock = modelSwapperInterfaceClassMock.return_value
notificationConsumer = DummyConsumer()
swapperMock.consumeModelSchedulerNotifications.return_value = (
notificationConsumer)
sc = SwapController(concurrency=3)
self.assertEqual(len(sc._slotAgents), 3)
# Run it in a thread
def runSwapControllerThread(sc, runResultQ):
r = sc.run()
runResultQ.put(r)
runResultQ = Queue.Queue()
scThread = threading.Thread(
target=runSwapControllerThread,
name="runSwapControllerThread",
args=(sc, runResultQ))
scThread.setDaemon(True)
scThread.start()
# Now stop it
sc.requestStopTS()
# Prod notification reader's consumer loop to detect the stop request and
# exit gracefully by adding a dummy message
notificationConsumer.q.put(None)
# There isn't much going on, it should stop immediately
scThread.join(timeout=5)
self.assertFalse(scThread.isAlive())
runResult = runResultQ.get_nowait()
self.assertIsNone(runResult)
def testModelPreemptionAndStop(self, slotAgentClassMock,
modelSwapperInterfaceClassMock):
# Test preemption of slots in SwapController
# Configure ModelSwapperInterface instance mock
swapperMock = modelSwapperInterfaceClassMock.return_value
notificationConsumer = DummyConsumer()
swapperMock.consumeModelSchedulerNotifications.return_value = (
notificationConsumer)
swapperMock.modelInputPending.side_effect = (
lambda modelID: not modelInputDescriptors[
modelID].requestBatchesQ.empty())
# Configure SlotAgent class mock to create dummy slot agent instances and
# add them to our list so that we can introspect them later
concurrency = 3
multiplier = 3
numModels = concurrency * multiplier
# Generate model IDs
modelIDs = [hex(i) for i in xrange(numModels)]
requestBatches = (
"firstBatch",
"secondBatch",
)
modelInputDescriptors = dict(
(modelID,
_ModelInputDescriptor(requestBatches=requestBatches,
consumeSizes=[1, 1]))
for modelID in modelIDs)
slotAgents = []
slotAgentClassMock.side_effect = (lambda slotID: slotAgents.append(
_DummySlotAgent(slotID, modelInputDescriptors.__getitem__)) or
slotAgents[-1])
# Run SwapController in a thread
sc = SwapController(concurrency=concurrency)
self.assertEqual(len(sc._slotAgents), concurrency)
self.assertEqual(len(slotAgents), concurrency)
def runSwapControllerThread(sc, runResultQ):
try:
g_logger.info("Swap Controller run-thread is running")
r = sc.run()
runResultQ.put(r)
finally:
g_logger.info("Swap Controller run-thread is exiting")
runResultQ = Queue.Queue()
scThread = threading.Thread(target=runSwapControllerThread,
name="runSwapControllerThread",
args=(sc, runResultQ))
scThread.setDaemon(True)
scThread.start()
# Prod SwapController to process all models
for modelID in modelIDs:
notificationConsumer.q.put(_createModelInputNotification(modelID))
# Wait for model input queues to drain
for modelID, desc in modelInputDescriptors.iteritems():
g_logger.info("Waiting for model=%s inputQ to be empty", modelID)
desc.requestBatchesQ.waitUntilEmpty(timeout=5)
self.assertTrue(desc.requestBatchesQ.empty())
g_logger.info("model=%s inputQ is empty", modelID)
# Verify that all SlotAgents are occupied
for sa in slotAgents:
self.assertIsNotNone(sa.modelID)
# Now stop SwapController
g_logger.info("Requesting SwapController to stop")
sc.requestStopTS()
# So that the notification reader thread detects stop request and exits:
notificationConsumer.q.put(_createModelInputNotification(modelID))
g_logger.info("Waiting for SwapController run-thread to stop")
scThread.join(timeout=5)
self.assertFalse(scThread.isAlive())
g_logger.info("SwapController run-thread stopped")
# Verify that SwapController.run() returned without error
self.assertIsNone(runResultQ.get_nowait())
# Verify that all slot agents were closed
for sa in slotAgents:
self.assertEqual(sa.numCloseCalls, 1)
# Verify that input data of all models was drained
for modelID, desc in modelInputDescriptors.iteritems():
g_logger.info("Verify empty input for model=%s", modelID)
self.assertEqual(desc.requestBatchesQ.qsize(), 0)
self.assertEqual(desc.requestBatchesProcessedQ.qsize(),
len(requestBatches))
# Verify that all slot agents did work and were closed
for sa in slotAgents:
g_logger.info(
"sa=%s: closeCalls=%s; startCalls=%s; stopCalls=%s; releaseCalls=%s",
sa.slotID, sa.numCloseCalls, sa.numStartModelCalls,
sa.numStopModelCalls, sa.numReleaseSlotCalls)
self.assertEqual(sa.numCloseCalls, 1)
self.assertEqual(sa.numStartModelCalls,
multiplier * len(requestBatches))
self.assertEqual(sa.numStopModelCalls,
multiplier * len(requestBatches))
self.assertEqual(sa.numReleaseSlotCalls,
multiplier * len(requestBatches))
def testSimpleSingleSuccessfulModelInputAndStop(
self, slotAgentClassMock, modelSwapperInterfaceClassMock):
# Instantiate SwapController instance, run it in a separate thread,
# feed some data for one model, then stop SwapController.
# Configure ModelSwapperInterface instance mock
swapperMock = modelSwapperInterfaceClassMock.return_value
notificationConsumer = DummyConsumer()
swapperMock.consumeModelSchedulerNotifications.return_value = (
notificationConsumer)
swapperMock.modelInputPending.return_value = False
# Configure SlotAgent class mock to create dummy slot agent instances and
# add them to our list so that we can introspect them later
modelID = "abcd"
requestBatches = (
"firstBatch",
"secondBatch",
)
modelInputDesc = _ModelInputDescriptor(requestBatches=requestBatches,
consumeSizes=[
2,
])
modelInputDescriptors = {modelID: modelInputDesc}
slotAgents = []
slotAgentClassMock.side_effect = (lambda slotID: slotAgents.append(
_DummySlotAgent(slotID, modelInputDescriptors.__getitem__)) or
slotAgents[-1])
# Run SwapController in a thread
concurrency = 3
sc = SwapController(concurrency=concurrency)
self.assertEqual(len(sc._slotAgents), concurrency)
self.assertEqual(len(slotAgents), concurrency)
def runSwapControllerThread(sc, runResultQ):
try:
g_logger.info("Swap Controller run-thread is running")
r = sc.run()
runResultQ.put(r)
except:
runResultQ.put(sys.exc_info()[1])
raise
finally:
g_logger.info("Swap Controller run-thread is exiting")
runResultQ = Queue.Queue()
scThread = threading.Thread(target=runSwapControllerThread,
name="runSwapControllerThread",
args=(sc, runResultQ))
scThread.setDaemon(True)
scThread.start()
# Prod SwapController to process model input
notificationConsumer.q.put(_createModelInputNotification(modelID))
# Wait for the model input queue to drain
g_logger.info("Waiting for model inputQ to be empty")
modelInputDesc.requestBatchesQ.waitUntilEmpty(timeout=5)
self.assertTrue(modelInputDesc.requestBatchesQ.empty())
g_logger.info("model inputQ is empty")
# Now stop SwapController
g_logger.info("Requesting SwapController to stop")
sc.requestStopTS()
# So that the notification reader thread detects stop request and exits:
notificationConsumer.q.put(_createModelInputNotification(modelID))
g_logger.info("Waiting for SwapController run-thread to stop")
scThread.join(timeout=5)
self.assertFalse(scThread.isAlive())
g_logger.info("SwapController run-thread stopped")
# Verify that SwapController.run() returned without error
self.assertIsNone(runResultQ.get_nowait())
# Verify that all slot agents were closed
for sa in slotAgents:
self.assertEqual(sa.numCloseCalls, 1)
# Verify that a single slot agent handled all the input data
targetSA = None
for sa in slotAgents:
if sa.numStartModelCalls > 0:
self.assertIsNone(targetSA)
targetSA = sa
self.assertEqual(sa.numStartModelCalls, 1)
self.assertEqual(sa.numStopModelCalls, 1)
self.assertEqual(
modelInputDesc.requestBatchesProcessedQ.qsize(),
len(requestBatches))
else:
self.assertEqual(sa.numStartModelCalls, 0)
self.assertEqual(sa.numStopModelCalls, 0)
self.assertIsNotNone(targetSA)
