def testProgramAbortOnInputReaderThreadCrash(
self, osExitMock, _slotAgentClassMock, modelSwapperInterfaceClassMock):
# Verify that a crash in notification reader thread results in a call
# to os._exit()
# Configure ModelSwapperInterface mock
swapperMock = modelSwapperInterfaceClassMock.return_value
swapperMock.initSchedulerNotification.side_effect = Exception(
"testProgramAbortOnInputReaderThreadCrash")
osExitArgQ = Queue.Queue()
osExitMock.side_effect = osExitArgQ.put
sc = SwapController(concurrency=1)
self.assertEqual(len(sc._slotAgents), 1)
# Request stop, so that the main loop will exit ASAP
sc.requestStopTS()
# Run SwapController in a thread
def runSwapControllerThread(sc, runResultQ):
try:
g_logger.info("Swap Controller run-thread is running")
try:
r = sc.run()
except Exception as e:
runResultQ.put(e)
else:
runResultQ.put(r)
finally:
g_logger.info("Swap Controller run-thread is exiting")
runResultQ = Queue.Queue()
scThread = threading.Thread(
target=runSwapControllerThread,
name="runSwapControllerThread",
args=(sc, runResultQ))
scThread.setDaemon(True)
scThread.start()
# Wait for os._exit to be called
self.assertEqual(
osExitArgQ.get(timeout=5),
SwapController._EXIT_CODE_ON_FAILURE_OF_NOTIFICATION_READER_THREAD)
# Wait for the run-thread to stop
g_logger.info("Waiting for SwapController run-thread to stop")
scThread.join(timeout=5)
self.assertFalse(scThread.isAlive())
g_logger.info("SwapController run-thread stopped")
runResult = runResultQ.get_nowait()
self.assertIsNone(runResult)
def testProgramAbortOnInputReaderThreadCrash(
self, osExitMock, _slotAgentClassMock, modelSwapperInterfaceClassMock):
# Verify that a crash in notification reader thread results in a call
# to os._exit()
# Configure ModelSwapperInterface mock
swapperMock = modelSwapperInterfaceClassMock.return_value
swapperMock.initSchedulerNotification.side_effect = Exception(
"testProgramAbortOnInputReaderThreadCrash")
osExitArgQ = Queue.Queue()
osExitMock.side_effect = osExitArgQ.put
sc = SwapController(concurrency=1)
self.assertEqual(len(sc._slotAgents), 1)
# Request stop, so that the main loop will exit ASAP
sc.requestStopTS()
# Run SwapController in a thread
def runSwapControllerThread(sc, runResultQ):
try:
g_logger.info("Swap Controller run-thread is running")
try:
r = sc.run()
except Exception as e:
runResultQ.put(e)
else:
runResultQ.put(r)
finally:
g_logger.info("Swap Controller run-thread is exiting")
runResultQ = Queue.Queue()
scThread = threading.Thread(
target=runSwapControllerThread,
name="runSwapControllerThread",
args=(sc, runResultQ))
scThread.setDaemon(True)
scThread.start()
# Wait for os._exit to be called
self.assertEqual(
osExitArgQ.get(timeout=5),
SwapController._EXIT_CODE_ON_FAILURE_OF_NOTIFICATION_READER_THREAD)
# Wait for the run-thread to stop
g_logger.info("Waiting for SwapController run-thread to stop")
scThread.join(timeout=5)
self.assertFalse(scThread.isAlive())
g_logger.info("SwapController run-thread stopped")
runResult = runResultQ.get_nowait()
self.assertIsNone(runResult)
def testCreateSwapControllerAndDeleteIt(self, **_kwargs):
# Instantiates, then deletes SwapController
sc = SwapController(concurrency=3)
self.assertEqual(len(sc._slotAgents), 3)
del sc
def __init__(self, concurrency):
self._logger = _getLogger()
self._concurrency = concurrency
self._signalPipeReadFD, self._signalPipeWriteFD = os.pipe()
# Make the write end non-blocking to prevent accidental deadlocking of the
# signal dispatcher
fcntl.fcntl(
self._signalPipeWriteFD, fcntl.F_SETFL,
fcntl.fcntl(self._signalPipeWriteFD, fcntl.F_GETFL)
| os.O_NONBLOCK)
# Register for signals of interest
self._signalsOfInterest = [
signal.SIGHUP, signal.SIGTERM, signal.SIGINT
]
for sig in self._signalsOfInterest:
signal.signal(sig, self._handleSignal)
# Create the slot agents and swap controller.
self._swapController = SwapController(concurrency=concurrency)
def testRunSwapControllerAndStopIt(
self, _slotAgentClassMock, modelSwapperInterfaceClassMock):
# Instantiate SwapController instance, run it in a separate thread,
# then stop SwapController.
# Configure ModelSwapperInterface instance mock
swapperMock = modelSwapperInterfaceClassMock.return_value
notificationConsumer = DummyConsumer()
swapperMock.consumeModelSchedulerNotifications.return_value = (
notificationConsumer)
sc = SwapController(concurrency=3)
self.assertEqual(len(sc._slotAgents), 3)
# Run it in a thread
def runSwapControllerThread(sc, runResultQ):
r = sc.run()
runResultQ.put(r)
runResultQ = Queue.Queue()
scThread = threading.Thread(
target=runSwapControllerThread,
name="runSwapControllerThread",
args=(sc, runResultQ))
scThread.setDaemon(True)
scThread.start()
# Now stop it
sc.requestStopTS()
# Prod notification reader's consumer loop to detect the stop request and
# exit gracefully by adding a dummy message
notificationConsumer.q.put(None)
# There isn't much going on, it should stop immediately
scThread.join(timeout=5)
self.assertFalse(scThread.isAlive())
runResult = runResultQ.get_nowait()
self.assertIsNone(runResult)
def testRunSwapControllerAndStopIt(
self, _slotAgentClassMock, modelSwapperInterfaceClassMock):
# Instantiate SwapController instance, run it in a separate thread,
# then stop SwapController.
# Configure ModelSwapperInterface instance mock
swapperMock = modelSwapperInterfaceClassMock.return_value
notificationConsumer = DummyConsumer()
swapperMock.consumeModelSchedulerNotifications.return_value = (
notificationConsumer)
sc = SwapController(concurrency=3)
self.assertEqual(len(sc._slotAgents), 3)
# Run it in a thread
def runSwapControllerThread(sc, runResultQ):
r = sc.run()
runResultQ.put(r)
runResultQ = Queue.Queue()
scThread = threading.Thread(
target=runSwapControllerThread,
name="runSwapControllerThread",
args=(sc, runResultQ))
scThread.setDaemon(True)
scThread.start()
# Now stop it
sc.requestStopTS()
# Prod notification reader's consumer loop to detect the stop request and
# exit gracefully by adding a dummy message
notificationConsumer.q.put(None)
# There isn't much going on, it should stop immediately
scThread.join(timeout=5)
self.assertFalse(scThread.isAlive())
runResult = runResultQ.get_nowait()
self.assertIsNone(runResult)
def testDetectNotificationReaderThreadTargetCallFailed(
self, _slotAgentClassMock, _modelSwapperInterfaceClassMock):
# Reproduce failure to invoke the SwapController's input-reader thread
# target and verify that SwapController's event loop raises the expected
# exception
sc = SwapController(concurrency=1)
self.assertEqual(len(sc._slotAgents), 1)
# Patch SwapController's input-thread object with one that will exhibit a
# failure while trying to call the thread target
def expectTwoArgs(_a, _b):
pass
t = threading.Thread(target=expectTwoArgs)
t.setDaemon(True)
patch.multiple(sc, _notificationReaderThread=t).start()
# Attempt to run it in a thread
def runSwapControllerThread(sc, runResultQ):
try:
g_logger.info("Swap Controller run-thread is running")
try:
r = sc.run()
except Exception as e:
runResultQ.put(e)
else:
runResultQ.put(r)
finally:
g_logger.info("Swap Controller run-thread is exiting")
runResultQ = Queue.Queue()
scThread = threading.Thread(
target=runSwapControllerThread,
name="runSwapControllerThread",
args=(sc, runResultQ))
scThread.setDaemon(True)
scThread.start()
# Wait for the run-thread to stop
g_logger.info("Waiting for SwapController run-thread to stop")
scThread.join(timeout=5)
self.assertFalse(scThread.isAlive())
g_logger.info("SwapController run-thread stopped")
# Confirm the expected exception
runResult = runResultQ.get_nowait()
self.assertIsInstance(runResult, AssertionError)
self.assertIn("Notification-reader thread failed to start in time",
runResult.args[0])
def __init__(self, concurrency):
self._logger = _getLogger()
self._concurrency = concurrency
self._signalPipeReadFD, self._signalPipeWriteFD = os.pipe()
# Make the write end non-blocking to prevent accidental deadlocking of the
# signal dispatcher
fcntl.fcntl(
self._signalPipeWriteFD,
fcntl.F_SETFL,
fcntl.fcntl(self._signalPipeWriteFD, fcntl.F_GETFL) | os.O_NONBLOCK)
# Register for signals of interest
self._signalsOfInterest = [signal.SIGHUP, signal.SIGTERM, signal.SIGINT]
for sig in self._signalsOfInterest:
signal.signal(sig, self._handleSignal)
# Create the slot agents and swap controller.
self._swapController = SwapController(concurrency=concurrency)
def testModelPreemptionAndStop(
self, slotAgentClassMock, modelSwapperInterfaceClassMock):
# Test preemption of slots in SwapController
# Configure ModelSwapperInterface instance mock
swapperMock = modelSwapperInterfaceClassMock.return_value
notificationConsumer = DummyConsumer()
swapperMock.consumeModelSchedulerNotifications.return_value = (
notificationConsumer)
swapperMock.modelInputPending.side_effect = (lambda modelID:
not modelInputDescriptors[modelID].requestBatchesQ.empty())
# Configure SlotAgent class mock to create dummy slot agent instances and
# add them to our list so that we can introspect them later
concurrency = 3
multiplier = 3
numModels = concurrency * multiplier
# Generate model IDs
modelIDs = [hex(i) for i in xrange(numModels)]
requestBatches = ("firstBatch", "secondBatch",)
modelInputDescriptors = dict((
modelID,
_ModelInputDescriptor(
requestBatches=requestBatches,
consumeSizes=[1, 1]))
for modelID in modelIDs)
slotAgents = []
slotAgentClassMock.side_effect = (lambda slotID:
slotAgents.append(
_DummySlotAgent(slotID, modelInputDescriptors.__getitem__))
or slotAgents[-1])
# Run SwapController in a thread
sc = SwapController(concurrency=concurrency)
self.assertEqual(len(sc._slotAgents), concurrency)
self.assertEqual(len(slotAgents), concurrency)
def runSwapControllerThread(sc, runResultQ):
try:
g_logger.info("Swap Controller run-thread is running")
r = sc.run()
runResultQ.put(r)
finally:
g_logger.info("Swap Controller run-thread is exiting")
runResultQ = Queue.Queue()
scThread = threading.Thread(
target=runSwapControllerThread,
name="runSwapControllerThread",
args=(sc, runResultQ))
scThread.setDaemon(True)
scThread.start()
# Prod SwapController to process all models
for modelID in modelIDs:
notificationConsumer.q.put(_createModelInputNotification(modelID))
# Wait for model input queues to drain
for modelID, desc in modelInputDescriptors.iteritems():
g_logger.info("Waiting for model=%s inputQ to be empty", modelID)
desc.requestBatchesQ.waitUntilEmpty(timeout=5)
self.assertTrue(desc.requestBatchesQ.empty())
g_logger.info("model=%s inputQ is empty", modelID)
# Verify that all SlotAgents are occupied
for sa in slotAgents:
self.assertIsNotNone(sa.modelID)
# Now stop SwapController
g_logger.info("Requesting SwapController to stop")
sc.requestStopTS()
# So that the notification reader thread detects stop request and exits:
notificationConsumer.q.put(_createModelInputNotification(modelID))
g_logger.info("Waiting for SwapController run-thread to stop")
scThread.join(timeout=5)
self.assertFalse(scThread.isAlive())
g_logger.info("SwapController run-thread stopped")
# Verify that SwapController.run() returned without error
self.assertIsNone(runResultQ.get_nowait())
# Verify that all slot agents were closed
for sa in slotAgents:
self.assertEqual(sa.numCloseCalls, 1)
# Verify that input data of all models was drained
for modelID, desc in modelInputDescriptors.iteritems():
g_logger.info("Verify empty input for model=%s", modelID)
self.assertEqual(desc.requestBatchesQ.qsize(), 0)
self.assertEqual(desc.requestBatchesProcessedQ.qsize(),
len(requestBatches))
# Verify that all slot agents did work and were closed
for sa in slotAgents:
g_logger.info(
"sa=%s: closeCalls=%s; startCalls=%s; stopCalls=%s; releaseCalls=%s",
sa.slotID,
sa.numCloseCalls,
sa.numStartModelCalls,
sa.numStopModelCalls,
sa.numReleaseSlotCalls)
self.assertEqual(sa.numCloseCalls, 1)
self.assertEqual(sa.numStartModelCalls, multiplier * len(requestBatches))
self.assertEqual(sa.numStopModelCalls, multiplier * len(requestBatches))
self.assertEqual(sa.numReleaseSlotCalls, multiplier * len(requestBatches))
def testSimpleSingleSuccessfulModelInputAndStop(
self, slotAgentClassMock, modelSwapperInterfaceClassMock):
# Instantiate SwapController instance, run it in a separate thread,
# feed some data for one model, then stop SwapController.
# Configure ModelSwapperInterface instance mock
swapperMock = modelSwapperInterfaceClassMock.return_value
notificationConsumer = DummyConsumer()
swapperMock.consumeModelSchedulerNotifications.return_value = (
notificationConsumer)
swapperMock.modelInputPending.return_value = False
# Configure SlotAgent class mock to create dummy slot agent instances and
# add them to our list so that we can introspect them later
modelID = "abcd"
requestBatches = ("firstBatch", "secondBatch",)
modelInputDesc = _ModelInputDescriptor(
requestBatches=requestBatches,
consumeSizes=[2,])
modelInputDescriptors = {
modelID: modelInputDesc
}
slotAgents = []
slotAgentClassMock.side_effect = (lambda slotID:
slotAgents.append(
_DummySlotAgent(slotID, modelInputDescriptors.__getitem__))
or slotAgents[-1])
# Run SwapController in a thread
concurrency = 3
sc = SwapController(concurrency=concurrency)
self.assertEqual(len(sc._slotAgents), concurrency)
self.assertEqual(len(slotAgents), concurrency)
def runSwapControllerThread(sc, runResultQ):
try:
g_logger.info("Swap Controller run-thread is running")
r = sc.run()
runResultQ.put(r)
except:
runResultQ.put(sys.exc_info()[1])
raise
finally:
g_logger.info("Swap Controller run-thread is exiting")
runResultQ = Queue.Queue()
scThread = threading.Thread(
target=runSwapControllerThread,
name="runSwapControllerThread",
args=(sc, runResultQ))
scThread.setDaemon(True)
scThread.start()
# Prod SwapController to process model input
notificationConsumer.q.put(_createModelInputNotification(modelID))
# Wait for the model input queue to drain
g_logger.info("Waiting for model inputQ to be empty")
modelInputDesc.requestBatchesQ.waitUntilEmpty(timeout=5)
self.assertTrue(modelInputDesc.requestBatchesQ.empty())
g_logger.info("model inputQ is empty")
# Now stop SwapController
g_logger.info("Requesting SwapController to stop")
sc.requestStopTS()
# So that the notification reader thread detects stop request and exits:
notificationConsumer.q.put(_createModelInputNotification(modelID))
g_logger.info("Waiting for SwapController run-thread to stop")
scThread.join(timeout=5)
self.assertFalse(scThread.isAlive())
g_logger.info("SwapController run-thread stopped")
# Verify that SwapController.run() returned without error
self.assertIsNone(runResultQ.get_nowait())
# Verify that all slot agents were closed
for sa in slotAgents:
self.assertEqual(sa.numCloseCalls, 1)
# Verify that a single slot agent handled all the input data
targetSA = None
for sa in slotAgents:
if sa.numStartModelCalls > 0:
self.assertIsNone(targetSA)
targetSA = sa
self.assertEqual(sa.numStartModelCalls, 1)
self.assertEqual(sa.numStopModelCalls, 1)
self.assertEqual(modelInputDesc.requestBatchesProcessedQ.qsize(),
len(requestBatches))
else:
self.assertEqual(sa.numStartModelCalls, 0)
self.assertEqual(sa.numStopModelCalls, 0)
self.assertIsNotNone(targetSA)
class ModelSchedulerService(object):
# Error code that will be passed to os._exit when the thread that runs
# SwapController experiences an unhandled exception
_ABORT_PROGRAM_ON_THREAD_EXCEPTION_EXIT_CODE = 1
def __init__(self, concurrency):
self._logger = _getLogger()
self._concurrency = concurrency
self._signalPipeReadFD, self._signalPipeWriteFD = os.pipe()
# Make the write end non-blocking to prevent accidental deadlocking of the
# signal dispatcher
fcntl.fcntl(
self._signalPipeWriteFD,
fcntl.F_SETFL,
fcntl.fcntl(self._signalPipeWriteFD, fcntl.F_GETFL) | os.O_NONBLOCK)
# Register for signals of interest
self._signalsOfInterest = [signal.SIGHUP, signal.SIGTERM, signal.SIGINT]
for sig in self._signalsOfInterest:
signal.signal(sig, self._handleSignal)
# Create the slot agents and swap controller.
self._swapController = SwapController(concurrency=concurrency)
def __enter__(self):
""" Context Manager protocol method. Allows a ModelSchedulerService instance
to be used in a "with" statement for automatic clean-up
Parameters:
------------------------------------------------------------------------
retval: self.
"""
return self
def __exit__(self, excType, excVal, excTb):
""" Context Manager protocol method. Allows a ModelSchedulerService instance
to be used in a "with" statement for automatic cleanup
Returns: False so as not to suppress the exception, if any
"""
self._close()
return False
def _close(self):
""" Gracefully stop the Model Scheduler """
self._logger.info("Closing...")
# Unregister from signal notifications
for sig in self._signalsOfInterest:
signal.signal(sig, signal.SIG_DFL)
os.close(self._signalPipeReadFD)
self._signalPipeReadFD = None
os.close(self._signalPipeWriteFD)
self._signalPipeWriteFD = None
def run(self):
"""
Returns: True if service should be restarted, False otherwise
"""
self._logger.info("Running: pid=%s", os.getpid())
quitPipeFileObj = os.fdopen(os.dup(self._signalPipeReadFD))
swapControllerThread = threading.Thread(
target=self._runSwapControllerThread,
name="%s-%s" % (self._swapController.__class__.__name__,
id(self._swapController)))
swapControllerThread.setDaemon(True)
swapControllerThread.start()
while True:
try:
signalnum = int(quitPipeFileObj.readline())
except IOError as e:
if e.errno != errno.EINTR:
raise
# System call was interrupted by signal - restart it
continue
else:
break
self._logger.info("Stopping service due to signal %s", signalnum)
# Call swap controller requestStopTS method and then join the thread
# running its run method.
self._swapController.requestStopTS()
swapControllerThread.join(_SWAP_CONTROLLER_JOIN_TIMEOUT)
assert not swapControllerThread.isAlive(), (
"Swap controller thread did not join in the allotted time "
"(%g seconds)." % _SWAP_CONTROLLER_JOIN_TIMEOUT)
return signalnum == signal.SIGHUP
@abortProgramOnAnyException(
exitCode=_ABORT_PROGRAM_ON_THREAD_EXCEPTION_EXIT_CODE, logger=_getLogger())
def _runSwapControllerThread(self):
self._swapController.run()
def _handleSignal(self, signalnum, _frame):
""" Handle system signal; write it to the pipe so that it may be processed
by the main thread.
"""
try:
os.write(self._signalPipeWriteFD, "%s\n" % (signalnum,))
except IOError as e:
if e.errno in [errno.EWOULDBLOCK, errno.EAGAIN]:
# Drop the signal if we were overwhelmed by signals to the point of
# running out of pipe buffer (this shouldn't happen)
pass
else:
raise
def testModelPreemptionAndStop(self, slotAgentClassMock,
modelSwapperInterfaceClassMock):
# Test preemption of slots in SwapController
# Configure ModelSwapperInterface instance mock
swapperMock = modelSwapperInterfaceClassMock.return_value
notificationConsumer = DummyConsumer()
swapperMock.consumeModelSchedulerNotifications.return_value = (
notificationConsumer)
swapperMock.modelInputPending.side_effect = (
lambda modelID: not modelInputDescriptors[
modelID].requestBatchesQ.empty())
# Configure SlotAgent class mock to create dummy slot agent instances and
# add them to our list so that we can introspect them later
concurrency = 3
multiplier = 3
numModels = concurrency * multiplier
# Generate model IDs
modelIDs = [hex(i) for i in xrange(numModels)]
requestBatches = (
"firstBatch",
"secondBatch",
)
modelInputDescriptors = dict(
(modelID,
_ModelInputDescriptor(requestBatches=requestBatches,
consumeSizes=[1, 1]))
for modelID in modelIDs)
slotAgents = []
slotAgentClassMock.side_effect = (lambda slotID: slotAgents.append(
_DummySlotAgent(slotID, modelInputDescriptors.__getitem__)) or
slotAgents[-1])
# Run SwapController in a thread
sc = SwapController(concurrency=concurrency)
self.assertEqual(len(sc._slotAgents), concurrency)
self.assertEqual(len(slotAgents), concurrency)
def runSwapControllerThread(sc, runResultQ):
try:
g_logger.info("Swap Controller run-thread is running")
r = sc.run()
runResultQ.put(r)
finally:
g_logger.info("Swap Controller run-thread is exiting")
runResultQ = Queue.Queue()
scThread = threading.Thread(target=runSwapControllerThread,
name="runSwapControllerThread",
args=(sc, runResultQ))
scThread.setDaemon(True)
scThread.start()
# Prod SwapController to process all models
for modelID in modelIDs:
notificationConsumer.q.put(_createModelInputNotification(modelID))
# Wait for model input queues to drain
for modelID, desc in modelInputDescriptors.iteritems():
g_logger.info("Waiting for model=%s inputQ to be empty", modelID)
desc.requestBatchesQ.waitUntilEmpty(timeout=5)
self.assertTrue(desc.requestBatchesQ.empty())
g_logger.info("model=%s inputQ is empty", modelID)
# Verify that all SlotAgents are occupied
for sa in slotAgents:
self.assertIsNotNone(sa.modelID)
# Now stop SwapController
g_logger.info("Requesting SwapController to stop")
sc.requestStopTS()
# So that the notification reader thread detects stop request and exits:
notificationConsumer.q.put(_createModelInputNotification(modelID))
g_logger.info("Waiting for SwapController run-thread to stop")
scThread.join(timeout=5)
self.assertFalse(scThread.isAlive())
g_logger.info("SwapController run-thread stopped")
# Verify that SwapController.run() returned without error
self.assertIsNone(runResultQ.get_nowait())
# Verify that all slot agents were closed
for sa in slotAgents:
self.assertEqual(sa.numCloseCalls, 1)
# Verify that input data of all models was drained
for modelID, desc in modelInputDescriptors.iteritems():
g_logger.info("Verify empty input for model=%s", modelID)
self.assertEqual(desc.requestBatchesQ.qsize(), 0)
self.assertEqual(desc.requestBatchesProcessedQ.qsize(),
len(requestBatches))
# Verify that all slot agents did work and were closed
for sa in slotAgents:
g_logger.info(
"sa=%s: closeCalls=%s; startCalls=%s; stopCalls=%s; releaseCalls=%s",
sa.slotID, sa.numCloseCalls, sa.numStartModelCalls,
sa.numStopModelCalls, sa.numReleaseSlotCalls)
self.assertEqual(sa.numCloseCalls, 1)
self.assertEqual(sa.numStartModelCalls,
multiplier * len(requestBatches))
self.assertEqual(sa.numStopModelCalls,
multiplier * len(requestBatches))
self.assertEqual(sa.numReleaseSlotCalls,
multiplier * len(requestBatches))
def testSimpleSingleSuccessfulModelInputAndStop(
self, slotAgentClassMock, modelSwapperInterfaceClassMock):
# Instantiate SwapController instance, run it in a separate thread,
# feed some data for one model, then stop SwapController.
# Configure ModelSwapperInterface instance mock
swapperMock = modelSwapperInterfaceClassMock.return_value
notificationConsumer = DummyConsumer()
swapperMock.consumeModelSchedulerNotifications.return_value = (
notificationConsumer)
swapperMock.modelInputPending.return_value = False
# Configure SlotAgent class mock to create dummy slot agent instances and
# add them to our list so that we can introspect them later
modelID = "abcd"
requestBatches = (
"firstBatch",
"secondBatch",
)
modelInputDesc = _ModelInputDescriptor(requestBatches=requestBatches,
consumeSizes=[
2,
])
modelInputDescriptors = {modelID: modelInputDesc}
slotAgents = []
slotAgentClassMock.side_effect = (lambda slotID: slotAgents.append(
_DummySlotAgent(slotID, modelInputDescriptors.__getitem__)) or
slotAgents[-1])
# Run SwapController in a thread
concurrency = 3
sc = SwapController(concurrency=concurrency)
self.assertEqual(len(sc._slotAgents), concurrency)
self.assertEqual(len(slotAgents), concurrency)
def runSwapControllerThread(sc, runResultQ):
try:
g_logger.info("Swap Controller run-thread is running")
r = sc.run()
runResultQ.put(r)
except:
runResultQ.put(sys.exc_info()[1])
raise
finally:
g_logger.info("Swap Controller run-thread is exiting")
runResultQ = Queue.Queue()
scThread = threading.Thread(target=runSwapControllerThread,
name="runSwapControllerThread",
args=(sc, runResultQ))
scThread.setDaemon(True)
scThread.start()
# Prod SwapController to process model input
notificationConsumer.q.put(_createModelInputNotification(modelID))
# Wait for the model input queue to drain
g_logger.info("Waiting for model inputQ to be empty")
modelInputDesc.requestBatchesQ.waitUntilEmpty(timeout=5)
self.assertTrue(modelInputDesc.requestBatchesQ.empty())
g_logger.info("model inputQ is empty")
# Now stop SwapController
g_logger.info("Requesting SwapController to stop")
sc.requestStopTS()
# So that the notification reader thread detects stop request and exits:
notificationConsumer.q.put(_createModelInputNotification(modelID))
g_logger.info("Waiting for SwapController run-thread to stop")
scThread.join(timeout=5)
self.assertFalse(scThread.isAlive())
g_logger.info("SwapController run-thread stopped")
# Verify that SwapController.run() returned without error
self.assertIsNone(runResultQ.get_nowait())
# Verify that all slot agents were closed
for sa in slotAgents:
self.assertEqual(sa.numCloseCalls, 1)
# Verify that a single slot agent handled all the input data
targetSA = None
for sa in slotAgents:
if sa.numStartModelCalls > 0:
self.assertIsNone(targetSA)
targetSA = sa
self.assertEqual(sa.numStartModelCalls, 1)
self.assertEqual(sa.numStopModelCalls, 1)
self.assertEqual(
modelInputDesc.requestBatchesProcessedQ.qsize(),
len(requestBatches))
else:
self.assertEqual(sa.numStartModelCalls, 0)
self.assertEqual(sa.numStopModelCalls, 0)
self.assertIsNotNone(targetSA)
class ModelSchedulerService(object):
# Error code that will be passed to os._exit when the thread that runs
# SwapController experiences an unhandled exception
_ABORT_PROGRAM_ON_THREAD_EXCEPTION_EXIT_CODE = 1
def __init__(self, concurrency):
self._logger = _getLogger()
self._concurrency = concurrency
self._signalPipeReadFD, self._signalPipeWriteFD = os.pipe()
# Make the write end non-blocking to prevent accidental deadlocking of the
# signal dispatcher
fcntl.fcntl(
self._signalPipeWriteFD, fcntl.F_SETFL,
fcntl.fcntl(self._signalPipeWriteFD, fcntl.F_GETFL)
| os.O_NONBLOCK)
# Register for signals of interest
self._signalsOfInterest = [
signal.SIGHUP, signal.SIGTERM, signal.SIGINT
]
for sig in self._signalsOfInterest:
signal.signal(sig, self._handleSignal)
# Create the slot agents and swap controller.
self._swapController = SwapController(concurrency=concurrency)
def __enter__(self):
""" Context Manager protocol method. Allows a ModelSchedulerService instance
to be used in a "with" statement for automatic clean-up
Parameters:
------------------------------------------------------------------------
retval: self.
"""
return self
def __exit__(self, excType, excVal, excTb):
""" Context Manager protocol method. Allows a ModelSchedulerService instance
to be used in a "with" statement for automatic cleanup
Returns: False so as not to suppress the exception, if any
"""
self._close()
return False
def _close(self):
""" Gracefully stop the Model Scheduler """
self._logger.info("Closing...")
# Unregister from signal notifications
for sig in self._signalsOfInterest:
signal.signal(sig, signal.SIG_DFL)
os.close(self._signalPipeReadFD)
self._signalPipeReadFD = None
os.close(self._signalPipeWriteFD)
self._signalPipeWriteFD = None
def run(self):
"""
Returns: True if service should be restarted, False otherwise
"""
self._logger.info("Running: pid=%s", os.getpid())
quitPipeFileObj = os.fdopen(os.dup(self._signalPipeReadFD))
swapControllerThread = threading.Thread(
target=self._runSwapControllerThread,
name="%s-%s" % (self._swapController.__class__.__name__,
id(self._swapController)))
swapControllerThread.setDaemon(True)
swapControllerThread.start()
while True:
try:
signalnum = int(quitPipeFileObj.readline())
except IOError as e:
if e.errno != errno.EINTR:
raise
# System call was interrupted by signal - restart it
continue
else:
break
self._logger.info("Stopping service due to signal %s", signalnum)
# Call swap controller requestStopTS method and then join the thread
# running its run method.
self._swapController.requestStopTS()
swapControllerThread.join(_SWAP_CONTROLLER_JOIN_TIMEOUT)
assert not swapControllerThread.isAlive(), (
"Swap controller thread did not join in the allotted time "
"(%g seconds)." % _SWAP_CONTROLLER_JOIN_TIMEOUT)
return signalnum == signal.SIGHUP
@abortProgramOnAnyException(
exitCode=_ABORT_PROGRAM_ON_THREAD_EXCEPTION_EXIT_CODE,
logger=_getLogger())
def _runSwapControllerThread(self):
self._swapController.run()
def _handleSignal(self, signalnum, _frame):
""" Handle system signal; write it to the pipe so that it may be processed
by the main thread.
"""
try:
os.write(self._signalPipeWriteFD, "%s\n" % (signalnum, ))
except IOError as e:
if e.errno in [errno.EWOULDBLOCK, errno.EAGAIN]:
# Drop the signal if we were overwhelmed by signals to the point of
# running out of pipe buffer (this shouldn't happen)
pass
else:
raise
