def _startDispatcher(self):
pid = self._fork('dispatcher', close=False,
criticalLogPath=self.cfg.getDispatcherLogPath())
if pid:
return
try:
disp = dispatcher.DispatcherServer(self.cfg, self.db)
disp._installSignalHandlers()
disp.serve_forever()
os._exit(0)
except SystemExit as err:
os._exit(err.args[0])
except Exception:
disp.exception("Failed to start dispatcher:")
finally:
os._exit(70)
def testDispatcherServer(self):
# Test only the server class, not the entire dispatcher
# hierarchy
db = mock.MockObject()
self.rmakeCfg.messageBusPort = None
server = dispatcher.DispatcherServer(self.rmakeCfg, db)
mock.mock(server, 'client')
cmd1 = self.makeCommandMessage(1, 1, self.getNVF('foo:source'))
trv = cmd1.getTrove()
cmd2 = self.makeCommandMessage(2, 2, self.getNVF('bar:source',
flavor='is:x86_64'),
[('foo:runtime', (None, None), (trv.getVersion(), parseFlavor('is:x86_64')))])
cmd3 = self.makeCommandMessage(3, 3, self.getNVF('foo:source',
flavor='bar is:x86'),
[('foo:runtime', (None, None), (trv.getVersion(), parseFlavor('bar is:x86')))])
# command 4 is also from job 2.
cmd4 = self.makeCommandMessage(4, 2, self.getNVF('bam:source'))
server.requestCommandAssignment(cmd1, cmd2, cmd3, cmd4)
server._assignQueuedCommands()
assert(server.listQueuedCommands() == [ cmd1.getMessageId(),
cmd2.getMessageId(),
cmd3.getMessageId(),
cmd4.getMessageId()])
node = self.makeRegisterNodeMessage().getNode()
# this assigns a command to this node as well.
server.nodeRegistered('session1', node)
server.client.assignCommand._mock.assertCalled(cmd1, node)
server._assignQueuedCommands()
assert(server.listQueuedCommands() == [ cmd2.getMessageId(),
cmd3.getMessageId(),
cmd4.getMessageId()])
assert(server.listAssignedCommands()
== [(cmd1.getCommandId(), node.sessionId)])
server.commandCompleted(cmd1.getCommandId())
assert(server.listAssignedCommands()
== [(cmd3.getCommandId(), node.sessionId)])
assert(server.listQueuedCommands() == [ cmd2.getMessageId(),
cmd4.getMessageId()])
server.commandErrored(cmd3.getCommandId())
assert(server.listQueuedCommands() == [ cmd2.getMessageId()])
assert(server.listAssignedCommands()
== [(cmd4.getCommandId(), node.sessionId)])
cmd5 = self.makeStopMessage(6, cmd2)
server.requestCommandAssignment(cmd5)
assert(not server.listQueuedCommands())
def startDispatcher(self, server):
pid = server._fork('Dispatcher')
if pid:
server._dispatcherPid = pid
return
try:
d = dispatcher.DispatcherServer(server.cfg, server.db)
try:
d._installSignalHandlers()
server._close()
server.db.reopen()
self._runServer(d, d.serve, 'Dispatcher')
except Exception, err:
d.error('Startup failed: %s: %s' %
(err, traceback.format_exc()))
finally:
os._exit(3)
def _setupMockDispatcher(self):
db = mock.MockObject()
self.rmakeCfg.messageBusPort = None
server = dispatcher.DispatcherServer(self.rmakeCfg, db)
sessionClient = server.client.getBusClient().getSession()
mock.mockMethod(sessionClient.poll)
mock.mockMethod(sessionClient.connect)
mock.mockMethod(sessionClient.disconnect)
mock.mockMethod(sessionClient.messageProcessor.sendMessage)
mock.mock(sessionClient, 'logger')
mock.mock(server.client.getBusClient(), 'logger')
sessionClient.handle_connect()
self._check(sessionClient, mbmessages.ConnectionRequest,
sessionClass='DSP')
m = mbmessages.ConnectedResponse()
m.set(sessionId='DSP-foo')
sessionClient.handle_message(m)
return server, sessionClient
def testThreshold(self):
db = mock.MockObject()
self.rmakeCfg.messageBusPort = None
server = dispatcher.DispatcherServer(self.rmakeCfg, db)
mock.mock(server, 'client')
cmd1 = self.makeCommandMessage(1, 1, self.getNVF('foo:source'))
cmd2 = self.makeCommandMessage(2, 2, self.getNVF('bar:source'))
server.requestCommandAssignment(cmd1, cmd2)
node = self.makeRegisterNodeMessage(loadThreshold=1).getNode()
server.nodeRegistered('session1', node)
nodeInfo = self.makeMachineInfo(loadavg=10)
server.nodeUpdated('session1', nodeInfo, ['CMD-1'])
# the command completed, but the load average for this node is
# too high, so we wait.
server.commandCompleted(cmd1.getCommandId())
assert(not server.listAssignedCommands())
nodeInfo = self.makeMachineInfo(loadavg=0.5)
# the load average is low again, so now the second job is assigned.
server.nodeUpdated('session1', nodeInfo, [])
assert(server.listAssignedCommands())
def testDispatcherRespectsChrootLimit(self):
db = mock.MockObject()
self.rmakeCfg.messageBusPort = None
server = dispatcher.DispatcherServer(self.rmakeCfg, db)
mock.mock(server, 'client')
node = self.makeRegisterNodeMessage(chrootLimit=1, slots=4).getNode()
server.nodeRegistered('session1', node)
cmd1 = self.makeResolveMessage(1, 1, self.getNVF('foo:source'))
cmd2 = self.makeResolveMessage(2, 2, self.getNVF('bar:source'))
cmd3 = self.makeCommandMessage(3, 2, self.getNVF('foo1:source'))
cmd4 = self.makeCommandMessage(4, 2, self.getNVF('bar2:source'))
server.requestCommandAssignment(cmd1, cmd2, cmd3, cmd4)
self.assertEquals(sorted(server.listAssignedCommands()),
[ ('CMD-1', 'session1'), ('CMD-2', 'session1'),
('CMD-3', 'session1')])
server.commandCompleted(cmd1.getCommandId())
self.assertEquals(sorted(server.listAssignedCommands()),
[ ('CMD-2', 'session1'), ('CMD-3', 'session1')])
server.commandCompleted(cmd3.getCommandId())
self.assertEquals(sorted(server.listAssignedCommands()),
[ ('CMD-2', 'session1'), ('CMD-4', 'session1')])
def testNodeMessagesOutOfSync(self):
db = mock.MockObject()
self.rmakeCfg.messageBusPort = None
server = dispatcher.DispatcherServer(self.rmakeCfg, db)
mock.mock(server, 'client')
cmd1 = self.makeCommandMessage(1, 1, self.getNVF('foo:source'))
cmd2 = self.makeCommandMessage(2, 2, self.getNVF('bar:source'))
server.requestCommandAssignment(cmd1, cmd2)
node = self.makeRegisterNodeMessage(loadThreshold=1,
loadavg=0.5).getNode()
server.nodeRegistered('session1', node)
self.assertEquals(server.listAssignedCommands(),
[ ('CMD-1', 'session1') ])
nodeInfo = self.makeMachineInfo(loadavg=1)
server.nodeUpdated('session1', nodeInfo, ['CMD-1'])
self.assertEquals(server.listAssignedCommands(),
[ ('CMD-1', 'session1') ])
server._logger.setQuietMode()
server.nodeUpdated('session1', nodeInfo, [])
assert(server.listAssignedCommands() == [])
server.nodeUpdated('session1', nodeInfo, ['CMD-3'])
assert(server.listAssignedCommands() == [])
