def test_start_stop(self):
self.guard._register_to_hub = MagicMock()
self.guard.heartbeat = MagicMock()
start_daemon_thread(self.guard.start)
if wait_condition_till_timeout(self.guard.is_alive, 5, False):
raise CouldNotStartException('guard')
self.assertTrue(self.guard.alive)
self.guard.heartbeat.assert_called()
self.guard.stop()
self.assertFalse(self.guard.alive)
def test_start_stop(self):
self.guard._register_to_hub = MagicMock()
self.guard.heartbeat = MagicMock()
start_daemon_thread(self.guard.start)
if wait_condition_till_timeout(self.guard.is_alive, 5, False):
raise CouldNotStartException('guard')
self.assertTrue(self.guard.alive)
self.guard.heartbeat.assert_called()
self.guard.stop()
self.assertFalse(self.guard.alive)
def test_register_to_hub(self):
reset()
start_daemon_thread(self.hub.start)
if wait_condition_till_timeout(self.hub.is_alive, 5, False):
raise CouldNotStartException('hub')
self.assertEqual(len(MACHINES), 0)
start_daemon_thread(self.guard.start)
if wait_condition_till_timeout(self.guard.is_alive, 5, False):
raise CouldNotStartException('guard')
self.assertEqual(len(MACHINES), 1)
self.assertEqual(MACHINES.keys(), [self.guard.machine.id])
def test_register_to_hub(self):
reset()
start_daemon_thread(self.hub.start)
if wait_condition_till_timeout(self.hub.is_alive, 5, False):
raise CouldNotStartException('hub')
self.assertEqual(len(MACHINES), 0)
start_daemon_thread(self.guard.start)
if wait_condition_till_timeout(self.guard.is_alive, 5, False):
raise CouldNotStartException('guard')
self.assertEqual(len(MACHINES), 1)
self.assertEqual(MACHINES.keys(), [self.guard.machine.id])
def test_heart_beat(self):
PLAN = {'q1': {self.guard.machine.id: 1}}
start_daemon_thread(self.hub.start)
if wait_condition_till_timeout(self.hub.is_alive, 5, False):
raise CouldNotStartException('hub')
start_daemon_thread(self.guard.start)
if wait_condition_till_timeout(self.guard.is_alive, 5, False):
raise CouldNotStartException('guard')
Hub.report_demand(InstructionType.WORKER, 'q1', 2)
_org_receive_order = commands.receive_order
commands.receive_order = MagicMock()
time.sleep(2)
commands.receive_order.assert_called()
commands.receive_order = _org_receive_order
def test_heart_beat(self):
PLAN = {'q1': {self.guard.machine.id: 1}}
start_daemon_thread(self.hub.start)
if wait_condition_till_timeout(self.hub.is_alive, 5, False):
raise CouldNotStartException('hub')
start_daemon_thread(self.guard.start)
if wait_condition_till_timeout(self.guard.is_alive, 5, False):
raise CouldNotStartException('guard')
Hub.report_demand(InstructionType.WORKER, 'q1', 2)
_org_receive_order = commands.receive_order
commands.receive_order = MagicMock()
time.sleep(2)
commands.receive_order.assert_called()
commands.receive_order = _org_receive_order
def test_worker_pool_grow(self):
planner = PlannerPool.get_or_create_planner(self.queue, self.app, self.strategy_name, self.labels)
planner.loop_interval = 0.1
planner.strategy.apply = PickableMock(return_value=self.worker_cnt)
start_daemon_thread(self.hub.start)
if wait_condition_till_timeout(self.hub.is_alive, 5, False):
CouldNotStartException('hub')
start_daemon_thread(self.guard.start)
if wait_condition_till_timeout(self.guard.is_alive, 5, False):
raise CouldNotStartException('guard')
self.guard.pool.get_or_create_pool_name = PickableMock(return_value=self.queue)
time.sleep(1)
self.guard.pool.grow.assert_called_with(self.queue, self.worker_cnt)
planner.stop()
def test_worker_pool_grow(self):
planner = PlannerPool.get_or_create_planner(self.queue, self.app,
self.strategy_name,
self.labels)
planner.loop_interval = 0.1
planner.strategy.apply = PickableMock(return_value=self.worker_cnt)
start_daemon_thread(self.hub.start)
if wait_condition_till_timeout(self.hub.is_alive, 5, False):
CouldNotStartException('hub')
start_daemon_thread(self.guard.start)
if wait_condition_till_timeout(self.guard.is_alive, 5, False):
raise CouldNotStartException('guard')
self.guard.pool.get_or_create_pool_name = PickableMock(
return_value=self.queue)
time.sleep(1)
self.guard.pool.grow.assert_called_with(self.queue, self.worker_cnt)
planner.stop()
def test_start_stop(self):
self.hub.analyze_demand = MagicMock()
self.hub.send_order_to_guard = MagicMock()
self.hub.load_balancing = MagicMock()
self.assertFalse(self.hub.alive)
self.assertFalse(self.hub.rpc_server.alive)
start_daemon_thread(self.hub.start)
if wait_condition_till_timeout(self.hub.is_alive, 5, False):
raise 'could not start hub'
self.assertTrue(self.hub.alive)
self.assertTrue(self.hub.rpc_server.alive)
self.hub.analyze_demand.assert_called()
self.hub.send_order_to_guard.assert_called()
self.hub.load_balancing.assert_not_called()
self.hub.stop()
self.assertFalse(self.hub.alive)
self.assertFalse(self.hub.rpc_server.alive)
def start(self):
if self.alive:
debug('[Planner] - [Already Start] - %s' % self)
else:
self.__shutdown.clear()
self.thread = start_daemon_thread(self.loop)
while not self.thread.is_alive():
continue
debug('[Planner] - [Start] - %s' % (self))
def start(self):
if self.alive:
debug('[Planner] - [Already Start] - %s' % self)
else:
self.__shutdown.clear()
self.thread = start_daemon_thread(self.loop)
while not self.thread.is_alive():
continue
debug('[Planner] - [Start] - %s' % (self))
def create_pool(self, queue, pool_cls=None, loglevel=None, logfile=None, pidfile=None, state_db=None):
kwargs = {
'autoscale': '200,1',
'queues': queue,
}
pool_cls = concurrency.get_implementation(pool_cls) or self.app.conf.CELERYD_POOL
hostname = '%s::%s' % (self.hostname, queue)
pool = self.app.Worker(
hostname=hostname,
pool_cls=pool_cls,
loglevel=loglevel,
logfile=logfile,
pidfile=node_format(pidfile, hostname),
state_db=node_format(state_db, hostname),
without_mingle=False,
**kwargs
)
start_daemon_thread(pool.start)
if self.check_pool_start(hostname):
return {'name': hostname, 'pool': pool}
else:
raise CouldNotStartException('Work Pool')
def force_quit(self):
start_daemon_thread(terminate_thread, args=(self.thread, ))
def force_quit(self):
start_daemon_thread(terminate_thread, args=(self.thread,))
def test_start_default_planner(self):
start_daemon_thread(self.hub.start)
if wait_condition_till_timeout(self.hub.is_alive, 5, False):
raise 'could not start hub'
self.assertEqual(PlannerPool.pool.keys(), self.app.conf.config['DEFAULT_QUEUE'].keys())