def run_from_argv(self, prog_name, argv, command=None):
from celery.bin.multi import MultiTool
argv.append("--cmd=%s worker --detach" % prog_name)
return MultiTool().execute_from_commandline(
[command] + argv,
prog_name,
)
def run_from_argv(self, prog_name, argv, command=None):
from celery.bin.multi import MultiTool
multi = MultiTool(quiet=self.quiet, no_color=self.no_color)
return multi.execute_from_commandline(
[command] + argv,
prog_name,
)
def test_range_prefix_not_used_in_named_range(self):
m = MultiTool()
range_prefix = 'worker'
_opt_parser, nodes = m._nodes_from_argv(
['a b c', '--range-prefix={}'.format(range_prefix)])
for i, node in enumerate(nodes, start=1):
assert not node.name.startswith(range_prefix)
def test_range_prefix_not_set(self):
m = MultiTool()
default_prefix = 'celery'
workers_count = 2
_opt_parser, nodes = m._nodes_from_argv(['{}'.format(workers_count)])
for i, node in enumerate(nodes, start=1):
assert node.name.startswith(default_prefix + str(i))
def test_splash(self):
x = MultiTool()
x.note = Mock()
x.nosplash = True
x.splash()
x.note.assert_not_called()
x.nosplash = False
x.splash()
x.note.assert_called()
def test_range_prefix(self):
m = MultiTool()
range_prefix = 'worker'
workers_count = 2
_opt_parser, nodes = m._nodes_from_argv([
'{}'.format(workers_count),
'--range-prefix={}'.format(range_prefix)
])
for i, node in enumerate(nodes, start=1):
assert node.name.startswith(range_prefix + str(i))
def setup(self):
self.fh = WhateverIO()
self.env = {}
self.t = MultiTool(env=self.env, fh=self.fh)
self.t.Cluster = Mock(name='Cluster')
self.t.carp = Mock(name='.carp')
self.t.usage = Mock(name='.usage')
self.t.splash = Mock(name='.splash')
self.t.say = Mock(name='.say')
self.t.ok = Mock(name='.ok')
self.cluster = self.t.Cluster.return_value
def test_Cluster(self):
m = MultiTool()
c = m.cluster_from_argv(['A', 'B', 'C'])
assert c.env is m.env
assert c.cmd == 'celery worker'
assert c.on_stopping_preamble == m.on_stopping_preamble
assert c.on_send_signal == m.on_send_signal
assert c.on_still_waiting_for == m.on_still_waiting_for
assert c.on_still_waiting_progress == m.on_still_waiting_progress
assert c.on_still_waiting_end == m.on_still_waiting_end
assert c.on_node_start == m.on_node_start
assert c.on_node_restart == m.on_node_restart
assert c.on_node_shutdown_ok == m.on_node_shutdown_ok
assert c.on_node_status == m.on_node_status
assert c.on_node_signal_dead == m.on_node_signal_dead
assert c.on_node_signal == m.on_node_signal
assert c.on_node_down == m.on_node_down
assert c.on_child_spawn == m.on_child_spawn
assert c.on_child_signalled == m.on_child_signalled
assert c.on_child_failure == m.on_child_failure
def setup(self):
self.fh = WhateverIO()
self.env = {}
self.t = MultiTool(env=self.env, fh=self.fh)
self.t.cluster_from_argv = Mock(name='cluster_from_argv')
self.t._cluster_from_argv = Mock(name='cluster_from_argv')
self.t.Cluster = Mock(name='Cluster')
self.t.carp = Mock(name='.carp')
self.t.usage = Mock(name='.usage')
self.t.splash = Mock(name='.splash')
self.t.say = Mock(name='.say')
self.t.ok = Mock(name='.ok')
self.cluster = self.t.Cluster.return_value
def _cluster_from_argv(argv):
p = self.t.OptionParser(argv)
p.parse()
return p, self.cluster
self.t.cluster_from_argv.return_value = self.cluster
self.t._cluster_from_argv.side_effect = _cluster_from_argv
def test_Cluster(self):
m = MultiTool()
c = m.Cluster(['A', 'B', 'C'])
self.assertListEqual(c.argv, ['A', 'B', 'C'])
self.assertIs(c.env, m.env)
self.assertEqual(c.cmd, 'celery worker')
self.assertEqual(c.on_stopping_preamble, m.on_stopping_preamble)
self.assertEqual(c.on_send_signal, m.on_send_signal)
self.assertEqual(c.on_still_waiting_for, m.on_still_waiting_for)
self.assertEqual(
c.on_still_waiting_progress,
m.on_still_waiting_progress,
)
self.assertEqual(c.on_still_waiting_end, m.on_still_waiting_end)
self.assertEqual(c.on_node_start, m.on_node_start)
self.assertEqual(c.on_node_restart, m.on_node_restart)
self.assertEqual(c.on_node_shutdown_ok, m.on_node_shutdown_ok)
self.assertEqual(c.on_node_status, m.on_node_status)
self.assertEqual(c.on_node_signal_dead, m.on_node_signal_dead)
self.assertEqual(c.on_node_signal, m.on_node_signal)
self.assertEqual(c.on_node_down, m.on_node_down)
self.assertEqual(c.on_child_spawn, m.on_child_spawn)
self.assertEqual(c.on_child_signalled, m.on_child_signalled)
self.assertEqual(c.on_child_failure, m.on_child_failure)
def run_from_argv(self, prog_name, argv):
from celery.bin.multi import MultiTool
return MultiTool().execute_from_commandline(argv, prog_name)
def setup(self):
self.fh = WhateverIO()
self.env = {}
self.t = MultiTool(env=self.env, fh=self.fh)
def setup(self):
self.fh = WhateverIO()
self.env = {}
with patch('celery.apps.multi.os.mkdir'):
self.t = MultiTool(env=self.env, fh=self.fh)
def handle(self, *args, **options):
from btw.settings._env import env
workers = get_defined_workers()
workers_by_name = {w.name: w for w in workers}
def get_worker_names(force_all=False):
get_all = force_all or options["all"]
worker_names = options["worker_names"]
if len(worker_names):
for name in worker_names:
if name not in workers_by_name:
raise CommandError("{0} is not a worker name"
.format(name))
elif get_all:
worker_names = [w.name for w in workers]
else:
raise CommandError("must specify a worker name or use --all")
return worker_names
cmd = options["command"]
if cmd == "start":
worker_names = get_worker_names()
running_workers = get_running_workers()
full_names = get_full_names(worker_names)
requests = []
for name in worker_names:
worker = workers_by_name[name]
if worker in running_workers:
self.error("{0} is already running.".format(name))
continue
full_name = full_names[name]
retcode = MultiTool().execute_from_commandline(
worker.start_cmd)
if retcode:
self.error("there was an error starting {0}"
.format(name))
# What we are doing here has more to do with waiting
# for the worker to start rather than actually
# checking the return value. It would be quite
# difficult to get into a situation where the
# environments do not coincide.
# We send the task directly to the worker so that we
# are sure *that* worker handles the request.
requests.append((worker,
get_btw_env.apply_async(
(),
queue=worker_direct(full_name))))
for worker, request in requests:
name = worker.name
result = request.get()
if result != env:
self.error(
("{0}: not using environment {1} "
"(uses environment {2})")
.format(name, env, result))
# We have to do this to unschedule the tasks that may
# have been scheduled by a previous worker. If we do
# not do this, we can end up with having gobs of tasks
# scheduled for future execution. However, we cannot
# do this when we stop the tasks. Why? Because the
# list of revoked tasks is held only in memory and
# will vanish when the workers are stopped.
self.revoke_scheduled(full_names[name])
if worker.start_task:
worker.start_task()
self.stdout.write("{0} has started.".format(name))
elif cmd == "names":
check_no_names(options, cmd)
check_no_all(options, cmd)
for name in get_worker_names(force_all=True):
self.stdout.write(name)
elif cmd == "lognames":
check_no_names(options, cmd)
check_no_all(options, cmd)
for w in workers:
self.stdout.write(w.logfile)
elif cmd == "stop":
worker_names = get_worker_names()
running_workers = get_running_workers()
for name in worker_names:
worker = workers_by_name[name]
if worker in running_workers:
retcode = MultiTool().execute_from_commandline(
worker.stop_cmd)
if retcode:
self.error("there was an error stopping {0}"
.format(name))
self.stdout.write("{0} has stopped.".format(name))
else:
self.stdout.write("{0} was not running.".format(name))
elif cmd == "ping":
check_no_names(options, cmd)
check_no_all(options, cmd)
full_names = get_full_names([w.name for w in workers])
for worker in workers:
self.stdout.write("Pinging worker %s... " %
worker.name, ending='')
status = worker_does_not_exist(worker)
if status:
self.stdout.write("failed: " + status)
continue
full_name = full_names[worker.name]
result = app.control.ping([full_name], timeout=0.5)
if result[0][full_name] == {'ok': 'pong'}:
self.stdout.write("passed")
else:
self.stdout.write("failed with response: " +
(repr(result[0].get(full_name)) or ""))
elif cmd == "check":
check_no_names(options, cmd)
check_no_all(options, cmd)
if not settings.CELERY_WORKER_DIRECT:
# We need CELERY_WORKER_DIRECT so that the next test will work.
raise CommandError("CELERY_WORKER_DIRECT must be True")
full_names = get_full_names([w.name for w in workers])
for worker in workers:
self.stdout.write("Checking worker %s... " %
worker.name, ending='')
status = worker_does_not_exist(worker)
if status:
self.stdout.write("failed: " + status)
continue
full_name = full_names[worker.name]
# We send the task directly to the worker so that we
# are sure *that* worker handles the request.
try:
result = get_btw_env.apply_async(
(),
queue=worker_direct(full_name)).get(timeout=60)
except TimeoutError:
self.stdout.write("failed: timed out")
continue
if result != env:
self.stdout.write(
("failed: not using environment {0} "
"(uses environment {1})")
.format(env, result))
continue
self.stdout.write("passed")
elif cmd == "revoke-scheduled":
self.revoke_scheduled()
else:
raise CommandError("bad command: " + cmd)
