def stop(self, worker):
# The Worker will call stop at shutdown only.
logging.warn('{0!r} is stopping. Attempting termination of current tasks...'.format(worker))
# Following code from worker.control.revoke
terminated = set()
# cleaning all reserved tasks since we are shutting down
signum = _signals.signum('TERM')
for request in [r for r in worker.state.reserved_requests]:
if request.id not in terminated:
terminated.add(request.id)
_logger.info('Terminating %s (%s)', request.id, signum)
request.terminate(worker.pool, signal=signum)
# Aborting currently running tasks, and triggering soft timeout exception to allow task to clean up.
signum = _signals.signum('USR1')
for request in [r for r in worker.state.active_requests]:
if request.id not in terminated:
terminated.add(request.id)
_logger.info('Terminating %s (%s)', request.id, signum)
request.terminate(worker.pool, signal=signum) # triggering SoftTimeoutException in Task
if terminated:
terminatedstr = ', '.join(terminated)
_logger.info('Tasks flagged as revoked: %s', terminatedstr)
self.node_proc.shutdown()
def terminate(self, pool, signal=None):
if self.time_start:
signal = _signals.signum(signal or 'TERM')
pool.terminate_job(self.worker_pid, signal)
self._announce_revoked('terminated', True, signal, False)
else:
self._terminate_on_ack = pool, signal
def revoke(state, task_id, terminate=False, signal=None, **kwargs):
"""Revoke task by task id."""
# supports list argument since 3.1
task_ids, task_id = set(maybe_list(task_id) or []), None
size = len(task_ids)
terminated = set()
revoked.update(task_ids)
if terminate:
signum = _signals.signum(signal or TERM_SIGNAME)
for request in _find_requests_by_id(task_ids):
if request.id not in terminated:
terminated.add(request.id)
logger.info('Terminating %s (%s)', request.id, signum)
request.terminate(state.consumer.pool, signal=signum)
if len(terminated) >= size:
break
if not terminated:
return ok('terminate: tasks unknown')
return ok('terminate: {0}'.format(', '.join(terminated)))
idstr = ', '.join(task_ids)
logger.info('Tasks flagged as revoked: %s', idstr)
return ok('tasks {0} flagged as revoked'.format(idstr))
def revoke(state, task_id, terminate=False, signal=None, **kwargs):
"""Revoke task by task id."""
# supports list argument since 3.1
task_ids, task_id = set(maybe_list(task_id) or []), None
size = len(task_ids)
terminated = set()
revoked.update(task_ids)
if terminate:
signum = _signals.signum(signal or 'TERM')
# reserved_requests changes size during iteration
# so need to consume the items first, then terminate after.
requests = set(_find_requests_by_id(
task_ids,
worker_state.reserved_requests,
))
for request in requests:
if request.id not in terminated:
terminated.add(request.id)
logger.info('Terminating %s (%s)', request.id, signum)
request.terminate(state.consumer.pool, signal=signum)
if len(terminated) >= size:
break
if not terminated:
return {'ok': 'terminate: tasks unknown'}
return {'ok': 'terminate: {0}'.format(', '.join(terminated))}
idstr = ', '.join(task_ids)
logger.info('Tasks flagged as revoked: %s', idstr)
return {'ok': 'tasks {0} flagged as revoked'.format(idstr)}
def revoke(state, task_id, terminate=False, signal=None, **kwargs):
"""Revoke task by task id (or list of ids).
Keyword Arguments:
terminate (bool): Also terminate the process if the task is active.
signal (str): Name of signal to use for terminate (e.g., ``KILL``).
"""
# pylint: disable=redefined-outer-name
# XXX Note that this redefines `terminate`:
# Outside of this scope that is a function.
# supports list argument since 3.1
task_ids, task_id = set(maybe_list(task_id) or []), None
size = len(task_ids)
terminated = set()
worker_state.revoked.update(task_ids)
if terminate:
signum = _signals.signum(signal or TERM_SIGNAME)
for request in _find_requests_by_id(task_ids):
if request.id not in terminated:
terminated.add(request.id)
logger.info('Terminating %s (%s)', request.id, signum)
request.terminate(state.consumer.pool, signal=signum)
if len(terminated) >= size:
break
if not terminated:
return ok('terminate: tasks unknown')
return ok('terminate: {0}'.format(', '.join(terminated)))
idstr = ', '.join(task_ids)
logger.info('Tasks flagged as revoked: %s', idstr)
return ok('tasks {0} flagged as revoked'.format(idstr))
def revoke(state, task_id, terminate=False, signal=None, **kwargs):
"""Revoke task by task id."""
# supports list argument since 3.1
task_ids, task_id = maybe_list(task_id) or [], None
to_terminate = set()
terminated = set()
for task_id in task_ids:
revoked.add(task_id)
if terminate:
to_terminate.add(task_id)
if to_terminate:
signum = _signals.signum(signal or "TERM")
_to_terminate = set()
# reserved_requests changes size during iteration
# so need to consume the items first, then terminate after.
requests = set(_find_requests_by_id(to_terminate, worker_state.reserved_requests))
for request in requests:
logger.info("Terminating %s (%s)", task_id, signum)
request.terminate(state.consumer.pool, signal=signum)
terminated.add(request.id)
if len(terminated) >= len(_to_terminate):
break
if not terminated:
return {"ok": "terminate: tasks unknown"}
return {"ok": "terminate: {0}".format(", ".join(terminated))}
idstr = ", ".join(task_ids)
logger.info("Tasks flagged as revoked: %s", idstr)
return {"ok": "tasks {0} flagged as revoked".format(idstr)}
def terminate(self, pool, signal=None):
if self.time_start:
signal = _signals.signum(signal or "TERM")
pool.terminate_job(self.worker_pid, signal)
send_revoked(self.task, signum=signal, terminated=True, expired=False)
else:
self._terminate_on_ack = pool, signal
def test_signum(self):
assert signals.signum(13) == 13
assert signals.signum('INT') == signal.SIGINT
assert signals.signum('SIGINT') == signal.SIGINT
with pytest.raises(TypeError):
signals.signum('int')
signals.signum(object())
def test_signum(self):
self.assertEqual(signals.signum(13), 13)
self.assertEqual(signals.signum('INT'), signal.SIGINT)
self.assertEqual(signals.signum('SIGINT'), signal.SIGINT)
with self.assertRaises(TypeError):
signals.signum('int')
signals.signum(object())
def test_signum(self):
self.assertEqual(signals.signum(13), 13)
self.assertEqual(signals.signum("INT"), signal.SIGINT)
self.assertEqual(signals.signum("SIGINT"), signal.SIGINT)
with self.assertRaises(TypeError):
signals.signum("int")
signals.signum(object())
def terminate(self, pool, signal=None):
signal = _signals.signum(signal or TERM_SIGNAME)
if self.time_start:
pool.terminate_job(self.worker_pid, signal)
self._announce_revoked('terminated', True, signal, False)
else:
self._terminate_on_ack = pool, signal
if self._apply_result is not None:
obj = self._apply_result() # is a weakref
if obj is not None:
obj.terminate(signal)
def terminate(self, pool, signal=None):
signal = _signals.signum(signal or TERM_SIGNAME)
if self.time_start:
pool.terminate_job(self.worker_pid, signal)
self._announce_revoked('terminated', True, signal, False)
else:
self._terminate_on_ack = pool, signal
if self._apply_result is not None:
obj = self._apply_result() # is a weakref
if obj is not None:
obj.terminate(signal)
def _find_sig_argument(self, args, default=signal.SIGTERM):
for arg in reversed(args):
if len(arg) == 2 and arg[0] == '-':
try:
return int(arg[1])
except ValueError:
pass
if arg[0] == '-':
try:
return signals.signum(arg[1:])
except (AttributeError, TypeError):
pass
return default
def revoke(panel, task_id, terminate=False, signal=None, **kwargs):
"""Revoke task by task id."""
revoked.add(task_id)
action = 'revoked'
if terminate:
signum = _signals.signum(signal or 'TERM')
for request in state.active_requests:
if request.id == task_id:
action = 'terminated (%s)' % (signum, )
request.terminate(panel.consumer.pool, signal=signum)
break
logger.info('Task %s %s.', task_id, action)
return {'ok': 'task %s %s' % (task_id, action)}
def revoke(panel, task_id, terminate=False, signal=None, **kwargs):
"""Revoke task by task id."""
revoked.add(task_id)
action = "revoked"
if terminate:
signum = _signals.signum(signal or "TERM")
for request in state.active_requests:
if request.id == task_id:
action = "terminated (%s)" % (signum, )
request.terminate(panel.consumer.pool, signal=signum)
break
logger.info("Task %s %s.", task_id, action)
return {"ok": "task %s %s" % (task_id, action)}
def revoke(panel, task_id, terminate=False, signal=None, **kwargs):
"""Revoke task by task id."""
revoked.add(task_id)
action = 'revoked'
if terminate:
signum = _signals.signum(signal or 'TERM')
for request in state.active_requests:
if request.id == task_id:
action = 'terminated (%s)' % (signum, )
request.terminate(panel.consumer.pool, signal=signum)
break
logger.info('Task %s %s.', task_id, action)
return {'ok': 'task %s %s' % (task_id, action)}
def revoke(panel, task_id, terminate=False, signal=None, **kwargs):
"""Revoke task by task id."""
revoked.add(task_id)
action = "revoked"
if terminate:
signum = _signals.signum(signal or "TERM")
for request in state.active_requests:
if request.task_id == task_id:
action = "terminated (%s)" % (signum, )
request.terminate(panel.consumer.pool, signal=signum)
break
panel.logger.info("Task %s %s." % (task_id, action))
return {"ok": "task %s %s" % (task_id, action)}
def revoke(panel, task_id, terminate=False, signal=None, **kwargs):
"""Revoke task by task id."""
revoked.add(task_id)
if terminate:
signum = _signals.signum(signal or 'TERM')
for request in state.reserved_requests:
if request.id == task_id:
logger.info('Terminating %s (%s)', task_id, signum)
request.terminate(panel.consumer.pool, signal=signum)
break
else:
return {'ok': 'terminate: task %s not found' % (task_id, )}
return {'ok': 'terminating %s (%s)' % (task_id, signal)}
logger.info('Revoking task %s', task_id)
return {'ok': 'revoking task %s' % (task_id, )}
def revoke(state, task_id, terminate=False, signal=None, **kwargs):
"""Revoke task by task id."""
revoked.add(task_id)
if terminate:
signum = _signals.signum(signal or "TERM")
for request in worker_state.reserved_requests:
if request.id == task_id:
logger.info("Terminating %s (%s)", task_id, signum)
request.terminate(state.consumer.pool, signal=signum)
break
else:
return {"ok": "terminate: task {0} not found".format(task_id)}
return {"ok": "terminating {0} ({1})".format(task_id, signal)}
logger.info("Revoking task %s", task_id)
return {"ok": "revoking task {0}".format(task_id)}
def revoke(panel, task_id, terminate=False, signal=None, **kwargs):
"""Revoke task by task id."""
revoked.add(task_id)
if terminate:
signum = _signals.signum(signal or 'TERM')
for request in state.reserved_requests:
if request.id == task_id:
logger.info('Terminating %s (%s)', task_id, signum)
request.terminate(panel.consumer.pool, signal=signum)
break
else:
return {'ok': 'terminate: task %s not found' % (task_id, )}
return {'ok': 'terminating %s (%s)' % (task_id, signal)}
logger.info('Revoking task %s', task_id)
return {'ok': 'revoking task %s' % (task_id, )}
def test_ignore(self, set):
signals.ignore("SIGINT")
set.assert_called_with(signals.signum("INT"), signals.ignored)
signals.ignore("SIGTERM")
set.assert_called_with(signals.signum("TERM"), signals.ignored)
def test_reset(self, set):
signals.reset('SIGINT')
set.assert_called_with(signals.signum('INT'), signals.default)
def test_ignore(self, set):
signals.ignore('SIGINT')
set.assert_called_with(signals.signum('INT'), signals.ignored)
signals.ignore('SIGTERM')
set.assert_called_with(signals.signum('TERM'), signals.ignored)
def test_ignore(self, set):
signals.ignore("SIGINT")
set.assert_called_with(signals.signum("INT"), signals.ignored)
signals.ignore("SIGTERM")
set.assert_called_with(signals.signum("TERM"), signals.ignored)
def test_ignore(self, set):
signals.ignore('SIGINT')
set.assert_called_with(signals.signum('INT'), signals.ignored)
signals.ignore('SIGTERM')
set.assert_called_with(signals.signum('TERM'), signals.ignored)
def test_reset(self, set):
signals.reset('SIGINT')
set.assert_called_with(signals.signum('INT'), signals.default)
def test_reset(self, set):
signals.reset("SIGINT")
set.assert_called_with(signals.signum("INT"), signals.default)
