def start(self):
blueprint = self.blueprint
while blueprint.state not in STOP_CONDITIONS:
maybe_shutdown()
if self.restart_count:
try:
self._restart_state.step()
except RestartFreqExceeded as exc:
crit('Frequent restarts detected: %r', exc, exc_info=1)
sleep(1)
self.restart_count += 1
try:
blueprint.start(self)
except self.connection_errors as exc:
# If we're not retrying connections, no need to catch
# connection errors
if not self.app.conf.broker_connection_retry:
raise
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
raise # Too many open files
maybe_shutdown()
if blueprint.state not in STOP_CONDITIONS:
if self.connection:
self.on_connection_error_after_connected(exc)
else:
self.on_connection_error_before_connected(exc)
self.on_close()
blueprint.restart(self)
def start(self):
blueprint = self.blueprint
while blueprint.state not in STOP_CONDITIONS:
maybe_shutdown()
if self.restart_count:
try:
self._restart_state.step()
except RestartFreqExceeded as exc:
crit('Frequent restarts detected: %r', exc, exc_info=1)
sleep(1)
self.restart_count += 1
try:
blueprint.start(self)
except self.connection_errors as exc:
# If we're not retrying connections, no need to catch
# connection errors
if not self.app.conf.broker_connection_retry:
raise
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
raise # Too many open files
maybe_shutdown()
if blueprint.state not in STOP_CONDITIONS:
if self.connection:
self.on_connection_error_after_connected(exc)
else:
self.on_connection_error_before_connected(exc)
self.on_close()
blueprint.restart(self)
def start(self):
blueprint = self.blueprint
while blueprint.state != CLOSE:
self.restart_count += 1
maybe_shutdown()
try:
blueprint.start(self)
except self.connection_errors as exc:
# If we're not retrying connections, no need to catch
# connection errors
if not self.app.conf.broker_connection_retry:
raise
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
raise # Too many open files
maybe_shutdown()
try:
self._restart_state.step()
except RestartFreqExceeded as exc:
crit('Frequent restarts detected: %r', exc, exc_info=1)
sleep(1)
if blueprint.state != CLOSE and self.connection:
warn(CONNECTION_RETRY, exc_info=True)
try:
self.connection.collect()
except Exception:
pass
self.on_close()
blueprint.restart(self)
def synloop(obj,
connection,
consumer,
blueprint,
hub,
qos,
heartbeat,
clock,
hbrate=2.0,
**kwargs):
"""Fallback blocking event loop for transports that doesn't support AIO."""
RUN = bootsteps.RUN
on_task_received = obj.create_task_handler()
perform_pending_operations = obj.perform_pending_operations
if getattr(obj.pool, 'is_green', False):
_enable_amqheartbeats(obj.timer, connection, rate=hbrate)
consumer.on_message = on_task_received
consumer.consume()
obj.on_ready()
while blueprint.state == RUN and obj.connection:
state.maybe_shutdown()
if qos.prev != qos.value:
qos.update()
try:
perform_pending_operations()
connection.drain_events(timeout=2.0)
except socket.timeout:
pass
except socket.error:
if blueprint.state == RUN:
raise
def start(self):
blueprint = self.blueprint
while blueprint.state != CLOSE:
self.restart_count += 1
maybe_shutdown()
try:
blueprint.start(self)
except self.connection_errors as exc:
# If we're not retrying connections, no need to catch
# connection errors
if not self.app.conf.broker_connection_retry:
raise
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
raise # Too many open files
maybe_shutdown()
try:
self._restart_state.step()
except RestartFreqExceeded as exc:
crit('Frequent restarts detected: %r', exc, exc_info=1)
sleep(1)
if blueprint.state != CLOSE and self.connection:
warn(CONNECTION_RETRY, exc_info=True)
try:
self.connection.collect()
except Exception:
pass
self.on_close()
blueprint.restart(self)
def start(self):
blueprint = self.blueprint
while blueprint.state not in STOP_CONDITIONS:
maybe_shutdown()
if self.restart_count:
try:
self._restart_state.step()
except RestartFreqExceeded as exc:
crit('Frequent restarts detected: %r', exc, exc_info=1)
sleep(1)
self.restart_count += 1
try:
blueprint.start(self)
except self.connection_errors as exc:
# If we're not retrying connections, we need to properly shutdown or terminate
# the Celery main process instead of abruptly aborting the process without any cleanup.
is_connection_loss_on_startup = self.restart_count == 0
connection_retry_type = self._get_connection_retry_type(
is_connection_loss_on_startup)
connection_retry = self.app.conf[connection_retry_type]
if not connection_retry:
crit(
f"Retrying to {'establish' if is_connection_loss_on_startup else 're-establish'} "
f"a connection to the message broker after a connection loss has "
f"been disabled (app.conf.{connection_retry_type}=False). Shutting down..."
)
raise WorkerShutdown(1) from exc
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
crit("Too many open files. Aborting...")
raise WorkerTerminate(1) from exc
maybe_shutdown()
if blueprint.state not in STOP_CONDITIONS:
if self.connection:
self.on_connection_error_after_connected(exc)
else:
self.on_connection_error_before_connected(exc)
self.on_close()
blueprint.restart(self)
def test_should_terminate(self):
state.should_terminate = True
with self.assertRaises(SystemTerminate):
state.maybe_shutdown()
def test_should_stop(self):
state.should_stop = True
with self.assertRaises(SystemExit):
state.maybe_shutdown()
def test_should_terminate(self, should_stop):
state.should_stop = should_stop
state.should_terminate = True
with pytest.raises(WorkerTerminate):
state.maybe_shutdown()
def test_should_stop(self):
state.should_stop = True
with pytest.raises(WorkerShutdown):
state.maybe_shutdown()
state.should_stop = 0
with pytest.raises(WorkerShutdown):
state.maybe_shutdown()
state.should_stop = False
try:
state.maybe_shutdown()
except SystemExit:
raise RuntimeError('should not have exited')
state.should_stop = None
try:
state.maybe_shutdown()
except SystemExit:
raise RuntimeError('should not have exited')
state.should_stop = 0
try:
state.maybe_shutdown()
except SystemExit as exc:
assert exc.code == 0
else:
raise RuntimeError('should have exited')
state.should_stop = 303
try:
state.maybe_shutdown()
except SystemExit as exc:
assert exc.code == 303
else:
raise RuntimeError('should have exited')
def test_should_stop(self):
state.should_stop = True
with self.assertRaises(WorkerShutdown):
state.maybe_shutdown()
state.should_stop = 0
with self.assertRaises(WorkerShutdown):
state.maybe_shutdown()
state.should_stop = False
try:
state.maybe_shutdown()
except SystemExit:
raise RuntimeError('should not have exited')
state.should_stop = None
try:
state.maybe_shutdown()
except SystemExit:
raise RuntimeError('should not have exited')
state.should_stop = 0
try:
state.maybe_shutdown()
except SystemExit as exc:
self.assertEqual(exc.code, 0)
else:
raise RuntimeError('should have exited')
state.should_stop = 303
try:
state.maybe_shutdown()
except SystemExit as exc:
self.assertEqual(exc.code, 303)
else:
raise RuntimeError('should have exited')
def test_should_stop(self):
state.should_stop = True
with self.assertRaises(WorkerShutdown):
state.maybe_shutdown()
def test_should_stop(self):
state.should_stop = True
with self.assertRaises(WorkerShutdown):
state.maybe_shutdown()
def test_should_terminate(self):
state.should_terminate = True
with pytest.raises(WorkerTerminate):
state.maybe_shutdown()
def test_should_stop(self):
state.should_stop = True
with pytest.raises(WorkerShutdown):
state.maybe_shutdown()
state.should_stop = 0
with pytest.raises(WorkerShutdown):
state.maybe_shutdown()
state.should_stop = False
try:
state.maybe_shutdown()
except SystemExit:
raise RuntimeError('should not have exited')
state.should_stop = None
try:
state.maybe_shutdown()
except SystemExit:
raise RuntimeError('should not have exited')
state.should_stop = 0
try:
state.maybe_shutdown()
except SystemExit as exc:
assert exc.code == 0
else:
raise RuntimeError('should have exited')
state.should_stop = 303
try:
state.maybe_shutdown()
except SystemExit as exc:
assert exc.code == 303
else:
raise RuntimeError('should have exited')
