def __init__(self,
handle_task,
init_callback=noop,
hostname=None,
pool=None,
app=None,
timer=None,
controller=None,
hub=None,
amqheartbeat=None,
worker_options=None,
disable_rate_limits=False,
**kwargs):
self.app = app_or_default(app)
self.controller = controller
self.init_callback = init_callback
self.hostname = hostname or socket.gethostname()
self.pool = pool
self.timer = timer or default_timer
self.strategies = {}
conninfo = self.app.connection()
self.connection_errors = conninfo.connection_errors
self.channel_errors = conninfo.channel_errors
self._restart_state = restart_state(maxR=5, maxT=1)
self._does_info = logger.isEnabledFor(logging.INFO)
self.handle_task = handle_task
self.amqheartbeat_rate = self.app.conf.BROKER_HEARTBEAT_CHECKRATE
self.disable_rate_limits = disable_rate_limits
# this contains a tokenbucket for each task type by name, used for
# rate limits, or None if rate limits are disabled for that task.
self.task_buckets = defaultdict(lambda: None)
self.reset_rate_limits()
if hub:
self.amqheartbeat = amqheartbeat
if self.amqheartbeat is None:
self.amqheartbeat = self.app.conf.BROKER_HEARTBEAT
self.hub = hub
self.hub.on_init.append(self.on_poll_init)
else:
self.hub = None
self.amqheartbeat = 0
if not hasattr(self, 'loop'):
self.loop = loops.asynloop if hub else loops.synloop
if _detect_environment() == 'gevent':
# there's a gevent bug that causes timeouts to not be reset,
# so if the connection timeout is exceeded once, it can NEVER
# connect again.
self.app.conf.BROKER_CONNECTION_TIMEOUT = None
self.steps = []
self.blueprint = self.Blueprint(
app=self.app,
on_close=self.on_close,
)
self.blueprint.apply(self, **dict(worker_options or {}, **kwargs))
def __init__(self, on_task_request,
init_callback=noop, hostname=None,
pool=None, app=None,
timer=None, controller=None, hub=None, amqheartbeat=None,
worker_options=None, disable_rate_limits=False,
initial_prefetch_count=2, prefetch_multiplier=1, **kwargs):
self.app = app
self.controller = controller
self.init_callback = init_callback
self.hostname = hostname or gethostname()
self.pid = os.getpid()
self.pool = pool
self.timer = timer
self.strategies = self.Strategies()
self.conninfo = self.app.connection_for_read()
self.connection_errors = self.conninfo.connection_errors
self.channel_errors = self.conninfo.channel_errors
self._restart_state = restart_state(maxR=5, maxT=1)
self._does_info = logger.isEnabledFor(logging.INFO)
self._limit_order = 0
self.on_task_request = on_task_request
self.on_task_message = set()
self.amqheartbeat_rate = self.app.conf.broker_heartbeat_checkrate
self.disable_rate_limits = disable_rate_limits
self.initial_prefetch_count = initial_prefetch_count
self.prefetch_multiplier = prefetch_multiplier
# this contains a tokenbucket for each task type by name, used for
# rate limits, or None if rate limits are disabled for that task.
self.task_buckets = defaultdict(lambda: None)
self.reset_rate_limits()
self.gevent_env = _detect_environment() == 'gevent'
self.hub = hub
if self.hub or getattr(self.pool, 'is_green', False) or self.gevent_env:
self.amqheartbeat = amqheartbeat
if self.amqheartbeat is None:
self.amqheartbeat = self.app.conf.broker_heartbeat
else:
self.amqheartbeat = 0
if not hasattr(self, 'loop'):
self.loop = loops.asynloop if hub else loops.synloop
if self.gevent_env:
# there's a gevent bug that causes timeouts to not be reset,
# so if the connection timeout is exceeded once, it can NEVER
# connect again.
self.app.conf.broker_connection_timeout = None
self._pending_operations = []
self.steps = []
self.blueprint = self.Blueprint(
steps=self.app.steps['consumer'],
on_close=self.on_close,
)
self.blueprint.apply(self, **dict(worker_options or {}, **kwargs))
def __init__(self, on_task_request,
init_callback=noop, hostname=None,
pool=None, app=None,
timer=None, controller=None, hub=None, amqheartbeat=None,
worker_options=None, disable_rate_limits=False,
initial_prefetch_count=2, prefetch_multiplier=1, **kwargs):
self.app = app
self.controller = controller
self.init_callback = init_callback
self.hostname = hostname or gethostname()
self.pid = os.getpid()
self.pool = pool
self.timer = timer
self.strategies = self.Strategies()
self.conninfo = self.app.connection_for_read()
self.connection_errors = self.conninfo.connection_errors
self.channel_errors = self.conninfo.channel_errors
self._restart_state = restart_state(maxR=5, maxT=1)
self._does_info = logger.isEnabledFor(logging.INFO)
self._limit_order = 0
self.on_task_request = on_task_request
self.on_task_message = set()
self.amqheartbeat_rate = self.app.conf.broker_heartbeat_checkrate
self.disable_rate_limits = disable_rate_limits
self.initial_prefetch_count = initial_prefetch_count
self.prefetch_multiplier = prefetch_multiplier
self._maximum_prefetch_restored = True
# this contains a tokenbucket for each task type by name, used for
# rate limits, or None if rate limits are disabled for that task.
self.task_buckets = defaultdict(lambda: None)
self.reset_rate_limits()
self.hub = hub
if self.hub or getattr(self.pool, 'is_green', False):
self.amqheartbeat = amqheartbeat
if self.amqheartbeat is None:
self.amqheartbeat = self.app.conf.broker_heartbeat
else:
self.amqheartbeat = 0
if not hasattr(self, 'loop'):
self.loop = loops.asynloop if hub else loops.synloop
if _detect_environment() == 'gevent':
# there's a gevent bug that causes timeouts to not be reset,
# so if the connection timeout is exceeded once, it can NEVER
# connect again.
self.app.conf.broker_connection_timeout = None
self._pending_operations = []
self.steps = []
self.blueprint = self.Blueprint(
steps=self.app.steps['consumer'],
on_close=self.on_close,
)
self.blueprint.apply(self, **dict(worker_options or {}, **kwargs))
def test_time_passed_resets_counter(self):
s = restart_state(100, 10)
s.R, s.T = 100, time()
with self.assertRaises(s.RestartFreqExceeded):
s.step()
s.R, s.T = 100, time()
s.step(time() + 20)
self.assertEqual(s.R, 1)
def test_time_passed_resets_counter(self):
s = restart_state(100, 10)
s.R, s.T = 100, time()
with self.assertRaises(s.RestartFreqExceeded):
s.step()
s.R, s.T = 100, time()
s.step(time() + 20)
self.assertEqual(s.R, 1)
def test_time_passed_resets_counter(self):
s = restart_state(100, 10)
s.R, s.T = 100, time()
with pytest.raises(s.RestartFreqExceeded):
s.step()
s.R, s.T = 100, time()
s.step(time() + 20)
assert s.R == 1
def test_time_passed_resets_counter(self):
s = restart_state(100, 10)
s.R, s.T = 100, time()
with pytest.raises(s.RestartFreqExceeded):
s.step()
s.R, s.T = 100, time()
s.step(time() + 20)
assert s.R == 1
def __init__(self,
ready_queue,
init_callback=noop,
hostname=None,
pool=None,
app=None,
timer=None,
controller=None,
hub=None,
amqheartbeat=None,
worker_options=None,
**kwargs):
self.app = app_or_default(app)
self.controller = controller
self.ready_queue = ready_queue
self.init_callback = init_callback
self.hostname = hostname or socket.gethostname()
self.pool = pool
self.timer = timer or default_timer
self.strategies = {}
conninfo = self.app.connection()
self.connection_errors = conninfo.connection_errors
self.channel_errors = conninfo.channel_errors
self._restart_state = restart_state(maxR=5, maxT=1)
self._does_info = logger.isEnabledFor(logging.INFO)
self._quick_put = self.ready_queue.put
self.amqheartbeat_rate = self.app.conf.BROKER_HEARTBEAT_CHECKRATE
if hub:
self.amqheartbeat = amqheartbeat
if self.amqheartbeat is None:
self.amqheartbeat = self.app.conf.BROKER_HEARTBEAT
self.hub = hub
self.hub.on_init.append(self.on_poll_init)
else:
self.hub = None
self.amqheartbeat = 0
if not hasattr(self, 'loop'):
self.loop = loops.asynloop if hub else loops.synloop
if _detect_environment() == 'gevent':
# there's a gevent bug that causes timeouts to not be reset,
# so if the connection timeout is exceeded once, it can NEVER
# connect again.
self.app.conf.BROKER_CONNECTION_TIMEOUT = None
self.steps = []
self.namespace = self.Namespace(
app=self.app,
on_close=self.on_close,
)
self.namespace.apply(self, **dict(worker_options or {}, **kwargs))
def __init__(self, on_task,
init_callback=noop, hostname=None,
pool=None, app=None,
timer=None, controller=None, hub=None, amqheartbeat=None,
worker_options=None, disable_rate_limits=False, **kwargs):
self.app = app
self.controller = controller
self.init_callback = init_callback
self.hostname = hostname or socket.gethostname()
self.pid = os.getpid()
self.pool = pool
self.timer = timer
self.strategies = self.Strategies()
conninfo = self.app.connection()
self.connection_errors = conninfo.connection_errors
self.channel_errors = conninfo.channel_errors
self._restart_state = restart_state(maxR=5, maxT=1)
self._does_info = logger.isEnabledFor(logging.INFO)
self.on_task = on_task
self.amqheartbeat_rate = self.app.conf.BROKER_HEARTBEAT_CHECKRATE
self.disable_rate_limits = disable_rate_limits
# this contains a tokenbucket for each task type by name, used for
# rate limits, or None if rate limits are disabled for that task.
self.task_buckets = defaultdict(lambda: None)
self.reset_rate_limits()
if hub:
self.amqheartbeat = amqheartbeat
if self.amqheartbeat is None:
self.amqheartbeat = self.app.conf.BROKER_HEARTBEAT
self.hub = hub
self.hub.on_init.append(self.on_poll_init)
else:
self.hub = None
self.amqheartbeat = 0
if not hasattr(self, 'loop'):
self.loop = loops.asynloop if hub else loops.synloop
if _detect_environment() == 'gevent':
# there's a gevent bug that causes timeouts to not be reset,
# so if the connection timeout is exceeded once, it can NEVER
# connect again.
self.app.conf.BROKER_CONNECTION_TIMEOUT = None
self.steps = []
self.blueprint = self.Blueprint(
app=self.app, on_close=self.on_close,
)
self.blueprint.apply(self, **dict(worker_options or {}, **kwargs))
def __init__(self, ready_queue,
init_callback=noop, hostname=None,
pool=None, app=None,
timer=None, controller=None, hub=None, amqheartbeat=None,
worker_options=None, **kwargs):
self.app = app_or_default(app)
self.controller = controller
self.ready_queue = ready_queue
self.init_callback = init_callback
self.hostname = hostname or socket.gethostname()
self.pool = pool
self.timer = timer or default_timer
self.strategies = {}
conninfo = self.app.connection()
self.connection_errors = conninfo.connection_errors
self.channel_errors = conninfo.channel_errors
self._restart_state = restart_state(maxR=5, maxT=1)
self._does_info = logger.isEnabledFor(logging.INFO)
self._quick_put = self.ready_queue.put
self.amqheartbeat_rate = self.app.conf.BROKER_HEARTBEAT_CHECKRATE
if hub:
self.amqheartbeat = amqheartbeat
if self.amqheartbeat is None:
self.amqheartbeat = self.app.conf.BROKER_HEARTBEAT
self.hub = hub
self.hub.on_init.append(self.on_poll_init)
else:
self.hub = None
self.amqheartbeat = 0
if not hasattr(self, 'loop'):
self.loop = loops.asynloop if hub else loops.synloop
if _detect_environment() == 'gevent':
# there's a gevent bug that causes timeouts to not be reset,
# so if the connection timeout is exceeded once, it can NEVER
# connect again.
self.app.conf.BROKER_CONNECTION_TIMEOUT = None
self.steps = []
self.namespace = self.Namespace(
app=self.app, on_close=self.on_close,
)
self.namespace.apply(self, **dict(worker_options or {}, **kwargs))
def test_raises(self):
s = restart_state(100, 1) # max 100 restarts in 1 second.
s.R = 99
s.step()
with self.assertRaises(s.RestartFreqExceeded):
s.step()
def test_raises(self):
s = restart_state(100, 1) # max 100 restarts in 1 second.
s.R = 99
s.step()
with self.assertRaises(s.RestartFreqExceeded):
s.step()
def __init__(self,
on_task_request,
init_callback=noop,
hostname=None,
pool=None,
app=None,
timer=None,
controller=None,
hub=None,
amqheartbeat=None,
worker_options=None,
disable_rate_limits=False,
initial_prefetch_count=2,
prefetch_multiplier=1,
**kwargs):
self.app = app
self.controller = controller
self.init_callback = init_callback
self.hostname = hostname or socket.gethostname()
self.pid = os.getpid()
self.pool = pool
self.timer = timer
self.strategies = self.Strategies()
conninfo = self.app.connection()
self.connection_errors = conninfo.connection_errors
self.channel_errors = conninfo.channel_errors
self._restart_state = restart_state(maxR=5, maxT=1)
self._does_info = logger.isEnabledFor(logging.INFO)
self.on_task_request = on_task_request
self.on_task_message = set()
self.amqheartbeat_rate = self.app.conf.BROKER_HEARTBEAT_CHECKRATE
self.disable_rate_limits = disable_rate_limits
self.initial_prefetch_count = initial_prefetch_count
self.prefetch_multiplier = prefetch_multiplier
# this contains a tokenbucket for each task type by name, used for
# rate limits, or None if rate limits are disabled for that task.
self.task_buckets = defaultdict(lambda: None)
self.reset_rate_limits()
self.hub = hub
if self.hub:
self.amqheartbeat = amqheartbeat
if self.amqheartbeat is None:
self.amqheartbeat = self.app.conf.BROKER_HEARTBEAT
else:
self.amqheartbeat = 0
if not hasattr(self, "loop"):
self.loop = loops.asynloop if hub else loops.synloop
if _detect_environment() == "gevent":
# there's a gevent bug that causes timeouts to not be reset,
# so if the connection timeout is exceeded once, it can NEVER
# connect again.
self.app.conf.BROKER_CONNECTION_TIMEOUT = None
self.steps = []
self.blueprint = self.Blueprint(
app=self.app,
on_close=self.on_close,
)
self.blueprint.apply(self, **dict(worker_options or {}, **kwargs))
