def rate_limit(state, task_name, rate_limit, **kwargs):
"""Tell worker(s) to modify the rate limit for a task by type.
See Also:
:attr:`celery.task.base.Task.rate_limit`.
Arguments:
task_name (str): Type of task to set rate limit for.
rate_limit (int, str): New rate limit.
"""
try:
timeutils.rate(rate_limit)
except ValueError as exc:
return nok('Invalid rate limit string: {0!r}'.format(exc))
try:
state.app.tasks[task_name].rate_limit = rate_limit
except KeyError:
logger.error('Rate limit attempt for unknown task %s',
task_name,
exc_info=True)
return nok('unknown task')
state.consumer.reset_rate_limits()
if not rate_limit:
logger.info('Rate limits disabled for tasks of type %s', task_name)
return ok('rate limit disabled successfully')
logger.info('New rate limit for tasks of type %s: %s.', task_name,
rate_limit)
return ok('new rate limit set successfully')
def rate_limit(panel, task_name, rate_limit, **kwargs):
"""Set new rate limit for a task type.
See :attr:`celery.task.base.Task.rate_limit`.
:param task_name: Type of task.
:param rate_limit: New rate limit.
"""
try:
timeutils.rate(rate_limit)
except ValueError as exc:
return {"error": "Invalid rate limit string: {0!r}".format(exc)}
try:
panel.app.tasks[task_name].rate_limit = rate_limit
except KeyError:
logger.error("Rate limit attempt for unknown task %s", task_name, exc_info=True)
return {"error": "unknown task"}
panel.consumer.reset_rate_limits()
if not rate_limit:
logger.info("Rate limits disabled for tasks of type %s", task_name)
return {"ok": "rate limit disabled successfully"}
logger.info("New rate limit for tasks of type %s: %s.", task_name, rate_limit)
return {"ok": "new rate limit set successfully"}
def rate_limit(state, task_name, rate_limit, **kwargs):
"""Set new rate limit for a task type.
See :attr:`celery.task.base.Task.rate_limit`.
:param task_name: Type of task.
:param rate_limit: New rate limit.
"""
try:
timeutils.rate(rate_limit)
except ValueError as exc:
return {'error': 'Invalid rate limit string: {0!r}'.format(exc)}
try:
state.app.tasks[task_name].rate_limit = rate_limit
except KeyError:
logger.error('Rate limit attempt for unknown task %s',
task_name,
exc_info=True)
return {'error': 'unknown task'}
state.consumer.reset_rate_limits()
if not rate_limit:
logger.info('Rate limits disabled for tasks of type %s', task_name)
return {'ok': 'rate limit disabled successfully'}
logger.info('New rate limit for tasks of type %s: %s.', task_name,
rate_limit)
return {'ok': 'new rate limit set successfully'}
def rate_limit(state, task_name, rate_limit, **kwargs):
"""Tell worker(s) to modify the rate limit for a task by type.
See Also:
:attr:`celery.task.base.Task.rate_limit`.
Arguments:
task_name (str): Type of task to set rate limit for.
rate_limit (int, str): New rate limit.
"""
try:
timeutils.rate(rate_limit)
except ValueError as exc:
return nok('Invalid rate limit string: {0!r}'.format(exc))
try:
state.app.tasks[task_name].rate_limit = rate_limit
except KeyError:
logger.error('Rate limit attempt for unknown task %s',
task_name, exc_info=True)
return nok('unknown task')
state.consumer.reset_rate_limits()
if not rate_limit:
logger.info('Rate limits disabled for tasks of type %s', task_name)
return ok('rate limit disabled successfully')
logger.info('New rate limit for tasks of type %s: %s.',
task_name, rate_limit)
return ok('new rate limit set successfully')
def rate_limit(state, task_name, rate_limit, **kwargs):
"""Set new rate limit for a task type.
See :attr:`celery.task.base.Task.rate_limit`.
:param task_name: Type of task.
:param rate_limit: New rate limit.
"""
try:
timeutils.rate(rate_limit)
except ValueError as exc:
return {'error': 'Invalid rate limit string: {0!r}'.format(exc)}
try:
state.app.tasks[task_name].rate_limit = rate_limit
except KeyError:
logger.error('Rate limit attempt for unknown task %s',
task_name, exc_info=True)
return {'error': 'unknown task'}
state.consumer.reset_rate_limits()
if not rate_limit:
logger.info('Rate limits disabled for tasks of type %s', task_name)
return {'ok': 'rate limit disabled successfully'}
logger.info('New rate limit for tasks of type %s: %s.',
task_name, rate_limit)
return {'ok': 'new rate limit set successfully'}
def test_conversion(self):
self.assertEqual(timeutils.rate(999), 999)
self.assertEqual(timeutils.rate("1456/s"), 1456)
self.assertEqual(timeutils.rate("100/m"), 100 / 60.0)
self.assertEqual(timeutils.rate("10/h"), 10 / 60.0 / 60.0)
for zero in (0, None, "0", "0/m", "0/h", "0/s"):
self.assertEqual(timeutils.rate(zero), 0)
def test_conversion(self):
self.assertEqual(timeutils.rate(999), 999)
self.assertEqual(timeutils.rate("1456/s"), 1456)
self.assertEqual(timeutils.rate("100/m"), 100 / 60.0)
self.assertEqual(timeutils.rate("10/h"), 10 / 60.0 / 60.0)
for zero in (0, None, "0", "0/m", "0/h", "0/s"):
self.assertEqual(timeutils.rate(zero), 0)
def test_conversion(self):
self.assertEqual(timeutils.rate(999), 999)
self.assertEqual(timeutils.rate('1456/s'), 1456)
self.assertEqual(timeutils.rate('100/m'), 100 / 60.0)
self.assertEqual(timeutils.rate('10/h'), 10 / 60.0 / 60.0)
for zero in (0, None, '0', '0/m', '0/h', '0/s'):
self.assertEqual(timeutils.rate(zero), 0)
def test_conversion(self):
self.assertEqual(timeutils.rate(999), 999)
self.assertEqual(timeutils.rate('1456/s'), 1456)
self.assertEqual(timeutils.rate('100/m'),
100 / 60.0)
self.assertEqual(timeutils.rate('10/h'),
10 / 60.0 / 60.0)
for zero in (0, None, '0', '0/m', '0/h', '0/s'):
self.assertEqual(timeutils.rate(zero), 0)
def rate_limit(panel, task_name, rate_limit, **kwargs):
"""Set new rate limit for a task type.
See :attr:`celery.task.base.Task.rate_limit`.
:param task_name: Type of task.
:param rate_limit: New rate limit.
"""
try:
timeutils.rate(rate_limit)
except ValueError, exc:
return {'error': 'Invalid rate limit string: %s' % exc}
def rate_limit(panel, task_name, rate_limit, **kwargs):
"""Set new rate limit for a task type.
See :attr:`celery.task.base.Task.rate_limit`.
:param task_name: Type of task.
:param rate_limit: New rate limit.
"""
try:
timeutils.rate(rate_limit)
except ValueError, exc:
return {"error": "Invalid rate limit string: %s" % exc}
def add_task_type(self, name):
task_type = self.task_registry[name]
limit = getattr(task_type, 'rate_limit', None)
limit = timeutils.rate(limit)
bucket = self.buckets[name] = (TokenBucket(limit, capacity=1)
if limit else None)
return bucket
def _context(self,
sig,
rate_limits=True,
events=True,
utc=True,
limit=None):
self.assertTrue(sig.type.Strategy)
reserved = Mock()
consumer = Mock()
consumer.task_buckets = defaultdict(lambda: None)
if limit:
bucket = TokenBucket(rate(limit), capacity=1)
consumer.task_buckets[sig.task] = bucket
consumer.controller.state.revoked = set()
consumer.disable_rate_limits = not rate_limits
consumer.event_dispatcher.enabled = events
s = sig.type.start_strategy(self.app, consumer, task_reserved=reserved)
self.assertTrue(s)
message = task_message_from_sig(
self.app,
sig,
utc=utc,
TaskMessage=self.get_message_class(),
)
message = self.prepare_message(message)
yield self.Context(sig, s, reserved, consumer, message)
def __init__(self, state, freq=1.0, maxrate=None,
cleanup_freq=3600.0, logger=None):
self.state = state
self.freq = freq
self.cleanup_freq = cleanup_freq
self.logger = logger
self.maxrate = maxrate and TokenBucket(rate(maxrate))
def __init__(self, state, freq=1.0, maxrate=None,
cleanup_freq=3600.0, logger=None):
self.state = state
self.freq = freq
self.cleanup_freq = cleanup_freq
self.logger = logger or log.get_default_logger(name="celery.cam")
self.maxrate = maxrate and TokenBucket(rate(maxrate))
def __init__(self, state, freq=1.0, maxrate=None,
cleanup_freq=3600.0, logger=None):
self.state = state
self.freq = freq
self.cleanup_freq = cleanup_freq
self.logger = logger or log.get_default_logger(name="celery.cam")
self.maxrate = maxrate and TokenBucket(rate(maxrate))
def add_task_type(self, name):
task_type = self.task_registry[name]
limit = getattr(task_type, 'rate_limit', None)
limit = timeutils.rate(limit)
bucket = self.buckets[name] = (
TokenBucket(limit, capacity=1) if limit else None
)
return bucket
def __init__(self, state, freq=1.0, maxrate=None, cleanup_freq=3600.0, timer=None, app=None):
self.app = app_or_default(app)
self.state = state
self.freq = freq
self.cleanup_freq = cleanup_freq
self.timer = timer or self.timer
self.logger = logger
self.maxrate = maxrate and TokenBucket(rate(maxrate))
def __init__(self, state, freq=1.0, maxrate=None,
cleanup_freq=3600.0, timer=None, app=None):
self.app = app_or_default(app)
self.state = state
self.freq = freq
self.cleanup_freq = cleanup_freq
self.timer = timer or self.timer
self.logger = logger
self.maxrate = maxrate and TokenBucket(rate(maxrate))
def test_conversion(self):
self.assertEqual(rate(999), 999)
self.assertEqual(rate(7.5), 7.5)
self.assertEqual(rate('2.5/s'), 2.5)
self.assertEqual(rate('1456/s'), 1456)
self.assertEqual(rate('100/m'), 100 / 60.0)
self.assertEqual(rate('10/h'), 10 / 60.0 / 60.0)
for zero in (0, None, '0', '0/m', '0/h', '0/s', '0.0/s'):
self.assertEqual(rate(zero), 0)
def test_conversion(self):
self.assertEqual(rate(999), 999)
self.assertEqual(rate(7.5), 7.5)
self.assertEqual(rate('2.5/s'), 2.5)
self.assertEqual(rate('1456/s'), 1456)
self.assertEqual(rate('100/m'),
100 / 60.0)
self.assertEqual(rate('10/h'),
10 / 60.0 / 60.0)
for zero in (0, None, '0', '0/m', '0/h', '0/s', '0.0/s'):
self.assertEqual(rate(zero), 0)
def update_bucket_for_type(self, task_name):
task_type = self.task_registry[task_name]
rate_limit = getattr(task_type, "rate_limit", None)
rate_limit = timeutils.rate(rate_limit)
if task_name in self.buckets:
task_queue = self._get_queue_for_type(task_name)
else:
task_queue = FastQueue()
if rate_limit:
task_queue = TokenBucketQueue(rate_limit, queue=task_queue)
self.buckets[task_name] = task_queue
return task_queue
def update_bucket_for_type(self, task_name):
task_type = self.task_registry[task_name]
rate_limit = getattr(task_type, 'rate_limit', None)
rate_limit = timeutils.rate(rate_limit)
task_queue = FastQueue()
if task_name in self.buckets:
task_queue = self._get_queue_for_type(task_name)
else:
task_queue = FastQueue()
if rate_limit:
task_queue = TokenBucketQueue(rate_limit, queue=task_queue)
self.buckets[task_name] = task_queue
return task_queue
def _context(self, sig, rate_limits=True, events=True, utc=True, limit=None):
self.assertTrue(sig.type.Strategy)
reserved = Mock()
consumer = Mock()
consumer.task_buckets = defaultdict(lambda: None)
if limit:
bucket = TokenBucket(rate(limit), capacity=1)
consumer.task_buckets[sig.task] = bucket
consumer.disable_rate_limits = not rate_limits
consumer.event_dispatcher.enabled = events
s = sig.type.start_strategy(self.app, consumer, task_reserved=reserved)
self.assertTrue(s)
message = Mock()
body = body_from_sig(self.app, sig, utc=utc)
yield self.Context(sig, s, reserved, consumer, message, body)
def _context(self, sig,
rate_limits=True, events=True, utc=True, limit=None):
self.assertTrue(sig.type.Strategy)
reserved = Mock()
consumer = Mock()
consumer.task_buckets = defaultdict(lambda: None)
if limit:
bucket = TokenBucket(rate(limit), capacity=1)
consumer.task_buckets[sig.task] = bucket
consumer.disable_rate_limits = not rate_limits
consumer.event_dispatcher.enabled = events
s = sig.type.start_strategy(self.c, consumer, task_reserved=reserved)
self.assertTrue(s)
message = Mock()
body = body_from_sig(self.c, sig, utc=utc)
yield self.Context(sig, s, reserved, consumer, message, body)
def _context(self, sig,
rate_limits=True, events=True, utc=True, limit=None):
self.assertTrue(sig.type.Strategy)
reserved = Mock()
consumer = Mock()
consumer.task_buckets = defaultdict(lambda: None)
if limit:
bucket = TokenBucket(rate(limit), capacity=1)
consumer.task_buckets[sig.task] = bucket
consumer.controller.state.revoked = set()
consumer.disable_rate_limits = not rate_limits
consumer.event_dispatcher.enabled = events
s = sig.type.start_strategy(self.app, consumer, task_reserved=reserved)
self.assertTrue(s)
message = task_message_from_sig(
self.app, sig, utc=utc, TaskMessage=self.get_message_class(),
)
message = self.prepare_message(message)
yield self.Context(sig, s, reserved, consumer, message)
def bucket_for_task(self, type):
limit = rate(getattr(type, 'rate_limit', None))
return TokenBucket(limit, capacity=1) if limit else None
def bucket_for_task(self, type):
limit = rate(getattr(type, 'rate_limit', None))
return TokenBucket(limit, capacity=1) if limit else None
def __init__(self):
self._buckets = defaultdict(
lambda: TokenBucket(rate(self.restart_max_rate)))
def __init__(self):
self._buckets = defaultdict(lambda: TokenBucket(
rate(self.restart_max_rate)))
