def rate_limit(state, task_name, rate_limit, **kwargs): """Tell worker(s) to modify the rate limit for a task by type. See Also: :attr:`celery.task.base.Task.rate_limit`. Arguments: task_name (str): Type of task to set rate limit for. rate_limit (int, str): New rate limit. """ try: timeutils.rate(rate_limit) except ValueError as exc: return nok('Invalid rate limit string: {0!r}'.format(exc)) try: state.app.tasks[task_name].rate_limit = rate_limit except KeyError: logger.error('Rate limit attempt for unknown task %s', task_name, exc_info=True) return nok('unknown task') state.consumer.reset_rate_limits() if not rate_limit: logger.info('Rate limits disabled for tasks of type %s', task_name) return ok('rate limit disabled successfully') logger.info('New rate limit for tasks of type %s: %s.', task_name, rate_limit) return ok('new rate limit set successfully')
def rate_limit(panel, task_name, rate_limit, **kwargs): """Set new rate limit for a task type. See :attr:`celery.task.base.Task.rate_limit`. :param task_name: Type of task. :param rate_limit: New rate limit. """ try: timeutils.rate(rate_limit) except ValueError as exc: return {"error": "Invalid rate limit string: {0!r}".format(exc)} try: panel.app.tasks[task_name].rate_limit = rate_limit except KeyError: logger.error("Rate limit attempt for unknown task %s", task_name, exc_info=True) return {"error": "unknown task"} panel.consumer.reset_rate_limits() if not rate_limit: logger.info("Rate limits disabled for tasks of type %s", task_name) return {"ok": "rate limit disabled successfully"} logger.info("New rate limit for tasks of type %s: %s.", task_name, rate_limit) return {"ok": "new rate limit set successfully"}
def rate_limit(state, task_name, rate_limit, **kwargs): """Set new rate limit for a task type. See :attr:`celery.task.base.Task.rate_limit`. :param task_name: Type of task. :param rate_limit: New rate limit. """ try: timeutils.rate(rate_limit) except ValueError as exc: return {'error': 'Invalid rate limit string: {0!r}'.format(exc)} try: state.app.tasks[task_name].rate_limit = rate_limit except KeyError: logger.error('Rate limit attempt for unknown task %s', task_name, exc_info=True) return {'error': 'unknown task'} state.consumer.reset_rate_limits() if not rate_limit: logger.info('Rate limits disabled for tasks of type %s', task_name) return {'ok': 'rate limit disabled successfully'} logger.info('New rate limit for tasks of type %s: %s.', task_name, rate_limit) return {'ok': 'new rate limit set successfully'}
def test_conversion(self): self.assertEqual(timeutils.rate(999), 999) self.assertEqual(timeutils.rate("1456/s"), 1456) self.assertEqual(timeutils.rate("100/m"), 100 / 60.0) self.assertEqual(timeutils.rate("10/h"), 10 / 60.0 / 60.0) for zero in (0, None, "0", "0/m", "0/h", "0/s"): self.assertEqual(timeutils.rate(zero), 0)
def test_conversion(self): self.assertEqual(timeutils.rate(999), 999) self.assertEqual(timeutils.rate('1456/s'), 1456) self.assertEqual(timeutils.rate('100/m'), 100 / 60.0) self.assertEqual(timeutils.rate('10/h'), 10 / 60.0 / 60.0) for zero in (0, None, '0', '0/m', '0/h', '0/s'): self.assertEqual(timeutils.rate(zero), 0)
def rate_limit(panel, task_name, rate_limit, **kwargs): """Set new rate limit for a task type. See :attr:`celery.task.base.Task.rate_limit`. :param task_name: Type of task. :param rate_limit: New rate limit. """ try: timeutils.rate(rate_limit) except ValueError, exc: return {'error': 'Invalid rate limit string: %s' % exc}
def rate_limit(panel, task_name, rate_limit, **kwargs): """Set new rate limit for a task type. See :attr:`celery.task.base.Task.rate_limit`. :param task_name: Type of task. :param rate_limit: New rate limit. """ try: timeutils.rate(rate_limit) except ValueError, exc: return {"error": "Invalid rate limit string: %s" % exc}
def add_task_type(self, name): task_type = self.task_registry[name] limit = getattr(task_type, 'rate_limit', None) limit = timeutils.rate(limit) bucket = self.buckets[name] = (TokenBucket(limit, capacity=1) if limit else None) return bucket
def _context(self, sig, rate_limits=True, events=True, utc=True, limit=None): self.assertTrue(sig.type.Strategy) reserved = Mock() consumer = Mock() consumer.task_buckets = defaultdict(lambda: None) if limit: bucket = TokenBucket(rate(limit), capacity=1) consumer.task_buckets[sig.task] = bucket consumer.controller.state.revoked = set() consumer.disable_rate_limits = not rate_limits consumer.event_dispatcher.enabled = events s = sig.type.start_strategy(self.app, consumer, task_reserved=reserved) self.assertTrue(s) message = task_message_from_sig( self.app, sig, utc=utc, TaskMessage=self.get_message_class(), ) message = self.prepare_message(message) yield self.Context(sig, s, reserved, consumer, message)
def __init__(self, state, freq=1.0, maxrate=None, cleanup_freq=3600.0, logger=None): self.state = state self.freq = freq self.cleanup_freq = cleanup_freq self.logger = logger self.maxrate = maxrate and TokenBucket(rate(maxrate))
def __init__(self, state, freq=1.0, maxrate=None, cleanup_freq=3600.0, logger=None): self.state = state self.freq = freq self.cleanup_freq = cleanup_freq self.logger = logger or log.get_default_logger(name="celery.cam") self.maxrate = maxrate and TokenBucket(rate(maxrate))
def add_task_type(self, name): task_type = self.task_registry[name] limit = getattr(task_type, 'rate_limit', None) limit = timeutils.rate(limit) bucket = self.buckets[name] = ( TokenBucket(limit, capacity=1) if limit else None ) return bucket
def __init__(self, state, freq=1.0, maxrate=None, cleanup_freq=3600.0, timer=None, app=None): self.app = app_or_default(app) self.state = state self.freq = freq self.cleanup_freq = cleanup_freq self.timer = timer or self.timer self.logger = logger self.maxrate = maxrate and TokenBucket(rate(maxrate))
def test_conversion(self): self.assertEqual(rate(999), 999) self.assertEqual(rate(7.5), 7.5) self.assertEqual(rate('2.5/s'), 2.5) self.assertEqual(rate('1456/s'), 1456) self.assertEqual(rate('100/m'), 100 / 60.0) self.assertEqual(rate('10/h'), 10 / 60.0 / 60.0) for zero in (0, None, '0', '0/m', '0/h', '0/s', '0.0/s'): self.assertEqual(rate(zero), 0)
def update_bucket_for_type(self, task_name): task_type = self.task_registry[task_name] rate_limit = getattr(task_type, "rate_limit", None) rate_limit = timeutils.rate(rate_limit) if task_name in self.buckets: task_queue = self._get_queue_for_type(task_name) else: task_queue = FastQueue() if rate_limit: task_queue = TokenBucketQueue(rate_limit, queue=task_queue) self.buckets[task_name] = task_queue return task_queue
def update_bucket_for_type(self, task_name): task_type = self.task_registry[task_name] rate_limit = getattr(task_type, 'rate_limit', None) rate_limit = timeutils.rate(rate_limit) task_queue = FastQueue() if task_name in self.buckets: task_queue = self._get_queue_for_type(task_name) else: task_queue = FastQueue() if rate_limit: task_queue = TokenBucketQueue(rate_limit, queue=task_queue) self.buckets[task_name] = task_queue return task_queue
def _context(self, sig, rate_limits=True, events=True, utc=True, limit=None): self.assertTrue(sig.type.Strategy) reserved = Mock() consumer = Mock() consumer.task_buckets = defaultdict(lambda: None) if limit: bucket = TokenBucket(rate(limit), capacity=1) consumer.task_buckets[sig.task] = bucket consumer.disable_rate_limits = not rate_limits consumer.event_dispatcher.enabled = events s = sig.type.start_strategy(self.app, consumer, task_reserved=reserved) self.assertTrue(s) message = Mock() body = body_from_sig(self.app, sig, utc=utc) yield self.Context(sig, s, reserved, consumer, message, body)
def _context(self, sig, rate_limits=True, events=True, utc=True, limit=None): self.assertTrue(sig.type.Strategy) reserved = Mock() consumer = Mock() consumer.task_buckets = defaultdict(lambda: None) if limit: bucket = TokenBucket(rate(limit), capacity=1) consumer.task_buckets[sig.task] = bucket consumer.disable_rate_limits = not rate_limits consumer.event_dispatcher.enabled = events s = sig.type.start_strategy(self.c, consumer, task_reserved=reserved) self.assertTrue(s) message = Mock() body = body_from_sig(self.c, sig, utc=utc) yield self.Context(sig, s, reserved, consumer, message, body)
def bucket_for_task(self, type): limit = rate(getattr(type, 'rate_limit', None)) return TokenBucket(limit, capacity=1) if limit else None
def __init__(self): self._buckets = defaultdict( lambda: TokenBucket(rate(self.restart_max_rate)))
def __init__(self): self._buckets = defaultdict(lambda: TokenBucket( rate(self.restart_max_rate)))