def testSampleNames(self):
"""Test `counters.Samples() getCounters API, etc"""
c = counters.samples(name='foo',
types=[counters.SampleType.COUNT],
windows=[100])
c.add(1)
self.assertEqual(c.getCounter('foo.count.100'), 1)
c = counters.samples(types=[counters.SampleType.COUNT], windows=[100])
c.add(1)
self.assertEqual(c.getCounter('count.100'), 1)
def testSampleNames(self):
"""Test `counters.Samples() getCounters API, etc"""
c = counters.samples(name='foo',
types=[counters.SampleType.COUNT], windows=[100])
c.add(1)
self.assertEquals(c.getCounter('foo.count.100'), 1)
c = counters.samples(
types=[counters.SampleType.COUNT], windows=[100])
c.add(1)
self.assertEquals(c.getCounter('count.100'), 1)
class PeriodicTask(VTask):
"""Task that executes `execute` at a specified interval
You must either override the `INTERVAL` (seconds) class attribute, or
pass a --{OPT_PREFIX}-interval in order for your task to run.
"""
INTERVAL = None
execute_duration_ms = samples(
windows=[60, 240],
types=[SampleType.AVG, SampleType.MAX, SampleType.MIN])
n_iterations = counter()
n_slow_iterations = counter()
n_try_later = counter()
interval = option(type=float,
metavar='SECONDS',
default=lambda cls: cls.INTERVAL,
help='How often this task should run [%(default)s] (s)')
def execute(self, context=None):
"""Override this to perform some custom action periodically."""
self.logger.debug('execute')
def initTask(self):
super(PeriodicTask, self).initTask()
assert self.interval is not None
# Register an event that we can more smartly wait on in case shutdown
# is requested while we would be `sleep()`ing
self.stop_event = Event()
def stop(self):
self.stop_event.set()
super(PeriodicTask, self).stop()
def _runloop(self):
t0 = time.time()
while not self.service._stop:
try:
self.execute()
except TryLater:
self.n_try_later.increment()
continue
self.n_iterations.increment()
self.execute_duration_ms.add((time.time() - t0) * 1000)
to_sleep = (t0 + self.interval) - time.time()
if to_sleep > 0:
if self.stop_event.wait(to_sleep):
return
else:
self.n_slow_iterations.increment()
t0 = time.time()
def testSamples(self):
c = counters.samples(
types=[counters.SampleType.COUNT, counters.SampleType.SUM],
windows=[100, 1000])
now = time.time()
c._now = self.mock.Mock()
# At t=0, add two values of 10.0
c._now.return_value = now
c.add(10.0)
c.add(10.0)
self.assertEquals(c.getCounter('count.100'), 2)
self.assertEquals(c.getCounter('count.1000'), 2)
self.assertEquals(c.getCounter('sum.100'), 20.0)
self.assertEquals(c.getCounter('sum.1000'), 20.0)
# Make sure there are only four counters
self.assertEquals(len(c.getCounters()), 4)
# At t=10, add one values of 10.0
c._now.return_value = now + 10
c.add(10.0)
self.assertEquals(c.getCounter('count.100'), 3)
self.assertEquals(c.getCounter('count.1000'), 3)
self.assertEquals(c.getCounter('sum.100'), 30.0)
self.assertEquals(c.getCounter('sum.1000'), 30.0)
# At t=101, 2 values should have fallen out of the 100 window
c._now.return_value = now + 101
self.assertEquals(c.getCounter('count.100'), 1)
self.assertEquals(c.getCounter('count.1000'), 3)
self.assertEquals(c.getCounter('sum.100'), 10.0)
self.assertEquals(c.getCounter('sum.1000'), 30.0)
# At t=1001, all values should be gone from the 100 window,
# but one value should remain in the 1000 window
c._now.return_value = now + 1001
self.assertEquals(c.getCounter('count.100'), 0)
self.assertEquals(c.getCounter('count.1000'), 1)
self.assertEquals(c.getCounter('sum.100'), 0.0)
self.assertEquals(c.getCounter('sum.1000'), 10.0)
# At t=1011, all values should be gone from all windows.
c._now.return_value = now + 1011
self.assertEquals(c.getCounter('count.100'), 0)
self.assertEquals(c.getCounter('count.1000'), 0, str((now, c.samples)))
self.assertEquals(c.getCounter('sum.100'), 0.0)
self.assertEquals(c.getCounter('sum.1000'), 0.0)
class PrintCountersTask(PeriodicTask):
INTERVAL = 6
execute_duration = samples(windows=[60],
types=[SampleType.MAX, SampleType.MIN])
def execute(self, *args, **kwargs):
hostcheck = self.service.requireTask(HostCheckTask)
self.logger.info("hostcheck.duration :: %s",
hostcheck.execute_duration.getCounters())
self.logger.info("this.duration :: %s",
self.execute_duration.getCounters())
def testSamples(self):
c = counters.samples(
types=[counters.SampleType.COUNT, counters.SampleType.SUM],
windows=[100, 1000])
now = time.time()
c._now = self.mock.Mock()
# At t=0, add two values of 10.0
c._now.return_value = now
c.add(10.0)
c.add(10.0)
self.assertEqual(c.getCounter('count.100'), 2)
self.assertEqual(c.getCounter('count.1000'), 2)
self.assertEqual(c.getCounter('sum.100'), 20.0)
self.assertEqual(c.getCounter('sum.1000'), 20.0)
# Make sure there are only four counters
self.assertEqual(len(c.getCounters()), 4)
# At t=10, add one values of 10.0
c._now.return_value = now + 10
c.add(10.0)
self.assertEqual(c.getCounter('count.100'), 3)
self.assertEqual(c.getCounter('count.1000'), 3)
self.assertEqual(c.getCounter('sum.100'), 30.0)
self.assertEqual(c.getCounter('sum.1000'), 30.0)
# At t=101, 2 values should have fallen out of the 100 window
c._now.return_value = now + 101
self.assertEqual(c.getCounter('count.100'), 1)
self.assertEqual(c.getCounter('count.1000'), 3)
self.assertEqual(c.getCounter('sum.100'), 10.0)
self.assertEqual(c.getCounter('sum.1000'), 30.0)
# At t=1001, all values should be gone from the 100 window,
# but one value should remain in the 1000 window
c._now.return_value = now + 1001
self.assertEqual(c.getCounter('count.100'), 0)
self.assertEqual(c.getCounter('count.1000'), 1)
self.assertEqual(c.getCounter('sum.100'), 0.0)
self.assertEqual(c.getCounter('sum.1000'), 10.0)
# At t=1011, all values should be gone from all windows.
c._now.return_value = now + 1011
self.assertEqual(c.getCounter('count.100'), 0)
self.assertEqual(c.getCounter('count.1000'), 0, str((now, c.samples)))
self.assertEqual(c.getCounter('sum.100'), 0.0)
self.assertEqual(c.getCounter('sum.1000'), 0.0)
class PeriodicTask(VTask):
"""Task that executes `execute` at a specified interval
You must either override the `INTERVAL` (seconds) class attribute, or
pass a --{OPT_PREFIX}-interval in order for your task to run.
"""
INTERVAL = None
execute_duration_ms = samples(windows=[60, 240],
types=[SampleType.AVG, SampleType.MAX, SampleType.MIN])
n_iterations = counter()
n_slow_iterations = counter()
n_try_later = counter()
interval = option(type=float, metavar='SECONDS',
default=lambda cls: cls.INTERVAL,
help='How often this task should run [%(default)s] (s)')
def execute(self, context=None):
"""Override this to perform some custom action periodically."""
self.logger.debug('execute')
def execute_async(self):
f = Future()
if self.running:
# There's a race condition here. If the task has thrown but the
# thread(s) haven't stopped yet, you can enqueue a future that will
# never complete.
self.__futures.put(f)
else:
# If the task has stopped (e.g., due to a previous error),
# fail the future now and don't insert it into the queue.
f.set_exception(RuntimeError("Worker not running"))
return f
def has_pending(self):
return self.__futures.qsize() > 0
def initTask(self):
# Register an event that we can more smartly wait on in case shutdown
# is requested while we would be `sleep()`ing
self.stop_event = Event()
self.__futures = queue.Queue()
super(PeriodicTask, self).initTask()
assert self.interval is not None, \
"INTERVAL must be defined on %s or --%s-interval passed" % \
(self.name, self.name)
def stop(self):
self.stop_event.set()
super(PeriodicTask, self).stop()
def _runloop(self):
timer = Timer()
timer.start()
while not self.service._stop:
try:
result = self.execute()
# On a successful result, notify all blocked futures.
# Use pop like this to avoid race conditions.
while self.__futures.qsize():
f = self.__futures.get()
f.set_result(result)
except TryLater as e:
if self._handle_try_later(e):
return
continue
except Exception as e:
# On unhandled exceptions, set the exception on any async
# blocked execute calls.
while self.__futures.qsize():
f = self.__futures.get()
f.set_exception(e)
raise
self.n_iterations.increment()
self.execute_duration_ms.add(timer.elapsed * 1000)
to_sleep = self.interval - timer.elapsed
if to_sleep > 0:
if self.stop_event.wait(to_sleep):
return
else:
self.n_slow_iterations.increment()
timer.start()
def _handle_try_later(self, e):
self.n_try_later.increment()
if e.after is not None:
self.logger.debug("TryLater (%s) thrown. Retrying in %.2fs",
e.message, e.after)
else:
self.logger.debug("TryLater (%s) thrown. Retrying now",
e.message)
return self.stop_event.wait(e.after)
class QueueTask(VTask):
"""Task that calls `execute` for all work put on its `queue`"""
MAX_ITEMS = 0
WORKERS = 1
max_items = option(type=int,
default=lambda cls: cls.MAX_ITEMS,
help='Set a bounded queue length. This may '
'cause unexpected deadlocks. [%(default)s]')
workers = option(type=int,
default=lambda cls: cls.WORKERS,
help='Number of threads to spawn to work on items from '
'its queue. [%(default)s]')
execute_duration_ms = samples(
windows=[60, 240],
types=[SampleType.AVG, SampleType.MAX, SampleType.MIN])
n_trylater = counter()
n_completed = counter()
n_unhandled = counter()
def execute(self, item, context):
"""Implement this in your QueueTask subclasses"""
raise NotImplementedError()
def _makeQueue(self):
"""Override this if you need a custom Queue implementation"""
return queue.Queue(maxsize=self.max_items)
def initTask(self):
super(QueueTask, self).initTask()
self.queue = self._makeQueue()
self.counters['queue_depth'] = \
CallbackCounter(lambda: self.queue.qsize())
self._shutdown_sentinel = object()
def stop(self):
super(QueueTask, self).stop()
self.queue.put(self._shutdown_sentinel)
def submit(self, item):
"""Enqueue `item` into this task's Queue. Returns a `Future`"""
future = Future()
work = ExecuteContext(item=item, future=future)
self.queue.put(work)
return future
def map(self, items, timeout=None):
"""Enqueues `items` into the queue"""
futures = map(self.submit, items)
return [f.result(timeout) for f in futures]
def _runloop(self):
while not self.service._stop:
try:
item = self.queue.get(timeout=1.0)
if item is self._shutdown_sentinel:
self.queue.put(item)
break
except queue.Empty:
continue
# Create an ExecuteContext if we didn't have one
if isinstance(item, ExecuteContext):
context = item
item = context.item
context.raw_wrapped = False
else:
context = ExecuteContext(item=item)
context.raw_wrapped = True
try:
context.start()
result = self.execute(item, context)
self.work_success(context, result)
except TryLater:
self.work_retry(context)
except Exception as ex:
self.work_fail(context, ex)
finally:
self.queue.task_done()
def work_success(self, context, result):
self.n_completed.increment()
self.execute_duration_ms.add(context.elapsed * 1000.0)
context.set_result(result)
self.work_done(context)
def work_retry(self, context):
self.n_trylater.increment()
context.attempt += 1
self.work_done(context)
self.queue.put(context)
def work_fail(self, context, exception):
self.n_unhandled.increment()
self.execute_duration_ms.add(context.elapsed * 1000.0)
handled = context.set_exception(exception)
self.work_done(context)
if not handled:
raise
def work_done(self, context):
pass
