def testSampleNames(self): """Test `counters.Samples() getCounters API, etc""" c = counters.samples(name='foo', types=[counters.SampleType.COUNT], windows=[100]) c.add(1) self.assertEqual(c.getCounter('foo.count.100'), 1) c = counters.samples(types=[counters.SampleType.COUNT], windows=[100]) c.add(1) self.assertEqual(c.getCounter('count.100'), 1)
def testSampleNames(self): """Test `counters.Samples() getCounters API, etc""" c = counters.samples(name='foo', types=[counters.SampleType.COUNT], windows=[100]) c.add(1) self.assertEquals(c.getCounter('foo.count.100'), 1) c = counters.samples( types=[counters.SampleType.COUNT], windows=[100]) c.add(1) self.assertEquals(c.getCounter('count.100'), 1)
class PeriodicTask(VTask): """Task that executes `execute` at a specified interval You must either override the `INTERVAL` (seconds) class attribute, or pass a --{OPT_PREFIX}-interval in order for your task to run. """ INTERVAL = None execute_duration_ms = samples( windows=[60, 240], types=[SampleType.AVG, SampleType.MAX, SampleType.MIN]) n_iterations = counter() n_slow_iterations = counter() n_try_later = counter() interval = option(type=float, metavar='SECONDS', default=lambda cls: cls.INTERVAL, help='How often this task should run [%(default)s] (s)') def execute(self, context=None): """Override this to perform some custom action periodically.""" self.logger.debug('execute') def initTask(self): super(PeriodicTask, self).initTask() assert self.interval is not None # Register an event that we can more smartly wait on in case shutdown # is requested while we would be `sleep()`ing self.stop_event = Event() def stop(self): self.stop_event.set() super(PeriodicTask, self).stop() def _runloop(self): t0 = time.time() while not self.service._stop: try: self.execute() except TryLater: self.n_try_later.increment() continue self.n_iterations.increment() self.execute_duration_ms.add((time.time() - t0) * 1000) to_sleep = (t0 + self.interval) - time.time() if to_sleep > 0: if self.stop_event.wait(to_sleep): return else: self.n_slow_iterations.increment() t0 = time.time()
def testSamples(self): c = counters.samples( types=[counters.SampleType.COUNT, counters.SampleType.SUM], windows=[100, 1000]) now = time.time() c._now = self.mock.Mock() # At t=0, add two values of 10.0 c._now.return_value = now c.add(10.0) c.add(10.0) self.assertEquals(c.getCounter('count.100'), 2) self.assertEquals(c.getCounter('count.1000'), 2) self.assertEquals(c.getCounter('sum.100'), 20.0) self.assertEquals(c.getCounter('sum.1000'), 20.0) # Make sure there are only four counters self.assertEquals(len(c.getCounters()), 4) # At t=10, add one values of 10.0 c._now.return_value = now + 10 c.add(10.0) self.assertEquals(c.getCounter('count.100'), 3) self.assertEquals(c.getCounter('count.1000'), 3) self.assertEquals(c.getCounter('sum.100'), 30.0) self.assertEquals(c.getCounter('sum.1000'), 30.0) # At t=101, 2 values should have fallen out of the 100 window c._now.return_value = now + 101 self.assertEquals(c.getCounter('count.100'), 1) self.assertEquals(c.getCounter('count.1000'), 3) self.assertEquals(c.getCounter('sum.100'), 10.0) self.assertEquals(c.getCounter('sum.1000'), 30.0) # At t=1001, all values should be gone from the 100 window, # but one value should remain in the 1000 window c._now.return_value = now + 1001 self.assertEquals(c.getCounter('count.100'), 0) self.assertEquals(c.getCounter('count.1000'), 1) self.assertEquals(c.getCounter('sum.100'), 0.0) self.assertEquals(c.getCounter('sum.1000'), 10.0) # At t=1011, all values should be gone from all windows. c._now.return_value = now + 1011 self.assertEquals(c.getCounter('count.100'), 0) self.assertEquals(c.getCounter('count.1000'), 0, str((now, c.samples))) self.assertEquals(c.getCounter('sum.100'), 0.0) self.assertEquals(c.getCounter('sum.1000'), 0.0)
class PrintCountersTask(PeriodicTask): INTERVAL = 6 execute_duration = samples(windows=[60], types=[SampleType.MAX, SampleType.MIN]) def execute(self, *args, **kwargs): hostcheck = self.service.requireTask(HostCheckTask) self.logger.info("hostcheck.duration :: %s", hostcheck.execute_duration.getCounters()) self.logger.info("this.duration :: %s", self.execute_duration.getCounters())
def testSamples(self): c = counters.samples( types=[counters.SampleType.COUNT, counters.SampleType.SUM], windows=[100, 1000]) now = time.time() c._now = self.mock.Mock() # At t=0, add two values of 10.0 c._now.return_value = now c.add(10.0) c.add(10.0) self.assertEqual(c.getCounter('count.100'), 2) self.assertEqual(c.getCounter('count.1000'), 2) self.assertEqual(c.getCounter('sum.100'), 20.0) self.assertEqual(c.getCounter('sum.1000'), 20.0) # Make sure there are only four counters self.assertEqual(len(c.getCounters()), 4) # At t=10, add one values of 10.0 c._now.return_value = now + 10 c.add(10.0) self.assertEqual(c.getCounter('count.100'), 3) self.assertEqual(c.getCounter('count.1000'), 3) self.assertEqual(c.getCounter('sum.100'), 30.0) self.assertEqual(c.getCounter('sum.1000'), 30.0) # At t=101, 2 values should have fallen out of the 100 window c._now.return_value = now + 101 self.assertEqual(c.getCounter('count.100'), 1) self.assertEqual(c.getCounter('count.1000'), 3) self.assertEqual(c.getCounter('sum.100'), 10.0) self.assertEqual(c.getCounter('sum.1000'), 30.0) # At t=1001, all values should be gone from the 100 window, # but one value should remain in the 1000 window c._now.return_value = now + 1001 self.assertEqual(c.getCounter('count.100'), 0) self.assertEqual(c.getCounter('count.1000'), 1) self.assertEqual(c.getCounter('sum.100'), 0.0) self.assertEqual(c.getCounter('sum.1000'), 10.0) # At t=1011, all values should be gone from all windows. c._now.return_value = now + 1011 self.assertEqual(c.getCounter('count.100'), 0) self.assertEqual(c.getCounter('count.1000'), 0, str((now, c.samples))) self.assertEqual(c.getCounter('sum.100'), 0.0) self.assertEqual(c.getCounter('sum.1000'), 0.0)
class PeriodicTask(VTask): """Task that executes `execute` at a specified interval You must either override the `INTERVAL` (seconds) class attribute, or pass a --{OPT_PREFIX}-interval in order for your task to run. """ INTERVAL = None execute_duration_ms = samples(windows=[60, 240], types=[SampleType.AVG, SampleType.MAX, SampleType.MIN]) n_iterations = counter() n_slow_iterations = counter() n_try_later = counter() interval = option(type=float, metavar='SECONDS', default=lambda cls: cls.INTERVAL, help='How often this task should run [%(default)s] (s)') def execute(self, context=None): """Override this to perform some custom action periodically.""" self.logger.debug('execute') def execute_async(self): f = Future() if self.running: # There's a race condition here. If the task has thrown but the # thread(s) haven't stopped yet, you can enqueue a future that will # never complete. self.__futures.put(f) else: # If the task has stopped (e.g., due to a previous error), # fail the future now and don't insert it into the queue. f.set_exception(RuntimeError("Worker not running")) return f def has_pending(self): return self.__futures.qsize() > 0 def initTask(self): # Register an event that we can more smartly wait on in case shutdown # is requested while we would be `sleep()`ing self.stop_event = Event() self.__futures = queue.Queue() super(PeriodicTask, self).initTask() assert self.interval is not None, \ "INTERVAL must be defined on %s or --%s-interval passed" % \ (self.name, self.name) def stop(self): self.stop_event.set() super(PeriodicTask, self).stop() def _runloop(self): timer = Timer() timer.start() while not self.service._stop: try: result = self.execute() # On a successful result, notify all blocked futures. # Use pop like this to avoid race conditions. while self.__futures.qsize(): f = self.__futures.get() f.set_result(result) except TryLater as e: if self._handle_try_later(e): return continue except Exception as e: # On unhandled exceptions, set the exception on any async # blocked execute calls. while self.__futures.qsize(): f = self.__futures.get() f.set_exception(e) raise self.n_iterations.increment() self.execute_duration_ms.add(timer.elapsed * 1000) to_sleep = self.interval - timer.elapsed if to_sleep > 0: if self.stop_event.wait(to_sleep): return else: self.n_slow_iterations.increment() timer.start() def _handle_try_later(self, e): self.n_try_later.increment() if e.after is not None: self.logger.debug("TryLater (%s) thrown. Retrying in %.2fs", e.message, e.after) else: self.logger.debug("TryLater (%s) thrown. Retrying now", e.message) return self.stop_event.wait(e.after)
class QueueTask(VTask): """Task that calls `execute` for all work put on its `queue`""" MAX_ITEMS = 0 WORKERS = 1 max_items = option(type=int, default=lambda cls: cls.MAX_ITEMS, help='Set a bounded queue length. This may ' 'cause unexpected deadlocks. [%(default)s]') workers = option(type=int, default=lambda cls: cls.WORKERS, help='Number of threads to spawn to work on items from ' 'its queue. [%(default)s]') execute_duration_ms = samples( windows=[60, 240], types=[SampleType.AVG, SampleType.MAX, SampleType.MIN]) n_trylater = counter() n_completed = counter() n_unhandled = counter() def execute(self, item, context): """Implement this in your QueueTask subclasses""" raise NotImplementedError() def _makeQueue(self): """Override this if you need a custom Queue implementation""" return queue.Queue(maxsize=self.max_items) def initTask(self): super(QueueTask, self).initTask() self.queue = self._makeQueue() self.counters['queue_depth'] = \ CallbackCounter(lambda: self.queue.qsize()) self._shutdown_sentinel = object() def stop(self): super(QueueTask, self).stop() self.queue.put(self._shutdown_sentinel) def submit(self, item): """Enqueue `item` into this task's Queue. Returns a `Future`""" future = Future() work = ExecuteContext(item=item, future=future) self.queue.put(work) return future def map(self, items, timeout=None): """Enqueues `items` into the queue""" futures = map(self.submit, items) return [f.result(timeout) for f in futures] def _runloop(self): while not self.service._stop: try: item = self.queue.get(timeout=1.0) if item is self._shutdown_sentinel: self.queue.put(item) break except queue.Empty: continue # Create an ExecuteContext if we didn't have one if isinstance(item, ExecuteContext): context = item item = context.item context.raw_wrapped = False else: context = ExecuteContext(item=item) context.raw_wrapped = True try: context.start() result = self.execute(item, context) self.work_success(context, result) except TryLater: self.work_retry(context) except Exception as ex: self.work_fail(context, ex) finally: self.queue.task_done() def work_success(self, context, result): self.n_completed.increment() self.execute_duration_ms.add(context.elapsed * 1000.0) context.set_result(result) self.work_done(context) def work_retry(self, context): self.n_trylater.increment() context.attempt += 1 self.work_done(context) self.queue.put(context) def work_fail(self, context, exception): self.n_unhandled.increment() self.execute_duration_ms.add(context.elapsed * 1000.0) handled = context.set_exception(exception) self.work_done(context) if not handled: raise def work_done(self, context): pass