class PeriodicTask(VTask):
"""Task that executes `execute` at a specified interval
You must either override the `INTERVAL` (seconds) class attribute, or
pass a --{OPT_PREFIX}-interval in order for your task to run.
"""
INTERVAL = None
execute_duration_ms = samples(
windows=[60, 240],
types=[SampleType.AVG, SampleType.MAX, SampleType.MIN])
n_iterations = counter()
n_slow_iterations = counter()
n_try_later = counter()
interval = option(type=float,
metavar='SECONDS',
default=lambda cls: cls.INTERVAL,
help='How often this task should run [%(default)s] (s)')
def execute(self, context=None):
"""Override this to perform some custom action periodically."""
self.logger.debug('execute')
def initTask(self):
super(PeriodicTask, self).initTask()
assert self.interval is not None
# Register an event that we can more smartly wait on in case shutdown
# is requested while we would be `sleep()`ing
self.stop_event = Event()
def stop(self):
self.stop_event.set()
super(PeriodicTask, self).stop()
def _runloop(self):
t0 = time.time()
while not self.service._stop:
try:
self.execute()
except TryLater:
self.n_try_later.increment()
continue
self.n_iterations.increment()
self.execute_duration_ms.add((time.time() - t0) * 1000)
to_sleep = (t0 + self.interval) - time.time()
if to_sleep > 0:
if self.stop_event.wait(to_sleep):
return
else:
self.n_slow_iterations.increment()
t0 = time.time()
class CommandTask(TwistedTask):
"""A task that provides a useful API for executing other commands.
Python's Popen() can be hard to use, especially if you are executing long
running child processes, and need to handle various stdout, stderr, and
process exit events asynchronously.
This particular implementation relies on Twisted's ProcessProtocol, but it
wraps it in a way that makes it mostly opaque.
"""
LOOPLESS = True
OPT_PREFIX = 'cmd'
kill_timeout = option(type=float,
default=10.0,
help="Default shutdown kill timeout for outstanding "
"commands [%(default)s]")
started = counter()
finished = counter()
def run(self,
command,
on_stdout=None,
on_stderr=None,
on_exit=None,
line_buffered=True,
kill_timeout=None,
env=None):
"""Call this function to start a new child process running `command`.
Additional callbacks, such as `on_stdout`, `on_stderr`, and `on_exit`,
can be provided, that will receive a variety of parameters on the
appropriate events.
Line buffering can be disabled by passing `line_buffered`=False.
Also, a custom `kill_timeout` (seconds) may be set that overrides the
task default, in the event that a shutdown is received and you want
to allow more time for the command to shut down."""
self.logger.debug("task starting %s...", command)
if isinstance(command, six.string_types):
command = command.split(" ")
# wrap on_exit with helper to remove registered comments
on_exit = functools.partial(self._procExited, on_exit)
proto = _ProcessProtocolAdapter(on_stdout,
on_stderr,
on_exit,
line_buffered=line_buffered)
if twisted.python.threadable.isInIOThread():
result = self.reactor.spawnProcess(proto,
executable=command[0],
args=command)
else:
result = twisted.internet.threads.blockingCallFromThread(
self.reactor,
self.reactor.spawnProcess,
proto,
executable=command[0],
args=command,
env=env)
self.outstanding[result] = kill_timeout
self.started.increment()
return result
def initTask(self):
super(CommandTask, self).initTask()
self.outstanding = {}
def _procExited(self, on_exit, proto, trans, reason):
self.logger.debug("%s closed for %s", trans, reason)
if on_exit is not None:
on_exit(reason)
self.outstanding.pop(trans)
self.finished.increment()
return None
def join(self):
"""Overridden to block for process workers to shutdown / be killed."""
# TODO: Conditions instead of sleep polling?
while len(self.outstanding) > 0:
time.sleep(0.250)
def _killOutstanding(self, trans):
if trans in self.outstanding:
self.logger.info("Sending SIGKILL to %s", trans)
trans.signalProcess(signal.SIGKILL)
def stop(self):
# twisted is pretty smart; the default signal handlers it installs
# propagate SIGTERM to its children, so while we don't need to manually
# TERM, we might still need to set some kill timeouts
super(CommandTask, self).stop()
for trans, kill_timeout in self.outstanding.items():
if kill_timeout is None:
kill_timeout = self.kill_timeout
self.logger.info("Enqueuing kill for %s in %.1fs", trans,
kill_timeout)
args = (kill_timeout, self._killOutstanding, trans)
if twisted.python.threadable.isInIOThread():
self.reactor.callLater(*args)
else:
self.reactor.callFromThread(self.reactor.callLater, *args)
def isDoneWithReactor(self):
"""Overridden to keep reactor running until all commands finish."""
return len(self.outstanding) == 0
class TornadoHTTPTask(TornadoTask):
"""A loopless task that implements an HTTP server using Tornado.
It is loopless because it depends on tornado's separate IOLoop task. You
will need to subclass this to do something more useful."""
LOOPLESS = True
OPT_PREFIX = 'http'
DEFAULT_PORT = 0
DEFAULT_HOST = ''
DEFAULT_SOCK = ''
requests = counter()
#latency = samples(windows=[60, 3600],
# types=[SampleType.AVG, SampleType.MIN, SampleType.MAX])
host = option(metavar='HOST',
default=lambda cls: cls.DEFAULT_HOST,
help='Address to bind server to [%(default)s]')
port = option(metavar='PORT',
default=lambda cls: cls.DEFAULT_PORT,
help='Port to run server on [%(default)s]')
sock = option(metavar='PATH',
default=lambda cls: cls.DEFAULT_SOCK,
help='Default path to use for local file socket '
'[%(default)s]')
group = option(name='sock-group',
metavar='GROUP',
default='',
help='Group to create unix files as [%(default)s]')
def getApplicationConfig(self):
"""Override this to register custom handlers / routes."""
return [
('/', HelloWorldHandler),
]
def initTask(self):
super(TornadoHTTPTask, self).initTask()
self.app = tornado.web.Application(self.getApplicationConfig(),
log_function=self.tornadoRequestLog)
self.server = tornado.httpserver.HTTPServer(self.app)
if self.sock:
assert self.host == self.DEFAULT_HOST, \
"Do not specify host *and* sock (%s, %s)" % \
(self.host, self.sock)
assert int(self.port) == self.DEFAULT_PORT, \
"Do not specify port *and* sock (%s, %s)" % \
(self.port, self.DEFAULT_PORT)
gid, mode = -1, 0o600
if self.group != '':
e = grp.getgrnam(self.group)
gid, mode = e.gr_gid, 0o660
sock = tornado.netutil.bind_unix_socket(self.sock, mode=mode)
if gid != -1:
os.chown(self.sock, -1, gid)
self.server.add_sockets([sock])
else:
self.server.listen(self.port, self.host)
self.bound_addrs = []
for sock in itervalues(self.server._sockets):
sockaddr = sock.getsockname()
self.bound_addrs.append(sockaddr)
self.logger.info("%s Server Started on %s (port %s)", self.name,
sockaddr[0], sockaddr[1])
@property
def bound_v4_addrs(self):
return [a[0] for a in self.bound_addrs if len(a) == 2]
@property
def bound_v6_addrs(self):
return [a[0] for a in self.bound_addrs if len(a) == 4]
def tornadoRequestLog(self, handler):
self.requests.increment()
def stop(self):
super(TornadoHTTPTask, self).stop()
self.server.stop()
class PeriodicTask(VTask):
"""Task that executes `execute` at a specified interval
You must either override the `INTERVAL` (seconds) class attribute, or
pass a --{OPT_PREFIX}-interval in order for your task to run.
"""
INTERVAL = None
execute_duration_ms = samples(windows=[60, 240],
types=[SampleType.AVG, SampleType.MAX, SampleType.MIN])
n_iterations = counter()
n_slow_iterations = counter()
n_try_later = counter()
interval = option(type=float, metavar='SECONDS',
default=lambda cls: cls.INTERVAL,
help='How often this task should run [%(default)s] (s)')
def execute(self, context=None):
"""Override this to perform some custom action periodically."""
self.logger.debug('execute')
def execute_async(self):
f = Future()
if self.running:
# There's a race condition here. If the task has thrown but the
# thread(s) haven't stopped yet, you can enqueue a future that will
# never complete.
self.__futures.put(f)
else:
# If the task has stopped (e.g., due to a previous error),
# fail the future now and don't insert it into the queue.
f.set_exception(RuntimeError("Worker not running"))
return f
def has_pending(self):
return self.__futures.qsize() > 0
def initTask(self):
# Register an event that we can more smartly wait on in case shutdown
# is requested while we would be `sleep()`ing
self.stop_event = Event()
self.__futures = queue.Queue()
super(PeriodicTask, self).initTask()
assert self.interval is not None, \
"INTERVAL must be defined on %s or --%s-interval passed" % \
(self.name, self.name)
def stop(self):
self.stop_event.set()
super(PeriodicTask, self).stop()
def _runloop(self):
timer = Timer()
timer.start()
while not self.service._stop:
try:
result = self.execute()
# On a successful result, notify all blocked futures.
# Use pop like this to avoid race conditions.
while self.__futures.qsize():
f = self.__futures.get()
f.set_result(result)
except TryLater as e:
if self._handle_try_later(e):
return
continue
except Exception as e:
# On unhandled exceptions, set the exception on any async
# blocked execute calls.
while self.__futures.qsize():
f = self.__futures.get()
f.set_exception(e)
raise
self.n_iterations.increment()
self.execute_duration_ms.add(timer.elapsed * 1000)
to_sleep = self.interval - timer.elapsed
if to_sleep > 0:
if self.stop_event.wait(to_sleep):
return
else:
self.n_slow_iterations.increment()
timer.start()
def _handle_try_later(self, e):
self.n_try_later.increment()
if e.after is not None:
self.logger.debug("TryLater (%s) thrown. Retrying in %.2fs",
e.message, e.after)
else:
self.logger.debug("TryLater (%s) thrown. Retrying now",
e.message)
return self.stop_event.wait(e.after)
class QueueTask(VTask):
"""Task that calls `execute` for all work put on its `queue`"""
MAX_ITEMS = 0
WORKERS = 1
max_items = option(type=int,
default=lambda cls: cls.MAX_ITEMS,
help='Set a bounded queue length. This may '
'cause unexpected deadlocks. [%(default)s]')
workers = option(type=int,
default=lambda cls: cls.WORKERS,
help='Number of threads to spawn to work on items from '
'its queue. [%(default)s]')
execute_duration_ms = samples(
windows=[60, 240],
types=[SampleType.AVG, SampleType.MAX, SampleType.MIN])
n_trylater = counter()
n_completed = counter()
n_unhandled = counter()
def execute(self, item, context):
"""Implement this in your QueueTask subclasses"""
raise NotImplementedError()
def _makeQueue(self):
"""Override this if you need a custom Queue implementation"""
return queue.Queue(maxsize=self.max_items)
def initTask(self):
super(QueueTask, self).initTask()
self.queue = self._makeQueue()
self.counters['queue_depth'] = \
CallbackCounter(lambda: self.queue.qsize())
self._shutdown_sentinel = object()
def stop(self):
super(QueueTask, self).stop()
self.queue.put(self._shutdown_sentinel)
def submit(self, item):
"""Enqueue `item` into this task's Queue. Returns a `Future`"""
future = Future()
work = ExecuteContext(item=item, future=future)
self.queue.put(work)
return future
def map(self, items, timeout=None):
"""Enqueues `items` into the queue"""
futures = map(self.submit, items)
return [f.result(timeout) for f in futures]
def _runloop(self):
while not self.service._stop:
try:
item = self.queue.get(timeout=1.0)
if item is self._shutdown_sentinel:
self.queue.put(item)
break
except queue.Empty:
continue
# Create an ExecuteContext if we didn't have one
if isinstance(item, ExecuteContext):
context = item
item = context.item
context.raw_wrapped = False
else:
context = ExecuteContext(item=item)
context.raw_wrapped = True
try:
context.start()
result = self.execute(item, context)
self.work_success(context, result)
except TryLater:
self.work_retry(context)
except Exception as ex:
self.work_fail(context, ex)
finally:
self.queue.task_done()
def work_success(self, context, result):
self.n_completed.increment()
self.execute_duration_ms.add(context.elapsed * 1000.0)
context.set_result(result)
self.work_done(context)
def work_retry(self, context):
self.n_trylater.increment()
context.attempt += 1
self.work_done(context)
self.queue.put(context)
def work_fail(self, context, exception):
self.n_unhandled.increment()
self.execute_duration_ms.add(context.elapsed * 1000.0)
handled = context.set_exception(exception)
self.work_done(context)
if not handled:
raise
def work_done(self, context):
pass