def __init__(self, config):
self.config = config
self.name = config.name
self.size = config.size
self.frequency = config.frequency
self.description = config.description
self.pool = QueuePool(config.size)
self.logging = Logging(config.name, self.pool.queue.logs)
self.__loop = True
self.greenlets = Greenlets([], [], [], [])
self.greenlets.metric.append(spawn(self.metricProducer))
self.__run = Event()
self.__run.clear()
self.__connections = {}
self.__children = {}
self.__parents = {}
self.__lookups = {}
self.__buildUplook()
self.stopped = True
def __init__(self, config):
self.config = config
self.name = config.name
self.size = config.size
self.frequency = config.frequency
self.pool = QueuePool(config.size)
self.logging = Logging(config.name, self.pool.queue.logs)
self.__loop = True
self.greenlets = Greenlets([], [], [], [])
self.greenlets.metric.append(spawn(self.metricProducer))
self.__run = Event()
self.__run.clear()
self.__connections = {}
self.__children = {}
self.__parents = {}
self.__lookups = {}
self.__buildUplook()
self.stopped = True
def __init__(self, config):
self.config = config
self.name = config.name
self.description = self.__getDescription(config)
self.pool = QueuePool(config.size)
self.logging = Logging(
name=config.name,
q=self.pool.queue._logs,
identification=self.config.identification
)
self.__loop = True
self.greenlets = Greenlets([], [], [], [])
self.greenlets.metric.append(spawn(self.__metricProducer))
self._run = Event()
self._run.clear()
self.stopped = True
# Setup the Jinja2 environment to render kwargs templates.
##########################################################
self.env_template = jinja2.Environment(
undefined=jinja2.StrictUndefined,
trim_blocks=True,
loader=jinja2.FileSystemLoader('/')
)
# Add the template functions to the template globals
####################################################
for key, value in self.config.template_functions.items():
self.env_template.globals.update({key: value.get})
# Store a copy of the raw/unmodified kwargs
##########################################
self.kwargs_raw = self.__getRawKwargs()
# Store a copy of kwargs with all templates replaced by a template instance
############################################################################
self.kwargs_template = self.__getTemplateKwargs(self.env_template, self.kwargs_raw)
# Store a copy of the rendered kwargs as an EasyDict instance
#############################################################
self.kwargs = self.__renderTemplateKwargs(self.kwargs_template)
# Do some sanity checks
#######################
self.__sanityChecks()
# Validate and setup module
# Methods are come from based class
###################################
self._moduleInitValidation()
self._moduleInitSetup()
self.version = self.__getVersion()
class Actor():
def __init__(self, config):
self.config = config
self.name = config.name
self.size = config.size
self.frequency = config.frequency
self.pool = QueuePool(config.size)
self.logging = Logging(config.name, self.pool.queue.logs)
self.__loop = True
self.greenlets = Greenlets([], [], [], [])
self.greenlets.metric.append(spawn(self.metricProducer))
self.__run = Event()
self.__run.clear()
self.__connections = {}
self.__children = {}
self.__parents = {}
self.__lookups = {}
self.__buildUplook()
self.stopped = True
def connect(self, source, destination_module, destination_queue):
'''Connects the <source> queue to the <destination> queue.
In fact, the source queue overwrites the destination queue.'''
if source in self.__children:
raise QueueConnected("Queue %s.%s is already connected to %s." % (self.name, source, self.__children[source]))
else:
self.__children[source] = "%s.%s" % (destination_module.name, destination_queue)
if destination_queue in destination_module.__parents:
raise QueueConnected("Queue %s.%s is already connected to %s" % (destination_module.name, destination_queue, destination_module.__parents[destination_queue]))
else:
destination_module.__parents[destination_queue] = "%s.%s" % (self.name, source)
if not self.pool.hasQueue(source):
self.pool.createQueue(source)
setattr(destination_module.pool.queue, destination_queue, self.pool.getQueue(source))
self.pool.getQueue(source).disableFallThrough()
self.logging.debug("Connected queue %s.%s to %s.%s" % (self.name, source, destination_module.name, destination_queue))
def createEvent(self):
'''Convenience function which returns an empty Wishbone event with the
current namespace already set.'''
return Wishbone_Event(self.name)
def doEventLookup(self, name):
(n, t, k) = name.split('.')
try:
return self.current_event.getHeaderValue(n, k)
except AttributeError:
return "You should use a dynamic lookup ~~ for header lookups. "
except KeyError:
self.logging.debug("There is no lookup value with name '%s'." % (name))
raise NoSuchValue
def getChildren(self, queue=None):
'''Returns the queue name <queue> is connected to.'''
if queue is None:
return [self.__children[q] for q in self.__children.keys()]
else:
return self.__children[queue]
def loop(self):
'''The global lock for this module'''
return self.__loop
def postHook(self):
pass
def preHook(self):
self.logging.debug("Initialized.")
def registerConsumer(self, function, queue):
'''Registers <function> to process all events in <queue>
Do not trap errors. When <function> fails then the event will be
submitted to the "failed" queue, If <function> succeeds to the
success queue.'''
self.greenlets.consumer.append(spawn(self.__consumer, function, queue))
def start(self):
'''Starts the module.'''
if hasattr(self, "preHook"):
self.logging.debug("preHook() found, executing")
self.preHook()
self.__run.set()
self.logging.debug("Started with max queue size of %s events and metrics interval of %s seconds." % (self.size, self.frequency))
self.stopped = False
def sendToBackground(self, function, *args, **kwargs):
'''Executes a function and sends it to the background.'''
self.greenlets.generic.append(spawn(function, *args, **kwargs))
def stop(self):
'''Stops the loop lock and waits until all registered consumers have exit otherwise kills them.'''
self.logging.info("Received stop. Initiating shutdown.")
self.__loop = False
for background_job in self.greenlets.metric:
kill(background_job)
for background_job in self.greenlets.generic:
kill(background_job)
for background_job in self.greenlets.consumer:
kill(background_job)
if hasattr(self, "postHook"):
self.logging.debug("postHook() found, executing")
self.postHook()
self.logging.debug("Exit.")
self.stopped = True
def submit(self, event, queue):
'''A convenience function which submits <event> to <queue>.'''
while self.loop():
try:
queue.put(event)
break
except QueueFull:
sleep(0.1)
def __consumer(self, function, queue):
'''Greenthread which applies <function> to each element from <queue>
'''
self.__run.wait()
while self.loop():
event = self.pool.queue.__dict__[queue].get()
self.current_event = event
event.initNamespace(self.name)
try:
function(event)
except Exception as err:
exc_type, exc_value, exc_traceback = exc_info()
event.setErrorValue(traceback.extract_tb(exc_traceback)[-1][1], str(exc_type), str(exc_value))
self.logging.error("%s" % (err))
self.submit(event, self.pool.queue.failed)
else:
self.submit(event, self.pool.queue.success)
def __buildUplook(self):
self.__current_event = {}
args = {}
for key, value in inspect.currentframe(2).f_locals.iteritems():
if key == "self" or isinstance(value, ActorConfig):
next
else:
args[key] = value
uplook = UpLook(**args)
for name in uplook.listFunctions():
if name not in self.config.lookup:
raise ModuleInitFailure("A lookup function '%s' was defined but no lookup function with that name registered." % (name))
else:
if isinstance(self.config.lookup[name], EventLookup):
uplook.registerLookup(name, self.doEventLookup)
else:
uplook.registerLookup(name, self.config.lookup[name])
self.uplook = uplook
self.kwargs = uplook.get()
def metricProducer(self):
'''A greenthread which collects the queue metrics at the defined interval.'''
self.__run.wait()
hostname = socket.gethostname()
while self.loop():
for queue in self.pool.listQueues(names=True):
for metric, value in self.pool.getQueue(queue).stats().iteritems():
event = Wishbone_Event(self.name)
event.data = Metric(time=time(), type="wishbone", source=hostname, name="module.%s.queue.%s.%s" % (self.name, queue, metric), value=value, unit="", tags=())
self.submit(event, self.pool.queue.metrics)
sleep(self.frequency)
class Actor():
def __init__(self, config):
self.config = config
self.name = config.name
self.size = config.size
self.frequency = config.frequency
self.description = config.description
self.pool = QueuePool(config.size)
self.logging = Logging(config.name, self.pool.queue.logs)
self.__loop = True
self.greenlets = Greenlets([], [], [], [])
self.greenlets.metric.append(spawn(self.metricProducer))
self.__run = Event()
self.__run.clear()
self.__connections = {}
self.__children = {}
self.__parents = {}
self.__lookups = {}
self.__buildUplook()
self.stopped = True
def connect(self, source, destination_module, destination_queue):
'''Connects the <source> queue to the <destination> queue.
In fact, the source queue overwrites the destination queue.'''
if source in self.__children:
raise QueueConnected("Queue %s.%s is already connected to %s." % (self.name, source, self.__children[source]))
else:
self.__children[source] = "%s.%s" % (destination_module.name, destination_queue)
if destination_queue in destination_module.__parents:
raise QueueConnected("Queue %s.%s is already connected to %s" % (destination_module.name, destination_queue, destination_module.__parents[destination_queue]))
else:
destination_module.__parents[destination_queue] = "%s.%s" % (self.name, source)
if not self.pool.hasQueue(source):
self.pool.createQueue(source)
setattr(destination_module.pool.queue, destination_queue, self.pool.getQueue(source))
self.pool.getQueue(source).disableFallThrough()
self.logging.debug("Connected queue %s.%s to %s.%s" % (self.name, source, destination_module.name, destination_queue))
def doEventLookup(self, name):
try:
return self.current_event.get(name)
except AttributeError:
return ""
except KeyError:
# No event has passed through this module. Most likely this is an
# input module which creates its own events. Therefor there is
# nothing to lookup yet and there for we rely up UpLook to return
# the default value for this lookup if any.
self.logging.debug("There is no lookup value with name '%s' Falling back to default lookup value if any." % (name))
raise NoSuchValue
def getChildren(self, queue=None):
'''Returns the queue name <queue> is connected to.'''
if queue is None:
return [self.__children[q] for q in list(self.__children.keys())]
else:
return self.__children[queue]
def loop(self):
'''The global lock for this module'''
return self.__loop
def postHook(self):
pass
def preHook(self):
self.logging.debug("Initialized.")
def registerConsumer(self, function, queue):
'''Registers <function> to process all events in <queue>
Do not trap errors. When <function> fails then the event will be
submitted to the "failed" queue, If <function> succeeds to the
success queue.'''
self.greenlets.consumer.append(spawn(self.__consumer, function, queue))
def start(self):
'''Starts the module.'''
if hasattr(self, "preHook"):
self.logging.debug("preHook() found, executing")
self.preHook()
self.__run.set()
self.logging.debug("Started with max queue size of %s events and metrics interval of %s seconds." % (self.size, self.frequency))
self.stopped = False
def sendToBackground(self, function, *args, **kwargs):
'''Executes a function and sends it to the background.
Background tasks are usually running indefinately. When such a
background task generates an error, it is automatically restarted and
an error is logged.
'''
def wrapIntoLoop():
while self.loop():
try:
function(*args, **kwargs)
# We want to break out of the loop if we get here because
# it's the intention of function() to exit without errors.
# Normally, background tasks run indefinately but in this
# case the user opted not to for some reason so we should
# obey that.
break
except Exception as err:
self.logging.error("Backgrounded function '%s' of module instance '%s' caused an error. This needs attention. Restarting it in 2 seconds. Reason: %s" % (
function.__name__,
self.name,
err)
)
sleep(2)
self.greenlets.generic.append(spawn(wrapIntoLoop))
def stop(self):
'''Stops the loop lock and waits until all registered consumers have exit otherwise kills them.'''
self.logging.info("Received stop. Initiating shutdown.")
self.__loop = False
for background_job in self.greenlets.metric:
kill(background_job)
for background_job in self.greenlets.generic:
kill(background_job)
for background_job in self.greenlets.consumer:
kill(background_job)
if hasattr(self, "postHook"):
self.logging.debug("postHook() found, executing")
self.postHook()
self.logging.debug("Exit.")
self.stopped = True
def submit(self, event, queue):
'''A convenience function which submits <event> to <queue>.'''
while self.loop():
try:
queue.put(event)
break
except QueueFull:
sleep(0.1)
def __consumer(self, function, queue):
'''Greenthread which applies <function> to each element from <queue>
'''
self.__run.wait()
while self.loop():
event = self.pool.queue.__dict__[queue].get()
self.current_event = event
try:
function(event)
except Exception as err:
exc_type, exc_value, exc_traceback = exc_info()
info = (traceback.extract_tb(exc_traceback)[-1][1], str(exc_type), str(exc_value))
if isinstance(event, Wishbone_Event):
event.set(info, "@errors.%s" % (self.name))
elif(event, Bulk):
event.error = info
self.logging.error("%s" % (err))
self.submit(event, self.pool.queue.failed)
else:
self.submit(event, self.pool.queue.success)
def __buildUplook(self):
self.__current_event = {}
args = {}
for key, value in list(inspect.getouterframes(inspect.currentframe())[2][0].f_locals.items()):
if key == "self" or isinstance(value, ActorConfig):
next
else:
args[key] = value
uplook = UpLook(**args)
for name in uplook.listFunctions():
if name not in self.config.lookup:
raise ModuleInitFailure("A lookup function '%s' was defined but no lookup function with that name registered." % (name))
else:
if self.config.lookup[name].__self__.__class__ == EventLookup:
uplook.registerLookup(name, self.doEventLookup)
else:
uplook.registerLookup(name, self.config.lookup[name])
self.uplook = uplook
self.kwargs = uplook.get()
def metricProducer(self):
'''A greenthread which collects the queue metrics at the defined interval.'''
self.__run.wait()
hostname = socket.gethostname()
while self.loop():
for queue in self.pool.listQueues(names=True):
for metric, value in list(self.pool.getQueue(queue).stats().items()):
metric = Metric(time=time(),
type="wishbone",
source=hostname,
name="module.%s.queue.%s.%s" % (self.name, queue, metric),
value=value,
unit="",
tags=())
event = Wishbone_Event(metric)
self.submit(event, self.pool.queue.metrics)
sleep(self.frequency)
def test_getQueue():
q = QueuePool(1)
q.createQueue("test")
assert isinstance(q.getQueue("test"), Queue)
def test_hasQueue():
q = QueuePool(1)
q.createQueue("test")
assert (q.hasQueue("test"))
def test_createQueue():
q = QueuePool(1)
q.createQueue("test")
assert (q.queue.test)
def test_listQueues():
q = QueuePool(1)
q.createQueue("hello")
assert sorted(list(q.listQueues(names=True))) == sorted(['hello', '_failed', '_success', '_logs', '_metrics'])
class Actor():
def __init__(self, config):
self.config = config
self.name = config.name
self.size = config.size
self.frequency = config.frequency
self.description = config.description
self.pool = QueuePool(config.size)
self.logging = Logging(config.name, self.pool.queue.logs)
self.__loop = True
self.greenlets = Greenlets([], [], [], [])
self.greenlets.metric.append(spawn(self.metricProducer))
self.__run = Event()
self.__run.clear()
self.__connections = {}
self.__children = {}
self.__parents = {}
self.__lookups = {}
self.__buildUplook()
self.stopped = True
def connect(self, source, destination_module, destination_queue):
'''Connects the <source> queue to the <destination> queue.
In fact, the source queue overwrites the destination queue.'''
if source in self.__children:
raise QueueConnected("Queue %s.%s is already connected to %s." %
(self.name, source, self.__children[source]))
else:
self.__children[source] = "%s.%s" % (destination_module.name,
destination_queue)
if destination_queue in destination_module.__parents:
raise QueueConnected(
"Queue %s.%s is already connected to %s" %
(destination_module.name, destination_queue,
destination_module.__parents[destination_queue]))
else:
destination_module.__parents[destination_queue] = "%s.%s" % (
self.name, source)
if not self.pool.hasQueue(source):
self.pool.createQueue(source)
setattr(destination_module.pool.queue, destination_queue,
self.pool.getQueue(source))
self.pool.getQueue(source).disableFallThrough()
self.logging.debug(
"Connected queue %s.%s to %s.%s" %
(self.name, source, destination_module.name, destination_queue))
def doEventLookup(self, name):
try:
return self.current_event.get(name)
except AttributeError:
return ""
except KeyError:
# No event has passed through this module. Most likely this is an
# input module which creates its own events. Therefor there is
# nothing to lookup yet and there for we rely up UpLook to return
# the default value for this lookup if any.
self.logging.debug(
"There is no lookup value with name '%s' Falling back to default lookup value if any."
% (name))
raise NoSuchValue
def getChildren(self, queue=None):
'''Returns the queue name <queue> is connected to.'''
if queue is None:
return [self.__children[q] for q in list(self.__children.keys())]
else:
return self.__children[queue]
def loop(self):
'''The global lock for this module'''
return self.__loop
def postHook(self):
pass
def preHook(self):
self.logging.debug("Initialized.")
def registerConsumer(self, function, queue):
'''Registers <function> to process all events in <queue>
Do not trap errors. When <function> fails then the event will be
submitted to the "failed" queue, If <function> succeeds to the
success queue.'''
self.greenlets.consumer.append(spawn(self.__consumer, function, queue))
def start(self):
'''Starts the module.'''
if hasattr(self, "preHook"):
self.logging.debug("preHook() found, executing")
self.preHook()
self.__run.set()
self.logging.debug(
"Started with max queue size of %s events and metrics interval of %s seconds."
% (self.size, self.frequency))
self.stopped = False
def sendToBackground(self, function, *args, **kwargs):
'''Executes a function and sends it to the background.
Background tasks are usually running indefinately. When such a
background task generates an error, it is automatically restarted and
an error is logged.
'''
def wrapIntoLoop():
while self.loop():
try:
function(*args, **kwargs)
# We want to break out of the loop if we get here because
# it's the intention of function() to exit without errors.
# Normally, background tasks run indefinately but in this
# case the user opted not to for some reason so we should
# obey that.
break
except Exception as err:
self.logging.error(
"Backgrounded function '%s' of module instance '%s' caused an error. This needs attention. Restarting it in 2 seconds. Reason: %s"
% (function.__name__, self.name, err))
sleep(2)
self.greenlets.generic.append(spawn(wrapIntoLoop))
def stop(self):
'''Stops the loop lock and waits until all registered consumers have exit otherwise kills them.'''
self.logging.info("Received stop. Initiating shutdown.")
self.__loop = False
for background_job in self.greenlets.metric:
kill(background_job)
for background_job in self.greenlets.generic:
kill(background_job)
for background_job in self.greenlets.consumer:
kill(background_job)
if hasattr(self, "postHook"):
self.logging.debug("postHook() found, executing")
self.postHook()
self.logging.debug("Exit.")
self.stopped = True
def submit(self, event, queue):
'''A convenience function which submits <event> to <queue>.'''
while self.loop():
try:
queue.put(event)
break
except QueueFull:
sleep(0.1)
def __consumer(self, function, queue):
'''Greenthread which applies <function> to each element from <queue>
'''
self.__run.wait()
while self.loop():
event = self.pool.queue.__dict__[queue].get()
self.current_event = event
try:
function(event)
except Exception as err:
exc_type, exc_value, exc_traceback = exc_info()
info = (traceback.extract_tb(exc_traceback)[-1][1],
str(exc_type), str(exc_value))
if isinstance(event, Wishbone_Event):
event.set(info, "@errors.%s" % (self.name))
elif (event, Bulk):
event.error = info
self.logging.error("%s" % (err))
self.submit(event, self.pool.queue.failed)
else:
self.submit(event, self.pool.queue.success)
def __buildUplook(self):
self.__current_event = {}
args = {}
for key, value in list(
inspect.getouterframes(
inspect.currentframe())[2][0].f_locals.items()):
if key == "self" or isinstance(value, ActorConfig):
next
else:
args[key] = value
uplook = UpLook(**args)
for name in uplook.listFunctions():
if name not in self.config.lookup:
raise ModuleInitFailure(
"A lookup function '%s' was defined but no lookup function with that name registered."
% (name))
else:
if self.config.lookup[name].__self__.__class__ == EventLookup:
uplook.registerLookup(name, self.doEventLookup)
else:
uplook.registerLookup(name, self.config.lookup[name])
self.uplook = uplook
self.kwargs = uplook.get()
def metricProducer(self):
'''A greenthread which collects the queue metrics at the defined interval.'''
self.__run.wait()
hostname = socket.gethostname()
while self.loop():
for queue in self.pool.listQueues(names=True):
for metric, value in list(
self.pool.getQueue(queue).stats().items()):
metric = Metric(time=time(),
type="wishbone",
source=hostname,
name="module.%s.queue.%s.%s" %
(self.name, queue, metric),
value=value,
unit="",
tags=())
event = Wishbone_Event(metric)
self.submit(event, self.pool.queue.metrics)
sleep(self.frequency)
class Actor(object):
"""A base class providing core Actor functionality.
The Actor base class is responsible for providing the base functionality,
setup and helper functions of a Wishbone module.
Args:
config (wishbone.actorconfig.ActorConfig): The ActorConfig object instance.
Attributes:
config (wishbone.actorconfig.ActorConfig): The ActorConfig object instance.
name (str): The name of the instance, derived from `config`.
description (str): The description of the actor based instance, derived from `config`.
pool (wishbone.pool.QueuePool): The Actor's queue pool.
Methods:
logging (wishbone.logging.Logging)
"""
def __init__(self, config):
self.config = config
self.name = config.name
self.description = self.__getDescription(config)
self.pool = QueuePool(config.size)
self.logging = Logging(
name=config.name,
q=self.pool.queue._logs,
identification=self.config.identification
)
self.__loop = True
self.greenlets = Greenlets([], [], [], [])
self.greenlets.metric.append(spawn(self.__metricProducer))
self._run = Event()
self._run.clear()
self.stopped = True
# Setup the Jinja2 environment to render kwargs templates.
##########################################################
self.env_template = jinja2.Environment(
undefined=jinja2.StrictUndefined,
trim_blocks=True,
loader=jinja2.FileSystemLoader('/')
)
# Add the template functions to the template globals
####################################################
for key, value in self.config.template_functions.items():
self.env_template.globals.update({key: value.get})
# Store a copy of the raw/unmodified kwargs
##########################################
self.kwargs_raw = self.__getRawKwargs()
# Store a copy of kwargs with all templates replaced by a template instance
############################################################################
self.kwargs_template = self.__getTemplateKwargs(self.env_template, self.kwargs_raw)
# Store a copy of the rendered kwargs as an EasyDict instance
#############################################################
self.kwargs = self.__renderTemplateKwargs(self.kwargs_template)
# Do some sanity checks
#######################
self.__sanityChecks()
# Validate and setup module
# Methods are come from based class
###################################
self._moduleInitValidation()
self._moduleInitSetup()
self.version = self.__getVersion()
def generateEvent(self, data={}, destination=None):
'''
Generates a new event.
This function can get overridden by
``wishbone.module.InputModule._generateNativeEvent``.
The provided ``data`` will be traversed in search of valid templates
which then will be rendered.
Args:
data (``data``): The payload to add to the event.
destination (None): The destination key to write the data to
Returns:
wishbone.event.Event: An event containing ``data`` as a payload.
'''
if destination in [None, "data"]:
event = Wishbone_Event(data)
event.renderField(destination, self.env_template)
else:
event = Wishbone_Event()
event.set(data, destination)
event.renderField(destination, self.env_template)
return event
def loop(self):
'''The global lock for this module.
Returns:
bool: True when module is in running mode. False if not.
'''
return self.__loop
def postHook(self):
'''
Is executed when module exits.
'''
self.logging.debug("Module has no postHook() method set.")
def preHook(self):
'''
Is executed when module starts. Can be overriden by the user.
'''
self.logging.debug("Module has no preHook() method set.")
def registerConsumer(self, function, queue):
'''
Registers <function> to process all events in <queue>
Don't not trap errors here. When <function> fails then the event will be
submitted to the "failed" queue, If <function> succeeds to the
success queue.
Registering ``function`` to consume ``queue`` will also apply all the
registered module functions against the events consumed from it.
Args:
function (``function``): The function which processes events
queue (str): The name of the queue from which ``function`` will
process the events.
Returns:
None
'''
self.greenlets.consumer.append(spawn(self._consumer, function, queue))
def renderEventKwargs(self, event, queue=None):
'''
Renders kwargs using the content of ``event`` and stores the result under
``event.kwargs``.
Args:
event (``wishbone.event.Event``): An Event instance
queue (str): The queue name so ``RenderKwargs`` can store the results
in the correct queue context.
Returns:
``wishbone.event.Event``: The provided event instance.
'''
event.kwargs = self.__renderKwargs.render(
queue_context=queue,
event_content=event.dump()
)
return event
def renderKwargs(self):
'''
Renders kwargs without making use of event content. This is typically
used when initiliazing a module and render the defined kwargs which do
not need a event data for rendering.
Returns:
None
'''
self.kwargs = self.__renderKwargs.render()
def start(self):
'''
Starts the module.
Returns:
None
'''
self.__postHook()
if hasattr(self, "preHook"):
self.logging.debug("preHook() found, executing")
self.preHook()
self.__validateAppliedFunctions()
self._run.set()
self.logging.debug(
"Started with max queue size of %s events and metrics interval of %s seconds." % (
self.config.size,
self.config.frequency
)
)
self.stopped = False
if not self.name.startswith("_"):
self.logging.debug("Started version %s" % (self.version))
def sendToBackground(self, function, *args, **kwargs):
'''
Executes a function and sends it to the background. Such a function
should never exit until ``self.loop`` returns ``False``.
This `method` wraps ``function`` again in a loop as long ``self.loop``
returns ``False`` so that ``function`` is restarted and an error is
logged.
Args:
function (``function``): The function which has to be executed.
*args: Variable length argument list.
**kwargs: Arbitrary keyword arguments.
'''
def wrapIntoLoop():
while self.loop():
try:
function(*args, **kwargs)
# We want to break out of the loop if we get here because
# it's the intention of function() to exit without errors.
# Normally, background tasks run indefinately but in this
# case the user opted not to for some reason so we should
# obey that.
break
except Exception as err:
if self.config.disable_exception_handling:
raise
self.logging.error("Backgrounded function '%s' of module instance '%s' caused an error. This needs attention. Restarting it in 2 seconds. Reason: %s" % (
function.__name__,
self.name,
err)
)
sleep(2)
self.greenlets.generic.append(spawn(wrapIntoLoop))
def stop(self):
'''
Makes ``self.loop`` return ``False`` and handles shutdown of of the
registered background jobs.
'''
self.logging.info("Received stop. Initiating shutdown.")
self.__loop = False
for background_job in self.greenlets.metric:
kill(background_job)
for background_job in self.greenlets.generic:
kill(background_job)
for background_job in self.greenlets.consumer:
kill(background_job)
if hasattr(self, "postHook"):
self.logging.debug("postHook() found, executing")
self.postHook()
self.logging.debug("Exit.")
self.stopped = True
def submit(self, event, queue):
'''
Submits <event> to the queue with name <queue>.
Args:
event (wishbone.event.Event): An event instance.
queue (str): The name of the queue
Returns:
None
'''
while self.loop():
try:
getattr(self.pool.queue, queue).put(event)
break
except AttributeError:
self.logging.error("No such queue %s. Event with uuid %s dropped." % (queue, event.get('uuid')))
break
except QueueFull:
self.logging.warning("Queue '%s' is full and stalls the event pipeline. You should probably look into this." % (queue))
sleep(0.1)
def _applyFunctions(self, queue, event):
'''
Executes and applies all registered module functions against the event.
Args:
queue (str): The name of the queue to which the function was registered.
event (wishbone.event.Event): The Wishbone
Returns:
wisbone.event.Event: The modified version of ``event``
'''
if queue in self.config.module_functions:
for f in self.config.module_functions[queue]:
try:
event = f.do(event)
except Exception as err:
if self.config.disable_exception_handling:
raise
self.logging.error("Function '%s' is skipped as it is causing an error. Reason: '%s'" % (f, err))
return event
def _consumer(self, function, queue):
'''
Greenthread which applies <function> to each element from <queue>
Args:
function (``function``): The function which has been registered to consume ``queue``.
queue (str): The name of the queue from which events have to be
consumed and processed by ``function``.
Returns:
None
'''
self._run.wait()
self.logging.debug("Function '%s' has been registered to consume queue '%s'" % (function.__name__, queue))
while self.loop():
event = self.pool.getQueue(queue).get()
if not event.has("tmp.%s" % (self.name)):
event.set({}, "tmp.%s" % (self.name))
# Render kwargs relative to the event's content and make these accessible under event.kwargs
event.renderKwargs(self.kwargs_template)
# Validate TTL
try:
event.decrementTTL()
except TTLExpired as err:
self.logging.warning("Event with UUID %s dropped. Reason: %s" % (event.get("uuid"), err))
continue
# Set the current event uuid to the logger object
self.logging.setCurrentEventID(event.get("uuid"))
# Apply all the defined queue functions to the event
event = self._applyFunctions(queue, event)
# Apply consumer function
try:
function(event)
except Exception as err:
if self.config.disable_exception_handling:
raise
exc_type, exc_value, exc_traceback = exc_info()
info = (traceback.extract_tb(exc_traceback)[-1][1], str(exc_type), str(exc_value))
event.set(info, "errors.%s" % (self.name))
self.logging.error("%s" % (err))
self.submit(event, "_failed")
else:
self.submit(event, "_success")
finally:
# Unset the current event uuid to the logger object
self.logging.setCurrentEventID(None)
def __getDescription(self, config):
'''
Gets the module description.
Args:
config (``wishbone.actorconfig.ActorConfig``): An ActorConfig instance
Returns:
str: The description of this actor instance.
'''
if config.description is None:
return self.__doc__.strip().split('\n')[0].strip('*')
else:
return config.description
def __getRawKwargs(self):
'''
Get the class paramaters of the class basing this class.
Returns (dict): A dict of the the raw kwargs
'''
kwargs = {}
for key, value in list(inspect.getouterframes(inspect.currentframe())[2][0].f_locals.items()):
if key == "self" or isinstance(value, ActorConfig):
next
else:
kwargs[key] = value
return kwargs
def __getTemplateKwargs(self, template_env, kwargs):
'''
Recurses through ``kwargs`` and returns a version of it in which all
strings are replaced by jinja2 template instances.
Args:
template_env (Jinja2.Environment instance): The Jinja2 environment instance to
derive templates from.
kwargs (dict): The dict of keyword/arguments.
'''
def recurse(data):
if isinstance(data, str):
try:
if len(list(template_env.parse(data).find_all(jinja2.nodes.Name))) > 0:
t = template_env.from_string(data)
return t
else:
return data
except Exception as err:
return data
elif isinstance(data, dict):
for key, value in data.items():
data[key] = recurse(value)
return data
elif isinstance(data, list):
for index, value in enumerate(data):
data[index] = recurse(value)
return data
else:
return data
return recurse(deepcopy(kwargs))
def __getVersion(self):
try:
return get_distribution(self.__module__.split('.')[0]).version
except Exception as err:
return "unknown"
def __metricProducer(self):
'''
A greenthread collecting the queue metrics at the defined interval.
'''
self._run.wait()
hostname = socket.gethostname()
while self.loop():
for queue in self.pool.listQueues(names=True):
for metric, value in list(self.pool.getQueue(queue).stats().items()):
event = Wishbone_Event({
"time": time(),
"type": "wishbone",
"source": hostname,
"name": "module.%s.queue.%s.%s" % (self.name, queue, metric),
"value": value,
"unit": "",
"tags": ()
})
self.submit(event, "_metrics")
sleep(self.config.frequency)
def __postHook(self):
'''
Is always executed when the module starts.
'''
self.logging.debug("Following template functions are available: %s" % ", ".join(
self.config.template_functions.keys()
)
)
def __renderTemplateKwargs(self, kwargs):
def recurse(data):
if isinstance(data, jinja2.environment.Template):
try:
return data.render()
except Exception as err:
return "#error: %s#" % (err)
# return self.template_strings[data]
elif isinstance(data, dict):
result = {}
for key, value in data.items():
result[key] = recurse(value)
return EasyDict(result)
elif isinstance(data, list):
result = []
for value in data:
result.append(recurse(value))
return result
else:
return data
rendered_kwargs = EasyDict(
recurse(
kwargs
)
)
return rendered_kwargs
def __validateAppliedFunctions(self):
'''
A validation routine which checks whether functions have been applied
to queues without a registered consumer. The effect of that would be
that the functions are never applied which is not what the user
wanted.
'''
queues_w_registered_consumers = [t.args[1] for t in self.greenlets.consumer]
for queue in self.config.module_functions.keys():
if queue not in queues_w_registered_consumers:
raise ModuleInitFailure("Failed to initialize module '%s'. You have functions defined on queue '%s' which doesn't have a registered consumer." % (self.name, queue))
def __sanityChecks(self):
'''
Does following validations:
- Validate if all template functions base ``TemplateFunction``
- Validate if all module functions base ``ModuleFunction``
- Validate if the module has attribute "MODULE_TYPE" indicating it's not an pre 3.0 module.
Args:
None
Returns:
None
Raises:
ModuleInitFailure: Raised when one of the components isn't correct.
'''
# Validate template functions
for n, f in self.config.template_functions.items():
if not isinstance(f, TemplateFunction):
raise ModuleInitFailure("Template function '%s' does not base TemplateFunction." % (n))
# Validate module functions
for name, functions in self.config.module_functions.items():
for function in functions:
if not isinstance(function, ModuleFunction):
raise ModuleInitFailure("Module function '%s' does not base ModuleFunction." % (name))
if not hasattr(self, "MODULE_TYPE"):
raise InvalidModule("Module instance '%s' seems to be of an incompatible old type." % (self.name))
