def __init__(self):
super().__init__()
self._queues = defaultdict(list)
self._queue_locks = {'null': Lock()}
self._meta_lock = Lock()
self._emit_job = None
self._backup_job = None
class ServiceProfileData(object):
'''Handles the communication between blocks on the ProfileData output block.
Use `with ServiceProfileData` to use the data. Put new data with put
'''
def __init__(self):
self._data = {}
self._lock = Lock()
def put(self, name, value):
'''Use this function to put new data in'''
spawn(self._put, name, value)
def _put(self, name, value):
with self._lock:
if name in self._data:
self._data[name] += value
else:
self._data[name] = value
def __enter__(self):
self._lock.aquire()
return self._data
def __exit__(self):
self._lock.release()
def __init__(self):
super().__init__()
self._queues = defaultdict(list)
self._queue_locks = {
'null': Lock()
}
self._meta_lock = Lock()
self._emit_job = None
self._backup_job = None
def _get_lock(self, grp="null"):
''' Returns the lock for a particular queue.
Note that we're maintaining a synchronized dictionary of locks
alongside our dict of queues.
'''
with self._meta_lock:
self._queue_locks[grp] = self._queue_locks.get(grp, Lock())
return self._queue_locks[grp]
def __init__(self):
super().__init__()
self._last_emission = None
self._cache = defaultdict(list)
self._cache_lock = Lock()
self._emission_job = None
class Queue(GroupBy, Block):
""" Queue block.
A NIO block for queueing up signals. As signals pile up,
the Queue block releases a configurable number at a configurable
interval. If incoming signals would overflow the queue, signals
are popped off the front as needed.
If a 'group_by' string is configured, incoming signals are divided
and grouped by the value of that attribute. The configured capacity
applies to *each* such queue, not the block as a whole.
"""
version = StringProperty(default='1.0')
interval = TimeDeltaProperty(title='Notification Interval')
backup_interval = TimeDeltaProperty(title='Backup Interval',
visible=False,
default={"minutes": 10})
capacity = IntProperty(default=100, title='Capacity')
group_by = ExpressionProperty(default='null', attr_default='null', title='Group By')
chunk_size = IntProperty(default=1, title='Chunk Size')
reload = BoolProperty(default=False, title='Auto-Reload?')
uniqueness = ExpressionProperty(title='Queue Uniqueness Expression',
attr_default=None)
update = BoolProperty(title='Update Non-Unique Signals', default=False)
def __init__(self):
super().__init__()
self._queues = defaultdict(list)
self._queue_locks = {
'null': Lock()
}
self._meta_lock = Lock()
self._emit_job = None
self._backup_job = None
def configure(self, context):
super().configure(context)
self._load()
def start(self):
super().start()
self._start_emit_job()
self._backup_job = Job(
self._backup,
self.backup_interval,
True
)
def stop(self):
if self._emit_job is not None:
self._emit_job.cancel()
self._backup_job.cancel()
self._backup()
def process_signals(self, signals):
self._logger.debug("Processing {} signals".format(len(signals)))
self.for_each_group(self._push_group, signals)
def pop(self, grp="null"):
''' Remove the top n signals from the specified queue.
Args:
grp (str): The queue from which to pop.
count (int): The number of signals to pop off.
reload (bool): If True, put popped signals back on queue.
Returns:
top_n (list): 'Count' signals from the front of the queue.
'''
count = self.chunk_size
reload = self.reload
# lock the queue we're popping from
self._logger.debug("pop: {} {} {}".format(grp, count, reload))
with self._get_lock(grp):
# check out the front of the queue
top_n = self._queues[grp][0:count]
self._logger.debug(
"Removing %d signals from %s_queue" % (len(top_n), grp))
self._queues[grp][:] = self._queues[grp][len(top_n):]
# If reloading, put signal back on queue.
if reload:
self._logger.debug("Reloading {}_queue".format(grp))
self._queues[grp].extend(top_n)
return top_n
def push(self, signal, grp):
''' Add a signal to the back of the queue.
Args:
signal (Signal): The signal to add.
grp (str): Group to add signal to.
Returns:
None
'''
queue = self._queues[grp]
# check for uniqueness if property is set
try:
unique_val = self.uniqueness(signal)
self._logger.debug(
"Testing uniqueness for signal: {}".format(unique_val))
except Exception as e:
unique_val = None
self._logger.warning(
"Uniqueness expression failed. Using value of None.")
if unique_val is not None:
for idx, sig in enumerate(queue):
try:
sig_val = self.uniqueness(sig)
except Exception as e:
sig_val = None
if sig_val == unique_val:
self._logger.debug(
"Signal {} already in {}_queue".format(sig_val, grp)
)
if self.update:
queue[idx] = signal
return
# pop one off the top of that queue if it's at capacity
if len(queue) == self.capacity:
self._logger.debug(
"Pushing signal and capactity of {}_signal is full: {}".format(
grp, self.capacity
)
)
queue.pop(0)
self._logger.debug("Appending signal to {}_queue".format(grp))
queue.append(signal)
def _push_group(self, signals, group):
# lock the queue before appending
with self._get_lock(group):
for signal in signals:
self.push(signal, group)
def _get_lock(self, grp="null"):
''' Returns the lock for a particular queue.
Note that we're maintaining a synchronized dictionary of locks
alongside our dict of queues.
'''
with self._meta_lock:
self._queue_locks[grp] = self._queue_locks.get(grp, Lock())
return self._queue_locks[grp]
def _start_emit_job(self):
if self.interval.total_seconds() >= 0:
self._emit_job = Job(
self.emit,
self.interval,
True
)
def emit(self):
''' Notify the configured number of signals from the front of the queue.
'''
signals_to_notify = self.for_each_group(self.pop)
if signals_to_notify:
self._logger.debug(
"Notifying {} signals".format(len(signals_to_notify))
)
self.notify_signals(signals_to_notify)
def _load(self):
prev_queues = self.persistence.load('queues')
# if persisted dictonary is not defaultdict, convert it
if prev_queues:
self._queues = defaultdict(list, prev_queues)
# build _groups for groupby mixin
self._groups = list(self._queues.keys())
def _backup(self):
''' Persist the current state of the queues using the persistence module.
'''
# store the serialized signals and save to disk
# grab the meta_lock so nobody else can interact with the queues during
# serialization
self._logger.debug("Persistence: backing up to file")
self._meta_lock.acquire()
self.persistence.store("queues", self._queues)
self._meta_lock.release()
self.persistence.save()
def _inspect_group(self, response, group):
response_group = {'count': 0, 'signals': []}
query = response.get('query', '{{ True }}')
ignored_signals = []
for signal in self._queues.get(group, []):
try:
eval = Evaluator(query, None).evaluate(signal)
except:
eval = False
if eval:
response_group['signals'].append(json.loads(json.dumps(signal.to_dict(), indent=4, separators=(',', ': '), default=str)))
response_group['count'] += 1
response['count'] +=1
else:
ignored_signals.append(signal)
response['groups'][group] = response_group
return response, ignored_signals
def view(self, query, group):
''' Command to view the signals that are in the queue.
If no group parameter is specified, all queues are returned.
'''
self._logger.debug("Command: view")
response = {}
response['query'] = query
response['group'] = group
response['count'] = 0
response['groups'] = {}
if group and group in self._queues:
# if group exists, return only the specified group
self._view_group(group, response)
elif not group:
# if no group is specifed in params return all groups
self.for_each_group(self._view_group,
kwargs={'response': response})
return response
def _view_group(self, group, response):
with self._get_lock(group):
response, _ = self._inspect_group(response, group)
def remove(self, query, group):
''' Remove signals from *group* where *query* is True.
Signals are not notified.
'''
self._logger.debug("Command: remove")
response = {}
response['query'] = query
response['group'] = group
response['count'] = 0
response['groups'] = {}
if group and group in self._queues:
# if group exists, remove from only only the specified group
self._remove_from_group(group, response, query)
elif not group:
# if no group is specifed in params return all groups
self.for_each_group(self._remove_from_group,
kwargs={'response': response,
'query': query})
return response
def _remove_from_group(self, group, response, query):
with self._get_lock(group):
response, signals = self._inspect_group(response, group)
# signals that don't match the query stay in the queue.
self._queues[group] = signals
def update_props(self, props):
''' Updates the *interval* property.
The next scheduled emit job with be canceled and a new repeatable emit
job is started.
'''
self._logger.debug("Command: update_props")
response = {}
if props is None or not isinstance(props, dict):
response['message'] = \
"'props' needs to be a dictionary: {}".format(props)
return response
# Update *interval*.
interval = props.get('interval')
if interval and isinstance(interval, dict) and \
(interval.get('days') or interval.get('seconds') \
or interval.get('microseconds')):
days = interval.get('days', 0)
seconds = interval.get('seconds', 0)
microseconds = interval.get('microseconds', 0)
interval = timedelta(days, seconds, microseconds)
response['interval'] = interval
response['prev_interval'] = self.interval
# cancel emit job and restart with new interval
if self._emit_job is not None:
self._emit_job.cancel()
self._start_emit_job()
self.interval = interval
self._logger.info('Interval has been updated to {}'.format(interval))
elif interval:
response['message'] = "'interval' needs to be a timedelta dict: {}".format(interval)
return response
class Queue(GroupBy, Block):
""" Queue block.
A NIO block for queueing up signals. As signals pile up,
the Queue block releases a configurable number at a configurable
interval. If incoming signals would overflow the queue, signals
are popped off the front as needed.
If a 'group_by' string is configured, incoming signals are divided
and grouped by the value of that attribute. The configured capacity
applies to *each* such queue, not the block as a whole.
"""
version = StringProperty(default='1.0')
interval = TimeDeltaProperty(title='Notification Interval')
backup_interval = TimeDeltaProperty(title='Backup Interval',
visible=False,
default={"minutes": 10})
capacity = IntProperty(default=100, title='Capacity')
group_by = ExpressionProperty(default='null',
attr_default='null',
title='Group By')
chunk_size = IntProperty(default=1, title='Chunk Size')
reload = BoolProperty(default=False, title='Auto-Reload?')
uniqueness = ExpressionProperty(title='Queue Uniqueness Expression',
attr_default=None)
update = BoolProperty(title='Update Non-Unique Signals', default=False)
def __init__(self):
super().__init__()
self._queues = defaultdict(list)
self._queue_locks = {'null': Lock()}
self._meta_lock = Lock()
self._emit_job = None
self._backup_job = None
def configure(self, context):
super().configure(context)
self._load()
def start(self):
super().start()
self._start_emit_job()
self._backup_job = Job(self._backup, self.backup_interval, True)
def stop(self):
if self._emit_job is not None:
self._emit_job.cancel()
self._backup_job.cancel()
self._backup()
def process_signals(self, signals):
self._logger.debug("Processing {} signals".format(len(signals)))
self.for_each_group(self._push_group, signals)
def pop(self, grp="null"):
''' Remove the top n signals from the specified queue.
Args:
grp (str): The queue from which to pop.
count (int): The number of signals to pop off.
reload (bool): If True, put popped signals back on queue.
Returns:
top_n (list): 'Count' signals from the front of the queue.
'''
count = self.chunk_size
reload = self.reload
# lock the queue we're popping from
self._logger.debug("pop: {} {} {}".format(grp, count, reload))
with self._get_lock(grp):
# check out the front of the queue
top_n = self._queues[grp][0:count]
self._logger.debug("Removing %d signals from %s_queue" %
(len(top_n), grp))
self._queues[grp][:] = self._queues[grp][len(top_n):]
# If reloading, put signal back on queue.
if reload:
self._logger.debug("Reloading {}_queue".format(grp))
self._queues[grp].extend(top_n)
return top_n
def push(self, signal, grp):
''' Add a signal to the back of the queue.
Args:
signal (Signal): The signal to add.
grp (str): Group to add signal to.
Returns:
None
'''
queue = self._queues[grp]
# check for uniqueness if property is set
try:
unique_val = self.uniqueness(signal)
self._logger.debug(
"Testing uniqueness for signal: {}".format(unique_val))
except Exception as e:
unique_val = None
self._logger.warning(
"Uniqueness expression failed. Using value of None.")
if unique_val is not None:
for idx, sig in enumerate(queue):
try:
sig_val = self.uniqueness(sig)
except Exception as e:
sig_val = None
if sig_val == unique_val:
self._logger.debug("Signal {} already in {}_queue".format(
sig_val, grp))
if self.update:
queue[idx] = signal
return
# pop one off the top of that queue if it's at capacity
if len(queue) == self.capacity:
self._logger.debug(
"Pushing signal and capactity of {}_signal is full: {}".format(
grp, self.capacity))
queue.pop(0)
self._logger.debug("Appending signal to {}_queue".format(grp))
queue.append(signal)
def _push_group(self, signals, group):
# lock the queue before appending
with self._get_lock(group):
for signal in signals:
self.push(signal, group)
def _get_lock(self, grp="null"):
''' Returns the lock for a particular queue.
Note that we're maintaining a synchronized dictionary of locks
alongside our dict of queues.
'''
with self._meta_lock:
self._queue_locks[grp] = self._queue_locks.get(grp, Lock())
return self._queue_locks[grp]
def _start_emit_job(self):
if self.interval.total_seconds() >= 0:
self._emit_job = Job(self.emit, self.interval, True)
def emit(self):
''' Notify the configured number of signals from the front of the queue.
'''
signals_to_notify = self.for_each_group(self.pop)
if signals_to_notify:
self._logger.debug("Notifying {} signals".format(
len(signals_to_notify)))
self.notify_signals(signals_to_notify)
def _load(self):
prev_queues = self.persistence.load('queues')
# if persisted dictonary is not defaultdict, convert it
if prev_queues:
self._queues = defaultdict(list, prev_queues)
# build _groups for groupby mixin
self._groups = list(self._queues.keys())
def _backup(self):
''' Persist the current state of the queues using the persistence module.
'''
# store the serialized signals and save to disk
# grab the meta_lock so nobody else can interact with the queues during
# serialization
self._logger.debug("Persistence: backing up to file")
self._meta_lock.acquire()
self.persistence.store("queues", self._queues)
self._meta_lock.release()
self.persistence.save()
def _inspect_group(self, response, group):
response_group = {'count': 0, 'signals': []}
query = response.get('query', '{{ True }}')
ignored_signals = []
for signal in self._queues.get(group, []):
try:
eval = Evaluator(query, None).evaluate(signal)
except:
eval = False
if eval:
response_group['signals'].append(
json.loads(
json.dumps(signal.to_dict(),
indent=4,
separators=(',', ': '),
default=str)))
response_group['count'] += 1
response['count'] += 1
else:
ignored_signals.append(signal)
response['groups'][group] = response_group
return response, ignored_signals
def view(self, query, group):
''' Command to view the signals that are in the queue.
If no group parameter is specified, all queues are returned.
'''
self._logger.debug("Command: view")
response = {}
response['query'] = query
response['group'] = group
response['count'] = 0
response['groups'] = {}
if group and group in self._queues:
# if group exists, return only the specified group
self._view_group(group, response)
elif not group:
# if no group is specifed in params return all groups
self.for_each_group(self._view_group,
kwargs={'response': response})
return response
def _view_group(self, group, response):
with self._get_lock(group):
response, _ = self._inspect_group(response, group)
def remove(self, query, group):
''' Remove signals from *group* where *query* is True.
Signals are not notified.
'''
self._logger.debug("Command: remove")
response = {}
response['query'] = query
response['group'] = group
response['count'] = 0
response['groups'] = {}
if group and group in self._queues:
# if group exists, remove from only only the specified group
self._remove_from_group(group, response, query)
elif not group:
# if no group is specifed in params return all groups
self.for_each_group(self._remove_from_group,
kwargs={
'response': response,
'query': query
})
return response
def _remove_from_group(self, group, response, query):
with self._get_lock(group):
response, signals = self._inspect_group(response, group)
# signals that don't match the query stay in the queue.
self._queues[group] = signals
def update_props(self, props):
''' Updates the *interval* property.
The next scheduled emit job with be canceled and a new repeatable emit
job is started.
'''
self._logger.debug("Command: update_props")
response = {}
if props is None or not isinstance(props, dict):
response['message'] = \
"'props' needs to be a dictionary: {}".format(props)
return response
# Update *interval*.
interval = props.get('interval')
if interval and isinstance(interval, dict) and \
(interval.get('days') or interval.get('seconds') \
or interval.get('microseconds')):
days = interval.get('days', 0)
seconds = interval.get('seconds', 0)
microseconds = interval.get('microseconds', 0)
interval = timedelta(days, seconds, microseconds)
response['interval'] = interval
response['prev_interval'] = self.interval
# cancel emit job and restart with new interval
if self._emit_job is not None:
self._emit_job.cancel()
self._start_emit_job()
self.interval = interval
self._logger.info(
'Interval has been updated to {}'.format(interval))
elif interval:
response[
'message'] = "'interval' needs to be a timedelta dict: {}".format(
interval)
return response
def __init__(self):
self._data = {}
self._lock = Lock()