class Bucket(object):
def __init__(self, key, size, age, logging, queue, looplock):
self.key = key
self.size = size
self.age = age
self.logging = logging
self.queue = queue
self.loop = looplock
self.bucket = None
self.createEmptyBucket()
self.logging.info("Created new bucket with aggregation key '%s'." %
(self.key))
def createEmptyBucket(self):
self.bucket = Bulk(self.size)
self.resetTimer()
def flushBucketTimer(self):
'''
Flushes the buffer when <bucket_age> has expired.
'''
while self.loop():
sleep(1)
self._timer -= 1
if self._timer == 0:
if self.bucket.size() > 0:
self.logging.debug("Bucket age expired after %s s." %
(self.age))
self.flush()
else:
self.resetTimer()
def flush(self):
'''
Flushes the buffer.
'''
self.logging.debug("Flushed bucket '%s' of size '%s'" %
(self.key, self.bucket.size()))
self.queue.put(self.bucket)
self.createEmptyBucket()
def resetTimer(self):
'''
Resets the buffer expiry countdown to its configured value.
'''
self._timer = self.age
class Bucket(object):
def __init__(self, key, size, age, logging, queue, looplock):
self.key = key
self.size = size
self.age = age
self.logging = logging
self.queue = queue
self.loop = looplock
self.bucket = None
self.createEmptyBucket()
self.logging.info("Created new bucket with aggregation key '%s'." % (self.key))
def createEmptyBucket(self):
self.bucket = Bulk(self.size)
self.resetTimer()
def flushBucketTimer(self):
'''
Flushes the buffer when <bucket_age> has expired.
'''
while self.loop():
sleep(1)
self._timer -= 1
if self._timer == 0:
if self.bucket.size() > 0:
self.logging.debug("Bucket age expired after %s s." % (self.age))
self.flush()
else:
self.resetTimer()
def flush(self):
'''
Flushes the buffer.
'''
self.logging.debug("Flushed bucket '%s' of size '%s'" % (self.key, self.bucket.size()))
self.queue.put(self.bucket)
self.createEmptyBucket()
def resetTimer(self):
'''
Resets the buffer expiry countdown to its configured value.
'''
self._timer = self.age
def consume(self, event):
if isinstance(event, Bulk):
data = Bulk.dumpFieldsAsString(self.kwargs.selection)
else:
data = event.get(self.kwargs.selection)
self.socket.sendto(str(data), (self.kwargs.host, self.kwargs.port))
def test_module_tcp_bulk():
tcpserver = TCPServer()
tcpserver.start()
actor_config = ActorConfig('tcpout', 100, 1, {}, "")
tcpout = TCPOut(actor_config)
tcpout.pool.queue.inbox.disableFallThrough()
tcpout.start()
event_one = Event('this_is_a_test_1')
event_two = Event('this_is_a_test_2')
bulk_event = Bulk()
bulk_event.append(event_one)
bulk_event.append(event_two)
tcpout.pool.queue.inbox.put(bulk_event)
sleep(1)
one = getter(tcpserver.q)
assert one == "this_is_a_test_1\nthis_is_a_test_2\n"
tcpserver.stop()
def createEmptyBucket(self):
self.bucket = Bulk(self.size)
self.resetTimer()
def createEmptyBucket(self):
self.bucket = Bulk(self.size)
self.resetTimer()
