Ejemplo n.º 1
0
class Bucket(object):
    def __init__(self, key, size, age, logging, queue, looplock):

        self.key = key
        self.size = size
        self.age = age
        self.logging = logging
        self.queue = queue
        self.loop = looplock

        self.bucket = None
        self.createEmptyBucket()
        self.logging.info("Created new bucket with aggregation key '%s'." %
                          (self.key))

    def createEmptyBucket(self):
        self.bucket = Bulk(self.size)
        self.resetTimer()

    def flushBucketTimer(self):
        '''
        Flushes the buffer when <bucket_age> has expired.
        '''

        while self.loop():
            sleep(1)
            self._timer -= 1
            if self._timer == 0:
                if self.bucket.size() > 0:
                    self.logging.debug("Bucket age expired after %s s." %
                                       (self.age))
                    self.flush()
                else:
                    self.resetTimer()

    def flush(self):
        '''
        Flushes the buffer.
        '''
        self.logging.debug("Flushed bucket '%s' of size '%s'" %
                           (self.key, self.bucket.size()))
        self.queue.put(self.bucket)
        self.createEmptyBucket()

    def resetTimer(self):
        '''
        Resets the buffer expiry countdown to its configured value.
        '''

        self._timer = self.age
Ejemplo n.º 2
0
class Bucket(object):

    def __init__(self, key, size, age, logging, queue, looplock):

        self.key = key
        self.size = size
        self.age = age
        self.logging = logging
        self.queue = queue
        self.loop = looplock

        self.bucket = None
        self.createEmptyBucket()
        self.logging.info("Created new bucket with aggregation key '%s'." % (self.key))

    def createEmptyBucket(self):
        self.bucket = Bulk(self.size)
        self.resetTimer()

    def flushBucketTimer(self):

        '''
        Flushes the buffer when <bucket_age> has expired.
        '''

        while self.loop():
            sleep(1)
            self._timer -= 1
            if self._timer == 0:
                if self.bucket.size() > 0:
                    self.logging.debug("Bucket age expired after %s s." % (self.age))
                    self.flush()
                else:
                    self.resetTimer()

    def flush(self):
        '''
        Flushes the buffer.
        '''
        self.logging.debug("Flushed bucket '%s' of size '%s'" % (self.key, self.bucket.size()))
        self.queue.put(self.bucket)
        self.createEmptyBucket()

    def resetTimer(self):
        '''
        Resets the buffer expiry countdown to its configured value.
        '''

        self._timer = self.age
Ejemplo n.º 3
0
    def consume(self, event):

        if isinstance(event, Bulk):
            data = Bulk.dumpFieldsAsString(self.kwargs.selection)
        else:
            data = event.get(self.kwargs.selection)

        self.socket.sendto(str(data), (self.kwargs.host, self.kwargs.port))
def test_module_tcp_bulk():

    tcpserver = TCPServer()
    tcpserver.start()

    actor_config = ActorConfig('tcpout', 100, 1, {}, "")
    tcpout = TCPOut(actor_config)
    tcpout.pool.queue.inbox.disableFallThrough()
    tcpout.start()

    event_one = Event('this_is_a_test_1')
    event_two = Event('this_is_a_test_2')

    bulk_event = Bulk()
    bulk_event.append(event_one)
    bulk_event.append(event_two)

    tcpout.pool.queue.inbox.put(bulk_event)

    sleep(1)
    one = getter(tcpserver.q)
    assert one == "this_is_a_test_1\nthis_is_a_test_2\n"
    tcpserver.stop()
Ejemplo n.º 5
0
 def createEmptyBucket(self):
     self.bucket = Bulk(self.size)
     self.resetTimer()
Ejemplo n.º 6
0
 def createEmptyBucket(self):
     self.bucket = Bulk(self.size)
     self.resetTimer()