Python Worker Examples

Example #1

 def test_reconnect(self):
     self.reader = FlakyReader(["baz"], self.config.buffer_size)
     self.client = Worker(self.reader, self.encoder, self.writer,
                          self.config)
     self.client.consume()
     self.assertTrue(self.writer.write.called)
     self.writer.write.assert_called_once_with(["baz"] *
                                               self.config.buffer_size)

Example #2

File: test_worker.py Project: citywander/kafka-influxdb

 def test_reconnect(self):
     """
     The worker should reconnect when the connection is interrupted.
     """
     reader = FlakyReader("baz", self.config.buffer_size)
     client = Worker(reader, self.encoder, self.writer, self.config)
     client.consume()
     self.assertTrue(self.writer.write.called)
     self.writer.write.assert_called_once_with(["baz"] *
                                               self.config.buffer_size)

Example #3

File: test_worker.py Project: mre/kafka-influxdb

 def test_reconnect(self):
     """
     The worker should reconnect when the connection is interrupted.
     """
     reader = FlakyReader("baz", self.config.buffer_size)
     client = Worker(reader, self.encoder, self.writer, self.config)
     client.consume()
     self.assertTrue(self.writer.write.called)
     self.writer.write.assert_called_once_with(
         ["baz"] * self.config.buffer_size)

Example #4

File: test_worker.py Project: mre/kafka-influxdb

 def test_flush(self):
     """
     Messages should be flushed to the writer when the buffer is full.
     """
     reader = DummyReader(["bar"], self.config.buffer_size)
     client = Worker(reader, self.encoder, self.writer, self.config)
     client.consume()
     self.assertTrue(self.writer.write.called)
     self.writer.write.assert_called_once_with(
         ["bar"] * self.config.buffer_size)

Example #5

File: test_worker.py Project: citywander/kafka-influxdb

 def test_flush(self):
     """
     Messages should be flushed to the writer when the buffer is full.
     """
     reader = DummyReader(["bar"], self.config.buffer_size)
     client = Worker(reader, self.encoder, self.writer, self.config)
     client.consume()
     self.assertTrue(self.writer.write.called)
     self.writer.write.assert_called_once_with(["bar"] *
                                               self.config.buffer_size)

Example #6

File: test_worker.py Project: mre/kafka-influxdb

 def test_buffer_timeout(self):
     """
     If the buffer has timed out flush to the writer.
     """
     reader = TimeoutReader(
         "bar", self.config.buffer_size, self.config.buffer_timeout)
     client = Worker(reader, self.encoder, self.writer, self.config)
     client.consume()
     self.assertTrue(self.writer.write.called)
     self.writer.write.assert_called_once_with(
         ["bar"] * int(self.config.buffer_size-1))

Example #7

File: test_worker.py Project: agrinessedge/backup-kafka-influxdb

 def test_buffer_timeout(self):
     """
     If the buffer has timed out flush to the writer.
     """
     reader = TimeoutReader("bar", self.config.buffer_size,
                            self.config.buffer_timeout)
     client = Worker(reader, self.encoder, self.writer, self.config)
     client.consume()
     self.assertTrue(self.writer.write.called)
     self.writer.write.assert_called_once_with(
         ["bar"] * int(self.config.buffer_size - 1))

Example #8

File: test_worker.py Project: mre/kafka-influxdb

    def test_create_database(self, _):
        """
        Retry creating the InfluxDB database in case of a connection error
        """
        reader = DummyReader(["bar"], self.config.buffer_size)
        writer_with_connection_error = Mock()
        writer_with_connection_error.create_database = Mock(
            side_effect=[ConnectionError, None])

        client = Worker(reader, self.encoder,
                        writer_with_connection_error, self.config)
        client.consume()
        self.assertEqual(
            writer_with_connection_error.create_database.call_count, 2)

Example #9

File: test_worker.py Project: agrinessedge/backup-kafka-influxdb

    def test_create_database(self, _):
        """
        Retry creating the InfluxDB database in case of a connection error
        """
        reader = DummyReader(["bar"], self.config.buffer_size)
        writer_with_connection_error = Mock()
        writer_with_connection_error.create_database = Mock(
            side_effect=[ConnectionError, None])

        client = Worker(reader, self.encoder, writer_with_connection_error,
                        self.config)
        client.consume()
        self.assertEqual(
            writer_with_connection_error.create_database.call_count, 2)

Example #10

File: __main__.py Project: vclang/kafka-influxdb

def start_consumer(config):
    """
    Start metrics consumer
    :param config:
    """
    logging.debug("Initializing Kafka Consumer")
    reader = load_reader(config.kafka_reader, config.kafka_host,
                         config.kafka_port, config.kafka_group,
                         config.kafka_topic)
    logging.debug("Initializing connection to InfluxDB at %s:%s",
                  config.influxdb_host, config.influxdb_port)
    writer = create_writer(config)
    logging.debug("Initializing message encoder: %s", config.encoder)
    encoder = load_encoder(config.encoder)
    client = Worker(reader, encoder, writer, config)
    client.consume()

Example #11

File: test_worker.py Project: Downlord/kafka-influxdb

class TestKafkaInfluxDB(unittest.TestCase):
    def setUp(self):
        self.config = Config(100)
        self.encoder = echo_encoder.Encoder()
        self.writer = DummyWriter()
        self.writer = Mock()
        self.writer.write.return_value = True

    def test_buffering(self):
        self.reader = DummyReader(["myhost.load.load.shortterm 0.05 1436357630"], self.config.buffer_size - 1)
        self.client = Worker(self.reader, self.encoder, self.writer, self.config)
        self.client.consume()
        self.assertFalse(self.writer.write.called)

    def test_flush(self):
        self.reader = DummyReader(["myhost.load.load.shortterm 0.05 1436357630"], self.config.buffer_size)
        self.client = Worker(self.reader, self.encoder, self.writer, self.config)
        self.client.consume()
        self.assertTrue(self.writer.write.called)

Example #12

File: test_worker.py Project: Omnistac/kafka-influxdb

class TestKafkaInfluxDB(unittest.TestCase):
    def setUp(self):
        self.config = Config(100)
        self.encoder = echo_encoder.Encoder()
        self.writer = DummyWriter()
        self.writer = Mock()
        self.writer.write.return_value = True

    def test_buffering(self):
        self.reader = DummyReader(["foo"], self.config.buffer_size - 1)
        self.client = Worker(self.reader, self.encoder, self.writer, self.config)
        self.client.consume()
        self.assertFalse(self.writer.write.called)

    def test_flush(self):
        self.reader = DummyReader(["bar"], self.config.buffer_size)
        self.client = Worker(self.reader, self.encoder, self.writer, self.config)
        self.client.consume()
        self.assertTrue(self.writer.write.called)

    def test_reconnect(self):
        self.reader = FlakyReader(["baz"], self.config.buffer_size)
        self.client = Worker(self.reader, self.encoder, self.writer, self.config)
        self.client.consume()
        self.assertTrue(self.writer.write.called)
        self.writer.write.assert_called_once_with(["baz"] * self.config.buffer_size)

Example #13

class TestKafkaInfluxDB(unittest.TestCase):
    def setUp(self):
        self.config = Config(100)
        self.encoder = echo_encoder.Encoder()
        self.writer = DummyWriter()
        self.writer = Mock()
        self.writer.write.return_value = True

    def test_buffering(self):
        self.reader = DummyReader(["foo"], self.config.buffer_size - 1)
        self.client = Worker(self.reader, self.encoder, self.writer,
                             self.config)
        self.client.consume()
        self.assertFalse(self.writer.write.called)

    def test_flush(self):
        self.reader = DummyReader(["bar"], self.config.buffer_size)
        self.client = Worker(self.reader, self.encoder, self.writer,
                             self.config)
        self.client.consume()
        self.assertTrue(self.writer.write.called)

    def test_reconnect(self):
        self.reader = FlakyReader(["baz"], self.config.buffer_size)
        self.client = Worker(self.reader, self.encoder, self.writer,
                             self.config)
        self.client.consume()
        self.assertTrue(self.writer.write.called)
        self.writer.write.assert_called_once_with(["baz"] *
                                                  self.config.buffer_size)

Example #14

File: test_worker.py Project: Omnistac/kafka-influxdb

 def test_reconnect(self):
     self.reader = FlakyReader(["baz"], self.config.buffer_size)
     self.client = Worker(self.reader, self.encoder, self.writer, self.config)
     self.client.consume()
     self.assertTrue(self.writer.write.called)
     self.writer.write.assert_called_once_with(["baz"] * self.config.buffer_size)

Example #15

File: test_worker.py Project: Omnistac/kafka-influxdb

 def test_flush(self):
     self.reader = DummyReader(["bar"], self.config.buffer_size)
     self.client = Worker(self.reader, self.encoder, self.writer, self.config)
     self.client.consume()
     self.assertTrue(self.writer.write.called)

Example #16

File: test_worker.py Project: Omnistac/kafka-influxdb

 def test_buffering(self):
     self.reader = DummyReader(["foo"], self.config.buffer_size - 1)
     self.client = Worker(self.reader, self.encoder, self.writer, self.config)
     self.client.consume()
     self.assertFalse(self.writer.write.called)

Example #17

File: test_worker.py Project: Downlord/kafka-influxdb

 def test_flush(self):
     self.reader = DummyReader(["myhost.load.load.shortterm 0.05 1436357630"], self.config.buffer_size)
     self.client = Worker(self.reader, self.encoder, self.writer, self.config)
     self.client.consume()
     self.assertTrue(self.writer.write.called)

Example #18

 def test_flush(self):
     self.reader = DummyReader(["bar"], self.config.buffer_size)
     self.client = Worker(self.reader, self.encoder, self.writer,
                          self.config)
     self.client.consume()
     self.assertTrue(self.writer.write.called)

Example #19

 def test_buffering(self):
     self.reader = DummyReader(["foo"], self.config.buffer_size - 1)
     self.client = Worker(self.reader, self.encoder, self.writer,
                          self.config)
     self.client.consume()
     self.assertFalse(self.writer.write.called)