def test_message_max_size(self):
proc = FirehoseProcessor("test")
proc.MAX_REQUEST_SIZE = 1
# Setup the mock
proc._client.put_record_batch.return_value = dict(FailedPutCount=0)
# Start and log
proc.start()
proc.process("a decently larger message")
proc.stop()
assert len(self.mock_boto.mock_calls) == 2
assert len(proc._client.put_record_batch.mock_calls) == 1
def test_batch_send_failure(self):
proc = FirehoseProcessor("test")
proc.MAX_RECORD_BATCH = 1
# Setup the mock
proc._client.put_record_batch.return_value = dict(FailedPutCount=1)
# Start and log
proc.start()
proc.process("a decently larger message")
proc.stop()
assert len(self.mock_boto.mock_calls) == 4
assert len(proc._client.put_record_batch.mock_calls) == 3
def test_message_max_batch(self):
proc = FirehoseProcessor("test")
proc.MAX_RECORD_BATCH = 1
# Setup the mock
proc._client.put_record_batch.return_value = dict(FailedPutCount=0)
# Start and log
proc.start()
proc.process("a decently larger message")
proc.stop()
eq_(len(self.mock_boto.mock_calls), 2)
eq_(len(proc._client.put_record_batch.mock_calls), 1)
def test_queue_timeout(self):
proc = FirehoseProcessor("test")
proc.MAX_INTERVAL = 0
proc.start()
proc.stop()
eq_(len(self.mock_boto.mock_calls), 1)
def test_queue_timeout(self):
proc = FirehoseProcessor("test")
proc.MAX_INTERVAL = 0
proc._records.get = mock_get = Mock()
proc._send_record_batch = mock_send = Mock()
mock_get.side_effect = (Queue.Empty, None)
proc.start()
proc.stop()
mock_send.assert_called()
def test_full_queue(self):
proc = FirehoseProcessor("test", 1)
proc.process("test")
assert proc._records.full() is True
proc.process("another")
assert proc._records.qsize() == 1
assert proc._records.get() == "test"
def test_full_queue(self):
proc = FirehoseProcessor("test", 1)
proc.process("test")
eq_(proc._records.full(), True)
proc.process("another")
eq_(proc._records.qsize(), 1)
eq_(proc._records.get(), "test")
