def test_compacted_topic_partition_rejects_old_message_after_initial(self):
        last_offset = 400
        message1 = mock.Mock()
        message1.value = "first-test"
        message1.partition_id = 0
        message1.offset = last_offset

        partition = mock.MagicMock()
        partition.id = 0
        op = OwnedPartition(partition, compacted_topic=True)
        op.enqueue_messages([message1])
        self.assertEqual(op.message_count, 1)
        op.consume()
        self.assertEqual(op.message_count, 0)
        self.assertEqual(op.last_offset_consumed, last_offset)

        message2 = mock.Mock()
        message2.value = "test"
        message2.partition_id = 0
        message2.offset = 20

        op.enqueue_messages([message2])
        self.assertEqual(op.message_count, 0)
        op.consume()
        self.assertEqual(op.last_offset_consumed, last_offset)
示例#2
0
    def test_partition_offset_counters(self):
        res = mock.Mock()
        res.offset = 400

        op = OwnedPartition(None)
        op.set_offset(res.offset)

        self.assertEqual(op.last_offset_consumed, res.offset)
        self.assertEqual(op.next_offset, res.offset + 1)
    def test_partition_offset_counters(self):
        res = mock.Mock()
        res.offset = 400

        op = OwnedPartition(None)
        op.set_offset(res.offset)

        self.assertEqual(op.last_offset_consumed, res.offset)
        self.assertEqual(op.next_offset, res.offset + 1)
示例#4
0
    def test_compacted_topic_partition_rejects_old_message_after_initial(self):
        last_offset = 400
        message1 = mock.Mock()
        message1.value = "first-test"
        message1.partition_id = 0
        message1.offset = last_offset

        partition = mock.MagicMock()
        partition.id = 0
        op = OwnedPartition(partition, compacted_topic=True)
        op.enqueue_messages([message1])
        self.assertEqual(op.message_count, 1)
        op.consume()
        self.assertEqual(op.message_count, 0)
        self.assertEqual(op.last_offset_consumed, last_offset)

        message2 = mock.Mock()
        message2.value = "test"
        message2.partition_id = 0
        message2.offset = 20

        op.enqueue_messages([message2])
        self.assertEqual(op.message_count, 0)
        op.consume()
        self.assertEqual(op.last_offset_consumed, last_offset)
    def test_partition_offset_fetch_request(self):
        topic = mock.Mock()
        topic.name = "test_topic"
        partition = mock.Mock()
        partition.topic = topic
        partition.id = 12345

        op = OwnedPartition(partition)

        request = op.build_offset_fetch_request()

        self.assertEqual(request.topic_name, topic.name)
        self.assertEqual(request.partition_id, partition.id)
示例#6
0
    def test_partition_offset_fetch_request(self):
        topic = mock.Mock()
        topic.name = "test_topic"
        partition = mock.Mock()
        partition.topic = topic
        partition.id = 12345

        op = OwnedPartition(partition)

        request = op.build_offset_fetch_request()

        self.assertEqual(request.topic_name, topic.name)
        self.assertEqual(request.partition_id, partition.id)
    def test_partition_saves_offset(self):
        msgval = "test"
        op = OwnedPartition(None)

        message = mock.Mock()
        message.value = msgval
        message.offset = 20

        op.enqueue_messages([message])
        self.assertEqual(op.message_count, 1)
        ret_message = op.consume()
        self.assertEqual(op.last_offset_consumed, message.offset)
        self.assertEqual(op.next_offset, message.offset + 1)
        self.assertNotEqual(ret_message, None)
        self.assertEqual(ret_message.value, msgval)
示例#8
0
    def test_partition_offset_commit_request(self):
        topic = mock.Mock()
        topic.name = "test_topic"
        partition = mock.Mock()
        partition.topic = topic
        partition.id = 12345

        op = OwnedPartition(partition)
        op.last_offset_consumed = 200

        request = op.build_offset_commit_request()

        self.assertEqual(request.topic_name, topic.name)
        self.assertEqual(request.partition_id, partition.id)
        self.assertEqual(request.offset, op.last_offset_consumed + 1)
        self.assertEqual(request.metadata, b'pykafka')
示例#9
0
    def test_partition_saves_offset(self):
        msgval = "test"
        partition = mock.MagicMock()
        op = OwnedPartition(partition)

        message = mock.Mock()
        message.value = msgval
        message.offset = 20

        op.enqueue_messages([message])
        self.assertEqual(op.message_count, 1)
        ret_message = op.consume()
        self.assertEqual(op.last_offset_consumed, message.offset)
        self.assertEqual(op.next_offset, message.offset + 1)
        self.assertNotEqual(ret_message, None)
        self.assertEqual(ret_message.value, msgval)
示例#10
0
    def test_partition_offset_commit_request(self):
        topic = mock.Mock()
        topic.name = "test_topic"
        partition = mock.Mock()
        partition.topic = topic
        partition.id = 12345

        op = OwnedPartition(partition)
        op.last_offset_consumed = 200

        request = op.build_offset_commit_request()

        self.assertEqual(request.topic_name, topic.name)
        self.assertEqual(request.partition_id, partition.id)
        self.assertEqual(request.offset, op.last_offset_consumed + 1)
        self.assertEqual(request.metadata, b'pykafka')
示例#11
0
    def test_partition_offset_commit_request(self):
        topic = mock.Mock()
        topic.name = "test_topic"
        partition = mock.Mock()
        partition.topic = topic
        partition.id = 12345

        op = OwnedPartition(partition)
        op.last_offset_consumed = 200

        request = op.build_offset_commit_request()

        self.assertEqual(request.topic_name, topic.name)
        self.assertEqual(request.partition_id, partition.id)
        self.assertEqual(request.offset, op.last_offset_consumed + 1)
        parsed_metadata = json.loads(get_string(request.metadata))
        self.assertEqual(parsed_metadata["consumer_id"], '')
        self.assertTrue(bool(parsed_metadata["hostname"]))
示例#12
0
    def test_partition_offset_commit_request(self):
        topic = mock.Mock()
        topic.name = "test_topic"
        partition = mock.Mock()
        partition.topic = topic
        partition.id = 12345

        op = OwnedPartition(partition)
        op.last_offset_consumed = 200

        request = op.build_offset_commit_request()

        self.assertEqual(request.topic_name, topic.name)
        self.assertEqual(request.partition_id, partition.id)
        self.assertEqual(request.offset, op.last_offset_consumed + 1)
        parsed_metadata = json.loads(get_string(request.metadata))
        self.assertEqual(parsed_metadata["consumer_id"], '')
        self.assertTrue(bool(parsed_metadata["hostname"]))
示例#13
0
    def test_partition_offset_commit_request(self):
        topic = mock.Mock()
        topic.name = "test_topic"
        partition = mock.Mock()
        partition.topic = topic
        partition.id = 12345

        op = OwnedPartition(partition)
        op.last_offset_consumed = 200

        rqtime = int(time.time())
        request = op.build_offset_commit_request()

        self.assertEqual(request.topic_name, topic.name)
        self.assertEqual(request.partition_id, partition.id)
        self.assertEqual(request.offset, op.last_offset_consumed)
        # sketchy, but it works because of second resolution
        self.assertEqual(request.timestamp, rqtime)
        self.assertEqual(request.metadata, 'pykafka')
示例#14
0
    def test_partition_rejects_old_message(self):
        last_offset = 400
        op = OwnedPartition(None)
        op.last_offset_consumed = last_offset

        message = mock.Mock()
        message.value = "test"
        message.offset = 20

        op.enqueue_messages([message])
        self.assertEqual(op.message_count, 0)
        op.consume()
        self.assertEqual(op.last_offset_consumed, last_offset)
示例#15
0
    def test_partition_rejects_old_message(self):
        last_offset = 400
        op = OwnedPartition(None)
        op.last_offset_consumed = last_offset

        message = mock.Mock()
        message.value = "test"
        message.offset = 20

        op.enqueue_messages([message])
        self.assertEqual(op.message_count, 0)
        op.consume()
        self.assertEqual(op.last_offset_consumed, last_offset)
示例#16
0
    def test_partition_consume_empty_queue(self):
        op = OwnedPartition(None)

        message = op.consume()
        self.assertEqual(message, None)
示例#17
0
    def test_partition_consume_empty_queue(self):
        op = OwnedPartition(None)

        message = op.consume()
        self.assertEqual(message, None)