Exemplo n.º 1
0
    async def test_memory_checkpoint(self):
        # first consumer
        checkpointer = MemoryCheckPointer(name="test")

        consumer_a = Consumer(stream_name=None,
                              checkpointer=checkpointer,
                              max_shard_consumers=1)

        self.patch_consumer_fetch(consumer_a)

        consumer_a.shards = [{"ShardId": "test-1"}, {"ShardId": "test-2"}]

        await consumer_a.fetch()

        shards = [s["ShardId"] for s in consumer_a.shards if s.get("stats")]

        # Expect only one shard assigned as max = 1
        self.assertEqual(["test-1"], shards)

        # second consumer (note: max_shard_consumers needs to be 2 as uses checkpointer to get allocated shards)

        consumer_b = Consumer(stream_name=None,
                              checkpointer=checkpointer,
                              max_shard_consumers=2)

        self.patch_consumer_fetch(consumer_b)

        consumer_b.shards = [{"ShardId": "test-1"}, {"ShardId": "test-2"}]

        await consumer_b.fetch()

        shards = [s["ShardId"] for s in consumer_b.shards if s.get("stats")]

        # Expect only one shard assigned as max = 1
        self.assertEqual(["test-2"], shards)
Exemplo n.º 2
0
    async def test_memory_checkpoint(self, *args):
        # *args pass through mock
        # first consumer
        checkpointer = MemoryCheckPointer(name="test")

        consumer_a = Consumer(
            stream_name=None,
            checkpointer=checkpointer,
            max_shard_consumers=1,
            endpoint_url=ENDPOINT_URL,
        )

        consumer_a.get_shard_iterator.return_value = True

        consumer_a.shards = [{
            "ShardId": "test-1",
            "SequenceNumberRange": {}
        }, {
            "ShardId": "test-2",
            "SequenceNumberRange": {}
        }]

        consumer_a.stream_status = consumer_a.ACTIVE
        consumer_a.shards_status = consumer_a.INITIALIZE

        await consumer_a.sync_shards()

        shards = [s["ShardId"] for s in consumer_a.shards if s.get("stats")]

        # Expect only one shard assigned as max = 1
        self.assertEqual(["test-1"], shards)

        # second consumer (note: max_shard_consumers needs to be 2 as uses checkpointer to get allocated shards)

        consumer_b = Consumer(
            stream_name=None,
            checkpointer=checkpointer,
            max_shard_consumers=2,
            endpoint_url=ENDPOINT_URL,
        )

        consumer_b.get_shard_iterator.return_value = True

        consumer_b.shards = [{
            "ShardId": "test-1",
            "SequenceNumberRange": {}
        }, {
            "ShardId": "test-2",
            "SequenceNumberRange": {}
        }]

        consumer_b.stream_status = consumer_b.ACTIVE
        consumer_b.shards_status = consumer_b.INITIALIZE

        await consumer_b.sync_shards()

        shards = [s["ShardId"] for s in consumer_b.shards if s.get("stats")]

        # Expect only one shard assigned as max = 2
        self.assertEqual(["test-1", "test-2"], shards)