def test_multiple_consumers(redis: StrictRedis, data): out = Stream().pluck(2) S = set() out.pluck("i").sink(S.add) stream, group = uuid(2) sources = set() for _ in range(3): con = uuid() source = Stream.from_redis_consumer_group( stream, group, con, count=1, timeout=0.1, ) source.connect(out) source.start() sources.add(source) for x in data: redis.xadd(stream, x) wait_for(lambda: len(S) == 50, 1) assert S == set(x["i"] for x in data) for s in sources: s.stop()
def test_replay(redis: StrictRedis, data): stream, group, con = uuid(3) maxsize = 5 source = Stream.from_redis_consumer_group( {stream: 0}, group, con, count=1, timeout=0.1 ) buffer = source.buffer(maxsize) L = buffer.rate_limit(0.1).sink_to_list() source.start() for x in data: redis.xadd(stream, x) wait_for(lambda: buffer.queue.qsize() == maxsize, 1) buffer.queue = Queue(maxsize) source.stop() wait_for(lambda: len(L) == 3, 1) source.start() wait_for(lambda: len(L) == 10, 2) assert set(x[2]["i"] for x in L) == set(x["i"] for x in data) source.stop()
def run_and_fail(): name = uuid() source = Stream.from_redis_consumer_group( stream, group, name, count=1, timeout=0.1, ) buffer = source.buffer(10) buffer.rate_limit(0.1).pluck(1).sink_to_redis_list(target) source.start() wait_for(lambda: buffer.queue.qsize() == 10, 3) buffer.queue = Queue(10) # lose data in the buffer, won't be ACKed source.stop() def pending_10(): cons = convert_bytes(redis.xpending(stream, group))["consumers"] for con in cons: if con["name"] == name and con["pending"] == 10: return True return False wait_for(pending_10, 1, period=0.1)
def test_claim(redis: StrictRedis, data): stream, group, target = uuid(3) for x in data: redis.xadd(stream, x) def run_and_fail(): name = uuid() source = Stream.from_redis_consumer_group( stream, group, name, count=1, timeout=0.1, ) buffer = source.buffer(10) buffer.rate_limit(0.1).pluck(1).sink_to_redis_list(target) source.start() wait_for(lambda: buffer.queue.qsize() == 10, 3) buffer.queue = Queue(10) # lose data in the buffer, won't be ACKed source.stop() def pending_10(): cons = convert_bytes(redis.xpending(stream, group))["consumers"] for con in cons: if con["name"] == name and con["pending"] == 10: return True return False wait_for(pending_10, 1, period=0.1) for _ in range(10): run_and_fail() source = Stream.from_redis_consumer_group( stream, group, uuid(), heartbeat_interval=0.1, claim_timeout=1, count=10, timeout=0.1, ) source.pluck(1).sink_to_redis_list(target) source.start() wait_for( lambda: redis.llen(target) == 500, 15, lambda: print(redis.llen(target)), period=0.1, ) source.stop()
def test_ack(redis: StrictRedis, data): stream, group, con = uuid(3) source = Stream.from_redis_consumer_group(stream, group, con, timeout=0.1) L = source.sink_to_list() for x in data: redis.xadd(stream, x) source.start() wait_for(lambda: len(L) == 3, 3, lambda: print(L)) sleep(0.05) # wait a bit for the last ack for _, messages in redis.xreadgroup(group, con, {stream: 0}): assert messages == [] source.stop()
import json import time from uuid import uuid4 from streamz import Stream def preprocess(x): source, _id, data = x data["_source"] = source data["_id"] = _id return json.dumps(data) sources = ["source-stream-1", "source-stream-2"] name = str(uuid4()) source = Stream.from_redis_consumer_group( sources, "my-group", name, heartbeat_interval=10, claim_timeout=120 ) ( source.map(preprocess) .partition(1000, timeout=3) .map(lambda x: "\n".join(x)) .sink_to_redis_list("preprocessed-data") ) if __name__ == "__main__": source.start() time.sleep(5)
import json import sys import time from streamz import Stream if len(sys.argv) == 2: _, name = sys.argv else: print("Usage: python consumer.py consumer-name") exit() def preprocess(x): source, _id, data = x data["_source"] = source data["_id"] = _id return json.dumps(data) sources = ["source-stream-1", "source-stream-2"] source = Stream.from_redis_consumer_group(sources, "my-group", name) (source.map(preprocess).partition(1000, timeout=3).map( lambda x: "\n".join(x)).sink_to_redis_list("preprocessed-data")) if __name__ == "__main__": source.start() time.sleep(5)