def test_multiple_consumers(redis: StrictRedis, data): out = Stream().pluck(2) S = set() out.pluck("i").sink(S.add) stream, group = uuid(2) sources = set() for _ in range(3): con = uuid() source = Stream.from_redis_consumer_group( stream, group, con, count=1, timeout=0.1, ) source.connect(out) source.start() sources.add(source) for x in data: redis.xadd(stream, x) wait_for(lambda: len(S) == 50, 1) assert S == set(x["i"] for x in data) for s in sources: s.stop()
def test_heartbeats(redis: StrictRedis): stream, group = uuid(2) redis.xgroup_create(stream, group, mkstream=True) interval = 0.1 timeout = 0.5 hearts = [] for _ in range(5): heart = Heart(stream, group, uuid(), interval=interval, timeout=timeout) hearts.append(heart) heart.start() S = set() sub = redis.pubsub() sub.subscribe(group) def predicate(): m = sub.get_message() if m is not None: S.add(m["data"]) return len(S) == 5 wait_for(predicate, 5, period=0.01) for h in hearts: h.stop()
def test_claim(redis: StrictRedis, data): stream, group, target = uuid(3) for x in data: redis.xadd(stream, x) def run_and_fail(): name = uuid() source = Stream.from_redis_consumer_group( stream, group, name, count=1, timeout=0.1, ) buffer = source.buffer(10) buffer.rate_limit(0.1).pluck(1).sink_to_redis_list(target) source.start() wait_for(lambda: buffer.queue.qsize() == 10, 3) buffer.queue = Queue(10) # lose data in the buffer, won't be ACKed source.stop() def pending_10(): cons = convert_bytes(redis.xpending(stream, group))["consumers"] for con in cons: if con["name"] == name and con["pending"] == 10: return True return False wait_for(pending_10, 1, period=0.1) for _ in range(10): run_and_fail() source = Stream.from_redis_consumer_group( stream, group, uuid(), heartbeat_interval=0.1, claim_timeout=1, count=10, timeout=0.1, ) source.pluck(1).sink_to_redis_list(target) source.start() wait_for( lambda: redis.llen(target) == 500, 15, lambda: print(redis.llen(target)), period=0.1, ) source.stop()
def test_replay(redis: StrictRedis, data): stream, group, con = uuid(3) maxsize = 5 source = Stream.from_redis_consumer_group( {stream: 0}, group, con, count=1, timeout=0.1 ) buffer = source.buffer(maxsize) L = buffer.rate_limit(0.1).sink_to_list() source.start() for x in data: redis.xadd(stream, x) wait_for(lambda: buffer.queue.qsize() == maxsize, 1) buffer.queue = Queue(maxsize) source.stop() wait_for(lambda: len(L) == 3, 1) source.start() wait_for(lambda: len(L) == 10, 2) assert set(x[2]["i"] for x in L) == set(x["i"] for x in data) source.stop()
def run_and_fail(): name = uuid() source = Stream.from_redis_consumer_group( stream, group, name, count=1, timeout=0.1, ) buffer = source.buffer(10) buffer.rate_limit(0.1).pluck(1).sink_to_redis_list(target) source.start() wait_for(lambda: buffer.queue.qsize() == 10, 3) buffer.queue = Queue(10) # lose data in the buffer, won't be ACKed source.stop() def pending_10(): cons = convert_bytes(redis.xpending(stream, group))["consumers"] for con in cons: if con["name"] == name and con["pending"] == 10: return True return False wait_for(pending_10, 1, period=0.1)
def test_stream(redis: StrictRedis, data): key = uuid() source = Stream() source.sink_to_redis_stream(key) for x in data: source.emit(x) assert redis.xlen(key) == 3
def test_group_consumer_ensure_group(redis: StrictRedis): s1, s2, group, con = uuid(4) consumer = GroupConsumer(redis, {s1: 0, s2: 0}, group, con) consumer.ensure_group() consumer.ensure_group() # idempotent assert redis.xinfo_groups(s1) != [] assert redis.xinfo_groups(s2) != []
def test_stream_maxlen(redis: StrictRedis, data): key = uuid() source = Stream() source.sink_to_redis_stream(key, maxlen=10, approximate=False) for x in data: source.emit(x) assert redis.xlen(key) == 10
def test_group_consumer_claim(redis: StrictRedis, data): stream, group, con = uuid(3) consumer = GroupConsumer(redis, {stream: 0}, group, con, count=10) for x in data: redis.xadd(stream, x) consumer.consume() consumer.name = uuid() # pretend do be a different consumer for _, messages in consumer.claim_pending(stream, con, count=3, min_idle_time=0): assert len(messages) == 3 for _, messages in consumer.claim_pending(stream, con, min_idle_time=0): assert len(messages) == 7
def test_from_redis_lists(redis: StrictRedis): name = uuid() source = Stream.from_redis_lists(name, timeout=0.1) L = source.pluck(1).map(int).sink_to_list() source.start() redis.rpush(name, *list(range(3))) wait_for(lambda: L == [0, 1, 2], 3) source.stop()
def test_consumer_default_id_zero(redis: StrictRedis, data): stream = uuid() consumer = Consumer(redis, stream, default_start_id="0") redis.xadd(stream, data[0]) # this will be consumed res = consumer.consume() assert just_data(res) == [data[0]] redis.xadd(stream, data[1]) res = consumer.consume() assert just_data(res) == [data[1]]
def test_list_left(redis: StrictRedis): key = uuid() source = Stream() source.sink_to_redis_list(key, right=False) for i in range(10): source.emit(i) assert redis.llen(key) == 10 assert int(redis.lpop(key)) == 9 assert int(redis.rpop(key)) == 0
def test_basic(redis: StrictRedis, data): stream = uuid() source = Stream.from_redis_streams(stream, timeout=0.1, default_start_id=0) L = source.sink_to_list() source.start() for x in data: redis.xadd(stream, x) wait_for(lambda: len(L) == 3, 2) assert [x[2] for x in L] == data source.stop()
def test_multiple(redis: StrictRedis): l1, l2 = uuid(2) source = Stream.from_redis_lists([l1, l2], timeout=0.1) L = source.pluck(1).map(int).sink_to_list() source.start() redis.rpush(l1, *list(range(3))) redis.rpush(l2, *list(range(3))) wait_for(lambda: len(L) == 6, 2) source.stop()
def test_consumer_multiple_streams(redis: StrictRedis, data): stream1, stream2 = uuid(2) consumer = Consumer(redis, {stream1: 0, stream2: 0}) for x in data: redis.xadd(stream1, x) redis.xadd(stream2, x) res = consumer.consume() part1, part2 = res assert [x for (_, x) in part1[1]] == data assert [x for (_, x) in part2[1]] == data
def test_check_dead(redis: StrictRedis, data): stream, group, con1, con2 = uuid(4) h = Heart(stream, group, con1, timeout=0.001) for x in data: redis.xadd(stream, x) GroupConsumer(redis, stream, group, con2).consume() h.redis = redis h.check_dead() h.check_dead() assert h.dead.get(timeout=1)[0] == con2
def test_ack(redis: StrictRedis, data): stream, group, con = uuid(3) source = Stream.from_redis_consumer_group(stream, group, con, timeout=0.1) L = source.sink_to_list() for x in data: redis.xadd(stream, x) source.start() wait_for(lambda: len(L) == 3, 3, lambda: print(L)) sleep(0.05) # wait a bit for the last ack for _, messages in redis.xreadgroup(group, con, {stream: 0}): assert messages == [] source.stop()
def test_group_consumer_claim_parallel(redis: StrictRedis, data): stream, group, con, c1, c2 = uuid(5) fail = GroupConsumer(redis, stream, group, con, count=10) for x in data: redis.xadd(stream, x) fail.consume() # do not ack con1 = GroupConsumer(redis, stream, group, c1, count=5) con2 = GroupConsumer(redis, stream, group, c2, count=5) r1 = con1.claim_pending(stream, con, 0) r2 = con2.claim_pending(stream, con, 0) assert len(r1[0][1] + r2[0][1]) == 10
def test_multiple(redis: StrictRedis, data): stream1, stream2 = uuid(2) source = Stream.from_redis_streams({stream1: 0, stream2: 0}, timeout=0.1) L1 = source.pluck(0).filter(lambda x: x == stream1).sink_to_list() L2 = source.pluck(0).filter(lambda x: x == stream2).sink_to_list() source.start() for x in data: redis.xadd(stream1, x) redis.xadd(stream2, x) wait_for(lambda: len(L1) == 3, 3) wait_for(lambda: len(L2) == 3, 3) assert L1 == [stream1] * 3 assert L2 == [stream2] * 3 source.stop()
def test_consumer_defaults(redis: StrictRedis, data): stream = uuid() consumer = Consumer(redis, stream) redis.xadd(stream, {"i": -1}) # this will not be consumed with ThreadPoolExecutor() as pool: future = pool.submit(consumer.consume) for x in data: redis.xadd(stream, x) res = future.result() assert len(res) > 0 assert just_data(res) == data[:1] res = consumer.consume() assert len(res) > 0 assert just_data(res) == data[1:]
def test_group_consumer_pending(redis: StrictRedis, data): stream, group, con = uuid(3) consumer = GroupConsumer(redis, {stream: 0}, group, con) for x in data: redis.xadd(stream, x) assert len(consumer.get_pending(stream, con, count=10)) == 0 res = consumer.consume() assert len(res) > 0 for stream, messages in res: consumer.ack(stream, *[_id for _id, _ in messages][:3]) assert len(consumer.get_pending(stream, con, count=10)) == 3 res = consumer.consume(pending=True) assert len(res) > 0 for stream, messages in res: assert [x for _, x in messages] == data[3:]
def test_group_consumer_consume(redis: StrictRedis, data): s1, s2, group, con = uuid(4) consumer = GroupConsumer(redis, {s1: 0, s2: 0}, group, con) for x in data: if int(x["i"]) % 2 == 0: redis.xadd(s1, x) else: redis.xadd(s2, x) res = consumer.consume() assert len(res) > 0 for _, messages in res: assert len(messages) == len(data) // 2 for x in data: redis.xadd(s1, x) res = consumer.consume() assert len(res) > 0 for _, messages in res: assert len(messages) == len(data)
def test_default_timeout(): stream, group, con = uuid(3) heart = Heart(stream, group, con, interval=10) assert heart.timeout == 100