def test_multiple_consumers(redis: StrictRedis, data):
out = Stream().pluck(2)
S = set()
out.pluck("i").sink(S.add)
stream, group = uuid(2)
sources = set()
for _ in range(3):
con = uuid()
source = Stream.from_redis_consumer_group(
stream,
group,
con,
count=1,
timeout=0.1,
)
source.connect(out)
source.start()
sources.add(source)
for x in data:
redis.xadd(stream, x)
wait_for(lambda: len(S) == 50, 1)
assert S == set(x["i"] for x in data)
for s in sources:
s.stop()
def test_heartbeats(redis: StrictRedis):
stream, group = uuid(2)
redis.xgroup_create(stream, group, mkstream=True)
interval = 0.1
timeout = 0.5
hearts = []
for _ in range(5):
heart = Heart(stream,
group,
uuid(),
interval=interval,
timeout=timeout)
hearts.append(heart)
heart.start()
S = set()
sub = redis.pubsub()
sub.subscribe(group)
def predicate():
m = sub.get_message()
if m is not None:
S.add(m["data"])
return len(S) == 5
wait_for(predicate, 5, period=0.01)
for h in hearts:
h.stop()
def test_claim(redis: StrictRedis, data):
stream, group, target = uuid(3)
for x in data:
redis.xadd(stream, x)
def run_and_fail():
name = uuid()
source = Stream.from_redis_consumer_group(
stream,
group,
name,
count=1,
timeout=0.1,
)
buffer = source.buffer(10)
buffer.rate_limit(0.1).pluck(1).sink_to_redis_list(target)
source.start()
wait_for(lambda: buffer.queue.qsize() == 10, 3)
buffer.queue = Queue(10) # lose data in the buffer, won't be ACKed
source.stop()
def pending_10():
cons = convert_bytes(redis.xpending(stream, group))["consumers"]
for con in cons:
if con["name"] == name and con["pending"] == 10:
return True
return False
wait_for(pending_10, 1, period=0.1)
for _ in range(10):
run_and_fail()
source = Stream.from_redis_consumer_group(
stream,
group,
uuid(),
heartbeat_interval=0.1,
claim_timeout=1,
count=10,
timeout=0.1,
)
source.pluck(1).sink_to_redis_list(target)
source.start()
wait_for(
lambda: redis.llen(target) == 500,
15,
lambda: print(redis.llen(target)),
period=0.1,
)
source.stop()
def test_replay(redis: StrictRedis, data):
stream, group, con = uuid(3)
maxsize = 5
source = Stream.from_redis_consumer_group(
{stream: 0}, group, con, count=1, timeout=0.1
)
buffer = source.buffer(maxsize)
L = buffer.rate_limit(0.1).sink_to_list()
source.start()
for x in data:
redis.xadd(stream, x)
wait_for(lambda: buffer.queue.qsize() == maxsize, 1)
buffer.queue = Queue(maxsize)
source.stop()
wait_for(lambda: len(L) == 3, 1)
source.start()
wait_for(lambda: len(L) == 10, 2)
assert set(x[2]["i"] for x in L) == set(x["i"] for x in data)
source.stop()
def run_and_fail():
name = uuid()
source = Stream.from_redis_consumer_group(
stream,
group,
name,
count=1,
timeout=0.1,
)
buffer = source.buffer(10)
buffer.rate_limit(0.1).pluck(1).sink_to_redis_list(target)
source.start()
wait_for(lambda: buffer.queue.qsize() == 10, 3)
buffer.queue = Queue(10) # lose data in the buffer, won't be ACKed
source.stop()
def pending_10():
cons = convert_bytes(redis.xpending(stream, group))["consumers"]
for con in cons:
if con["name"] == name and con["pending"] == 10:
return True
return False
wait_for(pending_10, 1, period=0.1)
def test_stream(redis: StrictRedis, data):
key = uuid()
source = Stream()
source.sink_to_redis_stream(key)
for x in data:
source.emit(x)
assert redis.xlen(key) == 3
def test_group_consumer_ensure_group(redis: StrictRedis):
s1, s2, group, con = uuid(4)
consumer = GroupConsumer(redis, {s1: 0, s2: 0}, group, con)
consumer.ensure_group()
consumer.ensure_group() # idempotent
assert redis.xinfo_groups(s1) != []
assert redis.xinfo_groups(s2) != []
def test_stream_maxlen(redis: StrictRedis, data):
key = uuid()
source = Stream()
source.sink_to_redis_stream(key, maxlen=10, approximate=False)
for x in data:
source.emit(x)
assert redis.xlen(key) == 10
def test_group_consumer_claim(redis: StrictRedis, data):
stream, group, con = uuid(3)
consumer = GroupConsumer(redis, {stream: 0}, group, con, count=10)
for x in data:
redis.xadd(stream, x)
consumer.consume()
consumer.name = uuid() # pretend do be a different consumer
for _, messages in consumer.claim_pending(stream,
con,
count=3,
min_idle_time=0):
assert len(messages) == 3
for _, messages in consumer.claim_pending(stream, con, min_idle_time=0):
assert len(messages) == 7
def test_from_redis_lists(redis: StrictRedis):
name = uuid()
source = Stream.from_redis_lists(name, timeout=0.1)
L = source.pluck(1).map(int).sink_to_list()
source.start()
redis.rpush(name, *list(range(3)))
wait_for(lambda: L == [0, 1, 2], 3)
source.stop()
def test_consumer_default_id_zero(redis: StrictRedis, data):
stream = uuid()
consumer = Consumer(redis, stream, default_start_id="0")
redis.xadd(stream, data[0]) # this will be consumed
res = consumer.consume()
assert just_data(res) == [data[0]]
redis.xadd(stream, data[1])
res = consumer.consume()
assert just_data(res) == [data[1]]
def test_list_left(redis: StrictRedis):
key = uuid()
source = Stream()
source.sink_to_redis_list(key, right=False)
for i in range(10):
source.emit(i)
assert redis.llen(key) == 10
assert int(redis.lpop(key)) == 9
assert int(redis.rpop(key)) == 0
def test_basic(redis: StrictRedis, data):
stream = uuid()
source = Stream.from_redis_streams(stream, timeout=0.1, default_start_id=0)
L = source.sink_to_list()
source.start()
for x in data:
redis.xadd(stream, x)
wait_for(lambda: len(L) == 3, 2)
assert [x[2] for x in L] == data
source.stop()
def test_multiple(redis: StrictRedis):
l1, l2 = uuid(2)
source = Stream.from_redis_lists([l1, l2], timeout=0.1)
L = source.pluck(1).map(int).sink_to_list()
source.start()
redis.rpush(l1, *list(range(3)))
redis.rpush(l2, *list(range(3)))
wait_for(lambda: len(L) == 6, 2)
source.stop()
def test_consumer_multiple_streams(redis: StrictRedis, data):
stream1, stream2 = uuid(2)
consumer = Consumer(redis, {stream1: 0, stream2: 0})
for x in data:
redis.xadd(stream1, x)
redis.xadd(stream2, x)
res = consumer.consume()
part1, part2 = res
assert [x for (_, x) in part1[1]] == data
assert [x for (_, x) in part2[1]] == data
def test_check_dead(redis: StrictRedis, data):
stream, group, con1, con2 = uuid(4)
h = Heart(stream, group, con1, timeout=0.001)
for x in data:
redis.xadd(stream, x)
GroupConsumer(redis, stream, group, con2).consume()
h.redis = redis
h.check_dead()
h.check_dead()
assert h.dead.get(timeout=1)[0] == con2
def test_ack(redis: StrictRedis, data):
stream, group, con = uuid(3)
source = Stream.from_redis_consumer_group(stream, group, con, timeout=0.1)
L = source.sink_to_list()
for x in data:
redis.xadd(stream, x)
source.start()
wait_for(lambda: len(L) == 3, 3, lambda: print(L))
sleep(0.05) # wait a bit for the last ack
for _, messages in redis.xreadgroup(group, con, {stream: 0}):
assert messages == []
source.stop()
def test_group_consumer_claim_parallel(redis: StrictRedis, data):
stream, group, con, c1, c2 = uuid(5)
fail = GroupConsumer(redis, stream, group, con, count=10)
for x in data:
redis.xadd(stream, x)
fail.consume() # do not ack
con1 = GroupConsumer(redis, stream, group, c1, count=5)
con2 = GroupConsumer(redis, stream, group, c2, count=5)
r1 = con1.claim_pending(stream, con, 0)
r2 = con2.claim_pending(stream, con, 0)
assert len(r1[0][1] + r2[0][1]) == 10
def test_multiple(redis: StrictRedis, data):
stream1, stream2 = uuid(2)
source = Stream.from_redis_streams({stream1: 0, stream2: 0}, timeout=0.1)
L1 = source.pluck(0).filter(lambda x: x == stream1).sink_to_list()
L2 = source.pluck(0).filter(lambda x: x == stream2).sink_to_list()
source.start()
for x in data:
redis.xadd(stream1, x)
redis.xadd(stream2, x)
wait_for(lambda: len(L1) == 3, 3)
wait_for(lambda: len(L2) == 3, 3)
assert L1 == [stream1] * 3
assert L2 == [stream2] * 3
source.stop()
def test_consumer_defaults(redis: StrictRedis, data):
stream = uuid()
consumer = Consumer(redis, stream)
redis.xadd(stream, {"i": -1}) # this will not be consumed
with ThreadPoolExecutor() as pool:
future = pool.submit(consumer.consume)
for x in data:
redis.xadd(stream, x)
res = future.result()
assert len(res) > 0
assert just_data(res) == data[:1]
res = consumer.consume()
assert len(res) > 0
assert just_data(res) == data[1:]
def test_group_consumer_pending(redis: StrictRedis, data):
stream, group, con = uuid(3)
consumer = GroupConsumer(redis, {stream: 0}, group, con)
for x in data:
redis.xadd(stream, x)
assert len(consumer.get_pending(stream, con, count=10)) == 0
res = consumer.consume()
assert len(res) > 0
for stream, messages in res:
consumer.ack(stream, *[_id for _id, _ in messages][:3])
assert len(consumer.get_pending(stream, con, count=10)) == 3
res = consumer.consume(pending=True)
assert len(res) > 0
for stream, messages in res:
assert [x for _, x in messages] == data[3:]
def test_group_consumer_consume(redis: StrictRedis, data):
s1, s2, group, con = uuid(4)
consumer = GroupConsumer(redis, {s1: 0, s2: 0}, group, con)
for x in data:
if int(x["i"]) % 2 == 0:
redis.xadd(s1, x)
else:
redis.xadd(s2, x)
res = consumer.consume()
assert len(res) > 0
for _, messages in res:
assert len(messages) == len(data) // 2
for x in data:
redis.xadd(s1, x)
res = consumer.consume()
assert len(res) > 0
for _, messages in res:
assert len(messages) == len(data)
def test_default_timeout():
stream, group, con = uuid(3)
heart = Heart(stream, group, con, interval=10)
assert heart.timeout == 100
