def test_multiple_consumers(redis: StrictRedis, data):
out = Stream().pluck(2)
S = set()
out.pluck("i").sink(S.add)
stream, group = uuid(2)
sources = set()
for _ in range(3):
con = uuid()
source = Stream.from_redis_consumer_group(
stream,
group,
con,
count=1,
timeout=0.1,
)
source.connect(out)
source.start()
sources.add(source)
for x in data:
redis.xadd(stream, x)
wait_for(lambda: len(S) == 50, 1)
assert S == set(x["i"] for x in data)
for s in sources:
s.stop()
def test_replay(redis: StrictRedis, data):
stream, group, con = uuid(3)
maxsize = 5
source = Stream.from_redis_consumer_group(
{stream: 0}, group, con, count=1, timeout=0.1
)
buffer = source.buffer(maxsize)
L = buffer.rate_limit(0.1).sink_to_list()
source.start()
for x in data:
redis.xadd(stream, x)
wait_for(lambda: buffer.queue.qsize() == maxsize, 1)
buffer.queue = Queue(maxsize)
source.stop()
wait_for(lambda: len(L) == 3, 1)
source.start()
wait_for(lambda: len(L) == 10, 2)
assert set(x[2]["i"] for x in L) == set(x["i"] for x in data)
source.stop()
def run_and_fail():
name = uuid()
source = Stream.from_redis_consumer_group(
stream,
group,
name,
count=1,
timeout=0.1,
)
buffer = source.buffer(10)
buffer.rate_limit(0.1).pluck(1).sink_to_redis_list(target)
source.start()
wait_for(lambda: buffer.queue.qsize() == 10, 3)
buffer.queue = Queue(10) # lose data in the buffer, won't be ACKed
source.stop()
def pending_10():
cons = convert_bytes(redis.xpending(stream, group))["consumers"]
for con in cons:
if con["name"] == name and con["pending"] == 10:
return True
return False
wait_for(pending_10, 1, period=0.1)
def test_claim(redis: StrictRedis, data):
stream, group, target = uuid(3)
for x in data:
redis.xadd(stream, x)
def run_and_fail():
name = uuid()
source = Stream.from_redis_consumer_group(
stream,
group,
name,
count=1,
timeout=0.1,
)
buffer = source.buffer(10)
buffer.rate_limit(0.1).pluck(1).sink_to_redis_list(target)
source.start()
wait_for(lambda: buffer.queue.qsize() == 10, 3)
buffer.queue = Queue(10) # lose data in the buffer, won't be ACKed
source.stop()
def pending_10():
cons = convert_bytes(redis.xpending(stream, group))["consumers"]
for con in cons:
if con["name"] == name and con["pending"] == 10:
return True
return False
wait_for(pending_10, 1, period=0.1)
for _ in range(10):
run_and_fail()
source = Stream.from_redis_consumer_group(
stream,
group,
uuid(),
heartbeat_interval=0.1,
claim_timeout=1,
count=10,
timeout=0.1,
)
source.pluck(1).sink_to_redis_list(target)
source.start()
wait_for(
lambda: redis.llen(target) == 500,
15,
lambda: print(redis.llen(target)),
period=0.1,
)
source.stop()
def test_ack(redis: StrictRedis, data):
stream, group, con = uuid(3)
source = Stream.from_redis_consumer_group(stream, group, con, timeout=0.1)
L = source.sink_to_list()
for x in data:
redis.xadd(stream, x)
source.start()
wait_for(lambda: len(L) == 3, 3, lambda: print(L))
sleep(0.05) # wait a bit for the last ack
for _, messages in redis.xreadgroup(group, con, {stream: 0}):
assert messages == []
source.stop()
import json
import time
from uuid import uuid4
from streamz import Stream
def preprocess(x):
source, _id, data = x
data["_source"] = source
data["_id"] = _id
return json.dumps(data)
sources = ["source-stream-1", "source-stream-2"]
name = str(uuid4())
source = Stream.from_redis_consumer_group(
sources, "my-group", name, heartbeat_interval=10, claim_timeout=120
)
(
source.map(preprocess)
.partition(1000, timeout=3)
.map(lambda x: "\n".join(x))
.sink_to_redis_list("preprocessed-data")
)
if __name__ == "__main__":
source.start()
time.sleep(5)
import json
import sys
import time
from streamz import Stream
if len(sys.argv) == 2:
_, name = sys.argv
else:
print("Usage: python consumer.py consumer-name")
exit()
def preprocess(x):
source, _id, data = x
data["_source"] = source
data["_id"] = _id
return json.dumps(data)
sources = ["source-stream-1", "source-stream-2"]
source = Stream.from_redis_consumer_group(sources, "my-group", name)
(source.map(preprocess).partition(1000, timeout=3).map(
lambda x: "\n".join(x)).sink_to_redis_list("preprocessed-data"))
if __name__ == "__main__":
source.start()
time.sleep(5)