def test_from_redis_lists(redis: StrictRedis):
name = uuid()
source = Stream.from_redis_lists(name, timeout=0.1)
L = source.pluck(1).map(int).sink_to_list()
source.start()
redis.rpush(name, *list(range(3)))
wait_for(lambda: L == [0, 1, 2], 3)
source.stop()
def test_multiple(redis: StrictRedis):
l1, l2 = uuid(2)
source = Stream.from_redis_lists([l1, l2], timeout=0.1)
L = source.pluck(1).map(int).sink_to_list()
source.start()
redis.rpush(l1, *list(range(3)))
redis.rpush(l2, *list(range(3)))
wait_for(lambda: len(L) == 6, 2)
source.stop()
import json
import time
from streamz import Stream
sources = ["data-source-1", "data-source-2"]
def preprocess(x):
source, data = x
data = json.reads(data)
data["_source"] = source
return json.dumps(data)
source = Stream.from_redis_lists(sources)
(
source.map(preprocess)
.partition(1000, timeout=3)
.map(lambda x: "\n".join(x))
.sink_to_redis_list("preprocessed-data")
)
if __name__ == "__main__":
source.start()
time.sleep(5)
