def test_double_buffer_to_event_model():
source = Stream(asynchronous=True)
t = FromEventStream("event", ("data", "det_image"), source, principle=True)
assert t.principle
ts = t
a = ts.map(slow_inc)
aa = ts.map(slow_inc)
n = a.zip(aa).SimpleToEventStream(("ct", ))
b = n
b.sink(print)
L = b.sink_to_list()
futures_L = []
tt = t.sink_to_list()
p = b.pluck(0).sink_to_list()
d = b.pluck(1).sink_to_list()
t0 = time.time()
for gg in y(10):
futures_L.append(gg)
yield source.emit(gg)
while len(L) < len(futures_L):
yield gen.sleep(.01)
t1 = time.time()
# check that this was faster than running in series
assert t1 - t0 > .5 * 10
assert tt
assert p == ["start", "descriptor"] + ["event"] * 10 + ["stop"]
assert d[1]["hints"] == {"analyzer": {"fields": ["ct"]}}
t0 = time.time()
for gg in y(10):
futures_L.append(gg)
yield source.emit(gg)
while len(L) < len(futures_L):
yield gen.sleep(.01)
print(len(L), len(futures_L))
t1 = time.time()
# check that this was faster than running in series
assert t1 - t0 > .5 * 10
assert tt
assert p == (["start", "descriptor"] + ["event"] * 10 + ["stop"]) * 2
assert d[14]["hints"] == {"analyzer": {"fields": ["ct"]}}
for i, j in zip([0, 1, 12], [13, 14, 25]):
assert p[i] == p[j]
assert d[i] != d[j]
def test_slow_to_event_model():
"""This doesn't use threads so it should be slower due to sleep"""
source = Stream(asynchronous=True)
t = FromEventStream("event", ("data", "det_image"), source, principle=True)
assert t.principle
a = t.map(slow_inc)
L = a.sink_to_list()
futures_L = a.sink_to_list()
n = a.SimpleToEventStream(("ct", ))
n.sink(print)
tt = t.sink_to_list()
p = n.pluck(0).sink_to_list()
d = n.pluck(1).sink_to_list()
t0 = time.time()
for gg in y(10):
yield source.emit(gg)
while len(L) < len(futures_L):
yield gen.sleep(.01)
t1 = time.time()
# check that this was faster than running in series
td = t1 - t0
ted = .5 * 10
assert td > ted
assert tt
assert p == ["start", "descriptor"] + ["event"] * 10 + ["stop"]
assert d[1]["hints"] == {"analyzer": {"fields": ["ct"]}}
def test_parent_nodes():
# build the graph
g1 = FromEventStream(
"event",
("data", "det_image"),
principle=True,
stream_name="g1",
asynchronous=True,
)
g11 = FromEventStream("event", ("data", "det_image"),
stream_name="g11",
asynchronous=True)
g2 = g1.zip(g11).starmap(op.mul, stream_name="mul")
g = g2.SimpleToEventStream(("img2", ))
l1 = g.sink_to_list()
# g.sink(print)
assert len(g.translation_nodes) == 2
print("start experiment")
# run the experiment
l0 = []
for yy in y(5):
l0.append(yy)
g11.update(yy)
g1.update(yy)
print(g11.start_uid)
assert len(l1[0][1]["parent_node_map"]) == 2
def test_replay_numpy(db):
# XXX: what to do if you have a source?
# build the graph
g1 = FromEventStream(
"event",
("data", "det_image"),
principle=True,
stream_name="g1",
asynchronous=True,
)
g2 = g1.map(np.exp, stream_name="mul")
g = g2.ToEventStream(("img2", ))
g.sink(print)
graph = g.graph
dbf = g.DBFriendly()
l1 = dbf.sink_to_list()
dbf.starsink(db.insert)
print("start experiment")
# run the experiment
l0 = []
for yy in y(5):
l0.append(yy)
db.insert(*yy)
yield g1.update(yy)
print("start replay")
# generate the replay
lg, parents, data, vs = replay(db, db[-1])
assert set(graph.nodes) == set(lg.nodes)
l2 = lg.nodes[list(nx.topological_sort(lg))[-1]]["stream"].sink_to_list()
# run the replay
lg.nodes[g1.uid]["stream"].sink(print)
print(graph.nodes)
print(parents)
for v in vs:
print(v["node"])
parents[v["node"]].update(data[v["uid"]])
# check that all the things are ok
assert len(l1) == len(l2)
assert len(l0) == len(l2)
for nd1, nd2 in zip(l0, l2):
assert nd1[0] == nd2[0]
if nd1[0] == "event":
assert (np.exp(
nd1[1]["data"]["det_image"]) == nd2[1]["data"]["img2"])
for nd1, nd2 in zip(l1, l2):
assert nd1[0] == nd2[0]
if nd1[0] == "event":
assert nd1[1]["data"]["img2"] == nd2[1]["data"]["img2"]
def test_replay_parallel(db):
print("build graph")
g1 = FromEventStream(
"event",
("data", "det_image"),
principle=True,
stream_name="g1",
asynchronous=True,
)
g2 = g1.scatter(backend="thread").map(op.mul, 5, stream_name="mul")
g = g2.ToEventStream(("img2", ))
graph = g.graph
dbf = g.buffer(10).gather().DBFriendly()
l1 = dbf.sink_to_list()
dbf.starsink(db.insert)
print("start experiment")
# run the experiment
l0 = []
for yy in y(5):
l0.append(yy)
db.insert(*yy)
yield g1.update(yy)
while len(l1) < len(l0):
yield gen.sleep(.01)
print("start replay")
# generate the replay
lg, parents, data, vs = replay(db, db[-1])
assert set(graph.nodes) == set(lg.nodes)
l2 = (lg.nodes[list(nx.topological_sort(lg))[-1]]["stream"].buffer(
10).gather().sink_to_list())
# run the replay
lg.nodes[g1.uid]["stream"].sink(print)
for v in vs:
parents[v["node"]].update(data[v["uid"]])
while len(l2) < len(l0):
yield gen.sleep(.01)
# check that all the things are ok
assert len(l1) == len(l2)
assert len(l0) == len(l2)
for nd1, nd2 in zip(l0, l2):
assert nd1[0] == nd2[0]
if nd1[0] == "event":
assert nd1[1]["data"]["det_image"] * 5 == nd2[1]["data"]["img2"]
for nd1, nd2 in zip(l1, l2):
assert nd1[0] == nd2[0]
if nd1[0] == "event":
assert nd1[1]["data"]["img2"] == nd2[1]["data"]["img2"]
async def test_slow_to_event_model_parallel_dask(c, s, a, b):
source = Stream(asynchronous=True)
t = FromEventStream("event", ("data", "det_image"), source, principle=True)
assert t.principle
ts = t.scatter(backend="dask")
# futures_L = t.sink_to_list()
a = ts.map(slow_inc)
n = a.SimpleToEventStream(("ct", ))
b = n.buffer(100).gather()
b.sink(print)
L = b.sink_to_list()
tt = t.sink_to_list()
p = b.pluck(0).sink_to_list()
d = b.pluck(1).sink_to_list()
t0 = time.time()
futures_L = []
for gg in y(10):
futures_L.append(gg)
await source.emit(gg)
while len(L) < len(futures_L):
await gen.sleep(.01)
t1 = time.time()
# check that this was faster than running in series
td = t1 - t0
ted = .5 * 10
assert td < ted
assert tt
assert p == ["start", "descriptor"] + ["event"] * 10 + ["stop"]
assert "uid" in d[0]
assert d[1]["hints"] == {"analyzer": {"fields": ["ct"]}}
assert d[1]["data_keys"]["ct"]["dtype"] == "number"
assert d[2]["data"]["ct"] == 2
def test_replay_export_test():
def y():
suid = str(uuid.uuid4())
yield ("start", {"uid": suid, "time": time.time()})
duid = str(uuid.uuid4())
yield (
"descriptor",
{
"uid": duid,
"run_start": suid,
"name": "primary",
"data_keys": {
"det_image": {
"dtype": "int",
"units": "arb"
}
},
"time": time.time(),
},
)
for i in range(5):
yield (
"event",
{
"uid": str(uuid.uuid4()),
"data": {
"det_image": i
},
"timestamps": {
"det_image": time.time()
},
"seq_num": i + 1,
"time": time.time(),
"descriptor": duid,
},
)
yield (
"stop",
{
"uid": str(uuid.uuid4()),
"time": time.time(),
"run_start": suid
},
)
print("build graph")
g1 = FromEventStream("event", ("data", "det_image"),
principle=True,
stream_name="g1")
g11 = FromEventStream("event", ("data", "det_image"), stream_name="g11")
g11_1 = g1.zip(g11)
g2 = g11_1.starmap(op.mul).map(np.log)
g = g2.SimpleToEventStream(("img2", ))
from pprint import pprint
g.sink(pprint)
L = g.sink_to_list()
print("run experiment")
for yy in y():
print(yy[0])
g11.update(yy)
g1.update(yy)
assert L[-1][1]["run_start"]
