def test_replay_numpy(db):
# XXX: what to do if you have a source?
# build the graph
g1 = FromEventStream(
"event",
("data", "det_image"),
principle=True,
stream_name="g1",
asynchronous=True,
)
g2 = g1.map(np.exp, stream_name="mul")
g = g2.ToEventStream(("img2", ))
g.sink(print)
graph = g.graph
dbf = g.DBFriendly()
l1 = dbf.sink_to_list()
dbf.starsink(db.insert)
print("start experiment")
# run the experiment
l0 = []
for yy in y(5):
l0.append(yy)
db.insert(*yy)
yield g1.update(yy)
print("start replay")
# generate the replay
lg, parents, data, vs = replay(db, db[-1])
assert set(graph.nodes) == set(lg.nodes)
l2 = lg.nodes[list(nx.topological_sort(lg))[-1]]["stream"].sink_to_list()
# run the replay
lg.nodes[g1.uid]["stream"].sink(print)
print(graph.nodes)
print(parents)
for v in vs:
print(v["node"])
parents[v["node"]].update(data[v["uid"]])
# check that all the things are ok
assert len(l1) == len(l2)
assert len(l0) == len(l2)
for nd1, nd2 in zip(l0, l2):
assert nd1[0] == nd2[0]
if nd1[0] == "event":
assert (np.exp(
nd1[1]["data"]["det_image"]) == nd2[1]["data"]["img2"])
for nd1, nd2 in zip(l1, l2):
assert nd1[0] == nd2[0]
if nd1[0] == "event":
assert nd1[1]["data"]["img2"] == nd2[1]["data"]["img2"]
def test_replay_parallel(db):
print("build graph")
g1 = FromEventStream(
"event",
("data", "det_image"),
principle=True,
stream_name="g1",
asynchronous=True,
)
g2 = g1.scatter(backend="thread").map(op.mul, 5, stream_name="mul")
g = g2.ToEventStream(("img2", ))
graph = g.graph
dbf = g.buffer(10).gather().DBFriendly()
l1 = dbf.sink_to_list()
dbf.starsink(db.insert)
print("start experiment")
# run the experiment
l0 = []
for yy in y(5):
l0.append(yy)
db.insert(*yy)
yield g1.update(yy)
while len(l1) < len(l0):
yield gen.sleep(.01)
print("start replay")
# generate the replay
lg, parents, data, vs = replay(db, db[-1])
assert set(graph.nodes) == set(lg.nodes)
l2 = (lg.nodes[list(nx.topological_sort(lg))[-1]]["stream"].buffer(
10).gather().sink_to_list())
# run the replay
lg.nodes[g1.uid]["stream"].sink(print)
for v in vs:
parents[v["node"]].update(data[v["uid"]])
while len(l2) < len(l0):
yield gen.sleep(.01)
# check that all the things are ok
assert len(l1) == len(l2)
assert len(l0) == len(l2)
for nd1, nd2 in zip(l0, l2):
assert nd1[0] == nd2[0]
if nd1[0] == "event":
assert nd1[1]["data"]["det_image"] * 5 == nd2[1]["data"]["img2"]
for nd1, nd2 in zip(l1, l2):
assert nd1[0] == nd2[0]
if nd1[0] == "event":
assert nd1[1]["data"]["img2"] == nd2[1]["data"]["img2"]
source = Stream()
(FromEventStream('event', 'motor1', upstream=source).scatter().map(
op.add,
1).buffer(8).gather().ToEventStream('result').DBFriendly().starsink(
db.insert))
RE = RunEngine()
RE.subscribe(lambda *x: source.emit(x))
RE(bp.count([hw.motor1], 1))
from shed_streaming.replay import replay
from rapidz.graph import _clean_text, readable_graph
# get the graph and data
graph, parents, data, vs = replay(db, db[-1])
# make a graph with human readable names
for k, v in graph.nodes.items():
v.update(label=_clean_text(str(v['stream'])).strip())
graph = readable_graph(graph)
# create a plot of the graph so we can look at it and figure out what
# the node names are
# the file will be named ``mystream.png``
graph.nodes['data motor1 FromEventStream']['stream'].visualize()
# print the results
graph.nodes['result ToEventStream'].sink(pprint)
# change the multiplication factor from 5 to 10
'descriptor': duid})
yield ('stop', {'uid': str(uuid.uuid4()),
'time': time.time(),
'run_start': suid})
print('build graph')
g1 = FromEventStream('event', ('data', 'det_image',), principle=True,
stream_name='g1')
g11 = FromEventStream('event', ('data', 'det_image',),
stream_name='g11')
g11_1 = g1.zip(g11)
g2 = g11_1.starmap(op.mul)
g = g2.ToEventStream(('img2',))
dbf = g.DBFriendly()
dbf.starsink(db.insert)
print('run experiment')
for yy in y():
db.insert(*yy)
g11.update(yy)
g1.update(yy)
print(db[-1]['stop'])
print('replay experiment')
rp = replay(db, db[-1])
lg = next(rp)
ts = list(nx.topological_sort(lg))
lg.nodes[ts[-1]]['stream'].sink(pprint)
next(rp)
print(db[-1]['stop'])