def test_slow_to_event_model():
"""This doesn't use threads so it should be slower due to sleep"""
source = Stream(asynchronous=True)
t = FromEventStream("event", ("data", "det_image"), source, principle=True)
assert t.principle
a = t.map(slow_inc)
L = a.sink_to_list()
futures_L = a.sink_to_list()
n = a.SimpleToEventStream(("ct", ))
n.sink(print)
tt = t.sink_to_list()
p = n.pluck(0).sink_to_list()
d = n.pluck(1).sink_to_list()
t0 = time.time()
for gg in y(10):
yield source.emit(gg)
while len(L) < len(futures_L):
yield gen.sleep(.01)
t1 = time.time()
# check that this was faster than running in series
td = t1 - t0
ted = .5 * 10
assert td > ted
assert tt
assert p == ["start", "descriptor"] + ["event"] * 10 + ["stop"]
assert d[1]["hints"] == {"analyzer": {"fields": ["ct"]}}
def test_last_cache(RE, hw):
source = Stream()
t = FromEventStream("event", ("data", "motor"), source, principle=True)
assert t.principle
n = ToEventStream(t, ("ct", ), data_key_md={
"ct": {
"units": "arb"
}
}).LastCache()
tt = t.sink_to_list()
names = n.pluck(0).sink_to_list()
docs = n.pluck(1).sink_to_list()
RE.subscribe(unstar(source.emit))
RE.subscribe(print)
RE(scan([hw.motor], hw.motor, 0, 9, 10))
assert len(docs) == 10 + 3 + 2
assert names[-3] == "descriptor"
assert names[-2] == "event"
assert tt
assert set(names) == {"start", "stop", "event", "descriptor"}
assert docs[1]["hints"] == {"analyzer": {"fields": ["ct"]}}
assert docs[1]["data_keys"]["ct"]["units"] == "arb"
assert docs[-1]["run_start"]
def test_to_event_model_new_api_multi_parent(RE, hw):
source = Stream()
t = FromEventStream("event", ("data", "motor"), source, principle=True)
t2 = FromEventStream("event", ("data", "motor"), source, principle=True)
assert t.principle
n = simple_to_event_stream_new_api({
t.zip(t2).pluck(0): {
"data_keys": {
"ct": {
"units": "arb",
"precision": 2
}
}
}
})
tt = t.sink_to_list()
p = n.pluck(0).sink_to_list()
d = n.pluck(1).sink_to_list()
RE.subscribe(unstar(source.emit))
RE.subscribe(print)
RE(scan([hw.motor], hw.motor, 0, 9, 10))
assert tt
assert set(p) == {"start", "stop", "event", "descriptor"}
assert d[1]["hints"] == {"analyzer": {"fields": ["ct"]}}
assert d[1]["data_keys"]["ct"]["units"] == "arb"
assert d[-1]["run_start"]
def test_to_event_model_new_api_clobber(RE, hw):
source = Stream()
t = FromEventStream("event", ("data", "motor"), source, principle=True)
assert t.principle
n = simple_to_event_stream_new_api(
{t: {
"data_keys": {
"ct": {
"units": "arb",
"dtype": "array"
}
}
}})
tt = t.sink_to_list()
p = n.pluck(0).sink_to_list()
d = n.pluck(1).sink_to_list()
RE.subscribe(unstar(source.emit))
RE.subscribe(print)
RE(scan([hw.motor], hw.motor, 0, 9, 10))
assert tt
assert set(p) == {"start", "stop", "event", "descriptor"}
assert d[1]["data_keys"]["ct"]["dtype"] == "array"
assert d[-1]["run_start"]
def test_from_event_model_all(RE, hw):
source = Stream()
t = FromEventStream("event", (), source, principle=True)
L = t.sink_to_list()
RE.subscribe(unstar(source.emit))
RE.subscribe(print)
RE(scan([hw.motor], hw.motor, 0, 9, 10))
assert len(L) == 10
for i, ll in enumerate(L):
assert i == ll["data"]["motor"]
def test_double_buffer_to_event_model():
source = Stream(asynchronous=True)
t = FromEventStream("event", ("data", "det_image"), source, principle=True)
assert t.principle
ts = t
a = ts.map(slow_inc)
aa = ts.map(slow_inc)
n = a.zip(aa).SimpleToEventStream(("ct", ))
b = n
b.sink(print)
L = b.sink_to_list()
futures_L = []
tt = t.sink_to_list()
p = b.pluck(0).sink_to_list()
d = b.pluck(1).sink_to_list()
t0 = time.time()
for gg in y(10):
futures_L.append(gg)
yield source.emit(gg)
while len(L) < len(futures_L):
yield gen.sleep(.01)
t1 = time.time()
# check that this was faster than running in series
assert t1 - t0 > .5 * 10
assert tt
assert p == ["start", "descriptor"] + ["event"] * 10 + ["stop"]
assert d[1]["hints"] == {"analyzer": {"fields": ["ct"]}}
t0 = time.time()
for gg in y(10):
futures_L.append(gg)
yield source.emit(gg)
while len(L) < len(futures_L):
yield gen.sleep(.01)
print(len(L), len(futures_L))
t1 = time.time()
# check that this was faster than running in series
assert t1 - t0 > .5 * 10
assert tt
assert p == (["start", "descriptor"] + ["event"] * 10 + ["stop"]) * 2
assert d[14]["hints"] == {"analyzer": {"fields": ["ct"]}}
for i, j in zip([0, 1, 12], [13, 14, 25]):
assert p[i] == p[j]
assert d[i] != d[j]
def test_from_event_model_stream_name2():
def data():
suid = str(uuid.uuid4())
duid = str(uuid.uuid4())
yield "start", {"hi": "world", "uid": suid}
yield "descriptor", {
"name": "hi",
"data_keys": {"ct"},
"uid": duid,
"run_start": suid,
}
for i in range(10):
yield "event", {
"uid": str(uuid.uuid4()),
"data": {
"ct": i
},
"descriptor": duid,
}
duid = str(uuid.uuid4())
yield "descriptor", {
"name": "not hi",
"data_keys": {"ct"},
"uid": duid,
"run_start": suid,
}
for i in range(100, 110):
yield "event", {
"uid": str(uuid.uuid4()),
"data": {
"ct": i
},
"descriptor": duid,
}
yield "stop", {"uid": str(uuid.uuid4()), "run_start": suid}
g = data()
source = Stream()
t = FromEventStream("event", ("data", "ct"),
source,
event_stream_name="not hi")
L = t.sink_to_list()
for gg in g:
source.emit(gg)
assert len(L) == 10
for i, ll in enumerate(L):
assert i + 100 == ll
def test_to_event_model_new_api_multi(RE, hw):
source = Stream()
stop = FromEventStream("stop", (), source)
t = FromEventStream("event", ("data", "motor"),
source,
principle=True,
stream_name="hi")
assert t.principle
tt = t.zip(stop)
n = simple_to_event_stream_new_api(
{
t: {
"data_keys": {
"ct": {
"units": "arb",
"precision": 2
}
}
},
tt: {
"name": "final",
"data_keys": {
"ct": {
"units": "arb",
"precision": 2
}
},
},
},
hello="world",
)
tt = t.sink_to_list()
p = n.pluck(0).sink_to_list()
d = n.pluck(1).sink_to_list()
RE.subscribe(unstar(source.emit))
RE.subscribe(print)
RE(scan([hw.motor], hw.motor, 0, 9, 10))
assert tt
assert set(p) == {"start", "stop", "event", "descriptor"}
assert d[0]["hello"] == "world"
assert d[1]["hints"] == {"analyzer": {"fields": ["ct"]}}
assert d[1]["data_keys"]["ct"]["units"] == "arb"
assert d[-3]["name"] == "final"
assert d[-1]["run_start"]
def test_to_event_model(RE, hw):
source = Stream()
t = FromEventStream("event", ("data", "motor"), source, principle=True)
assert t.principle
n = ToEventStream(t, ("ct", ), data_key_md={"ct": {"units": "arb"}})
tt = t.sink_to_list()
p = n.pluck(0).sink_to_list()
d = n.pluck(1).sink_to_list()
RE.subscribe(unstar(source.emit))
RE.subscribe(print)
RE(scan([hw.motor], hw.motor, 0, 9, 10))
assert tt
assert set(p) == {"start", "stop", "event", "descriptor"}
assert d[1]["hints"] == {"analyzer": {"fields": ["ct"]}}
assert d[1]["data_keys"]["ct"]["units"] == "arb"
assert d[-1]["run_start"]
def test_to_event_model_new_api_e_stop(RE, hw):
source = Stream()
t = FromEventStream("event", ("data", "motor"), source, principle=True)
assert t.principle
n = simple_to_event_stream_new_api(
{t: {
"data_keys": {
"ct": {
"units": "arb",
"precision": 2
}
}
}})
tt = t.sink_to_list()
p = n.pluck(0).sink_to_list()
d = n.pluck(1).sink_to_list()
def f(*x):
if x[0] == "stop":
return
source.emit(x)
RE.subscribe(f)
RE(scan([hw.motor], hw.motor, 0, 9, 10))
rs = d[0]["uid"]
assert tt
assert set(p) == {"start", "event", "descriptor"}
assert d[1]["hints"] == {"analyzer": {"fields": ["ct"]}}
assert d[1]["data_keys"]["ct"]["units"] == "arb"
ll = len(d)
RE(scan([hw.motor], hw.motor, 0, 9, 10))
assert d[ll]["run_start"] == rs
assert set(p) == {"start", "stop", "event", "descriptor"}
async def test_slow_to_event_model_parallel_dask(c, s, a, b):
source = Stream(asynchronous=True)
t = FromEventStream("event", ("data", "det_image"), source, principle=True)
assert t.principle
ts = t.scatter(backend="dask")
# futures_L = t.sink_to_list()
a = ts.map(slow_inc)
n = a.SimpleToEventStream(("ct", ))
b = n.buffer(100).gather()
b.sink(print)
L = b.sink_to_list()
tt = t.sink_to_list()
p = b.pluck(0).sink_to_list()
d = b.pluck(1).sink_to_list()
t0 = time.time()
futures_L = []
for gg in y(10):
futures_L.append(gg)
await source.emit(gg)
while len(L) < len(futures_L):
await gen.sleep(.01)
t1 = time.time()
# check that this was faster than running in series
td = t1 - t0
ted = .5 * 10
assert td < ted
assert tt
assert p == ["start", "descriptor"] + ["event"] * 10 + ["stop"]
assert "uid" in d[0]
assert d[1]["hints"] == {"analyzer": {"fields": ["ct"]}}
assert d[1]["data_keys"]["ct"]["dtype"] == "number"
assert d[2]["data"]["ct"] == 2
def test_to_event_model_new_api_no_data_keys(RE, hw):
source = Stream()
t = FromEventStream("event", ("data", ), source, principle=True)
assert t.principle
n = simple_to_event_stream_new_api({t: {}})
tt = t.sink_to_list()
p = n.pluck(0).sink_to_list()
d = n.pluck(1).sink_to_list()
RE.subscribe(unstar(source.emit))
RE.subscribe(print)
RE(scan([hw.motor], hw.motor, 0, 9, 10))
assert tt
assert set(p) == {"start", "stop", "event", "descriptor"}
assert d[1]["hints"] == {
"analyzer": {
"fields": ["motor", "motor_setpoint"]
}
}
assert d[1]["data_keys"]["motor"]
assert d[-1]["run_start"]