def clear_geo_gen(source, geometry_img_shape, **kwargs):
# If new calibration uid invalidate our current calibration cache
a = FromEventStream("start", ("detector_calibration_client_uid",), source)
move_to_first(a)
(
a.unique(history=1).sink(
lambda x: geometry_img_shape.lossless_buffer.clear()
)
)
def test_main_pipeline(
exp_db,
fast_tmp_dir,
start_uid3,
start_uid1,
start_uid2,
background,
exception,
pe2
):
namespace = link(
*pipeline_order, raw_source=Stream(stream_name="raw source"),
db=exp_db,
)
iq_em = ToEventStream(
namespace["mean"].combine_latest(namespace["q"], emit_on=0),
("iq", "q"))
iq_em.sink(print)
limg = []
move_to_first(namespace["bg_corrected_img"].sink(lambda x: limg.append(x)))
lbgc = namespace["mean"].sink_to_list()
lpdf = namespace["iq_comp"].sink_to_list()
t0 = time.time()
if background:
uid = start_uid1
elif pe2:
uid = start_uid2
else:
uid = -1
for nd in exp_db[uid].documents(fill=True):
name, doc = nd
if name == "start":
if exception:
doc["bt_wavelength"] = "bla"
nd = (name, doc)
try:
namespace["raw_source"].emit(nd)
except ValueError:
pass
t1 = time.time()
print(t1 - t0)
n_events = len(list(exp_db[-1].events()))
assert len(limg) == n_events
if exception:
assert_lbgc = 0
else:
assert_lbgc = n_events
assert len(lbgc) == assert_lbgc
assert len(lpdf) == assert_lbgc
assert iq_em.state == "stopped"
destroy_pipeline(namespace["raw_source"])
del namespace
limg.clear()
lbgc.clear()
lpdf.clear()
def test_main_pipeline(exp_db, fast_tmp_dir, start_uid3, start_uid1,
start_uid2, background, exception, pe2):
namespace = link(
*pipeline_order,
raw_source=Stream(stream_name="raw source"),
db=exp_db,
)
iq_em = ToEventStream(
namespace["mean"].combine_latest(namespace["q"], emit_on=0),
("iq", "q"))
iq_em.sink(print)
limg = []
move_to_first(namespace["bg_corrected_img"].sink(lambda x: limg.append(x)))
lbgc = namespace["mean"].sink_to_list()
lpdf = namespace["iq_comp"].sink_to_list()
t0 = time.time()
if background:
uid = start_uid1
elif pe2:
uid = start_uid2
else:
uid = -1
for nd in exp_db[uid].documents(fill=True):
name, doc = nd
if name == "start":
if exception:
doc["bt_wavelength"] = "bla"
nd = (name, doc)
try:
namespace["raw_source"].emit(nd)
except ValueError:
pass
t1 = time.time()
print(t1 - t0)
n_events = len(list(exp_db[-1].events()))
assert len(limg) == n_events
if exception:
assert_lbgc = 0
else:
assert_lbgc = n_events
assert len(lbgc) == assert_lbgc
assert len(lpdf) == assert_lbgc
assert iq_em.state == "stopped"
destroy_pipeline(namespace["raw_source"])
del namespace
limg.clear()
lbgc.clear()
lpdf.clear()
def tomo_event_stream(source,
rec,
sinogram,
*,
qoi_name,
rec_3D=None,
**kwargs):
raw_stripped = move_to_first(source.starmap(StripDepVar()))
rec_tes = SimpleToEventStream(
rec, (f"{qoi_name}_tomo", ),
analysis_stage="{}_tomo".format(qoi_name)).LastCache()
# If we have a 3D reconstruction translate it
if rec_3D:
rec_3D_tes = SimpleToEventStream(
rec_3D,
(f"{qoi_name}_tomo_3D", ),
analysis_stage="{}_tomo_3D".format(qoi_name),
).LastCache()
sinogram_tes = SimpleToEventStream(
sinogram,
(f"{qoi_name}_sinogram", ),
analysis_stage="{}_sinogram".format(qoi_name),
).LastCache()
return locals()
def test_align_interrupted(RE, hw):
a = Stream()
b = FromEventStream("event", ("data", "img"), a, principle=True).map(
op.add, 1
)
b.sink(print)
c = ToEventStream(b, ("out",))
z = move_to_first(a.AlignEventStreams(c))
sl = z.sink_to_list()
L = []
RE.subscribe(lambda *x: L.append(x))
RE(count([hw.img]))
for nd in L:
name, doc = nd
# cause an exception
if name == "event":
doc["data"]["img"] = "hi"
try:
a.emit((name, doc))
except TypeError:
pass
assert {"start", "stop"} == set(list(zip(*sl))[0])
# check that buffers are not cleared, yet
sl.clear()
# If there are elements in the buffer they need to be cleared when all
# start docs come in.
for nd in L:
name, doc = nd
# cause an exception
if name == "event":
doc["data"]["img"] = 1
a.emit((name, doc))
if name == "start":
# now buffers should be clear
assert not any(
[b for n, tb in z.true_buffers.items() for u, b in tb.items()]
)
assert {"start", "descriptor", "event", "stop"} == set(list(zip(*sl))[0])
# now buffers should be clear (as all docs were emitted)
assert not any(
[b for n, tb in z.true_buffers.items() for u, b in tb.items()]
)
def create_analysis_pipeline(
order,
stage_blacklist=(),
publisher=Publisher(glbl_dict["inbound_proxy_address"], prefix=b"an"),
**kwargs,
):
"""Create the analysis pipeline from an list of chunks and pipeline kwargs
Parameters
----------
order : list of functions
The list of pipeline chunk functions
kwargs : Any
The kwargs to pass to the pipeline creation
Returns
-------
namespace : dict
The namespace of the pipeline
"""
namespace = link(
*order, raw_source=Stream(stream_name="raw source"), **kwargs
)
source = namespace["source"]
# do inspection of pipeline for ToEventModel nodes, maybe?
# for analyzed data with independent data (vis and save)
# strip the dependant vars form the raw data
raw_stripped = move_to_first(source.starmap(StripDepVar()))
namespace.update(
to_event_stream_with_ind(
raw_stripped,
*[
node
for node in namespace.values()
if isinstance(node, SimpleToEventStream)
and node.md.get("analysis_stage", None) not in stage_blacklist
],
publisher=publisher,
)
)
return namespace
def create_analysis_pipeline(
order, inbound_proxy_address=glbl_dict["inbound_proxy_address"], **kwargs
):
"""Create the analysis pipeline from an list of chunks and pipeline kwargs
Parameters
----------
order : list of functions
The list of pipeline chunk functions
kwargs : Any
The kwargs to pass to the pipeline creation
Returns
-------
namespace : dict
The namespace of the pipeline
"""
namespace = link(
*order, raw_source=Stream(stream_name="raw source"), **kwargs
)
source = namespace["source"]
# do inspection of pipeline for ToEventModel nodes, maybe?
# for analyzed data with independent data (vis and save)
an_with_ind_pub = Publisher(inbound_proxy_address, prefix=b"an")
# strip the dependant vars form the raw data
raw_stripped = move_to_first(source.starmap(StripDepVar()))
namespace.update(
to_event_stream_with_ind(
raw_stripped,
*[
node
for node in namespace.values()
if isinstance(node, SimpleToEventStream)
],
publisher=an_with_ind_pub
)
)
return namespace
def test_align_buffering(RE, hw):
zz = {"data": False}
a = Stream()
b = FromEventStream(
"event",
("data", "motor"),
a.filter(lambda x: zz["data"]),
principle=True,
).map(op.add, 1)
c = ToEventStream(b, ("out",))
z = move_to_first(a.AlignEventStreams(c))
sl = z.sink_to_list()
RE.subscribe(lambda *x: a.emit(x))
RE(scan([hw.img], hw.motor, 0, 10, 10, md={"hello": "world"}))
zz["data"] = True
sl.clear()
RE(scan([hw.img], hw.motor, 0, 10, 10))
assert "hello" not in sl[0][1]
def tomo_event_stream(
source, rec, sinogram, *, qoi_name, rec_3D=None, **kwargs
):
raw_stripped = move_to_first(source.starmap(StripDepVar()))
rec_tes = SimpleToEventStream(
rec, (f"{qoi_name}_tomo",), analysis_stage="{}_tomo".format(qoi_name)
).LastCache()
# If we have a 3D reconstruction translate it
if rec_3D:
rec_3D_tes = SimpleToEventStream(
rec_3D,
(f"{qoi_name}_tomo_3D",),
analysis_stage="{}_tomo_3D".format(qoi_name),
).LastCache()
sinogram_tes = SimpleToEventStream(
sinogram,
(f"{qoi_name}_sinogram",),
analysis_stage="{}_sinogram".format(qoi_name),
).LastCache()
return locals()
def run_server(
prefix=None,
outbound_proxy_address=glbl_dict["outbound_proxy_address"],
inbound_proxy_address=glbl_dict["inbound_proxy_address"],
_publisher=None,
**kwargs
):
"""Start up the QOI server
Parameters
----------
prefix : bytes or list of bytes, optional
The Publisher channels to listen to. Defaults to
``[b"an", b"raw"]``
outbound_proxy_address : str, optional
The address and port of the zmq proxy. Defaults to
``glbl_dict["outbound_proxy_address"]``
inbound_proxy_address : str, optional
The inbound ip address for the ZMQ server. Defaults to the value
from the global dict
"""
if prefix is None:
prefix = [b"an", b"raw"]
d = RemoteDispatcher(outbound_proxy_address, prefix=prefix)
install_qt_kicker(loop=d.loop)
if _publisher is None:
an_with_ind_pub = Publisher(inbound_proxy_address, prefix=b"qoi")
else:
an_with_ind_pub = _publisher
raw_source = Stream()
# create amorphous pipeline
amorphous_ns = link(
*[amorphsivity_fem, amorphsivity_pipeline, amorphsivity_tem],
source=Stream(),
**kwargs
)
# Combine the data outputs with the raw independent data
amorphous_ns.update(
to_event_stream_with_ind(
move_to_first(raw_source.starmap(StripDepVar())),
*[
node
for node in amorphous_ns.values()
if isinstance(node, SimpleToEventStream)
],
publisher=an_with_ind_pub
)
)
rr = RunRouter(
[
lambda x: lambda *y: raw_source.emit(y)
if x["analysis_stage"] == "raw"
else None,
lambda x: lambda *y: amorphous_ns["source"].emit(y)
if x["analysis_stage"] == "pdf"
else None,
]
)
d.subscribe(rr)
print("Starting QOI Server")
d.start()
def clear_comp(source, iq_comp, **kwargs):
# Clear composition every start document
# FIXME: Needs to go after the iq_comp is defined
a = FromEventStream("start", (), source)
move_to_first(a)
(a.sink(lambda x: clear_combine_latest(iq_comp, 1)))
