def test_execution_order():
L = []
for i in range(5):
s = Stream()
b = s.pluck(1)
a = s.pluck(0)
l = a.combine_latest(b, emit_on=a).sink_to_list()
z = [(1, "red"), (2, "blue"), (3, "green")]
for zz in z:
s.emit(zz)
L.append((l, ))
for ll in L:
assert ll == L[0]
L2 = []
for i in range(5):
s = Stream()
a = s.pluck(0)
b = s.pluck(1)
l = a.combine_latest(b, emit_on=a).sink_to_list()
z = [(1, "red"), (2, "blue"), (3, "green")]
for zz in z:
s.emit(zz)
L2.append((l, ))
for ll, ll2 in zip(L, L2):
assert ll2 == L2[0]
assert ll != ll2
def test_pluck(backend):
source = Stream(asynchronous=True)
L = source.scatter(backend=backend).pluck(0).gather().sink_to_list()
for i in range(5):
yield source.emit((i, i))
assert L == list(range(5))
def pipeline():
pipeline = Stream()
b = pipeline.map(generate_data)
b.sink(plot_data)
c = b.map(fit_data)
c.sink(plot_fit)
return pipeline
def test_filter_args_kwargs():
def f(x, y, z=False):
print(y)
print(z)
return y and z
source = Stream()
L = source.filter(f, True, z=True).sink_to_list()
source.emit(1)
assert L[0] is 1
def test_starmap():
def add(x=0, y=0):
return x + y
source = Stream()
L = source.starmap(add).sink_to_list()
source.emit((1, 10))
assert L[0] == 11
def test_zip_latest_first():
a = Stream()
b = Stream()
c = a.zip_latest(b).starmap(op.sub)
d = a.zip_latest(b, first=True).starmap(op.add)
L = c.union(d).sink_to_list()
a.emit(1)
b.emit(1)
assert L == [2, 0]
def test_filter(backend):
source = Stream(asynchronous=True)
futures = scatter(source, backend=backend).filter(lambda x: x % 2 == 0)
futures_L = futures.sink_to_list()
L = futures.gather().sink_to_list()
for i in range(5):
yield source.emit(i)
assert L == [0, 2, 4]
assert all(isinstance(f, Future) for f in futures_L)
def test_map(backend):
source = Stream(asynchronous=True)
futures = scatter(source, backend=backend).map(inc)
futures_L = futures.sink_to_list()
L = futures.gather().sink_to_list()
for i in range(5):
yield source.emit(i)
assert L == [1, 2, 3, 4, 5]
assert all(isinstance(f, Future) for f in futures_L)
def test_sync_2(loop):
with cluster() as (s, [a, b]):
with Client(s["address"], loop=loop): # flake8: noqa
source = Stream()
L = source.scatter().map(inc).gather().sink_to_list()
for i in range(10):
source.emit(i)
assert len(L) == i + 1
assert L == list(map(inc, range(10)))
def test_starmap(c, s, a, b):
def add(x, y, z=0):
return x + y + z
source = Stream(asynchronous=True)
L = source.scatter().starmap(add, z=10).gather().sink_to_list()
for i in range(5):
yield source.emit((i, i))
assert L == [10, 12, 14, 16, 18]
def test_filter_map(c, s, a, b):
source = Stream(asynchronous=True)
futures = scatter(source).filter(lambda x: x % 2 == 0).map(inc)
futures_L = futures.sink_to_list()
L = futures.gather().sink_to_list()
for i in range(5):
yield source.emit(i)
assert L == [1, 3, 5]
assert all(isinstance(f, Future) for f in futures_L)
def test_scan(c, s, a, b):
source = Stream(asynchronous=True)
futures = scatter(source).map(inc).scan(add)
futures_L = futures.sink_to_list()
L = futures.gather().sink_to_list()
for i in range(5):
yield source.emit(i)
assert L == [1, 3, 6, 10, 15]
assert all(isinstance(f, Future) for f in futures_L)
def test_filter_starmap(backend):
source = Stream(asynchronous=True)
futures1 = scatter(source, backend=backend).filter(lambda x: x[1] % 2 == 0)
futures = futures1.starmap(add)
futures_L = futures.sink_to_list()
L = futures.gather().sink_to_list()
for i in range(5):
yield source.emit((i, i))
assert L == [0, 4, 8]
assert all(isinstance(f, Future) for f in futures_L)
def test_scan_state(c, s, a, b):
source = Stream(asynchronous=True)
def f(acc, i):
acc = acc + i
return acc, acc
L = scatter(source).scan(f, returns_state=True).gather().sink_to_list()
for i in range(3):
yield source.emit(i)
assert L == [0, 1, 3]
def test_filter_zip(c, s, a, b):
source = Stream(asynchronous=True)
s = scatter(source)
futures = s.filter(lambda x: x % 2 == 0).zip(s)
futures_L = futures.sink_to_list()
L = futures.gather().sink_to_list()
for i in range(5):
yield source.emit(i)
assert L == [(a, a) for a in [0, 2, 4]]
assert all(isinstance(f[0], Future) for f in futures_L)
def make_pipeline(_output_sinks=True):
# link the pipeline up
namespace = link(*(pipeline_order + [median_gen, std_gen, z_score_gen]),
**general_namespace)
polarization_array = namespace["polarization_array"]
mask = namespace["mask"]
mean = namespace["mean"]
q = namespace["q"]
geometry = namespace["geometry"]
dark_corrected_foreground = namespace["dark_corrected_foreground"]
dark_corrected_background = namespace["dark_corrected_background"]
mask_kwargs = namespace["mask_kwargs"]
mask_setting = namespace["mask_setting"]
median = namespace["median"]
std = namespace["std"]
z_score = namespace["z_score"]
# Modify graph
# create filename nodes
filename_source = Stream(stream_name="filename")
filename_node = filename_source.map(lambda x: os.path.splitext(x)[0])
# write out mask
mask.combine_latest(
filename_node,
emit_on=0).sink(lambda x: fit2d_save(np.flipud(x[0]), x[1]))
mask.combine_latest(
filename_node,
emit_on=0).sink(lambda x: np.save(x[1] + "_mask.npy", x[0]))
if _output_sinks:
outs = [q, mean, median, std]
out_tup = tuple([[] for _ in outs])
out_sinks = tuple([k.sink(L.append) for k, L in zip(outs, out_tup)])
(mean.zip(q).combine_latest(
filename_node, emit_on=0).map(lambda l: (*l[0], l[1])).sink(
lambda x: save_output(x[1], x[0], x[2], "Q")))
(median.zip(q).combine_latest(
filename_node, emit_on=0).map(lambda l: (*l[0], l[1])).sink(
lambda x: save_output(x[1], x[0], x[2] + "_median", "Q")))
(std.zip(q).combine_latest(
filename_node, emit_on=0).map(lambda l: (*l[0], l[1])).sink(
lambda x: save_output(x[1], x[0], x[2] + "_std", "Q")))
(z_score.combine_latest(
filename_node, emit_on=0).starsink(lambda img, n: tifffile.imsave(
n + "_zscore.tif", data=img.astype(np.float32))))
# If running from a terminal don't output stuff into lists (too much mem)
return locals()
def test_starmap_args(c, s, a, b):
def add(x, y, z=0):
return x + y + z
source = Stream(asynchronous=True)
futures = scatter(source).starmap(add, 10)
futures_L = futures.sink_to_list()
L = futures.gather().sink_to_list()
for i in range(5):
yield source.emit(i)
assert len(L) == len(futures_L)
assert L == [i + 10 for i in range(5)]
def test_buffer2(backend):
source = Stream(asynchronous=True)
futures = scatter(source, backend=backend)
futures_L = futures.sink_to_list()
L = futures.buffer(10).gather().sink_to_list()
for i in range(5):
yield source.emit(i)
assert len(futures_L) == 5
while len(L) < len(futures_L):
yield gen.sleep(.01)
assert L == [0, 1, 2, 3, 4]
assert all(isinstance(f, Future) for f in futures_L)
def test_sync(loop):
with cluster() as (s, [a, b]):
with Client(s["address"], loop=loop) as client: # flake8: noqa
source = Stream()
L = source.scatter().map(inc).gather().sink_to_list()
@gen.coroutine
def f():
for i in range(10):
yield source.emit(i, asynchronous=True)
sync(loop, f)
assert L == list(map(inc, range(10)))
def test_starmap(backend):
def add(x, y, z=0):
return x + y + z
source = Stream(asynchronous=True)
futures = scatter(source, backend=backend).starmap(add, z=10)
futures_L = futures.sink_to_list()
L = futures.gather().sink_to_list()
for i in range(5):
yield source.emit((i, i))
assert len(L) == len(futures_L)
assert L == [10, 12, 14, 16, 18]
def test_combine_latest_first():
a = Stream()
b = Stream()
c = a.zip(b)
z = c.starmap(op.add)
zz = z.combine_latest(b, emit_on=0, first=b)
L = zz.sink_to_list()
a.emit(1)
b.emit(1)
assert len(L) == 1
def test_unique_list():
source = Stream()
L = source.unique(history=1).sink_to_list()
source.emit(["a"])
source.emit(["a"])
source.emit(["b"])
assert L == [["a"], ["b"]]
def test_unique_dict():
source = Stream()
L = source.unique(history=1).sink_to_list()
source.emit({"a": 1})
source.emit({"a": 1})
source.emit({"b": 1})
assert L == [{"a": 1}, {"b": 1}]
def test_zip(c, s, a, b):
a = Stream(asynchronous=True)
b = Stream(asynchronous=True)
c = scatter(a).zip(scatter(b))
L = c.gather().sink_to_list()
yield a.emit(1)
yield b.emit("a")
yield a.emit(2)
yield b.emit("b")
assert L == [(1, "a"), (2, "b")]
def test_buffer(backend):
source = Stream(asynchronous=True)
L = (source.scatter(backend=backend).map(
slowinc, delay=0.5).buffer(5).gather().sink_to_list())
start = time.time()
for i in range(5):
yield source.emit(i)
end = time.time()
assert end - start < 0.5
for i in range(5, 10):
yield source.emit(i)
end2 = time.time()
assert end2 - start > (0.5 / 3)
while len(L) < 10:
yield gen.sleep(0.01)
assert time.time() - start < 5
assert L == list(map(inc, range(10)))
def test_double_scatter(backend):
source1 = Stream(asynchronous=True)
source2 = Stream(asynchronous=True)
sm = (source1.scatter(backend=backend).zip(
source2.scatter(backend=backend)).starmap(add))
futures_L = sm.sink_to_list()
r = sm.buffer(10).gather()
L = r.sink_to_list()
for i in range(5):
yield source1.emit(i)
yield source2.emit(i)
while len(L) < len(futures_L):
yield gen.sleep(.01)
assert L == [i + i for i in range(5)]
assert all(isinstance(f, Future) for f in futures_L)
def test_star_sink():
L = []
def add(x, y):
L.append(x + y)
source = Stream()
source.starsink(add)
source.emit((1, 10))
assert L[0] == 11
def test_combined_latest(backend):
def delay(x):
time.sleep(.5)
return x
source = Stream(asynchronous=True)
source2 = Stream(asynchronous=True)
futures = (source.scatter(backend=backend).map(delay).combine_latest(
source2.scatter(backend=backend), emit_on=1))
futures_L = futures.sink_to_list()
L = futures.buffer(10).gather().sink_to_list()
for i in range(5):
yield source.emit(i)
yield source.emit(i)
yield source2.emit(i)
while len(L) < len(futures_L):
yield gen.sleep(.01)
assert L == [(i, i) for i in range(5)]
def test_buffer_sync(loop):
with cluster() as (s, [a, b]):
with Client(s["address"], loop=loop) as c: # flake8: noqa
source = Stream()
buff = source.scatter().map(slowinc, delay=0.5).buffer(5)
L = buff.gather().sink_to_list()
start = time.time()
for i in range(5):
source.emit(i)
end = time.time()
assert end - start < 0.5
for i in range(5, 10):
source.emit(i)
end2 = time.time()
while len(L) < 10:
time.sleep(0.01)
assert time.time() - start < 5
assert L == list(map(inc, range(10)))
# dark_corrected_foreground, q,
# mean, tth, mask, pdf, fq, sq,
# pol_corrected_img, raw_background)
from xpdtools.pipelines.raw_pipeline import *
from xpdtools.pipelines.raw_pipeline import (
mask_setting, # noqa: F401
)
from xpdtools.tools import overlay_mask
image_name = glbl_dict['image_field']
db = glbl_dict['exp_db']
calibration_md_folder = {'folder': 'xpdAcq_calib_info.yml'}
filler = Filler(db=db)
# Build the general pipeline from the raw_pipeline
raw_source = Stream(stream_name='raw source')
# TODO: change this when new dark logic comes
# Check that the data isn't a dark
dk_uid = (FromEventStream(
'start', (), upstream=raw_source).map(lambda x: 'sc_dk_field_uid' in x))
# Fill the raw event stream
source = (
raw_source.combine_latest(dk_uid).filter(lambda x: x[1]).pluck(0)
# Filler returns None for resource/datum data
.starmap(filler).filter(lambda x: x is not None))
# Get all the documents
start_docs = FromEventStream('start', (), source)
descriptor_docs = FromEventStream('descriptor', (),
source,
event_stream_name='primary')
