def test_execution_order(): L = [] for i in range(5): s = Stream() b = s.pluck(1) a = s.pluck(0) l = a.combine_latest(b, emit_on=a).sink_to_list() z = [(1, "red"), (2, "blue"), (3, "green")] for zz in z: s.emit(zz) L.append((l, )) for ll in L: assert ll == L[0] L2 = [] for i in range(5): s = Stream() a = s.pluck(0) b = s.pluck(1) l = a.combine_latest(b, emit_on=a).sink_to_list() z = [(1, "red"), (2, "blue"), (3, "green")] for zz in z: s.emit(zz) L2.append((l, )) for ll, ll2 in zip(L, L2): assert ll2 == L2[0] assert ll != ll2
def test_pluck(backend): source = Stream(asynchronous=True) L = source.scatter(backend=backend).pluck(0).gather().sink_to_list() for i in range(5): yield source.emit((i, i)) assert L == list(range(5))
def pipeline(): pipeline = Stream() b = pipeline.map(generate_data) b.sink(plot_data) c = b.map(fit_data) c.sink(plot_fit) return pipeline
def test_filter_args_kwargs(): def f(x, y, z=False): print(y) print(z) return y and z source = Stream() L = source.filter(f, True, z=True).sink_to_list() source.emit(1) assert L[0] is 1
def test_starmap(): def add(x=0, y=0): return x + y source = Stream() L = source.starmap(add).sink_to_list() source.emit((1, 10)) assert L[0] == 11
def test_zip_latest_first(): a = Stream() b = Stream() c = a.zip_latest(b).starmap(op.sub) d = a.zip_latest(b, first=True).starmap(op.add) L = c.union(d).sink_to_list() a.emit(1) b.emit(1) assert L == [2, 0]
def test_filter(backend): source = Stream(asynchronous=True) futures = scatter(source, backend=backend).filter(lambda x: x % 2 == 0) futures_L = futures.sink_to_list() L = futures.gather().sink_to_list() for i in range(5): yield source.emit(i) assert L == [0, 2, 4] assert all(isinstance(f, Future) for f in futures_L)
def test_map(backend): source = Stream(asynchronous=True) futures = scatter(source, backend=backend).map(inc) futures_L = futures.sink_to_list() L = futures.gather().sink_to_list() for i in range(5): yield source.emit(i) assert L == [1, 2, 3, 4, 5] assert all(isinstance(f, Future) for f in futures_L)
def test_sync_2(loop): with cluster() as (s, [a, b]): with Client(s["address"], loop=loop): # flake8: noqa source = Stream() L = source.scatter().map(inc).gather().sink_to_list() for i in range(10): source.emit(i) assert len(L) == i + 1 assert L == list(map(inc, range(10)))
def test_starmap(c, s, a, b): def add(x, y, z=0): return x + y + z source = Stream(asynchronous=True) L = source.scatter().starmap(add, z=10).gather().sink_to_list() for i in range(5): yield source.emit((i, i)) assert L == [10, 12, 14, 16, 18]
def test_filter_map(c, s, a, b): source = Stream(asynchronous=True) futures = scatter(source).filter(lambda x: x % 2 == 0).map(inc) futures_L = futures.sink_to_list() L = futures.gather().sink_to_list() for i in range(5): yield source.emit(i) assert L == [1, 3, 5] assert all(isinstance(f, Future) for f in futures_L)
def test_scan(c, s, a, b): source = Stream(asynchronous=True) futures = scatter(source).map(inc).scan(add) futures_L = futures.sink_to_list() L = futures.gather().sink_to_list() for i in range(5): yield source.emit(i) assert L == [1, 3, 6, 10, 15] assert all(isinstance(f, Future) for f in futures_L)
def test_filter_starmap(backend): source = Stream(asynchronous=True) futures1 = scatter(source, backend=backend).filter(lambda x: x[1] % 2 == 0) futures = futures1.starmap(add) futures_L = futures.sink_to_list() L = futures.gather().sink_to_list() for i in range(5): yield source.emit((i, i)) assert L == [0, 4, 8] assert all(isinstance(f, Future) for f in futures_L)
def test_scan_state(c, s, a, b): source = Stream(asynchronous=True) def f(acc, i): acc = acc + i return acc, acc L = scatter(source).scan(f, returns_state=True).gather().sink_to_list() for i in range(3): yield source.emit(i) assert L == [0, 1, 3]
def test_filter_zip(c, s, a, b): source = Stream(asynchronous=True) s = scatter(source) futures = s.filter(lambda x: x % 2 == 0).zip(s) futures_L = futures.sink_to_list() L = futures.gather().sink_to_list() for i in range(5): yield source.emit(i) assert L == [(a, a) for a in [0, 2, 4]] assert all(isinstance(f[0], Future) for f in futures_L)
def make_pipeline(_output_sinks=True): # link the pipeline up namespace = link(*(pipeline_order + [median_gen, std_gen, z_score_gen]), **general_namespace) polarization_array = namespace["polarization_array"] mask = namespace["mask"] mean = namespace["mean"] q = namespace["q"] geometry = namespace["geometry"] dark_corrected_foreground = namespace["dark_corrected_foreground"] dark_corrected_background = namespace["dark_corrected_background"] mask_kwargs = namespace["mask_kwargs"] mask_setting = namespace["mask_setting"] median = namespace["median"] std = namespace["std"] z_score = namespace["z_score"] # Modify graph # create filename nodes filename_source = Stream(stream_name="filename") filename_node = filename_source.map(lambda x: os.path.splitext(x)[0]) # write out mask mask.combine_latest( filename_node, emit_on=0).sink(lambda x: fit2d_save(np.flipud(x[0]), x[1])) mask.combine_latest( filename_node, emit_on=0).sink(lambda x: np.save(x[1] + "_mask.npy", x[0])) if _output_sinks: outs = [q, mean, median, std] out_tup = tuple([[] for _ in outs]) out_sinks = tuple([k.sink(L.append) for k, L in zip(outs, out_tup)]) (mean.zip(q).combine_latest( filename_node, emit_on=0).map(lambda l: (*l[0], l[1])).sink( lambda x: save_output(x[1], x[0], x[2], "Q"))) (median.zip(q).combine_latest( filename_node, emit_on=0).map(lambda l: (*l[0], l[1])).sink( lambda x: save_output(x[1], x[0], x[2] + "_median", "Q"))) (std.zip(q).combine_latest( filename_node, emit_on=0).map(lambda l: (*l[0], l[1])).sink( lambda x: save_output(x[1], x[0], x[2] + "_std", "Q"))) (z_score.combine_latest( filename_node, emit_on=0).starsink(lambda img, n: tifffile.imsave( n + "_zscore.tif", data=img.astype(np.float32)))) # If running from a terminal don't output stuff into lists (too much mem) return locals()
def test_starmap_args(c, s, a, b): def add(x, y, z=0): return x + y + z source = Stream(asynchronous=True) futures = scatter(source).starmap(add, 10) futures_L = futures.sink_to_list() L = futures.gather().sink_to_list() for i in range(5): yield source.emit(i) assert len(L) == len(futures_L) assert L == [i + 10 for i in range(5)]
def test_buffer2(backend): source = Stream(asynchronous=True) futures = scatter(source, backend=backend) futures_L = futures.sink_to_list() L = futures.buffer(10).gather().sink_to_list() for i in range(5): yield source.emit(i) assert len(futures_L) == 5 while len(L) < len(futures_L): yield gen.sleep(.01) assert L == [0, 1, 2, 3, 4] assert all(isinstance(f, Future) for f in futures_L)
def test_sync(loop): with cluster() as (s, [a, b]): with Client(s["address"], loop=loop) as client: # flake8: noqa source = Stream() L = source.scatter().map(inc).gather().sink_to_list() @gen.coroutine def f(): for i in range(10): yield source.emit(i, asynchronous=True) sync(loop, f) assert L == list(map(inc, range(10)))
def test_starmap(backend): def add(x, y, z=0): return x + y + z source = Stream(asynchronous=True) futures = scatter(source, backend=backend).starmap(add, z=10) futures_L = futures.sink_to_list() L = futures.gather().sink_to_list() for i in range(5): yield source.emit((i, i)) assert len(L) == len(futures_L) assert L == [10, 12, 14, 16, 18]
def test_combine_latest_first(): a = Stream() b = Stream() c = a.zip(b) z = c.starmap(op.add) zz = z.combine_latest(b, emit_on=0, first=b) L = zz.sink_to_list() a.emit(1) b.emit(1) assert len(L) == 1
def test_unique_list(): source = Stream() L = source.unique(history=1).sink_to_list() source.emit(["a"]) source.emit(["a"]) source.emit(["b"]) assert L == [["a"], ["b"]]
def test_unique_dict(): source = Stream() L = source.unique(history=1).sink_to_list() source.emit({"a": 1}) source.emit({"a": 1}) source.emit({"b": 1}) assert L == [{"a": 1}, {"b": 1}]
def test_zip(c, s, a, b): a = Stream(asynchronous=True) b = Stream(asynchronous=True) c = scatter(a).zip(scatter(b)) L = c.gather().sink_to_list() yield a.emit(1) yield b.emit("a") yield a.emit(2) yield b.emit("b") assert L == [(1, "a"), (2, "b")]
def test_buffer(backend): source = Stream(asynchronous=True) L = (source.scatter(backend=backend).map( slowinc, delay=0.5).buffer(5).gather().sink_to_list()) start = time.time() for i in range(5): yield source.emit(i) end = time.time() assert end - start < 0.5 for i in range(5, 10): yield source.emit(i) end2 = time.time() assert end2 - start > (0.5 / 3) while len(L) < 10: yield gen.sleep(0.01) assert time.time() - start < 5 assert L == list(map(inc, range(10)))
def test_double_scatter(backend): source1 = Stream(asynchronous=True) source2 = Stream(asynchronous=True) sm = (source1.scatter(backend=backend).zip( source2.scatter(backend=backend)).starmap(add)) futures_L = sm.sink_to_list() r = sm.buffer(10).gather() L = r.sink_to_list() for i in range(5): yield source1.emit(i) yield source2.emit(i) while len(L) < len(futures_L): yield gen.sleep(.01) assert L == [i + i for i in range(5)] assert all(isinstance(f, Future) for f in futures_L)
def test_star_sink(): L = [] def add(x, y): L.append(x + y) source = Stream() source.starsink(add) source.emit((1, 10)) assert L[0] == 11
def test_combined_latest(backend): def delay(x): time.sleep(.5) return x source = Stream(asynchronous=True) source2 = Stream(asynchronous=True) futures = (source.scatter(backend=backend).map(delay).combine_latest( source2.scatter(backend=backend), emit_on=1)) futures_L = futures.sink_to_list() L = futures.buffer(10).gather().sink_to_list() for i in range(5): yield source.emit(i) yield source.emit(i) yield source2.emit(i) while len(L) < len(futures_L): yield gen.sleep(.01) assert L == [(i, i) for i in range(5)]
def test_buffer_sync(loop): with cluster() as (s, [a, b]): with Client(s["address"], loop=loop) as c: # flake8: noqa source = Stream() buff = source.scatter().map(slowinc, delay=0.5).buffer(5) L = buff.gather().sink_to_list() start = time.time() for i in range(5): source.emit(i) end = time.time() assert end - start < 0.5 for i in range(5, 10): source.emit(i) end2 = time.time() while len(L) < 10: time.sleep(0.01) assert time.time() - start < 5 assert L == list(map(inc, range(10)))
# dark_corrected_foreground, q, # mean, tth, mask, pdf, fq, sq, # pol_corrected_img, raw_background) from xpdtools.pipelines.raw_pipeline import * from xpdtools.pipelines.raw_pipeline import ( mask_setting, # noqa: F401 ) from xpdtools.tools import overlay_mask image_name = glbl_dict['image_field'] db = glbl_dict['exp_db'] calibration_md_folder = {'folder': 'xpdAcq_calib_info.yml'} filler = Filler(db=db) # Build the general pipeline from the raw_pipeline raw_source = Stream(stream_name='raw source') # TODO: change this when new dark logic comes # Check that the data isn't a dark dk_uid = (FromEventStream( 'start', (), upstream=raw_source).map(lambda x: 'sc_dk_field_uid' in x)) # Fill the raw event stream source = ( raw_source.combine_latest(dk_uid).filter(lambda x: x[1]).pluck(0) # Filler returns None for resource/datum data .starmap(filler).filter(lambda x: x is not None)) # Get all the documents start_docs = FromEventStream('start', (), source) descriptor_docs = FromEventStream('descriptor', (), source, event_stream_name='primary')