def test_raw_pipeline(mask_s): # link the pipeline up namespace = link(*pipeline_order, **g_namespace) is_calibration_img = namespace["is_calibration_img"] geo_input = namespace["geo_input"] img_counter = namespace["img_counter"] namespace["mask_setting"]["setting"] = mask_s pdf = namespace["pdf"] raw_background_dark = namespace["raw_background_dark"] raw_background = namespace["raw_background"] raw_foreground_dark = namespace["raw_foreground_dark"] composition = namespace["composition"] raw_foreground = namespace["raw_foreground"] sl = pdf.sink_to_list() L = namespace["geometry"].sink_to_list() ml = namespace["mask"].sink_to_list() is_calibration_img.emit(False) a = geo.getPyFAI() geo_input.emit(a) for s in [raw_background_dark, raw_background, raw_foreground_dark]: s.emit(np.zeros(img.shape)) composition.emit("Au") img_counter.emit(1) raw_foreground.emit(img) destroy_pipeline(raw_foreground) del namespace assert len(L) == 1 assert ml assert len(sl) == 1 sl.clear() L.clear() ml.clear()
def test_main_pipeline(exp_db, fast_tmp_dir, start_uid3, start_uid1, background, exception): namespace = link(*(pipeline_order + save_pipeline_order), raw_source=Stream(stream_name="raw source"), db=exp_db) namespace["save_kwargs"].update({"base_folder": fast_tmp_dir}) raw_source = namespace["raw_source"] t0 = time.time() if background: uid = start_uid1 else: uid = -1 for nd in exp_db[uid].documents(fill=True): name, doc = nd if name == "start": if exception: doc["bt_wavelength"] = "bla" nd = (name, doc) try: raw_source.emit(nd) except ValueError: pass if background: name = "kapton" else: name = "Au" t1 = time.time() print(t1 - t0) n_events = len(list(exp_db[-1].events())) for root, dirs, files in os.walk(fast_tmp_dir): level = root.replace(fast_tmp_dir, "").count(os.sep) indent = " " * 4 * level print("{}{}/".format(indent, os.path.basename(root))) subindent = " " * 4 * (level + 1) for f in files: print("{}{}".format(subindent, f)) time.sleep(2) print(os.listdir(fast_tmp_dir)) print(os.listdir(os.path.join(fast_tmp_dir, name))) assert name in os.listdir(fast_tmp_dir) if exception: output_list = ["dark_sub", "mask"] else: output_list = ["dark_sub", "mask", "iq", "itth", "pdf"] for f in output_list: assert f in os.listdir(os.path.join(fast_tmp_dir, name)) if f == "mask": assert (len(os.listdir(os.path.join(fast_tmp_dir, name, f))) == n_events * 2) else: assert (len(os.listdir(os.path.join(fast_tmp_dir, name, f))) == n_events) assert "{}_{:.6}.yaml".format(name, exp_db[uid].start["uid"][:6]) in os.listdir( os.path.join(fast_tmp_dir, name, "meta"))
def test_main_pipeline( exp_db, fast_tmp_dir, start_uid3, start_uid1, start_uid2, background, exception, pe2 ): namespace = link( *pipeline_order, raw_source=Stream(stream_name="raw source"), db=exp_db, ) iq_em = ToEventStream( namespace["mean"].combine_latest(namespace["q"], emit_on=0), ("iq", "q")) iq_em.sink(print) limg = [] move_to_first(namespace["bg_corrected_img"].sink(lambda x: limg.append(x))) lbgc = namespace["mean"].sink_to_list() lpdf = namespace["iq_comp"].sink_to_list() t0 = time.time() if background: uid = start_uid1 elif pe2: uid = start_uid2 else: uid = -1 for nd in exp_db[uid].documents(fill=True): name, doc = nd if name == "start": if exception: doc["bt_wavelength"] = "bla" nd = (name, doc) try: namespace["raw_source"].emit(nd) except ValueError: pass t1 = time.time() print(t1 - t0) n_events = len(list(exp_db[-1].events())) assert len(limg) == n_events if exception: assert_lbgc = 0 else: assert_lbgc = n_events assert len(lbgc) == assert_lbgc assert len(lpdf) == assert_lbgc assert iq_em.state == "stopped" destroy_pipeline(namespace["raw_source"]) del namespace limg.clear() lbgc.clear() lpdf.clear()
def tomo_callback_factory(doc, publisher, handler_reg, **kwargs): # TODO: Eventually extract from plan hints? if doc.get("tomo", {}).get("type", None) == "pencil": if len(doc['motors']) == 2: po = pencil_order else: po = d3_pencil_order return PencilTomoCallback( lambda **inner_kwargs: link(*po, **inner_kwargs), publisher, **kwargs, ) elif doc.get("tomo", {}).get("type", None) == "full_field": return FullFieldTomoCallback( lambda **inner_kwargs: link(*full_field_order, **inner_kwargs), publisher, handler_reg=handler_reg, **kwargs, )
def raw_pipeline_parallel(): # link the pipeline up gg_namespace = dict(g_namespace) s_ns = { k: v.scatter(backend="thread") for k, v in gg_namespace.items() if isinstance(v, Stream) } gg_namespace.update( {"_" + k: v for k, v in gg_namespace.items() if isinstance(v, Stream)}) gg_namespace.update(s_ns) namespace = link(*pipeline_order[:-1], **gg_namespace) geo_input = namespace["_geo_input"] composition = namespace["_composition"] raw_background_dark = namespace["_raw_background_dark"] raw_background = namespace["_raw_background"] raw_foreground_dark = namespace["_raw_foreground_dark"] raw_foreground = namespace["_raw_foreground"] print(type(namespace["raw_foreground"])) a = namespace["mean"] futures = a.sink_to_list() b = a.buffer(10) g = b.gather() # g.sink(lambda x: print("gathered data", time.time())) LL = g.map(lambda x: time.time()).sink_to_list() L = g.sink_to_list() a = geo.getPyFAI() geo_input.emit(a) composition.emit("Au1.0") for s in [raw_background_dark, raw_background, raw_foreground_dark]: s.emit(np.zeros(img.shape)) ii = 10 t0 = time.time() for i in range(ii): rimg = np.random.random(img.shape) raw_foreground.emit(img + rimg) while len(L) < ii: time.sleep(.01) time_diff = [LL[i] - LL[i - 1] for i in range(1, ii)] print(max(time_diff), min(time_diff), sum(time_diff) / len(time_diff)) # print([l - min(LL) for l in LL]) print([l - t0 for l in LL]) print(max([l - t0 for l in LL]) / ii) destroy_pipeline(raw_foreground) del namespace futures.clear() L.clear()
def make_pipeline(_output_sinks=True): # link the pipeline up namespace = link(*(pipeline_order + [median_gen, std_gen, z_score_gen]), **general_namespace) polarization_array = namespace["polarization_array"] mask = namespace["mask"] mean = namespace["mean"] q = namespace["q"] geometry = namespace["geometry"] dark_corrected_foreground = namespace["dark_corrected_foreground"] dark_corrected_background = namespace["dark_corrected_background"] mask_kwargs = namespace["mask_kwargs"] mask_setting = namespace["mask_setting"] median = namespace["median"] std = namespace["std"] z_score = namespace["z_score"] # Modify graph # create filename nodes filename_source = Stream(stream_name="filename") filename_node = filename_source.map(lambda x: os.path.splitext(x)[0]) # write out mask mask.combine_latest( filename_node, emit_on=0).sink(lambda x: fit2d_save(np.flipud(x[0]), x[1])) mask.combine_latest( filename_node, emit_on=0).sink(lambda x: np.save(x[1] + "_mask.npy", x[0])) if _output_sinks: outs = [q, mean, median, std] out_tup = tuple([[] for _ in outs]) out_sinks = tuple([k.sink(L.append) for k, L in zip(outs, out_tup)]) (mean.zip(q).combine_latest( filename_node, emit_on=0).map(lambda l: (*l[0], l[1])).sink( lambda x: save_output(x[1], x[0], x[2], "Q"))) (median.zip(q).combine_latest( filename_node, emit_on=0).map(lambda l: (*l[0], l[1])).sink( lambda x: save_output(x[1], x[0], x[2] + "_median", "Q"))) (std.zip(q).combine_latest( filename_node, emit_on=0).map(lambda l: (*l[0], l[1])).sink( lambda x: save_output(x[1], x[0], x[2] + "_std", "Q"))) (z_score.combine_latest( filename_node, emit_on=0).starsink(lambda img, n: tifffile.imsave( n + "_zscore.tif", data=img.astype(np.float32)))) # If running from a terminal don't output stuff into lists (too much mem) return locals()
def test_double_link(): def make_a(**kwargs): in_a = Stream() out_a = in_a.map(lambda x: x + 1) return locals() def make_b(out_a, **kwargs): out_b = out_a.map(lambda x: x * 2) return locals() def make_c(out_a, out_b, **kwargs): out_c = out_a.zip(out_b).map(sum) return locals() ab = link(make_a, make_b) abc = link(make_c, **ab) assert set(abc.keys()) == {'in_a', 'out_a', 'out_b', 'out_c'} L = ab["out_b"].sink_to_list() L2 = abc["out_c"].sink_to_list() for i in range(10): ab["in_a"].emit(i) assert L == [(i + 1) * 2 for i in range(10)] assert L2 == [((i + 1) * 2) + i + 1 for i in range(10)]
def test_main_pipeline(exp_db, fast_tmp_dir, start_uid3, start_uid1, start_uid2, background, exception, pe2): namespace = link( *pipeline_order, raw_source=Stream(stream_name="raw source"), db=exp_db, ) iq_em = ToEventStream( namespace["mean"].combine_latest(namespace["q"], emit_on=0), ("iq", "q")) iq_em.sink(print) limg = [] move_to_first(namespace["bg_corrected_img"].sink(lambda x: limg.append(x))) lbgc = namespace["mean"].sink_to_list() lpdf = namespace["iq_comp"].sink_to_list() t0 = time.time() if background: uid = start_uid1 elif pe2: uid = start_uid2 else: uid = -1 for nd in exp_db[uid].documents(fill=True): name, doc = nd if name == "start": if exception: doc["bt_wavelength"] = "bla" nd = (name, doc) try: namespace["raw_source"].emit(nd) except ValueError: pass t1 = time.time() print(t1 - t0) n_events = len(list(exp_db[-1].events())) assert len(limg) == n_events if exception: assert_lbgc = 0 else: assert_lbgc = n_events assert len(lbgc) == assert_lbgc assert len(lpdf) == assert_lbgc assert iq_em.state == "stopped" destroy_pipeline(namespace["raw_source"]) del namespace limg.clear() lbgc.clear() lpdf.clear()
def test_tiff_pipeline( exp_db, fast_tmp_dir, start_uid3, start_uid1, background, exception ): namespace = link( *pipeline_order, raw_source=Stream(stream_name="raw source"), db=exp_db ) namespace["save_kwargs"].update({"base_folder": fast_tmp_dir}) raw_source = namespace["raw_source"] t0 = time.time() if background: uid = start_uid1 else: uid = -1 for nd in exp_db[uid].documents(): name, doc = nd if name == "start": if exception: doc["bt_wavelength"] = "bla" nd = (name, doc) try: raw_source.emit(nd) except ValueError: pass if background: name = "kapton" else: name = "Au" t1 = time.time() print(t1 - t0) n_events = len(list(exp_db[-1].events())) for root, dirs, files in os.walk(fast_tmp_dir): level = root.replace(fast_tmp_dir, "").count(os.sep) indent = " " * 4 * level print("{}{}/".format(indent, os.path.basename(root))) subindent = " " * 4 * (level + 1) for f in files: print("{}{}".format(subindent, f)) print(os.listdir(fast_tmp_dir)) print(os.listdir(os.path.join(fast_tmp_dir, name))) assert name in os.listdir(fast_tmp_dir) for f in ["dark_sub"]: assert f in os.listdir(os.path.join(fast_tmp_dir, name)) assert len(os.listdir(os.path.join(fast_tmp_dir, name, f))) == n_events assert "{}_{:.6}.yaml".format( name, exp_db[uid].start["uid"][:6] ) in os.listdir(os.path.join(fast_tmp_dir, name, "meta"))
def test_link(): def make_a(): source = Stream() out_a = source.map(lambda x: x + 1) return locals() def make_b(out_a, **kwargs): out_b = out_a.map(lambda x: x * 2) return locals() ns = link(make_a, make_b) L = ns["out_b"].sink_to_list() for i in range(10): ns["source"].emit(i) assert L == [(i + 1) * 2 for i in range(10)]
def create_analysis_pipeline( order, stage_blacklist=(), publisher=Publisher(glbl_dict["inbound_proxy_address"], prefix=b"an"), **kwargs, ): """Create the analysis pipeline from an list of chunks and pipeline kwargs Parameters ---------- order : list of functions The list of pipeline chunk functions kwargs : Any The kwargs to pass to the pipeline creation Returns ------- namespace : dict The namespace of the pipeline """ namespace = link( *order, raw_source=Stream(stream_name="raw source"), **kwargs ) source = namespace["source"] # do inspection of pipeline for ToEventModel nodes, maybe? # for analyzed data with independent data (vis and save) # strip the dependant vars form the raw data raw_stripped = move_to_first(source.starmap(StripDepVar())) namespace.update( to_event_stream_with_ind( raw_stripped, *[ node for node in namespace.values() if isinstance(node, SimpleToEventStream) and node.md.get("analysis_stage", None) not in stage_blacklist ], publisher=publisher, ) ) return namespace
def test_raw_pipeline_parallel(n): # caplog.set_level(logging.CRITICAL) # link the pipeline up gg_namespace = dict(g_namespace) s_ns = { k: v.scatter(backend="thread") for k, v in gg_namespace.items() if isinstance(v, Stream) } gg_namespace.update( {"_" + k: v for k, v in gg_namespace.items() if isinstance(v, Stream)}) gg_namespace.update(s_ns) namespace = link(*pipeline_order, **gg_namespace) geo_input = namespace["_geo_input"] raw_background_dark = namespace["_raw_background_dark"] raw_background = namespace["_raw_background"] raw_foreground_dark = namespace["_raw_foreground_dark"] raw_foreground = namespace["_raw_foreground"] a = namespace[n] futures = a.sink_to_list() b = a.buffer(10) g = b.gather() g.sink(lambda x: print("gathered data", time.time())) L = g.sink_to_list() a = geo.getPyFAI() yield geo_input.emit(a) for s in [raw_background_dark, raw_background, raw_foreground_dark]: yield s.emit(np.zeros(img.shape)) ii = 2 for i in range(ii): rimg = np.random.random(img.shape) yield raw_foreground.emit(img + rimg) while len(L) < ii: yield gen.sleep(.01) destroy_pipeline(raw_foreground) del namespace futures.clear() L.clear()
def test_tomo_piecewise_pipeline(rand_size): ns = dict( qoi=Stream(), x=Stream(), th=Stream(), th_dim=Stream(), x_dim=Stream(), th_extents=Stream(), x_extents=Stream(), center=Stream(), ) x_linspace = np.linspace(0, 5, 6) th_linspace = np.linspace(0, 180, 6) ns["th_dimension"] = len(th_linspace) ns["x_dimension"] = len(x_linspace) ns.update(**link(*[tomo_prep, tomo_pipeline_piecewise], **ns)) L = ns["rec"].sink_to_list() ns["th_dim"].emit(len(th_linspace)) ns["x_dim"].emit(len(x_linspace)) ns["th_extents"].emit([0, 180]) ns["x_extents"].emit([x_linspace[0], x_linspace[-1]]) ns["center"].emit(2.5) # np.random.seed(42) for x in x_linspace: for th in th_linspace: ns["x"].emit(x) ns["th"].emit(th) ns["qoi"].emit(np.random.random(rand_size)) assert len(L) == len(x_linspace) * len(th_linspace) if rand_size: assert L[-1].shape == (*rand_size, len(x_linspace), len(th_linspace)) else: assert L[-1].shape == (len(x_linspace), len(th_linspace)) destroy_pipeline(ns["qoi"]) del ns L.clear()
def create_analysis_pipeline( order, inbound_proxy_address=glbl_dict["inbound_proxy_address"], **kwargs ): """Create the analysis pipeline from an list of chunks and pipeline kwargs Parameters ---------- order : list of functions The list of pipeline chunk functions kwargs : Any The kwargs to pass to the pipeline creation Returns ------- namespace : dict The namespace of the pipeline """ namespace = link( *order, raw_source=Stream(stream_name="raw source"), **kwargs ) source = namespace["source"] # do inspection of pipeline for ToEventModel nodes, maybe? # for analyzed data with independent data (vis and save) an_with_ind_pub = Publisher(inbound_proxy_address, prefix=b"an") # strip the dependant vars form the raw data raw_stripped = move_to_first(source.starmap(StripDepVar())) namespace.update( to_event_stream_with_ind( raw_stripped, *[ node for node in namespace.values() if isinstance(node, SimpleToEventStream) ], publisher=an_with_ind_pub ) ) return namespace
def test_qoi_pipeline(): # link the pipeline up namespace = link(*(pipeline_order + [max_intensity_mean, max_gr_mean]), **g_namespace) geometry = namespace["geometry"] mean_max = namespace["mean_max"] raw_background_dark = namespace["raw_background_dark"] raw_background = namespace["raw_background"] raw_foreground_dark = namespace["raw_foreground_dark"] raw_foreground = namespace["raw_foreground"] sl = mean_max.sink_to_list() geometry.emit(geo) for s in [raw_background_dark, raw_background, raw_foreground_dark]: s.emit(np.zeros(img.shape)) raw_foreground.emit(img) del namespace destroy_pipeline(raw_foreground) assert len(sl) == 1 sl.clear()
def test_extra_pipeline(): # link the pipeline up namespace = link(*(pipeline_order + [median_gen, std_gen, z_score_gen]), **g_namespace) geometry = namespace["geometry"] z_score = namespace["z_score"] raw_background_dark = namespace["raw_background_dark"] raw_background = namespace["raw_background"] raw_foreground_dark = namespace["raw_foreground_dark"] raw_foreground = namespace["raw_foreground"] sl = z_score.sink_to_list() geometry.emit(geo) for s in [raw_background_dark, raw_background, raw_foreground_dark]: s.emit(np.zeros(img.shape)) raw_foreground.emit(img) del namespace destroy_pipeline(raw_foreground) assert len(sl) == 1 sl.clear()
def start_analysis(save=True, vis=True, **kwargs): """Start analysis pipeline [Depreciated] Parameters ---------- mask_kwargs : dict The kwargs passed to the masking see xpdtools.tools.mask_img pdf_kwargs : dict The kwargs passed to the pdf generator, see xpdtools.tools.pdf_getter fq_kwargs : dict The kwargs passed to the fq generator, see xpdtools.tools.fq_getter mask_setting : dict The setting of the mask save_template : str The template string for file saving base_folder : str The base folder for saving files """ warn(DeprecationWarning("Use the server instead")) # TODO: also start up grave vis, maybe? d = RemoteDispatcher(glbl_dict["outbound_proxy_address"]) install_qt_kicker( loop=d.loop ) # This may need to be d._loop depending on tag order = pipeline_order if save: order += save_pipeline_order if vis: order += [vis_pipeline] namespace = link( *order, raw_source=Stream(stream_name="raw source"), **kwargs ) raw_source = namespace["raw_source"] d.subscribe(lambda *x: raw_source.emit(x)) print("Starting Analysis Server") d.start()
import numpy as np from tifffile import imread import pyFAI from xpdtools.pipelines.raw_pipeline import pipeline_order, namespace from profilehooks import profile from rapidz.link import link namespace["mask_setting"].update(setting="first") namespace = link(*pipeline_order, **namespace) # dark_corrected_background.sink(print) # pol_corrected_img_zip.sink(print) # mask.sink(print) # binner.sink(print) # mean.sink(print) geo = pyFAI.load("test.poni") img = imread("test.tiff") namespace["geometry"].emit(geo) namespace["composition"].emit("Au") for n in [ namespace["raw_background_dark"], namespace["raw_background"], namespace["raw_foreground_dark"], ]: n.emit(np.zeros(img.shape))
stream_name="raw_foreground", ) for image_name in image_names ] ).map(np.float32) raw_source.starsink(StartStopCallback()) return locals() pipeline_order = [ start_gen, image_process, calibration, clear_geo_gen, save_cal, scattering_correction, gen_mask, integration, pdf_gen, clear_comp, ] # If main print visualize pipeline if __name__ == "__main__": # pragma: no cover from rapidz import Stream from rapidz.link import link raw_source = Stream(stream_name="raw_source") ns = link(*pipeline_order, raw_source=raw_source) ns["raw_source"].visualize(source_node=True)
def astype(x, ret_type='float32'): return x.astype(ret_type) def pipeline(raw_source): b = (raw_source.map(astype).map(np.sum) # .sink(print) .ToEventStream(('sum', )).DBFriendly().starsink(db2.insert) ) return locals() namespace = link(pipeline, raw_source=FromEventStream('event', ('data', 'pe1_image'), principle=True)) # vis = False vis = True # source.visualize(source_node=True) # ''' for hdr in list((db[-1], )): for e in hdr.documents(fill=True): if e[0] == 'start': e[1].update(composition_string='EuTiO3') if e[0] == 'event' and vis: plt.pause(.1) if e[0] == 'event': if e[1]['seq_num'] > 3: continue
def run_server( prefix=None, outbound_proxy_address=glbl_dict["outbound_proxy_address"], inbound_proxy_address=glbl_dict["inbound_proxy_address"], _publisher=None, **kwargs ): """Start up the QOI server Parameters ---------- prefix : bytes or list of bytes, optional The Publisher channels to listen to. Defaults to ``[b"an", b"raw"]`` outbound_proxy_address : str, optional The address and port of the zmq proxy. Defaults to ``glbl_dict["outbound_proxy_address"]`` inbound_proxy_address : str, optional The inbound ip address for the ZMQ server. Defaults to the value from the global dict """ if prefix is None: prefix = [b"an", b"raw"] d = RemoteDispatcher(outbound_proxy_address, prefix=prefix) install_qt_kicker(loop=d.loop) if _publisher is None: an_with_ind_pub = Publisher(inbound_proxy_address, prefix=b"qoi") else: an_with_ind_pub = _publisher raw_source = Stream() # create amorphous pipeline amorphous_ns = link( *[amorphsivity_fem, amorphsivity_pipeline, amorphsivity_tem], source=Stream(), **kwargs ) # Combine the data outputs with the raw independent data amorphous_ns.update( to_event_stream_with_ind( move_to_first(raw_source.starmap(StripDepVar())), *[ node for node in amorphous_ns.values() if isinstance(node, SimpleToEventStream) ], publisher=an_with_ind_pub ) ) rr = RunRouter( [ lambda x: lambda *y: raw_source.emit(y) if x["analysis_stage"] == "raw" else None, lambda x: lambda *y: amorphous_ns["source"].emit(y) if x["analysis_stage"] == "pdf" else None, ] ) d.subscribe(rr) print("Starting QOI Server") d.start()
def astype(x, ret_type='float32'): return x.astype(ret_type) def pipeline(raw_source): b = ( raw_source.map(astype).map(np.sum) # .sink(print) .ToEventStream(('sum', )).DBFriendly().starsink(db2.insert)) return locals() namespace = link(pipeline, raw_source=FromEventStream('event', ('data', 'pe1_image'), principle=True)) # vis = False vis = True # source.visualize(source_node=True) # ''' for hdr in list((db[-1], )): for e in hdr.documents(fill=True): if e[0] == 'start': e[1].update(composition_string='EuTiO3') if e[0] == 'event' and vis: plt.pause(.1) if e[0] == 'event': if e[1]['seq_num'] > 3: continue