def test_raw_pipeline(mask_s):
# link the pipeline up
namespace = link(*pipeline_order, **g_namespace)
is_calibration_img = namespace["is_calibration_img"]
geo_input = namespace["geo_input"]
img_counter = namespace["img_counter"]
namespace["mask_setting"]["setting"] = mask_s
pdf = namespace["pdf"]
raw_background_dark = namespace["raw_background_dark"]
raw_background = namespace["raw_background"]
raw_foreground_dark = namespace["raw_foreground_dark"]
composition = namespace["composition"]
raw_foreground = namespace["raw_foreground"]
sl = pdf.sink_to_list()
L = namespace["geometry"].sink_to_list()
ml = namespace["mask"].sink_to_list()
is_calibration_img.emit(False)
a = geo.getPyFAI()
geo_input.emit(a)
for s in [raw_background_dark, raw_background, raw_foreground_dark]:
s.emit(np.zeros(img.shape))
composition.emit("Au")
img_counter.emit(1)
raw_foreground.emit(img)
destroy_pipeline(raw_foreground)
del namespace
assert len(L) == 1
assert ml
assert len(sl) == 1
sl.clear()
L.clear()
ml.clear()
def test_main_pipeline(
exp_db,
fast_tmp_dir,
start_uid3,
start_uid1,
start_uid2,
background,
exception,
pe2
):
namespace = link(
*pipeline_order, raw_source=Stream(stream_name="raw source"),
db=exp_db,
)
iq_em = ToEventStream(
namespace["mean"].combine_latest(namespace["q"], emit_on=0),
("iq", "q"))
iq_em.sink(print)
limg = []
move_to_first(namespace["bg_corrected_img"].sink(lambda x: limg.append(x)))
lbgc = namespace["mean"].sink_to_list()
lpdf = namespace["iq_comp"].sink_to_list()
t0 = time.time()
if background:
uid = start_uid1
elif pe2:
uid = start_uid2
else:
uid = -1
for nd in exp_db[uid].documents(fill=True):
name, doc = nd
if name == "start":
if exception:
doc["bt_wavelength"] = "bla"
nd = (name, doc)
try:
namespace["raw_source"].emit(nd)
except ValueError:
pass
t1 = time.time()
print(t1 - t0)
n_events = len(list(exp_db[-1].events()))
assert len(limg) == n_events
if exception:
assert_lbgc = 0
else:
assert_lbgc = n_events
assert len(lbgc) == assert_lbgc
assert len(lpdf) == assert_lbgc
assert iq_em.state == "stopped"
destroy_pipeline(namespace["raw_source"])
del namespace
limg.clear()
lbgc.clear()
lpdf.clear()
def raw_pipeline_parallel():
# link the pipeline up
gg_namespace = dict(g_namespace)
s_ns = {
k: v.scatter(backend="thread")
for k, v in gg_namespace.items() if isinstance(v, Stream)
}
gg_namespace.update(
{"_" + k: v
for k, v in gg_namespace.items() if isinstance(v, Stream)})
gg_namespace.update(s_ns)
namespace = link(*pipeline_order[:-1], **gg_namespace)
geo_input = namespace["_geo_input"]
composition = namespace["_composition"]
raw_background_dark = namespace["_raw_background_dark"]
raw_background = namespace["_raw_background"]
raw_foreground_dark = namespace["_raw_foreground_dark"]
raw_foreground = namespace["_raw_foreground"]
print(type(namespace["raw_foreground"]))
a = namespace["mean"]
futures = a.sink_to_list()
b = a.buffer(10)
g = b.gather()
# g.sink(lambda x: print("gathered data", time.time()))
LL = g.map(lambda x: time.time()).sink_to_list()
L = g.sink_to_list()
a = geo.getPyFAI()
geo_input.emit(a)
composition.emit("Au1.0")
for s in [raw_background_dark, raw_background, raw_foreground_dark]:
s.emit(np.zeros(img.shape))
ii = 10
t0 = time.time()
for i in range(ii):
rimg = np.random.random(img.shape)
raw_foreground.emit(img + rimg)
while len(L) < ii:
time.sleep(.01)
time_diff = [LL[i] - LL[i - 1] for i in range(1, ii)]
print(max(time_diff), min(time_diff), sum(time_diff) / len(time_diff))
# print([l - min(LL) for l in LL])
print([l - t0 for l in LL])
print(max([l - t0 for l in LL]) / ii)
destroy_pipeline(raw_foreground)
del namespace
futures.clear()
L.clear()
def test_main_pipeline(exp_db, fast_tmp_dir, start_uid3, start_uid1,
start_uid2, background, exception, pe2):
namespace = link(
*pipeline_order,
raw_source=Stream(stream_name="raw source"),
db=exp_db,
)
iq_em = ToEventStream(
namespace["mean"].combine_latest(namespace["q"], emit_on=0),
("iq", "q"))
iq_em.sink(print)
limg = []
move_to_first(namespace["bg_corrected_img"].sink(lambda x: limg.append(x)))
lbgc = namespace["mean"].sink_to_list()
lpdf = namespace["iq_comp"].sink_to_list()
t0 = time.time()
if background:
uid = start_uid1
elif pe2:
uid = start_uid2
else:
uid = -1
for nd in exp_db[uid].documents(fill=True):
name, doc = nd
if name == "start":
if exception:
doc["bt_wavelength"] = "bla"
nd = (name, doc)
try:
namespace["raw_source"].emit(nd)
except ValueError:
pass
t1 = time.time()
print(t1 - t0)
n_events = len(list(exp_db[-1].events()))
assert len(limg) == n_events
if exception:
assert_lbgc = 0
else:
assert_lbgc = n_events
assert len(lbgc) == assert_lbgc
assert len(lpdf) == assert_lbgc
assert iq_em.state == "stopped"
destroy_pipeline(namespace["raw_source"])
del namespace
limg.clear()
lbgc.clear()
lpdf.clear()
def test_pca_pipeline():
ns = dict(data=Stream(), start=Stream())
ns.update(pca_pipeline(**ns))
L = ns["scores"].sink_to_list()
# np.random.seed(42)
for i in range(10):
a = np.zeros(10)
a[i] = 1
ns["data"].emit(a)
assert len(L) == 10
assert L[-1].shape == (10, 9)
destroy_pipeline(ns["data"])
del ns
L.clear()
def test_tomo_pipeline_theta():
ns = dict(qoi=Stream(), theta=Stream(), center=Stream())
ns.update(tomo_pipeline_theta(**ns))
L = ns["rec"].sink_to_list()
# np.random.seed(42)
th_linspace = np.linspace(0, 180, 6)
ns["center"].emit(3)
for th in th_linspace:
ns["theta"].emit(th)
ns["qoi"].emit(np.random.random((6, 6)))
assert len(L) == 6
assert L[-1].shape == (6, 6, 6)
destroy_pipeline(ns["qoi"])
del ns
L.clear()
def test_tomo_piecewise_pipeline(rand_size):
ns = dict(
qoi=Stream(),
x=Stream(),
th=Stream(),
th_dim=Stream(),
x_dim=Stream(),
th_extents=Stream(),
x_extents=Stream(),
center=Stream(),
)
x_linspace = np.linspace(0, 5, 6)
th_linspace = np.linspace(0, 180, 6)
ns["th_dimension"] = len(th_linspace)
ns["x_dimension"] = len(x_linspace)
ns.update(**link(*[tomo_prep, tomo_pipeline_piecewise], **ns))
L = ns["rec"].sink_to_list()
ns["th_dim"].emit(len(th_linspace))
ns["x_dim"].emit(len(x_linspace))
ns["th_extents"].emit([0, 180])
ns["x_extents"].emit([x_linspace[0], x_linspace[-1]])
ns["center"].emit(2.5)
# np.random.seed(42)
for x in x_linspace:
for th in th_linspace:
ns["x"].emit(x)
ns["th"].emit(th)
ns["qoi"].emit(np.random.random(rand_size))
assert len(L) == len(x_linspace) * len(th_linspace)
if rand_size:
assert L[-1].shape == (*rand_size, len(x_linspace), len(th_linspace))
else:
assert L[-1].shape == (len(x_linspace), len(th_linspace))
destroy_pipeline(ns["qoi"])
del ns
L.clear()
def test_raw_pipeline_parallel(n):
# caplog.set_level(logging.CRITICAL)
# link the pipeline up
gg_namespace = dict(g_namespace)
s_ns = {
k: v.scatter(backend="thread")
for k, v in gg_namespace.items() if isinstance(v, Stream)
}
gg_namespace.update(
{"_" + k: v
for k, v in gg_namespace.items() if isinstance(v, Stream)})
gg_namespace.update(s_ns)
namespace = link(*pipeline_order, **gg_namespace)
geo_input = namespace["_geo_input"]
raw_background_dark = namespace["_raw_background_dark"]
raw_background = namespace["_raw_background"]
raw_foreground_dark = namespace["_raw_foreground_dark"]
raw_foreground = namespace["_raw_foreground"]
a = namespace[n]
futures = a.sink_to_list()
b = a.buffer(10)
g = b.gather()
g.sink(lambda x: print("gathered data", time.time()))
L = g.sink_to_list()
a = geo.getPyFAI()
yield geo_input.emit(a)
for s in [raw_background_dark, raw_background, raw_foreground_dark]:
yield s.emit(np.zeros(img.shape))
ii = 2
for i in range(ii):
rimg = np.random.random(img.shape)
yield raw_foreground.emit(img + rimg)
while len(L) < ii:
yield gen.sleep(.01)
destroy_pipeline(raw_foreground)
del namespace
futures.clear()
L.clear()
def test_qoi_pipeline():
# link the pipeline up
namespace = link(*(pipeline_order + [max_intensity_mean, max_gr_mean]),
**g_namespace)
geometry = namespace["geometry"]
mean_max = namespace["mean_max"]
raw_background_dark = namespace["raw_background_dark"]
raw_background = namespace["raw_background"]
raw_foreground_dark = namespace["raw_foreground_dark"]
raw_foreground = namespace["raw_foreground"]
sl = mean_max.sink_to_list()
geometry.emit(geo)
for s in [raw_background_dark, raw_background, raw_foreground_dark]:
s.emit(np.zeros(img.shape))
raw_foreground.emit(img)
del namespace
destroy_pipeline(raw_foreground)
assert len(sl) == 1
sl.clear()
def test_extra_pipeline():
# link the pipeline up
namespace = link(*(pipeline_order + [median_gen, std_gen, z_score_gen]),
**g_namespace)
geometry = namespace["geometry"]
z_score = namespace["z_score"]
raw_background_dark = namespace["raw_background_dark"]
raw_background = namespace["raw_background"]
raw_foreground_dark = namespace["raw_foreground_dark"]
raw_foreground = namespace["raw_foreground"]
sl = z_score.sink_to_list()
geometry.emit(geo)
for s in [raw_background_dark, raw_background, raw_foreground_dark]:
s.emit(np.zeros(img.shape))
raw_foreground.emit(img)
del namespace
destroy_pipeline(raw_foreground)
assert len(sl) == 1
sl.clear()
def main(
poni_file=None,
image_files=None,
bg_file=None,
mask_file=None,
polarization=.99,
edge=20,
lower_thresh=1.,
upper_thresh=None,
alpha=3.,
auto_type="median",
mask_settings="auto",
flip_input_mask=True,
bg_scale=1,
):
"""Run the data processing protocol taking raw images to background
subtracted I(Q) files.
The data processing steps included in this protocol are:
background subtraction, polarization correction, automated masking, and
pixel resolution integration
Parameters
----------
poni_file: str or None, optional
File generated from pyFAI's calibration, if None look in the
current working directory for the poni file, defaults to None.
image_files: str or None, optional
File to process, if None use all the valid files in the directory,
defaults to None.
bg_file: str or None, optional
Background image, if None no background subtraction is performed,
defaults to None.
mask_file: str or None, optional
Mask file to include in the data processing, if None don't use one,
defaults to None.
polarization: float, optional
The polzarization factor to use, defaults to .99, if None do not
perform polarization correction
edge: int, optional
The number of pixels from the edge to mask with an edge mask,
defaults to 20, if None no edge mask used
lower_thresh: float, optional
Threshold for lower threshold mask, all pixels with value less than
this value (after background subtraction if applicable), defaults
to 1. if None do not apply lower theshold mask
upper_thresh: float, optional
Threshold for upper threshold mask, all pixels with value greater
than this value (after background subtraction if applicable),
defaults to None if None do not apply upper theshold mask
alpha: float, optional
Number of standard deviations away from the ring mean to mask,
defaults to 3. if None do not apply automated masking
auto_type : {'median', 'mean'}, optional
The type of automasking to use, median is faster, mean is more
accurate. Defaults to 'median'.
mask_settings: {'auto', 'first', None}, optional
If auto mask every image, if first only mask first image, if None
mask no images. Defaults to None
flip_input_mask: bool, optional
If True flip the input mask up down, this helps when using fit2d
defaults to True.
bg_scale : float, optional
The scale for the image to image background subtraction, defaults
to 1
Returns
-------
q_l : list of ndarrays
The list of q values
mean_l : list of ndarrays
The list of mean values
median_l : list of ndarrays
The list of median values
std_l : list of ndarrays
The list of standard deviation values
"""
import pyFAI
ns = make_pipeline(_output_sinks)
ns["polarization_array"].args = (polarization, )
ns["dark_corrected_background"].args = (bg_scale, )
if mask_file:
if mask_file.endswith(".msk"):
# TODO: may need to flip this?
tmsk = read_fit2d_msk(mask_file)
else:
tmsk = np.load(mask_file)
if flip_input_mask:
tmsk = np.flipud(tmsk)
else:
tmsk = None
# update all the kwargs
ns["mask_kwargs"].update(
tmsk=tmsk,
edge=edge,
lower_thresh=lower_thresh,
upper_thresh=upper_thresh,
alpha=alpha,
auto_type=auto_type,
)
print(ns["mask_kwargs"])
ns["mask_setting"].update(setting=mask_settings)
# Load calibration
if poni_file is None:
poni_file = [f for f in os.listdir(".") if f.endswith(".poni")]
if len(poni_file) != 1:
raise RuntimeError("There can only be one poni file")
else:
poni_file = poni_file[0]
geo = pyFAI.load(poni_file)
bg = None
img_filenames = None
if image_files is None:
img_filenames = [
i for i in os.listdir(".")
if os.path.splitext(i)[-1] in img_extensions
]
# TODO: Test non tiff files
if all([
f.endswith(".tiff") or f.endswith(".tif")
for f in img_filenames
]):
imgs = (tifffile.imread(i) for i in img_filenames)
else:
imgs = (fabio.open(i).data.astype(float)
for i in os.listdir(".")
if os.path.splitext(i)[-1] in img_extensions)
else:
if isinstance(image_files, str):
image_files = (image_files, )
img_filenames = image_files
imgs = (fabio.open(i).data.astype(float) for i in image_files)
if bg_file is not None:
bg = fabio.open(bg_file).data.astype(float)
for k in ns.get("out_tup", []):
k.clear()
ns["geometry"].emit(geo)
for i, (fn, img) in enumerate(zip(img_filenames, imgs)):
ns["filename_source"].emit(fn)
if bg is None:
bg = np.zeros(img.shape)
ns["dark_corrected_background"].emit(bg)
ns["dark_corrected_foreground"].emit(img)
destroy_pipeline(ns["dark_corrected_foreground"])
res = tuple([tuple(x) for x in ns.get("out_tup", [])])
del ns
return res
