def test_main_fit2d_mask(fast_tmpdir):
poni_file = pyfai_poni
# Copy the image file to the temp dir
msk = np.random.randint(0, 2, 2048 * 2048, dtype=bool).reshape(
(2048, 2048)
)
fit2d_save(msk, "mask_test", str(fast_tmpdir))
dest_image_file = str(os.path.join(fast_tmpdir, "test.tiff"))
shutil.copy(image_file, dest_image_file)
print(dest_image_file)
# Copy the poni and image files to the temp dir
no_output_main(
poni_file,
dest_image_file,
edge=None,
lower_thresh=None,
alpha=None,
mask_file=os.path.join(str(fast_tmpdir), "mask_test.msk"),
flip_input_mask=True,
)
files = os.listdir(str(fast_tmpdir))
for ext in expected_outputs:
assert "test" + ext in files
msk2 = read_fit2d_msk(os.path.join(str(fast_tmpdir), "test.msk"))
assert_equal(msk, msk2)
def test_save_output_fit2d():
filename = "function_values"
msk = np.random.random_integers(
0, 1, (np.random.random_integers(0, 200),
np.random.random_integers(0, 200))).astype(bool)
fit2d_save(msk, filename, dir_path=None)
msk2 = read_fit2d_msk(filename)
assert_array_equal(msk2, msk)
os.remove("function_values.msk")
def test_save_output_fit2d(tmpdir):
t = tmpdir.join('function_values')
dir_path = t.dirname
filename = t.strpath
msk = np.random.randint(
0, 2,
(np.random.randint(0, 201), np.random.randint(0, 201))).astype(bool)
fit2d_save(msk, filename, dir_path=dir_path)
msk2 = read_fit2d_msk('%s.msk' % filename)
assert_array_equal(msk2, msk)
def disk_mask(tmp_dir, img_size):
mask = np.random.random_integers(0, 1, img_size).astype(bool)
dirn = tmp_dir
file_name_msk = os.path.join(dirn, 'mask_test' + '.msk')
assert ~os.path.exists(file_name_msk)
fit2d_save(mask, 'mask_test', dirn)
assert os.path.exists(file_name_msk)
file_name = os.path.join(dirn, 'mask_test' + '.npy')
assert ~os.path.exists(file_name)
np.save(file_name, mask)
assert os.path.exists(file_name)
yield (file_name_msk, file_name, mask)
def event(self, doc):
# fill the document
doc = super().event(doc)
for two_d_var in [
k for k, v in self.dep_shapes.items() if len(v) == 2
]:
for filename in self.filenames:
fit2d_save(
np.flipud(doc["data"][two_d_var]),
clean_template(pfmt.format(filename, ext="")),
)
np.save(
clean_template(pfmt.format(filename, ext="_mask.npy")),
doc["data"][two_d_var],
)
def event(self, doc):
# fill the document
doc = super().event(doc)
for two_d_var in [
k for k, v in self.dep_shapes.items() if len(v) == 2
]:
for filename in self.filenames:
fit2d_save(
np.flipud(doc["data"][two_d_var]),
clean_template(pfmt.format(filename, ext="")),
)
np.save(
clean_template(pfmt.format(filename, ext="_mask.npy")),
doc["data"][two_d_var],
)
def save_mask(filename_name_nodes, mask, **kwargs):
# Mask
d = mask.combine_latest(filename_name_nodes["mask_fit2d_name"],
first=mask,
emit_on=0)
(d.sink(lambda x: fit2d_save(np.flipud(x[0]), x[1])))
(d.sink(lambda x: np.save(x[1], x[0])))
return locals()
def save_mask(filename_name_nodes, mask, **kwargs):
# Mask
d = mask.combine_latest(
filename_name_nodes["mask_fit2d_name"], first=mask, emit_on=0
)
(d.sink(lambda x: fit2d_save(np.flipud(x[0]), x[1])))
(d.sink(lambda x: np.save(x[1], x[0])))
return locals()
def make_pipeline(_output_sinks=True):
# link the pipeline up
namespace = link(*(pipeline_order + [median_gen, std_gen, z_score_gen]),
**general_namespace)
polarization_array = namespace["polarization_array"]
mask = namespace["mask"]
mean = namespace["mean"]
q = namespace["q"]
geometry = namespace["geometry"]
dark_corrected_foreground = namespace["dark_corrected_foreground"]
dark_corrected_background = namespace["dark_corrected_background"]
mask_kwargs = namespace["mask_kwargs"]
mask_setting = namespace["mask_setting"]
median = namespace["median"]
std = namespace["std"]
z_score = namespace["z_score"]
# Modify graph
# create filename nodes
filename_source = Stream(stream_name="filename")
filename_node = filename_source.map(lambda x: os.path.splitext(x)[0])
# write out mask
mask.combine_latest(
filename_node,
emit_on=0).sink(lambda x: fit2d_save(np.flipud(x[0]), x[1]))
mask.combine_latest(
filename_node,
emit_on=0).sink(lambda x: np.save(x[1] + "_mask.npy", x[0]))
if _output_sinks:
outs = [q, mean, median, std]
out_tup = tuple([[] for _ in outs])
out_sinks = tuple([k.sink(L.append) for k, L in zip(outs, out_tup)])
(mean.zip(q).combine_latest(
filename_node, emit_on=0).map(lambda l: (*l[0], l[1])).sink(
lambda x: save_output(x[1], x[0], x[2], "Q")))
(median.zip(q).combine_latest(
filename_node, emit_on=0).map(lambda l: (*l[0], l[1])).sink(
lambda x: save_output(x[1], x[0], x[2] + "_median", "Q")))
(std.zip(q).combine_latest(
filename_node, emit_on=0).map(lambda l: (*l[0], l[1])).sink(
lambda x: save_output(x[1], x[0], x[2] + "_std", "Q")))
(z_score.combine_latest(
filename_node, emit_on=0).starsink(lambda img, n: tifffile.imsave(
n + "_zscore.tif", data=img.astype(np.float32))))
# If running from a terminal don't output stuff into lists (too much mem)
return locals()
from skbeam.io.save_powder_output import save_output
from streamz_ext import Stream
from xpdtools.pipelines.raw_pipeline import (polarization_array, mask, mean, q,
geometry,
dark_corrected_foreground,
dark_corrected_background,
z_score, std, median)
img_extensions = {'.tiff', '.edf', '.tif'}
# Modify graph
# create filename nodes
filename_source = Stream(stream_name='filename')
filename_node = (filename_source.map(lambda x: os.path.splitext(x)[0]))
# write out mask
mask.combine_latest(
filename_node, emit_on=0).sink(lambda x: fit2d_save(np.flipud(x[0]), x[1]))
mask.combine_latest(
filename_node, emit_on=0).sink(lambda x: np.save(x[1] + '_mask.npy', x[0]))
# write out chi
outs = [q, mean, median, std]
out_tup = tuple([[] for _ in outs])
out_sinks = tuple([k.sink(L.append) for k, L in zip(outs, out_tup)])
(mean.zip(q).combine_latest(filename_node,
emit_on=0).map(lambda l: (*l[0], l[1])).sink(
lambda x: save_output(x[1], x[0], x[2], 'Q')))
(median.zip(q).combine_latest(
filename_node, emit_on=0).map(lambda l: (*l[0], l[1])).sink(
lambda x: save_output(x[1], x[0], x[2] + '_median', 'Q')))
(std.zip(q).combine_latest(
def event(self, doc):
if self.dark_img is not None:
doc = next(self.db.fill_events([doc], self.descs))
if "cryostat_T" in doc["data"]:
doc["data"]["temperature"] = doc["data"].pop("cryostat_T")
# human readable timestamp
h_timestamp = _timestampstr(doc["time"])
# dark subtraction
img = doc["data"][self.image_data_key]
if str(img.dtype) == "uint16":
img = img.astype("float32")
if self.dark_img is not None:
img -= self.dark_img
if self.vis:
self.vis_callbacks["dark_sub_iq"]("event",
format_event(img=img))
if self.write_to_disk:
tiff_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage="dark_sub",
ext=".tiff",
)
tifffile.imsave(tiff_name, img)
if self.analysis_setting == "full":
# background correction
if self.background_img is not None:
img -= self.background_img
# get calibration
if self.is_calibration:
calibration, geo = img_calibration(img, self.wavelength,
self.calibrant,
self.detector)
_save_calib_param(
calibration,
h_timestamp,
os.path.join(self.calibration_md_folder,
"xpdAcq_calib_info.yml"),
)
if self.write_to_disk:
poni_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage="calib",
ext=".poni",
)
poni_saver(poni_name, calibration)
elif self.calibrant:
geo = load_geo(self.calibrant)
else:
geo = None
if geo:
img = polarization_correction(img, geo,
self.polarization_factor)
# Masking
if doc["seq_num"] == 1:
if (self.start_doc["sample_name"] == "Setup"
or self.mask_setting is None):
self.mask = np.ones(img.shape, dtype=bool)
else:
binner = generate_binner(geo, img.shape, self.mask)
self.mask = mask_img(img, binner,
**self.mask_kwargs)
if self.write_to_disk:
mask_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage="mask",
ext="",
)
fit2d_save(self.mask, mask_name)
if self.vis:
overlay = overlay_mask(img, self.mask)
self.vis_callbacks["masked_img"](
"event", format_event(overlay_mask=overlay))
# binner
binner = generate_binner(geo, img.shape, self.mask)
q, iq = (
binner.bin_centers,
np.nan_to_num(binner(img.flatten())),
)
if self.vis:
self.vis_callbacks["iq"]("event",
format_event(q=q, iq=iq))
self.vis_callbacks["zscore"](
"event",
format_event(img=z_score_image(img, binner)),
)
if self.write_to_disk:
iq_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage="iq_q",
ext="_Q.chi",
)
save_output(q, iq, iq_name, "Q")
tth = np.rad2deg(q_to_twotheta(q, self.wavelength))
if self.vis:
self.vis_callbacks["itth"]("event",
format_event(tth=tth,
iq=iq))
if self.write_to_disk:
itth_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage="iq_tth",
ext="_tth.chi",
)
save_output(tth, iq, itth_name, "2theta")
if self.composition:
fq_q, fq, fq_config = fq_getter(
q,
iq,
composition=self.composition,
**self.fq_kwargs)
if self.vis:
self.vis_callbacks["fq"]("event",
format_event(q=fq_q,
fq=fq))
r, gr, pdf_config = pdf_getter(
q,
iq,
composition=self.composition,
**self.pdf_kwargs)
if self.vis:
self.vis_callbacks["pdf"]("event",
format_event(r=r,
pdf=gr))
if self.write_to_disk:
pdf_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage="pdf",
ext=".gr",
)
pdf_saver(r, gr, pdf_config, pdf_name)
def event(self, doc):
if self.dark_img is not None:
doc = next(self.db.fill_events([doc], self.descs))
if 'cryostat_T' in doc['data']:
doc['data']['temperature'] = doc['data'].pop('cryostat_T')
# human readable timestamp
h_timestamp = _timestampstr(doc['time'])
# dark subtraction
img = doc['data'][self.image_data_key]
if str(img.dtype) == 'uint16':
img = img.astype('float32')
if self.dark_img is not None:
img -= self.dark_img
if self.vis:
self.vis_callbacks['dark_sub_iq']('event',
format_event(img=img))
if self.write_to_disk:
tiff_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage='dark_sub',
ext='.tiff')
tifffile.imsave(tiff_name, img)
if self.analysis_setting == 'full':
# background correction
if self.background_img is not None:
img -= self.background_img
# get calibration
if self.is_calibration:
calibration, geo = img_calibration(img, self.wavelength,
self.calibrant,
self.detector)
_save_calib_param(
calibration, h_timestamp,
os.path.join(self.calibration_md_folder,
'xpdAcq_calib_info.yml'))
if self.write_to_disk:
poni_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage='calib',
ext='.poni')
poni_saver(poni_name, calibration)
elif self.calibrant:
geo = load_geo(self.calibrant)
else:
geo = None
if geo:
img = polarization_correction(img, geo,
self.polarization_factor)
# Masking
if doc['seq_num'] == 1:
if (self.start_doc['sample_name'] == 'Setup'
or self.mask_setting is None):
self.mask = np.ones(img.shape, dtype=bool)
else:
binner = generate_binner(geo, img.shape, self.mask)
self.mask = mask_img(img, binner,
**self.mask_kwargs)
if self.write_to_disk:
mask_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage='mask',
ext='')
fit2d_save(self.mask, mask_name)
if self.vis:
overlay = overlay_mask(img, self.mask)
self.vis_callbacks['masked_img'](
'event', format_event(overlay_mask=overlay))
# binner
binner = generate_binner(geo, img.shape, self.mask)
q, iq = binner.bin_centers, np.nan_to_num(
binner(img.flatten()))
if self.vis:
self.vis_callbacks['iq']('event',
format_event(q=q, iq=iq))
self.vis_callbacks['zscore'](
'event',
format_event(img=z_score_image(img, binner)))
if self.write_to_disk:
iq_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage='iq_q',
ext='_Q.chi')
save_output(q, iq, iq_name, 'Q')
tth = np.rad2deg(q_to_twotheta(q, self.wavelength))
if self.vis:
self.vis_callbacks['itth']('event',
format_event(tth=tth,
iq=iq))
if self.write_to_disk:
itth_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage='iq_tth',
ext='_tth.chi')
save_output(tth, iq, itth_name, '2theta')
if self.composition:
fq_q, fq, fq_config = fq_getter(
q,
iq,
composition=self.composition,
**self.fq_kwargs)
if self.vis:
self.vis_callbacks['fq']('event',
format_event(q=fq_q,
fq=fq))
r, gr, pdf_config = pdf_getter(
q,
iq,
composition=self.composition,
**self.pdf_kwargs)
if self.vis:
self.vis_callbacks['pdf']('event',
format_event(r=r,
pdf=gr))
if self.write_to_disk:
pdf_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage='pdf',
ext='.gr')
pdf_saver(r, gr, pdf_config, pdf_name)