def auto_mask(img: ndarray,
ai: AzimuthalIntegrator,
mask_setting: dict = None) -> Tuple[ndarray, dict]:
"""Automatically generate the mask of the image.
Parameters
----------
img : ndarray
The 2D diffraction image array.
ai : AzimuthalIntegrator
The AzimuthalIntegrator instance.
mask_setting : dict
The user's modification to auto-masking settings.
Returns
-------
mask : ndarray
The mask as a boolean array. 0 are good pixels, 1 are masked out.
_mask_setting : dict
The whole mask_setting.
"""
_mask_setting = _AUTOMASK_SETTING.copy()
if mask_setting is not None:
_mask_setting.update(mask_setting)
binner = generate_binner(ai, img.shape)
mask = np.invert(mask_img(img, binner, **_mask_setting)).astype(int)
return mask, _mask_setting
def test_z_score_image():
b = generate_binner(geo, (2048, 2048))
img = np.ones((2048, 2048))
bad = np.unique(np.random.randint(0, 2048 * 2048, 1000))
urbad = np.unravel_index(bad, (2048, 2048))
img[urbad] = 10
z_score = z_score_image(img, b)
assert all(z_score[urbad] > 2)
def test_binned_outlier(mask_method):
b = generate_binner(geo, (2048, 2048))
img = np.ones((2048, 2048))
bad = np.unique(np.random.randint(0, 2048 * 2048, 1000))
urbad = np.unravel_index(bad, (2048, 2048))
img[urbad] = 10
mask = binned_outlier(img, b, mask_method=mask_method)
assert_equal(np.where(mask.ravel() == 0)[0], bad)
def test_mask_img(mask_method):
b = generate_binner(geo, (2048, 2048))
img = np.ones((2048, 2048))
bad = np.unique(np.random.randint(0, 2048 * 2048, 1000))
urbad = np.unravel_index(bad, (2048, 2048))
img[urbad] = 10
mask = mask_img(img, b, auto_type=mask_method, edge=None,
lower_thresh=None,
upper_thresh=None)
assert_equal(np.where(mask.ravel() == 0)[0], bad)
def test_generate_binner():
a = generate_binner(geo, (2048, 2048))
b = map_to_binner(*generate_map_bin(geo, (2048, 2048)))
assert a
assert b
assert_equal(a.flatcount, b.flatcount)
from tifffile import imread
from xpdtools.tools import binned_outlier, generate_binner, mask_img
from profilehooks import profile
import pyFAI
geo = pyFAI.load('test.poni')
img = imread('test.tiff')
bo = profile(binned_outlier, skip=1)
# bo = binned_outlier
binner = generate_binner(geo, img.shape)
a = binner.argsort_index
b = binner.flatcount
for i in range(2):
bo(
img,
binner,
# bs_width=None,
mask_method='mean',
)
# Median
# binned outlier
# median by itself .714
# median multithread .494
# numba median .270
# multithread numba .178
# mask_img
def event(self, doc):
if self.dark_img is not None:
doc = next(self.db.fill_events([doc], self.descs))
if "cryostat_T" in doc["data"]:
doc["data"]["temperature"] = doc["data"].pop("cryostat_T")
# human readable timestamp
h_timestamp = _timestampstr(doc["time"])
# dark subtraction
img = doc["data"][self.image_data_key]
if str(img.dtype) == "uint16":
img = img.astype("float32")
if self.dark_img is not None:
img -= self.dark_img
if self.vis:
self.vis_callbacks["dark_sub_iq"]("event",
format_event(img=img))
if self.write_to_disk:
tiff_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage="dark_sub",
ext=".tiff",
)
tifffile.imsave(tiff_name, img)
if self.analysis_setting == "full":
# background correction
if self.background_img is not None:
img -= self.background_img
# get calibration
if self.is_calibration:
calibration, geo = img_calibration(img, self.wavelength,
self.calibrant,
self.detector)
_save_calib_param(
calibration,
h_timestamp,
os.path.join(self.calibration_md_folder,
"xpdAcq_calib_info.yml"),
)
if self.write_to_disk:
poni_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage="calib",
ext=".poni",
)
poni_saver(poni_name, calibration)
elif self.calibrant:
geo = load_geo(self.calibrant)
else:
geo = None
if geo:
img = polarization_correction(img, geo,
self.polarization_factor)
# Masking
if doc["seq_num"] == 1:
if (self.start_doc["sample_name"] == "Setup"
or self.mask_setting is None):
self.mask = np.ones(img.shape, dtype=bool)
else:
binner = generate_binner(geo, img.shape, self.mask)
self.mask = mask_img(img, binner,
**self.mask_kwargs)
if self.write_to_disk:
mask_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage="mask",
ext="",
)
fit2d_save(self.mask, mask_name)
if self.vis:
overlay = overlay_mask(img, self.mask)
self.vis_callbacks["masked_img"](
"event", format_event(overlay_mask=overlay))
# binner
binner = generate_binner(geo, img.shape, self.mask)
q, iq = (
binner.bin_centers,
np.nan_to_num(binner(img.flatten())),
)
if self.vis:
self.vis_callbacks["iq"]("event",
format_event(q=q, iq=iq))
self.vis_callbacks["zscore"](
"event",
format_event(img=z_score_image(img, binner)),
)
if self.write_to_disk:
iq_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage="iq_q",
ext="_Q.chi",
)
save_output(q, iq, iq_name, "Q")
tth = np.rad2deg(q_to_twotheta(q, self.wavelength))
if self.vis:
self.vis_callbacks["itth"]("event",
format_event(tth=tth,
iq=iq))
if self.write_to_disk:
itth_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage="iq_tth",
ext="_tth.chi",
)
save_output(tth, iq, itth_name, "2theta")
if self.composition:
fq_q, fq, fq_config = fq_getter(
q,
iq,
composition=self.composition,
**self.fq_kwargs)
if self.vis:
self.vis_callbacks["fq"]("event",
format_event(q=fq_q,
fq=fq))
r, gr, pdf_config = pdf_getter(
q,
iq,
composition=self.composition,
**self.pdf_kwargs)
if self.vis:
self.vis_callbacks["pdf"]("event",
format_event(r=r,
pdf=gr))
if self.write_to_disk:
pdf_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage="pdf",
ext=".gr",
)
pdf_saver(r, gr, pdf_config, pdf_name)
first_mask = (img_cal_binner.filter(
lambda x, **kwargs: mask_setting['setting'] == 'first').zip(
img_counter).filter(lambda x: x[1] == 1).pluck(0).starmap(
mask_img, stream_name='mask', **mask_kwargs))
no_mask = (img_cal_binner.filter(
lambda x, **kwargs: mask_setting['setting'] == 'none').starmap(
lambda img, *_: np.ones(img.shape, dtype=bool)))
mask = all_mask.union(first_mask, no_mask)
# Integration
# TODO: try to get this to not call pyFAI again
binner = (mask.combine_latest(
geometry,
emit_on=0).starmap(lambda mask, geo: generate_binner(geo, mask=mask)))
f_img_binner = pol_corrected_img.map(np.ravel).combine_latest(binner,
emit_on=0)
mean = (f_img_binner.starmap(
lambda img, binner, **kwargs: binner(img, **kwargs), statistic='mean'))
median = (f_img_binner.starmap(
lambda img, binner, **kwargs: binner(img, **kwargs), statistic='median'))
std = (f_img_binner.starmap(
lambda img, binner, **kwargs: binner(img, **kwargs), statistic='std'))
q = binner.map(getattr, 'bin_centers')
tth = (q.combine_latest(wavelength,
emit_on=0).starmap(q_to_twotheta,
stream_name='tth')).map(np.rad2deg)
def test_generate_binner_mask():
b = generate_binner(
geo, (2048, 2048),
np.random.randint(0, 2, 2048 * 2048, dtype=bool).reshape((2048, 2048)))
assert b
def test_generate_binner():
b = generate_binner(geo, (2048, 2048))
assert b
def event(self, doc):
if self.dark_img is not None:
doc = next(self.db.fill_events([doc], self.descs))
if 'cryostat_T' in doc['data']:
doc['data']['temperature'] = doc['data'].pop('cryostat_T')
# human readable timestamp
h_timestamp = _timestampstr(doc['time'])
# dark subtraction
img = doc['data'][self.image_data_key]
if str(img.dtype) == 'uint16':
img = img.astype('float32')
if self.dark_img is not None:
img -= self.dark_img
if self.vis:
self.vis_callbacks['dark_sub_iq']('event',
format_event(img=img))
if self.write_to_disk:
tiff_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage='dark_sub',
ext='.tiff')
tifffile.imsave(tiff_name, img)
if self.analysis_setting == 'full':
# background correction
if self.background_img is not None:
img -= self.background_img
# get calibration
if self.is_calibration:
calibration, geo = img_calibration(img, self.wavelength,
self.calibrant,
self.detector)
_save_calib_param(
calibration, h_timestamp,
os.path.join(self.calibration_md_folder,
'xpdAcq_calib_info.yml'))
if self.write_to_disk:
poni_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage='calib',
ext='.poni')
poni_saver(poni_name, calibration)
elif self.calibrant:
geo = load_geo(self.calibrant)
else:
geo = None
if geo:
img = polarization_correction(img, geo,
self.polarization_factor)
# Masking
if doc['seq_num'] == 1:
if (self.start_doc['sample_name'] == 'Setup'
or self.mask_setting is None):
self.mask = np.ones(img.shape, dtype=bool)
else:
binner = generate_binner(geo, img.shape, self.mask)
self.mask = mask_img(img, binner,
**self.mask_kwargs)
if self.write_to_disk:
mask_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage='mask',
ext='')
fit2d_save(self.mask, mask_name)
if self.vis:
overlay = overlay_mask(img, self.mask)
self.vis_callbacks['masked_img'](
'event', format_event(overlay_mask=overlay))
# binner
binner = generate_binner(geo, img.shape, self.mask)
q, iq = binner.bin_centers, np.nan_to_num(
binner(img.flatten()))
if self.vis:
self.vis_callbacks['iq']('event',
format_event(q=q, iq=iq))
self.vis_callbacks['zscore'](
'event',
format_event(img=z_score_image(img, binner)))
if self.write_to_disk:
iq_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage='iq_q',
ext='_Q.chi')
save_output(q, iq, iq_name, 'Q')
tth = np.rad2deg(q_to_twotheta(q, self.wavelength))
if self.vis:
self.vis_callbacks['itth']('event',
format_event(tth=tth,
iq=iq))
if self.write_to_disk:
itth_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage='iq_tth',
ext='_tth.chi')
save_output(tth, iq, itth_name, '2theta')
if self.composition:
fq_q, fq, fq_config = fq_getter(
q,
iq,
composition=self.composition,
**self.fq_kwargs)
if self.vis:
self.vis_callbacks['fq']('event',
format_event(q=fq_q,
fq=fq))
r, gr, pdf_config = pdf_getter(
q,
iq,
composition=self.composition,
**self.pdf_kwargs)
if self.vis:
self.vis_callbacks['pdf']('event',
format_event(r=r,
pdf=gr))
if self.write_to_disk:
pdf_name = render_clean_makedir(
self.light_template,
human_timestamp=h_timestamp,
raw_event=doc,
raw_start=self.start_doc,
raw_descriptor=self.descriptor_doc,
analysis_stage='pdf',
ext='.gr')
pdf_saver(r, gr, pdf_config, pdf_name)
