def image_filter(raw_data, mask, display, gain_value, nsigma_b, nsigma_s,
global_threshold, min_local, kernel_size):
from dials.algorithms.image.threshold import DispersionThresholdDebug
from dials.array_family import flex
if display == 'image':
return raw_data
assert gain_value > 0
gain_map = [
flex.double(raw_data[i].accessor(), gain_value)
for i in range(len(raw_data))
]
kabsch_debug_list = []
for i_panel in range(len(raw_data)):
kabsch_debug_list.append(
DispersionThresholdDebug(raw_data[i_panel].as_double(),
mask[i_panel], gain_map[i_panel],
kernel_size, nsigma_b, nsigma_s,
global_threshold, min_local))
if display == 'mean':
display_data = [kabsch.mean() for kabsch in kabsch_debug_list]
elif display == 'variance':
display_data = [kabsch.variance() for kabsch in kabsch_debug_list]
elif display == 'dispersion':
display_data = [
kabsch.index_of_dispersion() for kabsch in kabsch_debug_list
]
elif display == 'sigma_b':
cv = [kabsch.index_of_dispersion() for kabsch in kabsch_debug_list]
display_data = [kabsch.cv_mask() for kabsch in kabsch_debug_list]
display_data = [mask.as_1d().as_double() for mask in display_data]
for i, mask in enumerate(display_data):
mask.reshape(cv[i].accessor())
elif display == 'sigma_s':
cv = [kabsch.index_of_dispersion() for kabsch in kabsch_debug_list]
display_data = [kabsch.value_mask() for kabsch in kabsch_debug_list]
display_data = [mask.as_1d().as_double() for mask in display_data]
for i, mask in enumerate(display_data):
mask.reshape(cv[i].accessor())
elif display == 'global_threshold':
cv = [kabsch.index_of_dispersion() for kabsch in kabsch_debug_list]
display_data = [kabsch.global_mask() for kabsch in kabsch_debug_list]
display_data = [mask.as_1d().as_double() for mask in display_data]
for i, mask in enumerate(display_data):
mask.reshape(cv[i].accessor())
elif display == 'threshold':
cv = [kabsch.index_of_dispersion() for kabsch in kabsch_debug_list]
display_data = [kabsch.final_mask() for kabsch in kabsch_debug_list]
display_data = [mask.as_1d().as_double() for mask in display_data]
for i, mask in enumerate(display_data):
mask.reshape(cv[i].accessor())
return display_data
def test_dispersion_debug(self):
from dials.algorithms.image.threshold import (
DispersionThresholdDebug,
dispersion,
dispersion_w_gain,
)
nsig_b = 3
nsig_s = 3
result1 = dispersion(
self.image, self.mask, self.size, nsig_b, nsig_s, self.min_count
)
debug = DispersionThresholdDebug(
self.image, self.mask, self.size, nsig_b, nsig_s, 0, self.min_count
)
result2 = debug.final_mask()
assert result1.all_eq(result2)
result3 = dispersion_w_gain(
self.image, self.mask, self.gain, self.size, nsig_b, nsig_s, self.min_count
)
debug = DispersionThresholdDebug(
self.image,
self.mask,
self.gain,
self.size,
nsig_b,
nsig_s,
0,
self.min_count,
)
result4 = debug.final_mask()
assert result3 == result4
def image_filter(
raw_data,
mask,
display,
gain_value,
nsigma_b,
nsigma_s,
global_threshold,
min_local,
kernel_size,
):
if display == "image":
return raw_data
assert gain_value > 0
gain_map = [flex.double(rd.accessor(), gain_value) for rd in raw_data]
kabsch_debug_list = [
DispersionThresholdDebug(
data.as_double(),
mask[i_panel],
gain_map[i_panel],
kernel_size,
nsigma_b,
nsigma_s,
global_threshold,
min_local,
)
for i_panel, data in enumerate(raw_data)
]
if display == "mean":
display_data = [kabsch.mean() for kabsch in kabsch_debug_list]
elif display == "variance":
display_data = [kabsch.variance() for kabsch in kabsch_debug_list]
elif display == "dispersion":
display_data = [kabsch.index_of_dispersion() for kabsch in kabsch_debug_list]
elif display == "sigma_b":
cv = [kabsch.index_of_dispersion() for kabsch in kabsch_debug_list]
display_data = (kabsch.cv_mask() for kabsch in kabsch_debug_list)
display_data = [_mask.as_1d().as_double() for _mask in display_data]
for i, _mask in enumerate(display_data):
_mask.reshape(cv[i].accessor())
elif display == "sigma_s":
cv = [kabsch.index_of_dispersion() for kabsch in kabsch_debug_list]
display_data = (kabsch.value_mask() for kabsch in kabsch_debug_list)
display_data = [_mask.as_1d().as_double() for _mask in display_data]
for i, _mask in enumerate(display_data):
_mask.reshape(cv[i].accessor())
elif display == "global_threshold":
cv = [kabsch.index_of_dispersion() for kabsch in kabsch_debug_list]
display_data = (kabsch.global_mask() for kabsch in kabsch_debug_list)
display_data = [_mask.as_1d().as_double() for _mask in display_data]
for i, _mask in enumerate(display_data):
_mask.reshape(cv[i].accessor())
elif display == "threshold":
cv = [kabsch.index_of_dispersion() for kabsch in kabsch_debug_list]
display_data = (kabsch.final_mask() for kabsch in kabsch_debug_list)
display_data = [_mask.as_1d().as_double() for _mask in display_data]
for i, _mask in enumerate(display_data):
_mask.reshape(cv[i].accessor())
return display_data
def estimate_gain(imageset,
kernel_size=(10, 10),
output_gain_map=None,
max_images=1):
detector = imageset.get_detector()
from dials.algorithms.image.threshold import DispersionThresholdDebug
gains = flex.double()
for image_no in range(len(imageset)):
raw_data = imageset.get_raw_data(image_no)
gain_value = 1
gain_map = [
flex.double(raw_data[i].accessor(), gain_value)
for i in range(len(detector))
]
mask = imageset.get_mask(image_no)
min_local = 0
# dummy values, shouldn't affect results
nsigma_b = 6
nsigma_s = 3
global_threshold = 0
kabsch_debug_list = [
DispersionThresholdDebug(
raw_data[i_panel].as_double(),
mask[i_panel],
gain_map[i_panel],
kernel_size,
nsigma_b,
nsigma_s,
global_threshold,
min_local,
) for i_panel in range(len(detector))
]
dispersion = flex.double()
for kabsch in kabsch_debug_list:
dispersion.extend(kabsch.index_of_dispersion().as_1d())
sorted_dispersion = flex.sorted(dispersion)
from libtbx.math_utils import nearest_integer as nint
q1 = sorted_dispersion[nint(len(sorted_dispersion) / 4)]
q2 = sorted_dispersion[nint(len(sorted_dispersion) / 2)]
q3 = sorted_dispersion[nint(len(sorted_dispersion) * 3 / 4)]
iqr = q3 - q1
print(f"q1, q2, q3: {q1:.2f}, {q2:.2f}, {q3:.2f}")
if iqr == 0.0:
raise Sorry(
"Unable to robustly estimate the variation of pixel values.")
inlier_sel = (sorted_dispersion >
(q1 - 1.5 * iqr)) & (sorted_dispersion <
(q3 + 1.5 * iqr))
sorted_dispersion = sorted_dispersion.select(inlier_sel)
gain = sorted_dispersion[nint(len(sorted_dispersion) / 2)]
print(f"Estimated gain: {gain:.2f}")
gains.append(gain)
if image_no == 0:
gain0 = gain
if image_no + 1 >= max_images:
break
if len(gains) > 1:
stats = flex.mean_and_variance(gains)
print("Average gain: %.2f +/- %.2f" %
(stats.mean(), stats.unweighted_sample_standard_deviation()))
if output_gain_map:
if len(gains) > 1:
raw_data = imageset.get_raw_data(0)
# write the gain map
gain_map = flex.double(flex.grid(raw_data[0].all()), gain0)
with open(output_gain_map, "wb") as fh:
pickle.dump(gain_map, fh, protocol=pickle.HIGHEST_PROTOCOL)
return gain0
def estimate_gain(imageset,
kernel_size=(10, 10),
output_gain_map=None,
max_images=1):
detector = imageset.get_detector()
from dials.algorithms.image.threshold import DispersionThresholdDebug
gains = flex.double()
for image_no in xrange(len(imageset)):
raw_data = imageset.get_raw_data(image_no)
#from IPython import embed; embed()
#this_data = raw_data[0]
#raw_data = (this_data + 80),
NSQ = 200
small_section = raw_data[0].matrix_copy_block(400, 400, NSQ, NSQ)
print("This small section", len(small_section), "mean ist",
flex.mean(small_section.as_double()))
raw_data = (small_section, )
gain_value = 1
gain_map = [
flex.double(raw_data[i].accessor(), gain_value)
for i in range(len(detector))
]
mask = imageset.get_mask(image_no)
mask = (mask[0].matrix_copy_block(400, 400, NSQ, NSQ)),
#from IPython import embed; embed()
min_local = 0
# dummy values, shouldn't affect results
nsigma_b = 6
nsigma_s = 3
global_threshold = 0
kabsch_debug_list = []
for i_panel in range(len(detector)):
kabsch_debug_list.append(
DispersionThresholdDebug(raw_data[i_panel].as_double(),
mask[i_panel], gain_map[i_panel],
kernel_size, nsigma_b, nsigma_s,
global_threshold, min_local))
dispersion = flex.double()
for ipix in range(5, NSQ - 15):
for spix in range(5, NSQ - 15):
data = small_section.matrix_copy_block(ipix, spix, 10,
10).as_double()
datasq = data * data
means = flex.mean(data)
var = flex.mean(datasq) - (means)**2
#print(ipix,spix,var,var/means)
dispersion.append(var / means)
if True:
dispersion = flex.double()
for kabsch in kabsch_debug_list:
a_section = kabsch.index_of_dispersion().matrix_copy_block(
5, 5, NSQ - 15, NSQ - 15)
print("mean of a_section", flex.mean(a_section))
dispersion.extend(a_section.as_1d())
#ST = flex.mean_and_variance(dispersion)
#from IPython import embed; embed()
sorted_dispersion = flex.sorted(dispersion)
from libtbx.math_utils import nearest_integer as nint
q1 = sorted_dispersion[nint(len(sorted_dispersion) / 4)]
q2 = sorted_dispersion[nint(len(sorted_dispersion) / 2)]
q3 = sorted_dispersion[nint(len(sorted_dispersion) * 3 / 4)]
iqr = q3 - q1
print("q1, q2, q3: %.2f, %.2f, %.2f" % (q1, q2, q3))
if iqr == 0.0:
raise Sorry(
'Unable to robustly estimate the variation of pixel values.')
inlier_sel = (sorted_dispersion >
(q1 - 1.5 * iqr)) & (sorted_dispersion <
(q3 + 1.5 * iqr))
sorted_dispersion = sorted_dispersion.select(inlier_sel)
gain = sorted_dispersion[nint(len(sorted_dispersion) / 2)]
print("Estimated gain: %.2f" % gain)
gains.append(gain)
if image_no == 0:
gain0 = gain
if image_no + 1 >= max_images:
break
if len(gains) > 1:
stats = flex.mean_and_variance(gains)
print("Average gain: %.2f +/- %.2f" %
(stats.mean(), stats.unweighted_sample_standard_deviation()))
if output_gain_map:
if len(gains) > 1:
raw_data = imageset.get_raw_data(0)
# write the gain map
import six.moves.cPickle as pickle
gain_map = flex.double(flex.grid(raw_data[0].all()), gain0)
with open(output_gain_map, "wb") as fh:
pickle.dump(gain_map, fh, protocol=pickle.HIGHEST_PROTOCOL)
if 0:
sel = flex.random_selection(population_size=len(sorted_dispersion),
sample_size=10000)
sorted_dispersion = sorted_dispersion.select(sel)
from matplotlib import pyplot
pyplot.scatter(range(len(sorted_dispersion)), sorted_dispersion)
pyplot.ylim(0, 10)
pyplot.show()
return gain0
def estimate_gain(raw_data,
offset=0,
algorithm="kabsch",
kernel_size=(10, 10),
output_gain_map=None,
max_images=1):
raw_data = (raw_data - offset),
from dials.algorithms.image.threshold import DispersionThresholdDebug
gains = flex.double()
if True:
NSQ = 200
ANCHOR = 400
small_section = raw_data[0].matrix_copy_block(ANCHOR, ANCHOR, NSQ, NSQ)
print("This small section", len(small_section), "mean is",
flex.mean(small_section.as_double()))
raw_data = (small_section, )
gain_value = 1
gain_map = [
flex.double(raw_data[i].accessor(), gain_value)
for i in range(len(raw_data))
]
mask = [
flex.bool(raw_data[i].accessor(), True)
for i in range(len(raw_data))
]
min_local = 0
# dummy values, shouldn't affect results
nsigma_b = 6
nsigma_s = 3
global_threshold = 0
kabsch_debug_list = []
for i_panel in range(1):
kabsch_debug_list.append(
DispersionThresholdDebug(raw_data[i_panel].as_double(),
mask[i_panel], gain_map[i_panel],
kernel_size, nsigma_b, nsigma_s,
global_threshold, min_local))
if algorithm != "kabsch":
dispersion = flex.double()
for ipix in range(5, NSQ - 15):
for spix in range(5, NSQ - 15):
data = small_section.matrix_copy_block(ipix, spix, 10,
10).as_double()
datasq = data * data
means = flex.mean(data)
var = flex.mean(datasq) - (means)**2
dispersion.append(var / means)
else:
dispersion = flex.double()
for kabsch in kabsch_debug_list:
a_section = kabsch.index_of_dispersion().matrix_copy_block(
5, 5, NSQ - 15, NSQ - 15)
print("mean of a_section", flex.mean(a_section))
dispersion.extend(a_section.as_1d())
#ST = flex.mean_and_variance(dispersion)
#from IPython import embed; embed()
sorted_dispersion = flex.sorted(dispersion)
from libtbx.math_utils import nearest_integer as nint
q1 = sorted_dispersion[nint(len(sorted_dispersion) / 4)]
q2 = sorted_dispersion[nint(len(sorted_dispersion) / 2)]
q3 = sorted_dispersion[nint(len(sorted_dispersion) * 3 / 4)]
iqr = q3 - q1
print("q1, q2, q3: %.2f, %.2f, %.2f" % (q1, q2, q3))
if iqr == 0.0:
raise Sorry(
'Unable to robustly estimate the variation of pixel values.')
inlier_sel = (sorted_dispersion >
(q1 - 1.5 * iqr)) & (sorted_dispersion <
(q3 + 1.5 * iqr))
sorted_dispersion = sorted_dispersion.select(inlier_sel)
gain = sorted_dispersion[nint(len(sorted_dispersion) / 2)]
print("Estimated gain %s: %.2f" % (algorithm, gain))
gains.append(gain)