def plot_cdf_manually(self, reflections, panel = None, ax = None, bounds = None):
colors = ['blue', 'green']
r = (reflections['xyzcal.mm']-reflections['xyzobs.mm.value']).norms()
h = flex.histogram(r)
sigma = h.slot_centers()[list(h.slots()).index(flex.max(h.slots()))] # mode
x_extent = max(r)
y_extent = len(r)
xobs = [i/x_extent for i in sorted(r)]
yobs = [i/y_extent for i in xrange(y_extent)]
obs = [(x, y) for x, y in zip(xobs, yobs)]
ncalc = 100
xcalc = [i/ncalc for i in xrange(ncalc)]
ycalc = [1-math.exp((-i**2)/(2*(sigma**2))) for i in xcalc]
calc = [(x, y) for x, y in zip(xcalc, ycalc)]
data = [flex.vec2_double(obs),
flex.vec2_double(calc)]
if bounds is None:
ax.set_xlim((-1,1))
ax.set_ylim((-1,1))
ax.set_title("%s Outlier SP Manually"%self.params.tag)
if bounds is not None:
data = [self.get_bounded_data(d, bounds) for d in data]
if ax is None:
fig = plt.figure()
ax = fig.add_subplot(111)
for subset,c in zip(data, colors):
ax.plot(subset.parts()[0], subset.parts()[1], '-', c=c)
def calc_2D_rmsd_and_displacements(reflections):
displacements = flex.vec2_double(reflections['xyzobs.px.value'].parts()[0], reflections['xyzobs.px.value'].parts()[1]) - \
flex.vec2_double(reflections['xyzcal.px'].parts()[0], reflections['xyzcal.px'].parts()[1])
rmsd = math.sqrt(flex.mean(displacements.dot( displacements )))
return rmsd,displacements
def _xyzcal_mm_to_px(self, experiments, reflections):
# set xyzcal.px field in reflections
reflections["xyzcal.px"] = flex.vec3_double(len(reflections))
for i, expt in enumerate(experiments):
imgset_sel = reflections["imageset_id"] == i
refined_reflections = reflections.select(imgset_sel)
panel_numbers = flex.size_t(refined_reflections["panel"])
xyzcal_mm = refined_reflections["xyzcal.mm"]
x_mm, y_mm, z_rad = xyzcal_mm.parts()
xy_cal_mm = flex.vec2_double(x_mm, y_mm)
xy_cal_px = flex.vec2_double(len(xy_cal_mm))
for i_panel in range(len(expt.detector)):
panel = expt.detector[i_panel]
sel = panel_numbers == i_panel
xy_cal_px.set_selected(
sel, panel.millimeter_to_pixel(xy_cal_mm.select(sel))
)
x_px, y_px = xy_cal_px.parts()
if expt.scan is not None:
z_px = expt.scan.get_array_index_from_angle(z_rad, deg=False)
else:
# must be a still image, z centroid not meaningful
z_px = z_rad
xyzcal_px = flex.vec3_double(x_px, y_px, z_px)
reflections["xyzcal.px"].set_selected(imgset_sel, xyzcal_px)
def calc_2D_rmsd_and_displacements(reflections):
displacements = flex.vec2_double(reflections['xyzobs.px.value'].parts()[0], reflections['xyzobs.px.value'].parts()[1]) - \
flex.vec2_double(reflections['xyzcal.px'].parts()[0], reflections['xyzcal.px'].parts()[1])
rmsd = math.sqrt(flex.mean(displacements.dot(displacements)))
return rmsd, displacements
def plot_histograms(self, reflections, panel = None, ax = None, bounds = None):
data = reflections['difference_vector_norms']
colors = ['b-', 'g-', 'g--', 'r-', 'b-', 'b--']
n_slots = 20
if self.params.residuals.histogram_max is None:
h = flex.histogram(data, n_slots=n_slots)
else:
h = flex.histogram(data.select(data <= self.params.residuals.histogram_max), n_slots=n_slots)
n = len(reflections)
rmsd_obs = math.sqrt((reflections['xyzcal.mm']-reflections['xyzobs.mm.value']).sum_sq()/n)
sigma = mode = h.slot_centers()[list(h.slots()).index(flex.max(h.slots()))]
mean_obs = flex.mean(data)
median = flex.median(data)
mean_rayleigh = math.sqrt(math.pi/2)*sigma
rmsd_rayleigh = math.sqrt(2)*sigma
data = flex.vec2_double([(i,j) for i, j in zip(h.slot_centers(), h.slots())])
n = len(data)
for i in [mean_obs, mean_rayleigh, mode, rmsd_obs, rmsd_rayleigh]:
data.extend(flex.vec2_double([(i, 0), (i, flex.max(h.slots()))]))
data = self.get_bounded_data(data, bounds)
tmp = [data[:n]]
for i in xrange(len(colors)):
tmp.append(data[n+(i*2):n+((i+1)*2)])
data = tmp
for d, c in zip(data, colors):
ax.plot(d.parts()[0], d.parts()[1], c)
if ax.get_legend() is None:
ax.legend([r"$\Delta$XY", "MeanObs", "MeanRayl", "Mode", "RMSDObs", "RMSDRayl"])
def matcher(reference, moving, params):
from annlib_ext import AnnAdaptor as ann_adaptor
from dials.array_family import flex
rxyz = reference['xyzobs.px.value'].parts()
mxyz = moving['xyzobs.px.value'].parts()
rxy = flex.vec2_double(rxyz[0], rxyz[1])
mxy = flex.vec2_double(mxyz[0], mxyz[1])
ann = ann_adaptor(rxy.as_double().as_1d(), 2)
ann.query(mxy.as_double().as_1d())
distances = flex.sqrt(ann.distances)
matches = (distances < params.far) & (distances >= params.close)
xyr = flex.vec2_double()
xym = flex.vec2_double()
for j in range(matches.size()):
if not matches[j]:
continue
xym.append(mxy[j])
xyr.append(rxy[ann.nn[j]])
# filter outliers - use IQR etc.
dxy = xym - xyr
dx, dy = dxy.parts()
iqx = IQR(dx.select(flex.sort_permutation(dx)))
iqy = IQR(dy.select(flex.sort_permutation(dy)))
keep_x = (dx > (iqx[0] - iqx[3])) & (dx < (iqx[2] + iqx[3]))
keep_y = (dy > (iqy[0] - iqy[3])) & (dy < (iqy[2] + iqy[3]))
keep = keep_x & keep_y
xyr = xyr.select(keep)
xym = xym.select(keep)
# compute Rt
R, t, d, n = Rt(xyr, xym)
# verify matches in original image coordinate system
from scitbx import matrix
import math
_R = matrix.sqr(R)
rmsd = 0.0
for j, _xym in enumerate(xym):
_xymm = _R * _xym + matrix.col(t)
rmsd += (matrix.col(xyr[j]) - _xymm).length()**2
assert abs(math.sqrt(rmsd / xym.size()) - d) < 1e-6
return R, t, d, n
def compute_Rt(reference, moving):
from dials.array_family import flex
rxyz = reference['xyzobs.px.value'].parts()
mxyz = moving['xyzobs.px.value'].parts()
rxy = flex.vec2_double(rxyz[0], rxyz[1])
mxy = flex.vec2_double(mxyz[0], mxyz[1])
# compute Rt
return Rt(rxy, mxy)
def pair_up(reference, moving, params, R0, t0):
from annlib_ext import AnnAdaptor as ann_adaptor
from dials.array_family import flex
rxyz = reference['xyzobs.px.value'].parts()
mxyz = moving['xyzobs.px.value'].parts()
# apply R0, t0 before performing matching - so should ideally be in almost
# right position
rxy = flex.vec2_double(rxyz[0], rxyz[1])
_mxy = flex.vec2_double(mxyz[0], mxyz[1])
mxy = flex.vec2_double()
for __mxy in _mxy:
mxy.append((R0 * __mxy + t0).elems)
ann = ann_adaptor(rxy.as_double().as_1d(), 2)
ann.query(mxy.as_double().as_1d())
distances = flex.sqrt(ann.distances)
matches = (distances < params.far)
rsel = flex.size_t()
msel = flex.size_t()
xyr = flex.vec2_double()
xym = flex.vec2_double()
for j in range(matches.size()):
if not matches[j]:
continue
msel.append(j)
rsel.append(ann.nn[j])
xym.append(mxy[j])
xyr.append(rxy[ann.nn[j]])
# filter outliers - use IQR etc.
dxy = xym - xyr
dx, dy = dxy.parts()
iqx = IQR(dx.select(flex.sort_permutation(dx)))
iqy = IQR(dy.select(flex.sort_permutation(dy)))
keep_x = (dx > (iqx[0] - iqx[3])) & (dx < (iqx[2] + iqx[3]))
keep_y = (dy > (iqy[0] - iqy[3])) & (dy < (iqy[2] + iqy[3]))
keep = keep_x & keep_y
return rsel.select(keep), msel.select(keep)
def get_pix_coords(wavelength, A, mill_arr, detector, delta_i=0.02):
""" Code copied from sim.py courtesy of Aaron and Tara """
s0=col((0,0,-1/wavelength))
q=flex.vec3_double([A*col(idx) for idx in mill_arr.indices().as_vec3_double()])
s0_hat=flex.vec3_double([s0.normalize()]*len(q))
q_hat=q.each_normalize()
#q_hat.cross(flex.vec3_double([s0_hat]*len(q_hat)))
e1_hat = q_hat.cross(s0_hat)
c0_hat = s0_hat.cross(e1_hat)
q_len_sq = flex.double([col(v).length_sq() for v in q])
a_side=q_len_sq*wavelength/2
b_side=flex.sqrt(q_len_sq)-a_side**2
#flex.vec3_double([sqrt(q.length_sq()-a_side**2 for idx in mill_arr)])
r_vec=flex.vec3_double(-a_side*s0_hat+b_side*c0_hat)
s1=r_vec+s0
EQ=q+s0
len_EQ=flex.double([col(v).length() for v in EQ])
ratio=len_EQ*wavelength
indices = flex.miller_index()
coords =flex.vec2_double()
for i in range(len(s1)):
if ratio[i] > 1 - delta_i and ratio[i] < 1 + delta_i:
indices.append(mill_arr.indices()[i])
pix = detector[0].get_ray_intersection_px(s1[i])
if detector[0].is_coord_valid(pix):
coords.append(pix)
return coords, indices
def correction_and_within_spot_sigma(params_version,
variance_within_spot=True):
absorption = KaptonAbsorption(
params_version[0],
params_version[1],
params_version[2],
params_version[3],
self.detector_dist_mm,
self.pixel_size_mm,
self.wavelength_ang,
*map(float, self.panel_size_px),
)
detector = self.expt.detector
# y_max = int(detector[0].millimeter_to_pixel(detector[0].get_image_size())[1])
absorption_corrections = flex.double()
absorption_sigmas = (
flex.double()
) # std dev of corrections for pixels within a spot, default sigma
if variance_within_spot:
mask_code = MaskCode.Foreground | MaskCode.Valid
for iref in range(len(self.reflections_sele)):
kapton_correction_vector = flex.double()
# foreground: integration mask
shoebox = self.reflections_sele[iref]["shoebox"]
foreground = ((shoebox.mask.as_1d()
& mask_code) == mask_code).iselection()
width = shoebox.xsize()
fast_coords = (foreground % width).as_int() # within spot
slow_coords = (foreground / width).as_int() # within spot
f_absolute = fast_coords + shoebox.bbox[
0] # relative to detector
s_absolute = slow_coords + shoebox.bbox[
2] # relative to detector
lab_coords = detector[0].get_lab_coord(
detector[0].pixel_to_millimeter(
flex.vec2_double(f_absolute.as_double(),
s_absolute.as_double())))
s1 = lab_coords.each_normalize()
kapton_correction_vector.extend(
absorption.abs_correction_flex(s1))
average_kapton_correction = flex.mean(
kapton_correction_vector)
absorption_corrections.append(average_kapton_correction)
try:
spot_px_stddev = flex.mean_and_variance(
kapton_correction_vector
).unweighted_sample_standard_deviation()
except Exception:
assert (len(kapton_correction_vector) == 1
), "stddev could not be calculated"
spot_px_stddev = 0
absorption_sigmas.append(spot_px_stddev)
return absorption_corrections, absorption_sigmas
else:
s1_flex = self.reflections_sele["s1"].each_normalize()
absorption_corrections = absorption.abs_correction_flex(
s1_flex)
return absorption_corrections, None
def get_pix_coords(wavelength, A, mill_arr, detector, delta_i=0.02):
""" Code copied from sim.py courtesy of Aaron and Tara """
s0 = col((0, 0, -1 / wavelength))
q = flex.vec3_double(
[A * col(idx) for idx in mill_arr.indices().as_vec3_double()])
s0_hat = flex.vec3_double([s0.normalize()] * len(q))
q_hat = q.each_normalize()
#q_hat.cross(flex.vec3_double([s0_hat]*len(q_hat)))
e1_hat = q_hat.cross(s0_hat)
c0_hat = s0_hat.cross(e1_hat)
q_len_sq = flex.double([col(v).length_sq() for v in q])
a_side = q_len_sq * wavelength / 2
b_side = flex.sqrt(q_len_sq) - a_side**2
#flex.vec3_double([sqrt(q.length_sq()-a_side**2 for idx in mill_arr)])
r_vec = flex.vec3_double(-a_side * s0_hat + b_side * c0_hat)
s1 = r_vec + s0
EQ = q + s0
len_EQ = flex.double([col(v).length() for v in EQ])
ratio = len_EQ * wavelength
indices = flex.miller_index()
coords = flex.vec2_double()
for i in range(len(s1)):
if ratio[i] > 1 - delta_i and ratio[i] < 1 + delta_i:
indices.append(mill_arr.indices()[i])
pix = detector[0].get_ray_intersection_px(s1[i])
if detector[0].is_coord_valid(pix):
coords.append(pix)
return coords, indices
def populate_pixel_positions(self):
assert 'xyzcal.px' in self.reflections, "no calculated spot positions"
self.frame['mapped_predictions'][0] = flex.vec2_double()
for i in range(len(self.reflections['xyzcal.px'])):
self.frame['mapped_predictions'][0].append(
tuple(self.reflections['xyzcal.px'][i][1::-1])
) # 1::-1 reverses the order taking only the first two elements first.
def test_calc_lookup_index():
pi = 3.141
theta_phi = flex.vec2_double([(0.001, -pi), (pi, pi), (0.001, pi), (pi, -pi)])
indices = calc_lookup_index(theta_phi, 1)
assert list(indices) == [0, 64799, 359, 64440]
indices = calc_lookup_index(theta_phi, 2)
assert list(indices) == [0, 259199, 719, 258480]
def plot_difference_vector_norms_histograms(self, reflections, panel = None, ax = None, bounds = None):
r = reflections['difference_vector_norms']*1000
h = flex.histogram(r, n_slots=50, data_min=0, data_max=100)
x_extent = max(r)
y_extent = len(r)
xobs = [i/x_extent for i in sorted(r)]
yobs = [i/y_extent for i in xrange(y_extent)]
obs = [(x, y) for x, y in zip(xobs, yobs)]
if bounds is None:
#ax.set_xlim((-1,1))
#ax.set_ylim((-1,1))
x = h.slot_centers().as_numpy_array()
y = h.slots().as_numpy_array()
ax.set_title("%s Residual norms histogram"%self.params.tag)
if bounds is not None:
d = flex.vec2_double(h.slot_centers(), h.slots().as_double())
data = self.get_bounded_data(d, bounds)
x, y = data.parts()
if ax is None:
fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(x, y, '-', c='blue')
def xy_to_q(x, y, detector, beam, oldmethod=False, panel_id=0):
"""
convert pixel coords to q-vectors
:param x,y: pixel coordinates of spots as separate lists/arrays
x should be the fast-scan coord
:param detector: dxtbx detector model
:param beam: dxtbx beam model
:param method1: whether to use method 1 below..
:return: the Q-vectors corresponding to the spots
"""
panel = detector[panel_id]
if oldmethod:
pixsizeFS, pixsizeSS = panel.get_pixel_size()[0]
orig = np.array(panel.get_origin())
ss = np.array(panel.get_slow_axis())
fs = np.array(panel.get_fast_axis())
s0 = np.array(beam.get_s0()) # this is already scaled by 1/wavelength
pix_mm = np.vstack([
orig + fs * i * pixsizeFS + ss * j * pixsizeSS
for i, j in zip(x, y)
])
s1 = pix_mm / np.linalg.norm(pix_mm,
axis=1)[:, None] / beam.get_wavelength()
q_vecs = (s1 - s0)
else:
pix_mm = panel.pixel_to_millimeter(flex.vec2_double(zip(x, y)))
coords = panel.get_lab_coord(pix_mm)
coords = coords / coords.norms()
q_vecs = coords * (1. / beam.get_wavelength()) - beam.get_s0()
# I like to return as numpy array...
q_vecs = q_vecs.as_double().as_numpy_array().reshape((len(q_vecs), 3))
return q_vecs
def test_sort():
table = flex.reflection_table()
table["a"] = flex.int([2, 4, 3, 1, 5, 6])
table["b"] = flex.vec2_double([(3, 2), (3, 1), (1, 3), (4, 5), (4, 3),
(2, 0)])
table["c"] = flex.miller_index([(3, 2, 1), (3, 1, 1), (2, 4, 2), (2, 1, 1),
(1, 1, 1), (1, 1, 2)])
table.sort("a")
assert list(table["a"]) == [1, 2, 3, 4, 5, 6]
table.sort("b")
assert list(table["b"]) == [(1, 3), (2, 0), (3, 1), (3, 2), (4, 3), (4, 5)]
table.sort("c")
assert list(table["c"]) == [
(1, 1, 1),
(1, 1, 2),
(2, 1, 1),
(2, 4, 2),
(3, 1, 1),
(3, 2, 1),
]
table.sort("c", order=(1, 2, 0))
assert list(table["c"]) == [
(1, 1, 1),
(2, 1, 1),
(3, 1, 1),
(1, 1, 2),
(3, 2, 1),
(2, 4, 2),
]
def get_bounded_data(self, data, bounds):
assert len(bounds) == 4
x = [b[0] for b in bounds]
y = [b[1] for b in bounds]
left = sorted(x)[1]
right = sorted(x)[2]
top = sorted(y)[2]
bottom = sorted(y)[1]
origin = col((left, bottom))
scale_x = right-left
scale_y = top-bottom
scale = min(scale_x, scale_y)
data_max_x = flex.max(data.parts()[0])
data_min_x = flex.min(data.parts()[0])
data_max_y = flex.max(data.parts()[1])
data_min_y = flex.min(data.parts()[1])
data_scale_x = data_max_x - data_min_x
data_scale_y = data_max_y - data_min_y
if data_scale_x == 0 or data_scale_y == 0:
print "WARNING bad scale"
return data
return flex.vec2_double(data.parts()[0] * (scale/abs(data_scale_x)),
data.parts()[1] * (scale/abs(data_scale_y))) + origin
def random_positions(n, amount):
from scitbx.random import variate, normal_distribution
from dials.array_family import flex
g = variate(normal_distribution(mean=0, sigma=amount))
xy = flex.vec2_double(n)
for j in range(n):
xy[j] = (next(g), next(g))
return xy
def exercise_polygon():
from dials.algorithms.polygon import polygon
x = 1
y = 1
vertices = [(0, 0), (2, 0), (2, 2), (0, 2)]
poly = polygon(vertices)
assert poly.is_inside(x, y)
poly = polygon([(3, 5), (40, 90), (80, 70), (50, 50), (70, 20)])
for p in [(42, 40), (16, 25), (64, 67), (16, 30), (48, 45), (21, 30)]:
assert poly.is_inside(p[0], p[1])
for p in [(59, 4), (70, 15), (57, 14), (21, 78), (37, 100), (88, 89)]:
assert not poly.is_inside(p[0], p[1])
if 0:
# for visual confirmation of algorithm
from scitbx.array_family import flex
inside_points = flex.vec2_double()
outside_points = flex.vec2_double()
import random
x_max = 100
y_max = 100
for i in range(1000):
x = random.randint(0, x_max)
y = random.randint(0, y_max)
is_inside = poly.is_inside(x, y)
if is_inside:
inside_points.append((x, y))
else:
outside_points.append((x, y))
from matplotlib import pyplot
from matplotlib.patches import Polygon
v = poly.vertices + poly.vertices[:1]
fig = pyplot.figure()
ax = fig.add_subplot(111)
ax.add_patch(Polygon(poly.vertices, closed=True, fill=False))
inside_x, inside_y = inside_points.parts()
outside_x, outside_y = outside_points.parts()
ax.scatter(inside_x, inside_y, marker="+", c="r")
ax.scatter(outside_x, outside_y, marker="+", c="b")
pyplot.show()
def plot_obs_colored_by_data(self, data, reflections, panel = None, ax = None, bounds = None):
assert panel is not None and ax is not None and bounds is not None
norm, cmap, color_vals, sm = self.get_normalized_colors(data)
mm_panel_coords = flex.vec2_double(reflections['xyzobs.mm.value'].parts()[0], reflections['xyzobs.mm.value'].parts()[1])
lab_coords = panel.get_lab_coord(mm_panel_coords)
ax.scatter(lab_coords.parts()[0], lab_coords.parts()[1], c = data, norm=norm, cmap = cmap, linewidths=0, s=self.params.dot_size)
return sm, color_vals
def correction_and_within_spot_sigma(params_version, variance_within_spot=True):
# instantiate Kapton absorption class here
absorption = KaptonTape_2019(
params_version[0],
params_version[1],
params_version[2],
params_version[3],
self.wavelength_ang,
)
# *map(float, self.panel_size_px)) #
detector = self.expt.detector
absorption_corrections = flex.double()
absorption_sigmas = (
flex.double()
) # std dev of corrections for pixels within a spot, default sigma
if variance_within_spot:
mask_code = MaskCode.Foreground | MaskCode.Valid
for iref in range(len(self.reflections_sele)):
kapton_correction_vector = flex.double()
# foreground: integration mask
shoebox = self.reflections_sele[iref]["shoebox"]
foreground = (
(shoebox.mask.as_1d() & mask_code) == mask_code
).iselection()
f_absolute, s_absolute, z_absolute = (
shoebox.coords().select(foreground).parts()
)
panel_number = self.reflections_sele[iref]["panel"]
lab_coords = detector[panel_number].get_lab_coord(
detector[panel_number].pixel_to_millimeter(
flex.vec2_double(f_absolute, s_absolute)
)
)
s1 = lab_coords.each_normalize()
# Real step right here
kapton_correction_vector.extend(absorption.abs_correction_flex(s1))
#
average_kapton_correction = flex.mean(kapton_correction_vector)
absorption_corrections.append(average_kapton_correction)
try:
spot_px_stddev = flex.mean_and_variance(
kapton_correction_vector
).unweighted_sample_standard_deviation()
except Exception:
assert (
len(kapton_correction_vector) == 1
), "stddev could not be calculated"
spot_px_stddev = 0
absorption_sigmas.append(spot_px_stddev)
return absorption_corrections, absorption_sigmas
else:
s1_flex = self.reflections_sele["s1"].each_normalize()
absorption_corrections = absorption.abs_correction_flex(s1_flex)
return absorption_corrections, None
def exercise_polygon():
from dials.algorithms.polygon import polygon
x = 1
y = 1
vertices = [(0,0), (2,0), (2,2), (0,2)]
poly = polygon(vertices)
assert poly.is_inside(x,y)
poly = polygon([(3,5), (40,90), (80,70), (50,50), (70,20)])
for p in [(42,40), (16,25), (64,67), (16,30), (48, 45), (21,30)]:
assert poly.is_inside(p[0], p[1])
for p in [(59,4), (70,15), (57,14), (21,78), (37,100), (88,89)]:
assert not poly.is_inside(p[0], p[1])
if 0:
# for visual confirmation of algorithm
from scitbx.array_family import flex
inside_points = flex.vec2_double()
outside_points = flex.vec2_double()
import random
x_max = 100
y_max = 100
for i in range(1000):
x = random.randint(0,x_max)
y = random.randint(0,y_max)
is_inside = poly.is_inside(x, y)
if is_inside:
inside_points.append((x,y))
else:
outside_points.append((x,y))
from matplotlib import pyplot
from matplotlib.patches import Polygon
v = poly.vertices + poly.vertices[:1]
fig = pyplot.figure()
ax = fig.add_subplot(111)
ax.add_patch(Polygon(poly.vertices, closed=True, fill=False))
inside_x, inside_y = inside_points.parts()
outside_x, outside_y = outside_points.parts()
ax.scatter(inside_x, inside_y, marker='+', c='r')
ax.scatter(outside_x, outside_y, marker='+', c='b')
pyplot.show()
def sim_rois(self,
rois=None,
reset=True,
cuda=False,
omp=False,
add_water=False,
boost=1,
add_spots=True,
show_params=False):
if show_params:
self.SIM2.show_params()
#if self.crystal_size_mm is not None:
print(" Mosaic domain size mm = %.3g" %
np.power(self.mosaic_domain_volume, 1 / 3.))
print(" Spot scale = %.3g" % self.SIM2.spot_scale)
if rois is None:
rois = [self.FULL_ROI]
if add_spots:
for roi in rois:
self.SIM2.region_of_interest = roi
if cuda:
self.SIM2.add_nanoBragg_spots_cuda()
elif omp:
from boost.python import streambuf # will deposit printout into dummy StringIO as side effect
from six.moves import StringIO
self.SIM2.add_nanoBragg_spots_nks(streambuf(StringIO()))
else:
self.SIM2.add_nanoBragg_spots()
self.SIM2.raw_pixels = self.SIM2.raw_pixels * boost
if add_water: # add water for full panel, ignoring ROI
water_scatter = flex.vec2_double([(0, 2.57), (0.0365, 2.58),
(0.07, 2.8), (0.12, 5),
(0.162, 8), (0.2, 6.75),
(0.18, 7.32), (0.216, 6.75),
(0.236, 6.5), (0.28, 4.5),
(0.3, 4.3), (0.345, 4.36),
(0.436, 3.77), (0.5, 3.17)])
self.SIM2.Fbg_vs_stol = water_scatter
self.SIM2.amorphous_density_gcm3 = 1
self.SIM2.amorphous_molecular_weight_Da = 18
self.SIM2.amorphous_sample_thick_mm = self.amorphous_sample_thick_mm
self.SIM2.region_of_interest = self.FULL_ROI
self.SIM2.add_background()
img = self.SIM2.raw_pixels.as_numpy_array()
if reset:
self.SIM2.raw_pixels *= 0
self.SIM2.region_of_interest = self.FULL_ROI
return img
def __init__(self, strong_spots, experiment=None):
self.strong = strong_spots
assert 'bbox' in self.strong and 'shoebox' in self.strong, \
"Spotfinder shoeboxes are required for spot length calculations."
assert experiment, \
"Supply one experiment object."
self.det = experiment.detector
self.beam = experiment.beam
self.s0 = matrix.col(self.beam.get_unit_s0())
self.panel_s0_intersections = flex.vec2_double([
self.det[i].get_ray_intersection_px(self.s0)
for i in range(len(self.det))
])
def sim_rois(self,
rois,
reset=True,
cuda=False,
omp=False,
add_water=False,
add_noise=False,
water_par=None,
noise_par=None,
boost=1,
add_spots=True):
if add_spots:
for roi in rois:
self.SIM2.region_of_interest = roi
if cuda:
self.SIM2.add_nanoBragg_spots_cuda()
elif omp:
from boost.python import streambuf # will deposit printout into dummy StringIO as side effect
from six.moves import StringIO
self.SIM2.add_nanoBragg_spots_nks(streambuf(StringIO()))
else:
self.SIM2.add_nanoBragg_spots()
self.SIM2.raw_pixels = self.SIM2.raw_pixels * boost
if add_water:
water_scatter = flex.vec2_double([(0, 2.57), (0.0365, 2.58),
(0.07, 2.8), (0.12, 5),
(0.162, 8), (0.2, 6.75),
(0.18, 7.32), (0.216, 6.75),
(0.236, 6.5), (0.28, 4.5),
(0.3, 4.3), (0.345, 4.36),
(0.436, 3.77), (0.5, 3.17)])
self.SIM2.Fbg_vs_stol = water_scatter
self.SIM2.amorphous_sample_thick_mm = 0.004 # typical GDVN
self.SIM2.amorphous_density_gcm3 = 1
self.SIM2.amorphous_molecular_weight_Da = 18
self.SIM2.exposure_s = 1.0 # multiplies flux x exposure
self.SIM2.region_of_interest = self.FULL_ROI
self.SIM2.add_background()
if add_noise:
self.SIM2.add_noise()
img = self.SIM2.raw_pixels.as_numpy_array()
if reset:
self.SIM2.raw_pixels *= 0
self.SIM2.region_of_interest = self.FULL_ROI
return img
def set_general_variables(self):
self.frame = load(self.raw)
self.detector = self.frame.get_detector()
self.beam = self.frame.get_beam()
self.s0 = self.beam.get_s0()
self.gonio = self.frame.get_goniometer()
self.scan = self.frame.get_scan()
self.lab_coordinates = flex.vec3_double()
for panel in self.detector:
self.beam_center_mm_x, self.beam_center_mm_y = col(panel.get_beam_centre(self.s0))
pixels = flex.vec2_double(panel.get_image_size())
mms = panel.pixel_to_millimeter(pixels)
self.lab_coordinates.extend(panel.get_lab_coord(mms))
self.Isizex, self.Isizey = panel.get_image_size()
self.beam_center_x, self.beam_center_y = col(panel.get_beam_centre_px(self.s0))
self.detector_distance = panel.get_distance()
thrshim_min, thrshim_max = panel.get_trusted_range()
self.pixel_size = panel.get_pixel_size()[0]
self.raw_data = self.frame.get_raw_data()
if thrshim_min < 0 :
self.thrshim_min = int(0)
else:
self.thrshim_min = thrshim_min
if thrshim_max > 32767:
self.thrshim_max = int(32767)
else:
self.thrshim_max = int(thrshim_max)
self.polarization_fraction = self.beam.get_polarization_fraction()
self.polarization_offset = 0.0
self.cassette_x = 0.0
self.cassette_y = 0.0
self.windim_xmax = int(self.Isizex)-100 # right border for processed image (pixels)
self.windim_xmin = 100 # left border for processed image (pixels)
self.windim_ymax = int(self.Isizey)-100 # top border for processed image (pixels)
self.windim_ymin = 100 # bottom border for processed image (pixels)
### beamstop borders
self.punchim_xmax = int(self.Isizex) # right border of beam stop shadow (pixels)
self.punchim_xmin = int(self.beam_center_x)-80 # left border of beam stop shadow (pixels)
self.punchim_ymax = int(self.beam_center_y)+100 # top border of beam stop shadow (pixels)
self.punchim_ymin = int(self.beam_center_y)-40 # bottom border of beam stop shadow (pixels)
self.mode_filter_footprint = int(20)
return
def del_op(A):
from dials.array_family import flex
del_A = flex.vec2_double(A.accessor())
for j in range(A.all()[0]):
for i in range(A.all()[1]):
del_x = 0
del_y = 0
if i > 0:
del_x = A[j, i] - A[j, i - 1]
if j > 0:
del_y = A[j, i] - A[j - 1, i]
del_A[j, i] = (del_y, del_x)
return del_A
def plot_obs_colored_by_deltapsi(self, reflections, panel = None, ax = None, bounds = None):
assert panel is not None and ax is not None and bounds is not None
data = reflections['delpsical.rad'] * (180/math.pi)
norm, cmap, color_vals, sm = self.get_normalized_colors(data, vmin=-0.1, vmax=0.1)
deltas = (reflections['xyzcal.mm']-reflections['xyzobs.mm.value'])*self.delta_scalar
x, y = panel.get_image_size_mm()
offset = col((x, y, 0))/2
deltas += offset
mm_panel_coords = flex.vec2_double(deltas.parts()[0], deltas.parts()[1])
lab_coords = panel.get_lab_coord(mm_panel_coords)
ax.scatter(lab_coords.parts()[0], lab_coords.parts()[1], c = data, norm=norm, cmap = cmap, linewidths=0, s=self.params.dot_size)
return sm, color_vals
def del_op(A):
from dials.array_family import flex
del_A = flex.vec2_double(A.accessor())
for j in range(A.all()[0]):
for i in range(A.all()[1]):
del_x = 0
del_y = 0
if i > 0:
del_x = A[j,i] - A[j,i-1]
if j < 0:
del_y = A[j,i] - A[j-1,i]
del_A[j,i] = (del_y, del_x)
return del_A
def plot_radial_displacements_vs_deltapsi(self, reflections, panel = None, ax = None, bounds = None):
assert panel is not None and ax is not None and bounds is not None
data = reflections['difference_vector_norms']
norm, cmap, color_vals, sm = self.get_normalized_colors(data)
a = reflections['delpsical.rad']*180/math.pi
b = reflections['radial_displacements']
fake_coords = flex.vec2_double(a, b) * self.delta_scalar
x, y = panel.get_image_size_mm()
offset = col((x, y))/2
lab_coords = fake_coords + panel.get_lab_coord(offset)[0:2]
ax.scatter(lab_coords.parts()[0], lab_coords.parts()[1], c = data, norm=norm, cmap = cmap, linewidths=0, s=self.params.dot_size)
return sm, color_vals
def test_sort():
from dials.array_family import flex
table = flex.reflection_table()
table['a'] = flex.int([2, 4, 3, 1, 5])
table['b'] = flex.vec2_double([(3, 2), (3, 1), (1, 3), (4, 5), (4, 3)])
table['c'] = flex.miller_index([(3,2,1), (3,1,1), (2,4,2), (2,1,1), (1,1,1)])
table.sort("a")
assert list(table['a']) == [1, 2, 3, 4, 5]
table.sort("b")
assert list(table['b']) == [(1,3), (3,1), (3,2), (4,3), (4,5)]
table.sort("c")
assert list(table['c']) == [(1,1,1),(2,1,1),(2,4,2),(3,1,1),(3,2,1)]
table.sort("c", order=(1,2,0))
assert list(table['c']) == [(1, 1, 1), (2, 1, 1), (3, 1, 1), (3, 2, 1), (2, 4, 2)]
def __init__(self, strong_spots, experiment=None, datablock=None):
self.strong = strong_spots
assert 'bbox' in self.strong and 'shoebox' in self.strong, \
"Spotfinder shoeboxes are required for spot length calculations."
assert experiment or datablock, \
"Supply one experiment or datablock object."
if datablock:
imgset = datablock._imagesets[0]
self.det = imgset.get_detector()
self.beam = imgset.get_beam()
else:
self.det = experiment.detector
self.beam = experiment.beam
self.s0 = matrix.col(self.beam.get_unit_s0())
self.panel_s0_intersections = flex.vec2_double([
self.det[i].get_ray_intersection_px(self.s0)
for i in xrange(len(self.det))
])
def extract_spot_data(reflections, experiments, max_two_theta):
"""
From the spot positions, extract reciprocal space X, Y and angle positions
for each reflection up to the scattering angle max_two_theta
"""
# Map reflections to reciprocal space
reflections.centroid_px_to_mm(experiments)
# Calculate scattering vectors
reflections["s1"] = flex.vec3_double(len(reflections))
reflections["2theta"] = flex.double(len(reflections))
panel_numbers = flex.size_t(reflections["panel"])
for i, expt in enumerate(experiments):
if "imageset_id" in reflections:
sel_expt = reflections["imageset_id"] == i
else:
sel_expt = reflections["id"] == i
for i_panel in range(len(expt.detector)):
sel = sel_expt & (panel_numbers == i_panel)
x, y, _ = reflections["xyzobs.mm.value"].select(sel).parts()
s1 = expt.detector[i_panel].get_lab_coord(flex.vec2_double(x, y))
s1 = s1 / s1.norms() * (1 / expt.beam.get_wavelength())
tt = s1.angle(expt.beam.get_s0(), deg=True)
reflections["s1"].set_selected(sel, s1)
reflections["2theta"].set_selected(sel, tt)
# Filter reflections
full_len = len(reflections)
reflections = reflections.select(reflections["2theta"] <= max_two_theta)
if len(reflections) < full_len:
logger.info(
f"{len(reflections)} reflections with 2θ ≤ {max_two_theta}° selected from {full_len} total"
)
x, y, _ = reflections["s1"].parts()
_, _, angle = reflections["xyzobs.mm.value"].parts()
arr = flumpy.to_numpy(x)
arr = np.c_[x, flumpy.to_numpy(
-y
)] # Y is inverted to match calculation in edtools.find_rotation_axis
arr = np.c_[arr, flumpy.to_numpy(angle)]
return arr
def get_experiment_xvectors(experiments):
beam = experiments[0].beam
detector = experiments[0].detector
x = []
for panel in detector:
lab_coordinates = flex.vec3_double()
pixels = flex.vec2_double(panel.get_image_size())
mms = panel.pixel_to_millimeter(pixels)
lab_coordinates.extend(panel.get_lab_coord(mms))
# generate s1 vectors
s1 = lab_coordinates.each_normalize() * (1 / beam.get_wavelength())
# Generate x vectors
x.append((s1 - beam.get_s0()).as_double())
return (x)
def calc_2theta(reflections, experiments):
"""Calculate and return 2theta angles in radians"""
twotheta = flex.double(len(reflections), 0.)
for iexp, exp in enumerate(experiments):
isel = (reflections['id'] == iexp).iselection()
sub_ref = reflections.select(isel)
s0 = matrix.col(exp.beam.get_s0())
for ipanel in range(len(exp.detector)):
sel = (sub_ref['panel'] == ipanel)
panel_ref = sub_ref.select(sel)
x, y, phi = panel_ref['xyzobs.mm.value'].parts()
s1 = exp.detector[ipanel].get_lab_coord(flex.vec2_double(x,y))
s1 = s1/s1.norms() * s0.length()
sub_isel = isel.select(sel)
twotheta.set_selected(sub_isel, s1.angle(s0))
return twotheta
def calc_2theta(reflections, experiments):
"""Calculate and return 2theta angles in radians"""
twotheta = flex.double(len(reflections), 0.0)
for iexp, exp in enumerate(experiments):
isel = (reflections["id"] == iexp).iselection()
sub_ref = reflections.select(isel)
s0 = matrix.col(exp.beam.get_s0())
for ipanel in range(len(exp.detector)):
sel = sub_ref["panel"] == ipanel
panel_ref = sub_ref.select(sel)
x, y, phi = panel_ref["xyzobs.mm.value"].parts()
s1 = exp.detector[ipanel].get_lab_coord(flex.vec2_double(x, y))
s1 = s1 / s1.norms() * s0.length()
sub_isel = isel.select(sel)
twotheta.set_selected(sub_isel, s1.angle(s0))
return twotheta
def tst_sort(self):
from dials.array_family import flex
table = flex.reflection_table()
table['a'] = flex.int([2, 4, 3, 1, 5])
table['b'] = flex.vec2_double([(3, 2), (3, 1), (1, 3), (4, 5), (4, 3)])
table['c'] = flex.miller_index([(3,2,1), (3,1,1), (2,4,2), (2,1,1), (1,1,1)])
table.sort("a")
assert list(table['a']) == [1, 2, 3, 4, 5]
table.sort("b")
assert list(table['b']) == [(1,3), (3,1), (3,2), (4,3), (4,5)]
table.sort("c")
assert list(table['c']) == [(1,1,1),(2,1,1),(2,4,2),(3,1,1),(3,2,1)]
table.sort("c", order=(1,2,0))
assert list(table['c']) == [(1, 1, 1), (2, 1, 1), (3, 1, 1), (3, 2, 1), (2, 4, 2)]
print "OK"
def test_to_from_msgpack(tmpdir):
from dials.model.data import Shoebox
def gen_shoebox():
shoebox = Shoebox(0, (0, 4, 0, 3, 0, 1))
shoebox.allocate()
for k in range(1):
for j in range(3):
for i in range(4):
shoebox.data[k, j, i] = i + j + k + 0.1
shoebox.mask[k, j, i] = i % 2
shoebox.background[k, j, i] = i * j + 0.2
return shoebox
def compare(a, b):
assert a.is_consistent()
assert b.is_consistent()
assert a.panel == b.panel
assert a.bbox == b.bbox
for aa, bb in zip(a.data, b.data):
if abs(aa - bb) > 1e-9:
return False
for aa, bb in zip(a.background, b.background):
if abs(aa - bb) > 1e-9:
return False
for aa, bb in zip(a.mask, b.mask):
if aa != bb:
return False
return True
# The columns as lists
c1 = list(range(10))
c2 = list(range(10))
c3 = ["a", "b", "c", "d", "e", "f", "g", "i", "j", "k"]
c4 = [True, False, True, False, True] * 2
c5 = list(range(10))
c6 = [(i + 1, i + 2) for i in range(10)]
c7 = [(i + 1, i + 2, i + 3) for i in range(10)]
c8 = [tuple(i + j for j in range(9)) for i in range(10)]
c9 = [tuple(i + j for j in range(6)) for i in range(10)]
c10 = [(i + 1, i + 2, i + 3) for i in range(10)]
c11 = [gen_shoebox() for i in range(10)]
# Create a table with some elements
table = flex.reflection_table()
table["col1"] = flex.int(c1)
table["col2"] = flex.double(c2)
table["col3"] = flex.std_string(c3)
table["col4"] = flex.bool(c4)
table["col5"] = flex.size_t(c5)
table["col6"] = flex.vec2_double(c6)
table["col7"] = flex.vec3_double(c7)
table["col8"] = flex.mat3_double(c8)
table["col9"] = flex.int6(c9)
table["col10"] = flex.miller_index(c10)
table["col11"] = flex.shoebox(c11)
obj = table.as_msgpack()
new_table = flex.reflection_table.from_msgpack(obj)
assert new_table.is_consistent()
assert new_table.nrows() == 10
assert new_table.ncols() == 11
assert all(tuple(a == b for a, b in zip(new_table["col1"], c1)))
assert all(tuple(a == b for a, b in zip(new_table["col2"], c2)))
assert all(tuple(a == b for a, b in zip(new_table["col3"], c3)))
assert all(tuple(a == b for a, b in zip(new_table["col4"], c4)))
assert all(tuple(a == b for a, b in zip(new_table["col5"], c5)))
assert all(tuple(a == b for a, b in zip(new_table["col6"], c6)))
assert all(tuple(a == b for a, b in zip(new_table["col7"], c7)))
assert all(tuple(a == b for a, b in zip(new_table["col8"], c8)))
assert all(tuple(a == b for a, b in zip(new_table["col9"], c9)))
assert all(tuple(a == b for a, b in zip(new_table["col10"], c10)))
assert all(tuple(compare(a, b) for a, b in zip(new_table["col11"], c11)))
table.as_msgpack_file(tmpdir.join("reflections.mpack").strpath)
new_table = flex.reflection_table.from_msgpack_file(
tmpdir.join("reflections.mpack").strpath)
assert new_table.is_consistent()
assert new_table.nrows() == 10
assert new_table.ncols() == 11
assert all(tuple(a == b for a, b in zip(new_table["col1"], c1)))
assert all(tuple(a == b for a, b in zip(new_table["col2"], c2)))
assert all(tuple(a == b for a, b in zip(new_table["col3"], c3)))
assert all(tuple(a == b for a, b in zip(new_table["col4"], c4)))
assert all(tuple(a == b for a, b in zip(new_table["col5"], c5)))
assert all(tuple(a == b for a, b in zip(new_table["col6"], c6)))
assert all(tuple(a == b for a, b in zip(new_table["col7"], c7)))
assert all(tuple(a == b for a, b in zip(new_table["col8"], c8)))
assert all(tuple(a == b for a, b in zip(new_table["col9"], c9)))
assert all(tuple(a == b for a, b in zip(new_table["col10"], c10)))
assert all(tuple(compare(a, b) for a, b in zip(new_table["col11"], c11)))
print(len(selection), len(reflections))
from matplotlib import pylab
pylab.hist(I_obs, bins=50)
pylab.show()
# from matplotlib import pylab
# s0 = matrix.col(experiments[0].beam.get_s0())
# D = [matrix.col(s2).length()/s0.length() for s2 in s2_obs]
# pylab.hist(D, bins=50)
# pylab.show()
angles = []
for s1, s2 in zip(s1_obs, s2_obs):
a = matrix.col(s1).angle(matrix.col(s2), deg=True)
angles.append(a)
print("Mean angle between s1 and s2 %f degrees " %
(sum(angles) / len(angles)))
# Do the ray intersection
reflections["intensity.sum.value"] = I_obs
reflections["s1_obs"] = s1_obs
reflections["s1"] = s2_obs
reflections["xyzobs.px"] = flex.vec2_double([
experiments[0].detector[0].get_ray_intersection_px(s1) for s1 in s1_obs
])
# Do the refinement
refiner = ProfileRefiner(experiments[0], reflections)
def index(self):
# most of this is the same as dials.algorithms.indexing.indexer.indexer_base.index(), with some stills
# specific modifications (don't re-index after choose best orientation matrix, but use the indexing from
# choose best orientation matrix, also don't use macrocycles) of refinement after indexing.
# 2017 update: do accept multiple lattices per shot
if self.params.refinement_protocol.n_macro_cycles > 1:
raise Sorry(
"For stills, please set refinement_protocol.n_macro_cycles = 1"
)
experiments = ExperimentList()
had_refinement_error = False
have_similar_crystal_models = False
while True:
self.d_min = self.params.refinement_protocol.d_min_start
if had_refinement_error or have_similar_crystal_models:
break
max_lattices = self.params.multiple_lattice_search.max_lattices
if max_lattices is not None and len(experiments) >= max_lattices:
break
if len(experiments) > 0:
cutoff_fraction = \
self.params.multiple_lattice_search.recycle_unindexed_reflections_cutoff
d_spacings = 1 / self.reflections['rlp'].norms()
d_min_indexed = flex.min(
d_spacings.select(self.indexed_reflections))
min_reflections_for_indexing = \
cutoff_fraction * len(self.reflections.select(d_spacings > d_min_indexed))
crystal_ids = self.reflections.select(
d_spacings > d_min_indexed)['id']
if (crystal_ids
== -1).count(True) < min_reflections_for_indexing:
logger.info(
"Finish searching for more lattices: %i unindexed reflections remaining."
% (min_reflections_for_indexing))
break
n_lattices_previous_cycle = len(experiments)
# index multiple lattices per shot
if len(experiments) == 0:
experiments.extend(self.find_lattices())
if len(experiments) == 0:
raise Sorry("No suitable lattice could be found.")
else:
try:
new = self.find_lattices()
experiments.extend(new)
except Exception as e:
logger.info("Indexing remaining reflections failed")
logger.debug(
"Indexing remaining reflections failed, exception:\n" +
str(e))
# reset reflection lattice flags
# the lattice a given reflection belongs to: a value of -1 indicates
# that a reflection doesn't belong to any lattice so far
self.reflections['id'] = flex.int(len(self.reflections), -1)
self.index_reflections(experiments, self.reflections)
if len(experiments) == n_lattices_previous_cycle:
# no more lattices found
break
if not self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None:
# now apply the space group symmetry only after the first indexing
# need to make sure that the symmetrized orientation is similar to the P1 model
target_space_group = self.target_symmetry_primitive.space_group(
)
for i_cryst, cryst in enumerate(experiments.crystals()):
if i_cryst >= n_lattices_previous_cycle:
new_cryst, cb_op_to_primitive = self.apply_symmetry(
cryst, target_space_group)
if self.cb_op_primitive_inp is not None:
new_cryst = new_cryst.change_basis(
self.cb_op_primitive_inp)
logger.info(new_cryst.get_space_group().info())
cryst.update(new_cryst)
cryst.set_space_group(
self.params.known_symmetry.space_group.group())
for i_expt, expt in enumerate(experiments):
if expt.crystal is not cryst:
continue
if not cb_op_to_primitive.is_identity_op():
miller_indices = self.reflections[
'miller_index'].select(
self.reflections['id'] == i_expt)
miller_indices = cb_op_to_primitive.apply(
miller_indices)
self.reflections['miller_index'].set_selected(
self.reflections['id'] == i_expt,
miller_indices)
if self.cb_op_primitive_inp is not None:
miller_indices = self.reflections[
'miller_index'].select(
self.reflections['id'] == i_expt)
miller_indices = self.cb_op_primitive_inp.apply(
miller_indices)
self.reflections['miller_index'].set_selected(
self.reflections['id'] == i_expt,
miller_indices)
# discard nearly overlapping lattices on the same shot
if len(experiments) > 1:
from dials.algorithms.indexing.compare_orientation_matrices \
import difference_rotation_matrix_axis_angle
cryst_b = experiments.crystals()[-1]
have_similar_crystal_models = False
for i_a, cryst_a in enumerate(experiments.crystals()[:-1]):
R_ab, axis, angle, cb_op_ab = \
difference_rotation_matrix_axis_angle(cryst_a, cryst_b)
min_angle = self.params.multiple_lattice_search.minimum_angular_separation
if abs(angle) < min_angle: # degrees
logger.info(
"Crystal models too similar, rejecting crystal %i:"
% (len(experiments)))
logger.info(
"Rotation matrix to transform crystal %i to crystal %i"
% (i_a + 1, len(experiments)))
logger.info(R_ab)
logger.info("Rotation of %.3f degrees" % angle +
" about axis (%.3f, %.3f, %.3f)" % axis)
#show_rotation_matrix_differences([cryst_a, cryst_b])
have_similar_crystal_models = True
del experiments[-1]
break
if have_similar_crystal_models:
break
self.indexed_reflections = (self.reflections['id'] > -1)
if self.d_min is None:
sel = self.reflections['id'] <= -1
else:
sel = flex.bool(len(self.reflections), False)
lengths = 1 / self.reflections['rlp'].norms()
isel = (lengths >= self.d_min).iselection()
sel.set_selected(isel, True)
sel.set_selected(self.reflections['id'] > -1, False)
self.unindexed_reflections = self.reflections.select(sel)
reflections_for_refinement = self.reflections.select(
self.indexed_reflections)
if len(self.params.stills.isoforms) > 0:
logger.info("")
logger.info("#" * 80)
logger.info("Starting refinement")
logger.info("#" * 80)
logger.info("")
import copy
isoform_experiments = ExperimentList()
isoform_reflections = flex.reflection_table()
# Note, changes to params after initial indexing. Cannot use tie to target when fixing the unit cell.
self.all_params.refinement.reflections.outlier.algorithm = "null"
self.all_params.refinement.parameterisation.crystal.fix = "cell"
self.all_params.refinement.parameterisation.crystal.unit_cell.restraints.tie_to_target = []
for expt_id, experiment in enumerate(experiments):
reflections = reflections_for_refinement.select(
reflections_for_refinement['id'] == expt_id)
reflections['id'] = flex.int(len(reflections), 0)
refiners = []
for isoform in self.params.stills.isoforms:
iso_experiment = copy.deepcopy(experiment)
crystal = iso_experiment.crystal
if isoform.lookup_symbol != crystal.get_space_group(
).type().lookup_symbol():
logger.info(
"Crystal isoform lookup_symbol %s does not match isoform %s lookup_symbol %s"
% (crystal.get_space_group().type(
).lookup_symbol(), isoform.name,
isoform.lookup_symbol))
continue
crystal.set_B(isoform.cell.fractionalization_matrix())
logger.info("Refining isoform %s" % isoform.name)
refiners.append(
e_refine(params=self.all_params,
experiments=ExperimentList(
[iso_experiment]),
reflections=reflections,
graph_verbose=False))
if len(refiners) == 0:
raise Sorry(
"No isoforms had a lookup symbol that matched")
positional_rmsds = [
math.sqrt(P.rmsds()[0]**2 + P.rmsds()[1]**2)
for P in refiners
]
logger.info("Positional rmsds for all isoforms:" +
str(positional_rmsds))
minrmsd_mm = min(positional_rmsds)
minindex = positional_rmsds.index(minrmsd_mm)
logger.info(
"The smallest rmsd is %5.1f um from isoform %s" %
(1000. * minrmsd_mm,
self.params.stills.isoforms[minindex].name))
if self.params.stills.isoforms[
minindex].rmsd_target_mm is not None:
logger.info("Asserting %f < %f" %
(minrmsd_mm, self.params.stills.
isoforms[minindex].rmsd_target_mm))
assert minrmsd_mm < self.params.stills.isoforms[
minindex].rmsd_target_mm
logger.info("Acceptable rmsd for isoform %s." %
(self.params.stills.isoforms[minindex].name))
if len(self.params.stills.isoforms) == 2:
logger.info(
"Rmsd gain over the other isoform %5.1f um." %
(1000. *
abs(positional_rmsds[0] - positional_rmsds[1])))
R = refiners[minindex]
# Now one last check to see if direct beam is out of bounds
if self.params.stills.isoforms[
minindex].beam_restraint is not None:
from scitbx import matrix
refined_beam = matrix.col(
R.get_experiments()
[0].detector[0].get_beam_centre_lab(
experiments[0].beam.get_s0())[0:2])
known_beam = matrix.col(
self.params.stills.isoforms[minindex].
beam_restraint)
logger.info(
"Asserting difference in refined beam center and expected beam center %f < %f"
%
((refined_beam - known_beam).length(), self.params.
stills.isoforms[minindex].rmsd_target_mm))
assert (refined_beam - known_beam
).length() < self.params.stills.isoforms[
minindex].rmsd_target_mm
# future--circle of confusion could be given as a separate length in mm instead of reusing rmsd_target
experiment = R.get_experiments()[0]
experiment.crystal.identified_isoform = self.params.stills.isoforms[
minindex].name
isoform_experiments.append(experiment)
reflections['id'] = flex.int(len(reflections), expt_id)
isoform_reflections.extend(reflections)
experiments = isoform_experiments
reflections_for_refinement = isoform_reflections
try:
refined_experiments, refined_reflections = self.refine(
experiments, reflections_for_refinement)
except Exception as e:
s = str(e)
if len(experiments) == 1:
raise Sorry(e)
had_refinement_error = True
logger.info("Refinement failed:")
logger.info(s)
del experiments[-1]
break
# sanity check for unrealistic unit cell volume increase during refinement
# usually this indicates too many parameters are being refined given the
# number of observations provided.
if not self.params.refinement_protocol.disable_unit_cell_volume_sanity_check:
for orig_expt, refined_expt in zip(experiments,
refined_experiments):
uc1 = orig_expt.crystal.get_unit_cell()
uc2 = refined_expt.crystal.get_unit_cell()
volume_change = abs(uc1.volume() -
uc2.volume()) / uc1.volume()
cutoff = 0.5
if volume_change > cutoff:
msg = "\n".join((
"Unrealistic unit cell volume increase during refinement of %.1f%%.",
"Please try refining fewer parameters, either by enforcing symmetry",
"constraints (space_group=) and/or disabling experimental geometry",
"refinement (detector.fix=all and beam.fix=all). To disable this",
"sanity check set disable_unit_cell_volume_sanity_check=True."
)) % (100 * volume_change)
raise Sorry(msg)
self.refined_reflections = refined_reflections.select(
refined_reflections['id'] > -1)
for i, imageset in enumerate(self.imagesets):
ref_sel = self.refined_reflections.select(
self.refined_reflections['imageset_id'] == i)
ref_sel = ref_sel.select(ref_sel['id'] >= 0)
for i_expt in set(ref_sel['id']):
expt = refined_experiments[i_expt]
imageset.set_detector(expt.detector)
imageset.set_beam(expt.beam)
imageset.set_goniometer(expt.goniometer)
imageset.set_scan(expt.scan)
expt.imageset = imageset
if not (self.all_params.refinement.parameterisation.beam.fix
== 'all' and
self.all_params.refinement.parameterisation.detector.fix
== 'all'):
# Experimental geometry may have changed - re-map centroids to
# reciprocal space
spots_mm = self.reflections
self.reflections = flex.reflection_table()
for i, imageset in enumerate(self.imagesets):
spots_sel = spots_mm.select(spots_mm['imageset_id'] == i)
self.map_centroids_to_reciprocal_space(
spots_sel, imageset.get_detector(),
imageset.get_beam(), imageset.get_goniometer())
self.reflections.extend(spots_sel)
# update for next cycle
experiments = refined_experiments
self.refined_experiments = refined_experiments
if not 'refined_experiments' in locals():
raise Sorry("None of the experiments could refine.")
# discard experiments with zero reflections after refinement
id_set = set(self.refined_reflections['id'])
if len(id_set) < len(self.refined_experiments):
filtered_refined_reflections = flex.reflection_table()
for i in xrange(len(self.refined_experiments)):
if i not in id_set:
del self.refined_experiments[i]
for old, new in zip(sorted(id_set), range(len(id_set))):
subset = self.refined_reflections.select(
self.refined_reflections['id'] == old)
subset['id'] = flex.int(len(subset), new)
filtered_refined_reflections.extend(subset)
self.refined_reflections = filtered_refined_reflections
if len(self.refined_experiments) > 1:
from dials.algorithms.indexing.compare_orientation_matrices \
import show_rotation_matrix_differences
show_rotation_matrix_differences(
self.refined_experiments.crystals(), out=info_handle)
logger.info("Final refined crystal models:")
for i, crystal_model in enumerate(self.refined_experiments.crystals()):
n_indexed = 0
for i_expt in experiments.where(crystal=crystal_model):
n_indexed += (self.reflections['id'] == i).count(True)
logger.info("model %i (%i reflections):" % (i + 1, n_indexed))
logger.info(crystal_model)
if 'xyzcal.mm' in self.refined_reflections: # won't be there if refine_all_candidates = False and no isoforms
self.refined_reflections['xyzcal.px'] = flex.vec3_double(
len(self.refined_reflections))
for i, imageset in enumerate(self.imagesets):
imgset_sel = self.refined_reflections['imageset_id'] == i
# set xyzcal.px field in self.refined_reflections
refined_reflections = self.refined_reflections.select(
imgset_sel)
panel_numbers = flex.size_t(refined_reflections['panel'])
xyzcal_mm = refined_reflections['xyzcal.mm']
x_mm, y_mm, z_rad = xyzcal_mm.parts()
xy_cal_mm = flex.vec2_double(x_mm, y_mm)
xy_cal_px = flex.vec2_double(len(xy_cal_mm))
for i_panel in range(len(imageset.get_detector())):
panel = imageset.get_detector()[i_panel]
sel = (panel_numbers == i_panel)
isel = sel.iselection()
ref_panel = refined_reflections.select(
panel_numbers == i_panel)
xy_cal_px.set_selected(
sel, panel.millimeter_to_pixel(xy_cal_mm.select(sel)))
x_px, y_px = xy_cal_px.parts()
scan = imageset.get_scan()
if scan is not None:
z_px = scan.get_array_index_from_angle(z_rad, deg=False)
else:
# must be a still image, z centroid not meaningful
z_px = z_rad
xyzcal_px = flex.vec3_double(x_px, y_px, z_px)
self.refined_reflections['xyzcal.px'].set_selected(
imgset_sel, xyzcal_px)
def run(self):
''' Parse the options. '''
from dials.util.options import flatten_experiments, flatten_reflections
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
self.params = params
experiments = flatten_experiments(params.input.experiments)
# Find all detector objects
detectors = experiments.detectors()
# Verify inputs
if len(params.input.reflections) == len(detectors) and len(detectors) > 1:
# case for passing in multiple images on the command line
assert len(params.input.reflections) == len(detectors)
reflections = flex.reflection_table()
for expt_id in xrange(len(detectors)):
subset = params.input.reflections[expt_id].data
subset['id'] = flex.int(len(subset), expt_id)
reflections.extend(subset)
else:
# case for passing in combined experiments and reflections
reflections = flatten_reflections(params.input.reflections)[0]
detector = detectors[0]
#from dials.algorithms.refinement.prediction import ExperimentsPredictor
#ref_predictor = ExperimentsPredictor(experiments, force_stills=experiments.all_stills())
print "N reflections total:", len(reflections)
if params.residuals.exclude_outliers:
reflections = reflections.select(reflections.get_flags(reflections.flags.used_in_refinement))
print "N reflections used in refinement:", len(reflections)
print "Reporting only on those reflections used in refinement"
if self.params.residuals.i_sigi_cutoff is not None:
sel = (reflections['intensity.sum.value']/flex.sqrt(reflections['intensity.sum.variance'])) >= self.params.residuals.i_sigi_cutoff
reflections = reflections.select(sel)
print "After filtering by I/sigi cutoff of %f, there are %d reflections left"%(self.params.residuals.i_sigi_cutoff,len(reflections))
reflections['difference_vector_norms'] = (reflections['xyzcal.mm']-reflections['xyzobs.mm.value']).norms()
n = len(reflections)
rmsd = self.get_weighted_rmsd(reflections)
print "Dataset RMSD (microns)", rmsd * 1000
if params.tag is None:
tag = ''
else:
tag = '%s '%params.tag
# set up delta-psi ratio heatmap
p = flex.int() # positive
n = flex.int() # negative
for i in set(reflections['id']):
exprefls = reflections.select(reflections['id']==i)
p.append(len(exprefls.select(exprefls['delpsical.rad']>0)))
n.append(len(exprefls.select(exprefls['delpsical.rad']<0)))
plt.hist2d(p, n, bins=30)
cb = plt.colorbar()
cb.set_label("N images")
plt.title(r"%s2D histogram of pos vs. neg $\Delta\Psi$ per image"%tag)
plt.xlabel(r"N reflections with $\Delta\Psi$ > 0")
plt.ylabel(r"N reflections with $\Delta\Psi$ < 0")
self.delta_scalar = 50
# Iterate through the detectors, computing detector statistics at the per-panel level (IE one statistic per panel)
# Per panel dictionaries
rmsds = {}
refl_counts = {}
transverse_rmsds = {}
radial_rmsds = {}
ttdpcorr = {}
pg_bc_dists = {}
mean_delta_two_theta = {}
# per panelgroup flex arrays
pg_rmsds = flex.double()
pg_r_rmsds = flex.double()
pg_t_rmsds = flex.double()
pg_refls_count = flex.int()
pg_refls_count_d = {}
table_header = ["PG id", "RMSD","Radial", "Transverse", "N refls"]
table_header2 = ["","(um)","RMSD (um)","RMSD (um)",""]
table_data = []
table_data.append(table_header)
table_data.append(table_header2)
# Compute a set of radial and transverse displacements for each reflection
print "Setting up stats..."
tmp = flex.reflection_table()
# Need to construct a variety of vectors
for panel_id, panel in enumerate(detector):
panel_refls = reflections.select(reflections['panel'] == panel_id)
bcl = flex.vec3_double()
tto = flex.double()
ttc = flex.double()
# Compute the beam center in lab space (a vector pointing from the origin to where the beam would intersect
# the panel, if it did intersect the panel)
for expt_id in set(panel_refls['id']):
beam = experiments[expt_id].beam
s0 = beam.get_s0()
expt_refls = panel_refls.select(panel_refls['id'] == expt_id)
beam_centre = panel.get_beam_centre_lab(s0)
bcl.extend(flex.vec3_double(len(expt_refls), beam_centre))
obs_x, obs_y, _ = expt_refls['xyzobs.px.value'].parts()
cal_x, cal_y, _ = expt_refls['xyzcal.px'].parts()
tto.extend(flex.double([panel.get_two_theta_at_pixel(s0, (obs_x[i], obs_y[i])) for i in xrange(len(expt_refls))]))
ttc.extend(flex.double([panel.get_two_theta_at_pixel(s0, (cal_x[i], cal_y[i])) for i in xrange(len(expt_refls))]))
panel_refls['beam_centre_lab'] = bcl
panel_refls['two_theta_obs'] = tto * (180/math.pi)
panel_refls['two_theta_cal'] = ttc * (180/math.pi) #+ (0.5*panel_refls['delpsical.rad']*panel_refls['two_theta_obs'])
# Compute obs in lab space
x, y, _ = panel_refls['xyzobs.mm.value'].parts()
c = flex.vec2_double(x, y)
panel_refls['obs_lab_coords'] = panel.get_lab_coord(c)
# Compute deltaXY in panel space. This vector is relative to the panel origin
x, y, _ = (panel_refls['xyzcal.mm'] - panel_refls['xyzobs.mm.value']).parts()
# Convert deltaXY to lab space, subtracting off of the panel origin
panel_refls['delta_lab_coords'] = panel.get_lab_coord(flex.vec2_double(x,y)) - panel.get_origin()
tmp.extend(panel_refls)
reflections = tmp
# The radial vector points from the center of the reflection to the beam center
radial_vectors = (reflections['obs_lab_coords'] - reflections['beam_centre_lab']).each_normalize()
# The transverse vector is orthogonal to the radial vector and the beam vector
transverse_vectors = radial_vectors.cross(reflections['beam_centre_lab']).each_normalize()
# Compute the raidal and transverse components of each deltaXY
reflections['radial_displacements'] = reflections['delta_lab_coords'].dot(radial_vectors)
reflections['transverse_displacements'] = reflections['delta_lab_coords'].dot(transverse_vectors)
# Iterate through the detector at the specified hierarchy level
for pg_id, pg in enumerate(iterate_detector_at_level(detector.hierarchy(), 0, params.hierarchy_level)):
pg_msd_sum = 0
pg_r_msd_sum = 0
pg_t_msd_sum = 0
pg_refls = 0
pg_delpsi = flex.double()
pg_deltwotheta = flex.double()
for p in iterate_panels(pg):
panel_id = id_from_name(detector, p.get_name())
panel_refls = reflections.select(reflections['panel'] == panel_id)
n = len(panel_refls)
pg_refls += n
delta_x = panel_refls['xyzcal.mm'].parts()[0] - panel_refls['xyzobs.mm.value'].parts()[0]
delta_y = panel_refls['xyzcal.mm'].parts()[1] - panel_refls['xyzobs.mm.value'].parts()[1]
tmp = flex.sum((delta_x**2)+(delta_y**2))
pg_msd_sum += tmp
r = panel_refls['radial_displacements']
t = panel_refls['transverse_displacements']
pg_r_msd_sum += flex.sum_sq(r)
pg_t_msd_sum += flex.sum_sq(t)
pg_delpsi.extend(panel_refls['delpsical.rad']*180/math.pi)
pg_deltwotheta.extend(panel_refls['two_theta_obs'] - panel_refls['two_theta_cal'])
bc = col(pg.get_beam_centre_lab(s0))
ori = get_center(pg)
pg_bc_dists[pg.get_name()] = (ori-bc).length()
if len(pg_deltwotheta) > 0:
mean_delta_two_theta[pg.get_name()] = flex.mean(pg_deltwotheta)
else:
mean_delta_two_theta[pg.get_name()] = 0
if pg_refls == 0:
pg_rmsd = pg_r_rmsd = pg_t_rmsd = 0
else:
pg_rmsd = math.sqrt(pg_msd_sum/pg_refls) * 1000
pg_r_rmsd = math.sqrt(pg_r_msd_sum/pg_refls) * 1000
pg_t_rmsd = math.sqrt(pg_t_msd_sum/pg_refls) * 1000
pg_rmsds.append(pg_rmsd)
pg_r_rmsds.append(pg_r_rmsd)
pg_t_rmsds.append(pg_t_rmsd)
pg_refls_count.append(pg_refls)
pg_refls_count_d[pg.get_name()] = pg_refls
table_data.append(["%d"%pg_id, "%.1f"%pg_rmsd, "%.1f"%pg_r_rmsd, "%.1f"%pg_t_rmsd, "%6d"%pg_refls])
refl_counts[pg.get_name()] = pg_refls
if pg_refls == 0:
rmsds[p.get_name()] = -1
radial_rmsds[p.get_name()] = -1
transverse_rmsds[p.get_name()] = -1
ttdpcorr[pg.get_name()] = -1
else:
rmsds[pg.get_name()] = pg_rmsd
radial_rmsds[pg.get_name()] = pg_r_rmsd
transverse_rmsds[pg.get_name()] = pg_t_rmsd
lc = flex.linear_correlation(pg_delpsi, pg_deltwotheta)
ttdpcorr[pg.get_name()] = lc.coefficient()
r1 = ["Weighted mean"]
r2 = ["Weighted stddev"]
if len(pg_rmsds) > 1:
stats = flex.mean_and_variance(pg_rmsds, pg_refls_count.as_double())
r1.append("%.1f"%stats.mean())
r2.append("%.1f"%stats.gsl_stats_wsd())
stats = flex.mean_and_variance(pg_r_rmsds, pg_refls_count.as_double())
r1.append("%.1f"%stats.mean())
r2.append("%.1f"%stats.gsl_stats_wsd())
stats = flex.mean_and_variance(pg_t_rmsds, pg_refls_count.as_double())
r1.append("%.1f"%stats.mean())
r2.append("%.1f"%stats.gsl_stats_wsd())
else:
r1.extend([""]*3)
r2.extend([""]*3)
r1.append("")
r2.append("")
table_data.append(r1)
table_data.append(r2)
table_data.append(["Mean", "", "", "", "%8.1f"%flex.mean(pg_refls_count.as_double())])
from libtbx import table_utils
print "Detector statistics. Angles in degrees, RMSDs in microns"
print table_utils.format(table_data,has_header=2,justify='center',delim=" ")
self.histogram(reflections, '%sDifference vector norms (mm)'%tag)
if params.show_plots:
if self.params.tag is None:
t = ""
else:
t = "%s "%self.params.tag
self.image_rmsd_histogram(reflections, tag)
# Plots! these are plots with callbacks to draw on individual panels
self.detector_plot_refls(detector, reflections, '%sOverall positional displacements (mm)'%tag, show=False, plot_callback=self.plot_obs_colored_by_deltas)
self.detector_plot_refls(detector, reflections, '%sRadial positional displacements (mm)'%tag, show=False, plot_callback=self.plot_obs_colored_by_radial_deltas)
self.detector_plot_refls(detector, reflections, '%sTransverse positional displacements (mm)'%tag, show=False, plot_callback=self.plot_obs_colored_by_transverse_deltas)
self.detector_plot_refls(detector, reflections, r'%s$\Delta\Psi$'%tag, show=False, plot_callback=self.plot_obs_colored_by_deltapsi, colorbar_units=r"$\circ$")
self.detector_plot_refls(detector, reflections, r'%s$\Delta$XY*%s'%(tag, self.delta_scalar), show=False, plot_callback=self.plot_deltas)
self.detector_plot_refls(detector, reflections, '%sSP Manual CDF'%tag, show=False, plot_callback=self.plot_cdf_manually)
self.detector_plot_refls(detector, reflections, r'%s$\Delta$XY Histograms'%tag, show=False, plot_callback=self.plot_histograms)
self.detector_plot_refls(detector, reflections, r'%sRadial displacements vs. $\Delta\Psi$, colored by $\Delta$XY'%tag, show=False, plot_callback=self.plot_radial_displacements_vs_deltapsi)
self.detector_plot_refls(detector, reflections, r'%sDistance vector norms'%tag, show=False, plot_callback=self.plot_difference_vector_norms_histograms)
# Plot intensity vs. radial_displacement
fig = plt.figure()
panel_id = 15
panel_refls = reflections.select(reflections['panel'] == panel_id)
a = panel_refls['radial_displacements']
b = panel_refls['intensity.sum.value']
sel = (a > -0.2) & (a < 0.2) & (b < 50000)
plt.hist2d(a.select(sel), b.select(sel), bins=100)
plt.title("%s2D histogram of intensity vs. radial displacement for panel %d"%(tag, panel_id))
plt.xlabel("Radial displacement (mm)")
plt.ylabel("Intensity")
ax = plt.colorbar()
ax.set_label("Counts")
# Plot delta 2theta vs. deltapsi
n_bins = 10
bin_size = len(reflections)//n_bins
bin_low = []
bin_high = []
data = flex.sorted(reflections['two_theta_obs'])
for i in xrange(n_bins):
bin_low = data[i*bin_size]
if (i+1)*bin_size >= len(reflections):
bin_high = data[-1]
else:
bin_high = data[(i+1)*bin_size]
refls = reflections.select((reflections['two_theta_obs'] >= bin_low) &
(reflections['two_theta_obs'] <= bin_high))
a = refls['delpsical.rad']*180/math.pi
b = refls['two_theta_obs'] - refls['two_theta_cal']
fig = plt.figure()
sel = (a > -0.2) & (a < 0.2) & (b > -0.05) & (b < 0.05)
plt.hist2d(a.select(sel), b.select(sel), bins=50, range = [[-0.2, 0.2], [-0.05, 0.05]])
cb = plt.colorbar()
cb.set_label("N reflections")
plt.title(r'%sBin %d (%.02f, %.02f 2$\Theta$) $\Delta2\Theta$ vs. $\Delta\Psi$. Showing %d of %d refls'%(tag,i,bin_low,bin_high,len(a.select(sel)),len(a)))
plt.xlabel(r'$\Delta\Psi \circ$')
plt.ylabel(r'$\Delta2\Theta \circ$')
# Plot delta 2theta vs. 2theta
a = reflections['two_theta_obs']#[:71610]
b = reflections['two_theta_obs'] - reflections['two_theta_cal']
fig = plt.figure()
limits = -0.05, 0.05
sel = (b > limits[0]) & (b < limits[1])
plt.hist2d(a.select(sel), b.select(sel), bins=100, range=((0,50), limits))
plt.clim((0,100))
cb = plt.colorbar()
cb.set_label("N reflections")
plt.title(r'%s$\Delta2\Theta$ vs. 2$\Theta$. Showing %d of %d refls'%(tag,len(a.select(sel)),len(a)))
plt.xlabel(r'2$\Theta \circ$')
plt.ylabel(r'$\Delta2\Theta \circ$')
# calc the trendline
z = np.polyfit(a.select(sel), b.select(sel), 1)
print 'y=%.7fx+(%.7f)'%(z[0],z[1])
# Plots with single values per panel
self.detector_plot_dict(detector, refl_counts, u"%s N reflections"%t, u"%6d", show=False)
self.detector_plot_dict(detector, rmsds, "%s Positional RMSDs (microns)"%t, u"%4.1f", show=False)
self.detector_plot_dict(detector, radial_rmsds, "%s Radial RMSDs (microns)"%t, u"%4.1f", show=False)
self.detector_plot_dict(detector, transverse_rmsds, "%s Transverse RMSDs (microns)"%t, u"%4.1f", show=False)
self.detector_plot_dict(detector, ttdpcorr, r"%s $\Delta2\Theta$ vs. $\Delta\Psi$ CC"%t, u"%5.3f", show=False)
self.plot_unitcells(experiments)
self.plot_data_by_two_theta(reflections, tag)
# Plot data by panel group
sorted_values = sorted(pg_bc_dists.values())
vdict = {}
for k in pg_bc_dists:
vdict[pg_bc_dists[k]] = k
sorted_keys = [vdict[v] for v in sorted_values if vdict[v] in rmsds]
x = [sorted_values[i] for i in xrange(len(sorted_values)) if pg_bc_dists.keys()[i] in rmsds]
self.plot_multi_data(x,
[[pg_refls_count_d[k] for k in sorted_keys],
([rmsds[k] for k in sorted_keys],
[radial_rmsds[k] for k in sorted_keys],
[transverse_rmsds[k] for k in sorted_keys]),
[radial_rmsds[k]/transverse_rmsds[k] for k in sorted_keys],
[mean_delta_two_theta[k] for k in sorted_keys]],
"Panel group distance from beam center (mm)",
["N reflections",
("Overall RMSD",
"Radial RMSD",
"Transverse RMSD"),
"R/T RMSD ratio",
"Delta two theta"],
["N reflections",
"RMSD (microns)",
"R/T RMSD ratio",
"Delta two theta (degrees)"],
"%sData by panelgroup"%tag)
if self.params.save_pdf:
pp = PdfPages('residuals_%s.pdf'%(tag.strip()))
for i in plt.get_fignums():
pp.savefig(plt.figure(i))
pp.close()
else:
plt.show()
def generate_mask(imageset: ImageSet,
params: libtbx.phil.scope_extract) -> Tuple[flex.bool]:
"""Generate a mask based on the input parameters.
Args:
imageset (ImageSet): The imageset for which to generate a mask
params (libtbx.phil.scope_extract): The phil parameters for mask generation. This
should be an extract of `dials.util.masking.phil_scope`
"""
# Get the detector and beam
detector = imageset.get_detector()
beam = imageset.get_beam()
# Create the mask for each panel
masks = []
for index, panel in enumerate(detector):
mask = flex.bool(flex.grid(reversed(panel.get_image_size())), True)
# Add a border around the image
if params.border > 0:
logger.debug(f"Generating border mask:\n border = {params.border}")
border = params.border
height, width = mask.all()
borderx = flex.bool(flex.grid(border, width), False)
bordery = flex.bool(flex.grid(height, border), False)
mask[0:border, :] = borderx
mask[-border:, :] = borderx
mask[:, 0:border] = bordery
mask[:, -border:] = bordery
# Apply the untrusted regions
for region in params.untrusted:
if region.panel is None:
region.panel = 0
if region.panel == index:
if not any([
region.circle, region.rectangle, region.polygon,
region.pixel
]):
mask[:, :] = flex.bool(flex.grid(mask.focus()), False)
continue
if region.circle is not None:
xc, yc, radius = region.circle
logger.debug("Generating circle mask:\n" +
f" panel = {region.panel}\n" +
f" xc = {xc}\n" + f" yc = {yc}\n" +
f" radius = {radius}")
mask_untrusted_circle(mask, xc, yc, radius)
if region.rectangle is not None:
x0, x1, y0, y1 = region.rectangle
logger.debug("Generating rectangle mask:\n" +
f" panel = {region.panel}\n" +
f" x0 = {x0}\n" + f" y0 = {y0}\n" +
f" x1 = {x1}\n" + f" y1 = {y1}")
mask_untrusted_rectangle(mask, x0, x1, y0, y1)
if region.polygon is not None:
assert (len(region.polygon) %
2 == 0), "Polygon must contain 2D coords"
vertices = []
for i in range(int(len(region.polygon) / 2)):
x = region.polygon[2 * i]
y = region.polygon[2 * i + 1]
vertices.append((x, y))
polygon = flex.vec2_double(vertices)
logger.debug(
f"Generating polygon mask:\n panel = {region.panel}\n"
+ "\n".join(f" coord = {vertex}"
for vertex in vertices))
mask_untrusted_polygon(mask, polygon)
if region.pixel is not None:
mask[region.pixel] = False
# Generate high and low resolution masks
if params.d_min is not None:
logger.debug(
f"Generating high resolution mask:\n d_min = {params.d_min}")
_apply_resolution_mask(mask, beam, panel, 0, params.d_min)
if params.d_max is not None:
logger.debug(
f"Generating low resolution mask:\n d_max = {params.d_max}")
d_max = params.d_max
d_inf = max(d_max + 1, 1e9)
_apply_resolution_mask(mask, beam, panel, d_max, d_inf)
try:
# Mask out the resolution range
for drange in params.resolution_range:
d_min = min(drange)
d_max = max(drange)
assert d_min < d_max, "d_min must be < d_max"
logger.debug("Generating resolution range mask:\n" +
f" d_min = {d_min}\n" + f" d_max = {d_max}")
_apply_resolution_mask(mask, beam, panel, d_min, d_max)
except TypeError:
# Catch the default value None of params.resolution_range
if any(params.resolution_range):
raise
# Mask out the resolution ranges for the ice rings
for drange in generate_ice_ring_resolution_ranges(
beam, panel, params.ice_rings):
d_min = min(drange)
d_max = max(drange)
assert d_min < d_max, "d_min must be < d_max"
logger.debug("Generating ice ring mask:\n" +
f" d_min = {d_min:.4f}\n" + f" d_max = {d_max:.4f}")
_apply_resolution_mask(mask, beam, panel, d_min, d_max)
# Add to the list
masks.append(mask)
# Return the mask
return tuple(masks)
def tv_alg(image, mask, tolerance=1e-3, max_iter=10):
from dials.array_family import flex
from scitbx import matrix
# Set lambda
L = flex.double(image.accessor(), 0)
for j in range(image.all()[0]):
for i in range(image.all()[1]):
if mask[j,i]:
L[j,i] = 100
else:
L[j,i] = 0
gamma = 1
F = image
UPREV = flex.double(image.accessor(), 0)
D = flex.vec2_double(image.accessor(), (0,0))
B = flex.vec2_double(image.accessor(), (0,0))
for num_iter in range(max_iter):
U = UPREV
del_U = del_op(U)
# Solve D subproblem
for j in range(image.all()[0]):
for i in range(image.all()[1]):
d = matrix.col(del_U[j,i]) + matrix.col(B[j,i])
dn = d.length()
if dn == 0:
D[j,i] = (0,0)
else:
D[j,i] = (d / dn) * max([dn-1.0/gamma, 0])
# Solve U subproblem
div_db = div_op(D - B)
delta_U = delta_op(U)
UCURR = flex.double(image.accessor(), 0)
for j in range(image.all()[0]):
for i in range(image.all()[1]):
if L[j,i] == 0:
UCURR[j,i] = 0
else:
const = gamma / L[j,i]
UCURR[j,i] = F[j,i] - const*div_db[j,i]+ const*delta_U[j,i]
# Do the update
B = B + del_U - D
# Break if reached convergence
from math import sqrt
l2norm = sqrt(sum([u**2 for u in (UCURR - UPREV)]))
print l2norm
if l2norm < tolerance:
break
UPREV = UCURR
from matplotlib import pylab
#pylab.imshow(UCURR.as_numpy_array())
#pylab.show()
return UCURR
from __future__ import absolute_import, division, print_function
from simtbx.nanoBragg import nanoBragg, nanoBragg_beam
from dials.array_family import flex
import numpy as np
"""Purpose of the test: compare nanoBragg background two ways:
1) single channel
2) multiple channels
Overall photon fluence is the same in both simulations.
Results will be nearly identical if the multiple channel bandpass is small,
and if the spectrum is even (tophat), not irregular (random).
"""
water = flex.vec2_double([(0, 2.57), (0.0365, 2.58), (0.07, 2.8), (0.12, 5),
(0.162, 8), (0.18, 7.32), (0.2, 6.75), (0.216, 6.75),
(0.236, 6.5), (0.28, 4.5), (0.3, 4.3), (0.345, 4.36),
(0.436, 3.77), (0.5, 3.17)])
def gaussian(x, mu, sig):
return np.exp(-np.power(x - mu, 2.) / (2 * np.power(sig, 2.)))
class run_background_simulation:
def __init__(self):
self.SIM = nanoBragg()
self.SIM.progress_meter = False
self.SIM.Fbg_vs_stol = water
self.SIM.amorphous_sample_thick_mm = 0.1
self.SIM.amorphous_density_gcm3 = 1
self.SIM.amorphous_molecular_weight_Da = 18
self.total_flux = self.SIM.flux = 1e12
n_indexed = 0
for i_expt in experiments.where(crystal=crystal_model):
n_indexed += (self.reflections['id'] == i).count(True)
info("model %i (%i reflections):" %(i+1, n_indexed))
info(crystal_model)
self.refined_reflections['xyzcal.px'] = flex.vec3_double(
len(self.refined_reflections))
for i, imageset in enumerate(self.imagesets):
imgset_sel = self.refined_reflections['imageset_id'] == i
# set xyzcal.px field in self.refined_reflections
refined_reflections = self.refined_reflections.select(imgset_sel)
panel_numbers = flex.size_t(refined_reflections['panel'])
xyzcal_mm = refined_reflections['xyzcal.mm']
x_mm, y_mm, z_rad = xyzcal_mm.parts()
xy_cal_mm = flex.vec2_double(x_mm, y_mm)
xy_cal_px = flex.vec2_double(len(xy_cal_mm))
for i_panel in range(len(imageset.get_detector())):
panel = imageset.get_detector()[i_panel]
sel = (panel_numbers == i_panel)
isel = sel.iselection()
ref_panel = refined_reflections.select(panel_numbers == i_panel)
xy_cal_px.set_selected(
sel, panel.millimeter_to_pixel(xy_cal_mm.select(sel)))
x_px, y_px = xy_cal_px.parts()
scan = imageset.get_scan()
if scan is not None:
z_px = scan.get_array_index_from_angle(z_rad, deg=False)
else:
# must be a still image, z centroid not meaningful
z_px = z_rad
# raise ValueError,"Output datasze file must be specified."
else:
datasize = args.pop(datasizeidx).split("=")[1]
import copy, os
import dxtbx
from dxtbx.model.experiment_list import ExperimentListFactory
from dials.array_family import flex
experiments = ExperimentListFactory.from_json_file(json, check_format=False)
beam = experiments[0].beam
detector = experiments[0].detector
lab_coordinates = flex.vec3_double()
for panel in detector:
pixels = flex.vec2_double(panel.get_image_size())
mms = panel.pixel_to_millimeter(pixels)
lab_coordinates.extend(panel.get_lab_coord(mms))
# generate s1 vectors
s1 = lab_coordinates.each_normalize() * (1/beam.get_wavelength())
# Generate x vectors
x = np.asarray(s1 - beam.get_s0())
# DATAsize = np.asarray(detector[0].get_image_size())
# np.save(xvectors,x)
# np.save(datasize,DATAsize)
x.astype('float32').tofile(xvectors)
def populate_pixel_positions(self):
assert self.reflections.has_key('xyzcal.px'), "no calculated spot positions"
self.frame['mapped_predictions'][0] = flex.vec2_double()
for i in xrange(len(self.reflections['xyzcal.px'])):
self.frame['mapped_predictions'][0].append(tuple(self.reflections['xyzcal.px'][i][1::-1])) # 1::-1 reverses the order taking only the first two elements first.
def model_reflection_rt0(reflection, experiment, params):
import math
import random
from scitbx import matrix
from dials.array_family import flex
still = (experiment.goniometer is None) or (experiment.scan is None)
d2r = math.pi / 180.0
hkl = reflection['miller_index']
xyz = reflection['xyzcal.px']
xyz_mm = reflection['xyzcal.mm']
panel = reflection['panel']
p = experiment.detector[panel]
s0 = matrix.col(experiment.beam.get_s0())
if params.debug:
print 'hkl = %d %d %d' % hkl
print 'xyz px = %f %f %f' % xyz
print 'xyz mm = %f %f %f' % xyz_mm
if reflection['entering']:
print 'entering'
else:
print 'exiting'
resolution = p.get_resolution_at_pixel(s0, reflection['xyzobs.px.value'][0:2])
print "Resolution = %.2f"% resolution
if still:
Amat = matrix.sqr(experiment.crystal.get_A())
p0_star = Amat * hkl
angle, s1 = predict_still_delpsi_and_s1(p0_star, experiment)
assert angle
if params.debug:
print 'delpsi angle = %f' % angle
else:
Amat = matrix.sqr(experiment.crystal.get_A_at_scan_point(int(xyz[2])))
p0_star = Amat * hkl
angles = predict_angles(p0_star, experiment)
assert(angles)
if params.debug:
print 'angles = %f %f' % angles
angle = angles[0] if (abs(angles[0] - xyz_mm[2]) <
abs(angles[1] - xyz_mm[2])) else angles[1]
# FIX DQE for this example *** NOT PORTABLE ***
n = matrix.col(p.get_normal())
s1 = matrix.col(reflection['s1'])
t = p.get_thickness() / math.cos(s1.angle(n))
if params.debug and 'dqe' in reflection:
print 'old dqe = %f' % reflection['dqe']
reflection['dqe'] = (1.0 - math.exp(-p.get_mu() * t * 0.1))
if params.debug:
print 'dqe = %f' % reflection['dqe']
if params.physics:
i0 = reflection['intensity.sum.value'] / reflection['dqe']
else:
i0 = reflection['intensity.sum.value']
if params.min_isum:
if i0 < params.min_isum:
return
s1 = reflection['s1']
if not still:
a = matrix.col(experiment.goniometer.get_rotation_axis())
if params.debug:
print 's1 = %f %f %f' % s1
pixels = reflection['shoebox']
pixels.flatten()
data = pixels.data
dz, dy, dx = data.focus()
# since now 2D data
data.reshape(flex.grid(dy, dx))
if params.show:
print 'Observed reflection (flattened in Z):'
print
for j in range(dy):
for i in range(dx):
print '%5d' % data[(j, i)],
print
if params.sigma_m is None:
sigma_m = 0
elif params.sigma_m > 0:
sigma_m = params.sigma_m * d2r
else:
sigma_m = experiment.profile.sigma_m() * d2r
if params.sigma_b is None:
sigma_b = 0
elif params.sigma_b > 0:
sigma_b = params.sigma_b * d2r
elif experiment.profile is not None:
sigma_b = experiment.profile.sigma_b() * d2r
r0 = xyz_mm[2]
detector = experiment.detector
if params.whole_panel:
whole_panel = flex.double(flex.grid(p.get_image_size()[1], p.get_image_size()[0]))
all_pix = flex.vec2_double()
all_iw = flex.double()
patch = flex.double(dy * dx, 0)
patch.reshape(flex.grid(dy, dx))
bbox = reflection['bbox']
if params.interference_weighting.enable:
# Kroon-Batenburg 2015 equation 7
uc = experiment.crystal.get_unit_cell()
lmbda = experiment.beam.get_wavelength()
if params.interference_weighting.ncell:
if params.interference_weighting.domain_size_angstroms:
raise RuntimeError, "Only specify ncell or domain_size_angstroms, not both"
ncell = params.interference_weighting.ncell
else:
if params.interference_weighting.domain_size_angstroms:
diameter = params.interference_weighting.domain_size_angstroms
elif hasattr(experiment.crystal, "_ML_domain_size_ang"):
diameter = experiment.crystal._ML_domain_size_ang
else:
diameter = None
if diameter is None:
ncell = 25
else:
# assume spherical crystallite
volume = (math.pi*4/3)*((diameter/2)**3)
ncell = volume / uc.volume()
if params.debug:
print "ncell: %.1f"%ncell
d = uc.d(hkl)
theta = uc.two_theta(hkl, lmbda)/2
iw_s = ncell * (abs(hkl[0]) + abs(hkl[1]) + abs(hkl[2]))
iw_B = 2*math.pi*d*math.cos(theta)/lmbda
iw_term1 = (1/(2*(math.sin(theta)**2))) * (1/iw_s)
scale = params.scale
if params.nrays is None:
nrays = int(round(i0 * scale))
else:
nrays = params.nrays
if params.show:
print '%d rays' % (nrays)
for i in range(nrays):
if params.real_space_beam_simulation.enable:
# all values in mm
if params.real_space_beam_simulation.source_shape == 'square':
source_length = params.real_space_beam_simulation.source_dimension
sx = (random.random() * source_length) - (source_length / 2)
sy = (random.random() * source_length) - (source_length / 2)
elif params.real_space_beam_simulation.source_shape == 'circle':
source_radius = params.real_space_beam_simulation.source_dimension
v = matrix.col((random.random() * source_radius, 0, 0))
v = v.rotate(matrix.col((0,0,1)), random.random() * 2.0 * math.pi)
sx = v[0]
sy = v[1]
else:
raise RuntimeError, "Invalid choice for source_shape"
if params.real_space_beam_simulation.illuminated_crystal_volume.shape == 'sphere':
crystal_radius = params.real_space_beam_simulation.illuminated_crystal_volume.sphere.radius
v = get_random_axis() * crystal_radius * random.random()
elif params.real_space_beam_simulation.illuminated_crystal_volume.shape == 'rectangular_parallelepiped':
width = params.real_space_beam_simulation.illuminated_crystal_volume.rectangular_parallelepiped.width
depth = params.real_space_beam_simulation.illuminated_crystal_volume.rectangular_parallelepiped.depth
cx = (random.random() * width) - (width / 2)
cy = (random.random() * width) - (width / 2)
cz = (random.random() * depth) - (depth / 2)
v = matrix.col((cx, cy, cz))
else:
raise RuntimeError, "Invalid choice for illuminated_crystal_volume.shape"
# construct a vector from a random point on the source to a random point in the crystal (note the minus sign!)
source_to_crystal = -params.real_space_beam_simulation.source_to_crystal
real_space_beam_vector = matrix.col((sx, sy, source_to_crystal)) + v
# construct the randomized reciprocal space beam vector
b = real_space_beam_vector.normalize() * (1/experiment.beam.get_wavelength())
if params.sigma_l:
l_scale = random.gauss(1, params.sigma_l)
b *= l_scale
else:
if params.sigma_l:
l_scale = random.gauss(1, params.sigma_l)
b = random_vector_cone(s0 * l_scale, sigma_b)
else:
b = random_vector_cone(s0, sigma_b)
if params.sigma_m_rotates_relp:
if params.sigma_cell:
cell_scale = random.gauss(1, params.sigma_cell)
p0 = random_vector_cone(cell_scale * Amat * hkl, sigma_m)
else:
p0 = random_vector_cone(Amat * hkl, sigma_m)
else:
axis = get_random_axis()
rotation = random.gauss(0, sigma_m)
R = axis.axis_and_angle_as_r3_rotation_matrix(rotation)
if params.sigma_cell:
cell_scale = random.gauss(1, params.sigma_cell)
p0 = cell_scale * R * Amat * hkl
else:
p0 = R * Amat * hkl
if params.rs_node_size > 0:
ns = params.rs_node_size
import random
dp0 = matrix.col((random.gauss(0, ns),
random.gauss(0, ns),
random.gauss(0, ns)))
p0 += dp0
if still:
result = predict_still_delpsi_and_s1(p0, experiment, b, s1_rotated = not params.interference_weighting.enable)
if result is None:
# scattered ray ended up in blind region
continue
epsilon, s1 = result
else:
angles = predict_angles(p0, experiment, b)
if angles is None:
# scattered ray ended up in blind region
continue
r = angles[0] if reflection['entering'] else angles[1]
p = p0.rotate(a, r)
s1 = p + b
if params.interference_weighting.enable:
# Rest of Kroon-Batenburg eqn 7
iw = iw_term1 * (math.sin(iw_s*iw_B*epsilon)**2) / ((iw_B*epsilon)**2)
else:
iw = 1
if params.physics:
model_path_through_sensor(detector, reflection, s1, patch, scale)
else:
try:
xy = p.get_ray_intersection(s1)
except RuntimeError, e:
# Not on the detector
continue
all_pix.append(xy)
all_iw.append(iw)
# FIXME DO NOT USE THIS FUNCTION EVENTUALLY...
x, y = detector[panel].millimeter_to_pixel(xy)
x = int(round(x))
y = int(round(y))
if params.whole_panel:
if x < 0 or x >= whole_panel.focus()[1]:
continue
if y < 0 or y >= whole_panel.focus()[0]:
continue
whole_panel[(y, x)] += 1.0 * iw / scale
if x < bbox[0] or x >= bbox[1]:
continue
if y < bbox[2] or y >= bbox[3]:
continue
x -= bbox[0]
y -= bbox[2]
# FIXME in here try to work out probability distribution along path
# length through the detector sensitive surface i.e. deposit fractional
# counts along pixels (and allow for DQE i.e. photon passing right through
# the detector)
patch[(y, x)] += 1.0 * iw / scale
# import dxtbx
t0 = time()
# img = FormatSMVADSCNoDateStamp(imgname)
img = dxtbx.load(imgname)
detector = img.get_detector()
beam = img.get_beam()
scan = img.get_scan()
gonio = img.get_goniometer()
print "transform pixel numbers to mm positions and rotational degrees"
print "Creating pixel map..."
t0 = time()
lab_coordinates = flex.vec3_double()
for panel in detector:
pixels = flex.vec2_double(panel.get_image_size())
mms = panel.pixel_to_millimeter(pixels)
lab_coordinates.extend(panel.get_lab_coord(mms))
# generate s1 vectors
s1_vectors = lab_coordinates.each_normalize() * (1 / beam.get_wavelength())
# Generate x vectors
x_vectors = np.asarray(s1_vectors - beam.get_s0())
# print "there are ",x_vectors.size()," elements in x_vectors"
tel = time() - t0
print "done creating pixel map (", tel, " sec)"
workdir = "tmpdir_common"
# if (os.path.isdir(workdir)):
# command = 'rm -r {0}'.format(workdir)
def generate(self, imageset):
"""Generate the mask."""
# Get the detector and beam
detector = imageset.get_detector()
beam = imageset.get_beam()
# Get the first image
image = imageset.get_raw_data(0)
assert len(detector) == len(image)
# Create the mask for each panel
masks = []
for index, (im, panel) in enumerate(zip(image, detector)):
# Build a trusted mask by looking for pixels that are always outside
# the trusted range. This identifies bad pixels, but does not include
# pixels that are overloaded on some images.
if self.params.use_trusted_range:
warnings.warn(
"Checking for hot pixels using the trusted_range is"
" deprecated. https://github.com/dials/dials/issues/1156",
FutureWarning,
)
trusted_mask = None
low, high = panel.get_trusted_range()
# Take 10 evenly-spaced images from the imageset. Pixels outside
# the trusted mask on all of these images are considered bad and
# masked. https://github.com/dials/dials/issues/1061
stride = max(int(len(imageset) / 10), 1)
image_indices = range(0, len(imageset), stride)
for image_index in image_indices:
image_data = imageset.get_raw_data(
image_index)[index].as_double()
frame_mask = (image_data > low) & (image_data < high)
if trusted_mask is None:
trusted_mask = frame_mask
else:
trusted_mask = trusted_mask | frame_mask
if trusted_mask.count(False) == 0:
break
mask = trusted_mask
else:
mask = flex.bool(flex.grid(im.all()), True)
# Add a border around the image
if self.params.border > 0:
logger.info("Generating border mask:")
logger.info(" border = %d" % self.params.border)
border = self.params.border
height, width = mask.all()
borderx = flex.bool(flex.grid(border, width), False)
bordery = flex.bool(flex.grid(height, border), False)
mask[0:border, :] = borderx
mask[-border:, :] = borderx
mask[:, 0:border] = bordery
mask[:, -border:] = bordery
# Apply the untrusted regions
for region in self.params.untrusted:
if region.panel is None:
region.panel = 0
if region.panel == index:
if not any([
region.circle, region.rectangle, region.polygon,
region.pixel
]):
mask[:, :] = flex.bool(flex.grid(mask.focus()), False)
continue
if region.circle is not None:
xc, yc, radius = region.circle
logger.info("Generating circle mask:")
logger.info(" panel = %d" % region.panel)
logger.info(" xc = %d" % xc)
logger.info(" yc = %d" % yc)
logger.info(" radius = %d" % radius)
mask_untrusted_circle(mask, xc, yc, radius)
if region.rectangle is not None:
x0, x1, y0, y1 = region.rectangle
logger.info("Generating rectangle mask:")
logger.info(" panel = %d" % region.panel)
logger.info(" x0 = %d" % x0)
logger.info(" y0 = %d" % y0)
logger.info(" x1 = %d" % x1)
logger.info(" y1 = %d" % y1)
mask_untrusted_rectangle(mask, x0, x1, y0, y1)
if region.polygon is not None:
assert (len(region.polygon) %
2 == 0), "Polygon must contain 2D coords"
vertices = []
for i in range(int(len(region.polygon) / 2)):
x = region.polygon[2 * i]
y = region.polygon[2 * i + 1]
vertices.append((x, y))
polygon = flex.vec2_double(vertices)
logger.info("Generating polygon mask:")
logger.info(" panel = %d" % region.panel)
for vertex in vertices:
logger.info(" coord = (%d, %d)" % (vertex))
mask_untrusted_polygon(mask, polygon)
if region.pixel is not None:
mask[region.pixel] = False
# Generate high and low resolution masks
if self.params.d_min is not None:
logger.info("Generating high resolution mask:")
logger.info(" d_min = %f" % self.params.d_min)
_apply_resolution_mask(mask, beam, panel, 0, self.params.d_min)
if self.params.d_max is not None:
logger.info("Generating low resolution mask:")
logger.info(" d_max = %f" % self.params.d_max)
d_max = self.params.d_max
d_inf = max(d_max + 1, 1e9)
_apply_resolution_mask(mask, beam, panel, d_max, d_inf)
try:
# Mask out the resolution range
for drange in self.params.resolution_range:
d_min = min(drange)
d_max = max(drange)
assert d_min < d_max, "d_min must be < d_max"
logger.info("Generating resolution range mask:")
logger.info(" d_min = %f" % d_min)
logger.info(" d_max = %f" % d_max)
_apply_resolution_mask(mask, beam, panel, d_min, d_max)
except TypeError:
# Catch the default value None of self.params.resolution_range
if any(self.params.resolution_range):
raise
# Mask out the resolution ranges for the ice rings
for drange in generate_ice_ring_resolution_ranges(
beam, panel, self.params.ice_rings):
d_min = min(drange)
d_max = max(drange)
assert d_min < d_max, "d_min must be < d_max"
logger.info("Generating ice ring mask:")
logger.info(" d_min = %f" % d_min)
logger.info(" d_max = %f" % d_max)
_apply_resolution_mask(mask, beam, panel, d_min, d_max)
# Add to the list
masks.append(mask)
# Return the mask
return tuple(masks)
def generate(self, imageset):
''' Generate the mask. '''
from dials.util import ResolutionMaskGenerator
from dials.util import mask_untrusted_rectangle
from dials.util import mask_untrusted_circle
from dials.util import mask_untrusted_polygon
from dials.util import mask_untrusted_resolution_range
from dials.array_family import flex
from math import floor, ceil
from logging import info
# Get the detector and beam
detector = imageset.get_detector()
beam = imageset.get_beam()
# Get the first image
image = imageset.get_raw_data(0)
assert(len(detector) == len(image))
# Create the mask for each image
masks = []
for index, (im, panel) in enumerate(zip(image, detector)):
# The image width height
height, width = im.all()
# Create the basic mask from the trusted range
if self.params.use_trusted_range:
low, high = panel.get_trusted_range()
imd = im.as_double()
mask = (imd > low) & (imd < high)
else:
mask = flex.bool(flex.grid(im.all()), True)
# Add a border around the image
if self.params.border > 0:
info("Generating border mask:")
info(" border = %d" % self.params.border)
border = self.params.border
height, width = mask.all()
borderx = flex.bool(flex.grid(border, width), False)
bordery = flex.bool(flex.grid(height, border), False)
mask[0:border,:] = borderx
mask[-border:,:] = borderx
mask[:,0:border] = bordery
mask[:,-border:] = bordery
# Apply the untrusted regions
for region in self.params.untrusted:
if region.panel == index:
if region.circle is not None:
xc, yc, radius = region.circle
info("Generating circle mask:")
info(" panel = %d" % region.panel)
info(" xc = %d" % xc)
info(" yc = %d" % yc)
info(" radius = %d" % radius)
mask_untrusted_circle(mask, xc, yc, radius)
if region.rectangle is not None:
x0, x1, y0, y1 = region.rectangle
info("Generating rectangle mask:")
info(" panel = %d" % region.panel)
info(" x0 = %d" % x0)
info(" y0 = %d" % y0)
info(" x1 = %d" % x1)
info(" y1 = %d" % y1)
mask_untrusted_rectangle(mask, x0, x1, y0, y1)
if region.polygon is not None:
assert len(region.polygon) % 2 == 0, "Polygon must contain 2D coords"
vertices = []
for i in range(int(len(region.polygon)/2)):
x = region.polygon[2*i]
y = region.polygon[2*i+1]
vertices.append((x,y))
polygon = flex.vec2_double(vertices)
info("Generating polygon mask:")
info(" panel = %d" % region.panel)
for vertex in vertices:
info(" coord = (%d, %d)" % (vertex))
mask_untrusted_polygon(mask, polygon)
# Create the resolution mask generator
class ResolutionMaskGeneratorGetter(object):
def __init__(self, beam, panel):
self.beam = beam
self.panel = panel
self.result = None
def __call__(self):
if self.result is None:
self.result = ResolutionMaskGenerator(beam, panel)
return self.result
get_resolution_mask_generator = ResolutionMaskGeneratorGetter(beam, panel)
# Generate high and low resolution masks
if self.params.d_min is not None:
info("Generating high resolution mask:")
info(" d_min = %f" % self.params.d_min)
get_resolution_mask_generator().apply(mask, 0, self.params.d_min)
if self.params.d_max is not None:
info("Generating low resolution mask:")
info(" d_max = %f" % self.params.d_max)
d_min = self.params.d_max
d_max = max(d_min + 1, 1e9)
get_resolution_mask_generator().apply(mask, d_min, d_max)
# Mask out the resolution range
for drange in self.params.resolution_range:
d_min=min(drange)
d_max=max(drange)
assert d_min < d_max, "d_min must be < d_max"
info("Generating resolution range mask:")
info(" d_min = %f" % d_min)
info(" d_max = %f" % d_max)
get_resolution_mask_generator().apply(mask, d_min, d_max)
# Mask out the resolution ranges for the ice rings
for drange in generate_ice_ring_resolution_ranges(
beam, panel, self.params.ice_rings):
d_min=min(drange)
d_max=max(drange)
assert d_min < d_max, "d_min must be < d_max"
info("Generating ice ring mask:")
info(" d_min = %f" % d_min)
info(" d_max = %f" % d_max)
get_resolution_mask_generator().apply(mask, d_min, d_max)
# Add to the list
masks.append(mask)
# Return the mask
return tuple(masks)
