def get_pix_coords(wavelength, A, mill_arr, detector, delta_i=0.02):
""" Code copied from sim.py courtesy of Aaron and Tara """
s0=col((0,0,-1/wavelength))
q=flex.vec3_double([A*col(idx) for idx in mill_arr.indices().as_vec3_double()])
s0_hat=flex.vec3_double([s0.normalize()]*len(q))
q_hat=q.each_normalize()
#q_hat.cross(flex.vec3_double([s0_hat]*len(q_hat)))
e1_hat = q_hat.cross(s0_hat)
c0_hat = s0_hat.cross(e1_hat)
q_len_sq = flex.double([col(v).length_sq() for v in q])
a_side=q_len_sq*wavelength/2
b_side=flex.sqrt(q_len_sq)-a_side**2
#flex.vec3_double([sqrt(q.length_sq()-a_side**2 for idx in mill_arr)])
r_vec=flex.vec3_double(-a_side*s0_hat+b_side*c0_hat)
s1=r_vec+s0
EQ=q+s0
len_EQ=flex.double([col(v).length() for v in EQ])
ratio=len_EQ*wavelength
indices = flex.miller_index()
coords =flex.vec2_double()
for i in range(len(s1)):
if ratio[i] > 1 - delta_i and ratio[i] < 1 + delta_i:
indices.append(mill_arr.indices()[i])
pix = detector[0].get_ray_intersection_px(s1[i])
if detector[0].is_coord_valid(pix):
coords.append(pix)
return coords, indices
def large_reflection_table():
"""Generate reflection table to test the basis and target function."""
# these miller_idx/d_values don't make physical sense, but I didn't want to
# have to write the tests for lots of reflections.
reflections = flex.reflection_table()
reflections["intensity"] = flex.double(
[75.0, 10.0, 100.0, 25.0, 50.0, 100.0, 25.0, 20.0, 300.0, 10.0])
reflections["variance"] = flex.double(
[50.0, 10.0, 100.0, 50.0, 10.0, 100.0, 50.0, 10.0, 100.0, 10.0])
reflections["inverse_scale_factor"] = flex.double(10, 1.0)
reflections["miller_index"] = flex.miller_index([
(1, 0, 0),
(0, 0, 1),
(1, 0, 0),
(1, 0, 0),
(0, 0, 1),
(1, 0, 0),
(0, 4, 0),
(0, 0, 1),
(1, 0, 0),
(0, 4, 0),
]) # don't change
reflections["d"] = flex.double(
[2.0, 0.8, 2.0, 2.0, 0.8, 2.0, 2.0, 0.8, 2.0, 1.0]) # don't change
reflections["partiality"] = flex.double(10, 1.0)
reflections["xyzobs.px.value"] = flex.vec3_double(
[(0.0, 0.0, i) for i in range(0, 45, 5)] + [(0.0, 0.0, 59.0)])
reflections["s1"] = flex.vec3_double([(0.0, 0.1, 1.0)] * 2 +
[(0.0, 0.1, 20.0)] * 8)
return reflections
def generate_reflections(self):
"""Use reeke_model to generate indices of reflections near to the Ewald
sphere that might be observed on a still image. Build a reflection_table
of these."""
from cctbx.sgtbx import space_group_info
space_group_type = space_group_info("P 1").group().type()
# create a ReekeIndexGenerator
UB = self.crystal.get_A()
axis = self.goniometer.get_rotation_axis()
s0 = self.beam.get_s0()
dmin = 1.5
# use the same UB at the beginning and end - the margin parameter ensures
# we still have indices close to the Ewald sphere generated
from dials.algorithms.spot_prediction import ReekeIndexGenerator
r = ReekeIndexGenerator(UB, UB, space_group_type, axis, s0, dmin=1.5, margin=1)
# generate indices
hkl = r.to_array()
nref = len(hkl)
# create a reflection table
from dials.array_family import flex
table = flex.reflection_table()
table['flags'] = flex.size_t(nref, 0)
table['id'] = flex.int(nref, 0)
table['panel'] = flex.size_t(nref, 0)
table['miller_index'] = flex.miller_index(hkl)
table['entering'] = flex.bool(nref, True)
table['s1'] = flex.vec3_double(nref)
table['xyzcal.mm'] = flex.vec3_double(nref)
table['xyzcal.px'] = flex.vec3_double(nref)
return table
def __call__(self, experiments, reflections, add_correction_column=False):
result = flex.reflection_table()
for expt_id, experiment in enumerate(experiments):
refls = reflections.select(reflections['id'] == expt_id)
beam = experiment.beam
# Remove the need for pixel size within cxi.merge. Allows multipanel detector with dissimilar panels.
# Relies on new frame extractor code called by dials.stills_process that writes s0, s1 and polarization normal
# vectors all to the integration pickle. Future path (IE THIS CODE): use dials json and reflection file.
s0_vec = matrix.col(beam.get_s0()).normalize()
s0_polar_norm = beam.get_polarization_normal()
s1_vec = refls['s1']
Ns1 = len(s1_vec)
# project the s1_vector onto the plane normal to s0. Get result by subtracting the
# projection of s1 onto s0, which is (s1.dot.s0_norm)s0_norm
s0_norm = flex.vec3_double(Ns1,s0_vec)
s1_proj = (s1_vec.dot(s0_norm))*s0_norm
s1_in_normal_plane = s1_vec - s1_proj
# Now want the polar angle between the projected s1 and the polarization normal
s0_polar_norms = flex.vec3_double(Ns1,s0_polar_norm)
dotprod = (s1_in_normal_plane.dot(s0_polar_norms))
costheta = dotprod/(s1_in_normal_plane.norms())
theta = flex.acos(costheta)
cos_two_polar_angle = flex.cos(2.0*theta)
# gives same as old answer to ~1% but not exact. Not sure why, should not matter.
tt_vec = experiment.crystal.get_unit_cell().two_theta(miller_indices = refls['miller_index'],
wavelength = beam.get_wavelength())
cos_tt_vec = flex.cos(tt_vec)
sin_tt_vec = flex.sin(tt_vec)
cos_sq_tt_vec = cos_tt_vec * cos_tt_vec
sin_sq_tt_vec = sin_tt_vec * sin_tt_vec
P_nought_vec = 0.5 * (1. + cos_sq_tt_vec)
F_prime = -1.0 # Hard-coded value defines the incident polarization axis
P_prime = 0.5 * F_prime * cos_two_polar_angle * sin_sq_tt_vec
# added as a diagnostic
#prange=P_nought_vec - P_prime
#other_F_prime = 1.0
#otherP_prime = 0.5 * other_F_prime * cos_two_polar_angle * sin_sq_tt_vec
#otherprange=P_nought_vec - otherP_prime
#diff2 = flex.abs(prange - otherprange)
#print >> out, "mean diff is",flex.mean(diff2), "range",flex.min(diff2), flex.max(diff2)
# done
correction = 1 / ( P_nought_vec - P_prime )
refls['intensity.sum.value'] = refls['intensity.sum.value'] * correction
refls['intensity.sum.variance'] = refls['intensity.sum.variance'] * correction**2 # propagated error
# This corrects observations for polarization assuming 100% polarization on
# one axis (thus the F_prime = -1.0 rather than the perpendicular axis, 1.0)
# Polarization model as described by Kahn, Fourme, Gadet, Janin, Dumas & Andre
# (1982) J. Appl. Cryst. 15, 330-337, equations 13 - 15.
if add_correction_column:
refls['polarization_correction'] = correction
result.extend(refls)
return result
def test_calc_crystal_frame_vectors(test_reflection_table, mock_exp):
"""Test the namesake function, to check that the vectors are correctly rotated
into the crystal frame."""
rt, exp = test_reflection_table, mock_exp
s0_vec = (1.0, 0.0, 0.0)
s1_vec = (1.0 / sqrt(2.0), 0.0, 1.0 / sqrt(2.0))
reflection_table = calc_crystal_frame_vectors(rt, exp)
assert list(reflection_table["s0"]) == list(
flex.vec3_double([s0_vec, s0_vec, s0_vec])
)
assert approx_equal(
list(reflection_table["s0c"]),
list(
flex.vec3_double(
[s0_vec, (1.0 / sqrt(2.0), -1.0 / sqrt(2.0), 0.0), (0.0, -1.0, 0.0)]
)
),
)
assert approx_equal(
list(reflection_table["s1c"]),
list(
flex.vec3_double(
[
s1_vec,
(1.0 / 2.0, -1.0 / 2.0, 1.0 / sqrt(2.0)),
(0.0, -1.0 / sqrt(2.0), 1.0 / sqrt(2.0)),
]
)
),
)
def generated_refl(not_integrated=False, idval=0):
"""Generate a test reflection table."""
reflections = flex.reflection_table()
reflections["intensity.prf.value"] = flex.double([1.0, 10.0, 100.0, 1.0])
reflections["intensity.prf.variance"] = flex.double(
[1.0, 10.0, 100.0, 1.0])
reflections["intensity.sum.value"] = flex.double(
[12.0, 120.0, 1200.0, 21.0])
reflections["intensity.sum.variance"] = flex.double(
[12.0, 120.0, 2100.0, 1.0])
# reflections['inverse_scale_factor'] = flex.double(4, 1.0)
reflections["miller_index"] = flex.miller_index([(1, 0, 0), (0, 0, 1),
(2, 0, 0), (2, 2, 2)
]) # don't change
reflections["d"] = flex.double([0.8, 2.0, 2.0, 0.0]) # don't change
reflections["partiality"] = flex.double([1.0, 1.0, 1.0, 1.0])
reflections["xyzobs.px.value"] = flex.vec3_double([(0.0, 0.0, 0.0),
(0.0, 0.0, 5.0),
(0.0, 0.0, 10.0),
(0.0, 0.0, 10.0)])
reflections["s1"] = flex.vec3_double([(0.0, 0.1, 1.0), (0.0, 0.1, 1.0),
(0.0, 0.1, 1.0), (0.0, 0.1, 1.0)])
if not_integrated:
reflections.set_flags(flex.bool([False, False, False, False]),
reflections.flags.integrated)
else:
reflections.set_flags(flex.bool([True, True, False, False]),
reflections.flags.integrated)
reflections.set_flags(flex.bool([False, False, True, True]),
reflections.flags.bad_for_scaling)
reflections["id"] = flex.int(4, idval)
reflections.experiment_identifiers()[idval] = str(idval)
return reflections
def generate_reflections(self):
"""Use reeke_model to generate indices of reflections near to the Ewald
sphere that might be observed on a still image. Build a reflection_table
of these."""
from cctbx.sgtbx import space_group_info
space_group_type = space_group_info("P 1").group().type()
# create a ReekeIndexGenerator
UB = self.crystal.get_U() * self.crystal.get_B()
axis = self.goniometer.get_rotation_axis()
s0 = self.beam.get_s0()
dmin = 1.5
# use the same UB at the beginning and end - the margin parameter ensures
# we still have indices close to the Ewald sphere generated
from dials.algorithms.spot_prediction import ReekeIndexGenerator
r = ReekeIndexGenerator(UB, UB, space_group_type, axis, s0, dmin=1.5, margin=1)
# generate indices
hkl = r.to_array()
nref = len(hkl)
# create a reflection table
from dials.array_family import flex
table = flex.reflection_table()
table['flags'] = flex.size_t(nref, 0)
table['id'] = flex.int(nref, 0)
table['panel'] = flex.size_t(nref, 0)
table['miller_index'] = flex.miller_index(hkl)
table['entering'] = flex.bool(nref, True)
table['s1'] = flex.vec3_double(nref)
table['xyzcal.mm'] = flex.vec3_double(nref)
table['xyzcal.px'] = flex.vec3_double(nref)
return table
def generated_refl_2(exclude_refl=True):
"""Generate a reflection table."""
# these miller_idx/d_values don't make physical sense, but I didn't want to
# have to write the tests for lots of reflections.
reflections = flex.reflection_table()
reflections["intensity"] = flex.double([1.0, 10.0, 100.0, 1.0])
reflections["variance"] = flex.double([1.0, 10.0, 100.0, 1.0])
reflections["miller_index"] = flex.miller_index([(1, 0, 0), (0, 0, 1),
(2, 0, 0), (2, 2, 2)
]) # don't change
reflections["d"] = flex.double([0.8, 2.0, 2.0, 0.0]) # don't change
reflections["d"] = flex.double([0.8, 2.0, 2.1, 0.1])
reflections["Esq"] = flex.double([1.0, 1.0, 1.0, 1.0])
reflections["inverse_scale_factor"] = flex.double([1.0, 1.0, 1.0, 1.0])
reflections["id"] = flex.int(4, 0)
reflections["xyzobs.px.value"] = flex.vec3_double([(0.0, 0.0, 0.0),
(0.0, 0.0, 5.0),
(0.0, 0.0, 10.0),
(0.0, 0.0, 10.0)])
reflections["s1"] = flex.vec3_double([(0.0, 0.1, 1.0), (0.0, 0.1, 1.0),
(0.0, 0.1, 1.0), (0.0, 0.1, 1.0)])
if exclude_refl:
integrated_list = flex.bool([True, True, False, False])
bad_list = flex.bool([False, False, True, True])
else:
integrated_list = flex.bool(4, True)
bad_list = flex.bool(4, False)
reflections.set_flags(integrated_list, reflections.flags.integrated)
reflections.set_flags(bad_list, reflections.flags.bad_for_scaling)
return reflections
def refls_to_q(refls, detector, beam, update_table=False):
orig_vecs = {}
fs_vecs = {}
ss_vecs = {}
u_pids = set([r['panel'] for r in refls])
for pid in u_pids:
orig_vecs[pid] = np.array(detector[pid].get_origin())
fs_vecs[pid] = np.array(detector[pid].get_fast_axis())
ss_vecs[pid] = np.array(detector[pid].get_slow_axis())
s1_vecs = []
q_vecs = []
for r in refls:
pid = r['panel']
i_fs, i_ss, _ = r['xyzobs.px.value']
panel = detector[pid]
orig = orig_vecs[pid] #panel.get_origin()
fs = fs_vecs[pid] #panel.get_fast_axis()
ss = ss_vecs[pid] #panel.get_slow_axis()
fs_pixsize, ss_pixsize = panel.get_pixel_size()
s1 = orig + i_fs * fs * fs_pixsize + i_ss * ss * ss_pixsize # scattering vector
s1 = s1 / np.linalg.norm(s1) / beam.get_wavelength()
s1_vecs.append(s1)
q_vecs.append(s1 - beam.get_s0())
if update_table:
refls['s1'] = flex.vec3_double(tuple(map(tuple, s1_vecs)))
refls['rlp'] = flex.vec3_double(tuple(map(tuple, q_vecs)))
return np.vstack(q_vecs)
def generated_refl_for_splitting_1():
""""Make a reflection table."""
reflections = flex.reflection_table()
reflections["intensity"] = flex.double([1.0, 2.0, 3.0, 4.0, 5.0, 6.0])
reflections["variance"] = flex.double(6, 1.0)
reflections["inverse_scale_factor"] = flex.double(6, 1.0)
reflections["miller_index"] = flex.miller_index(
[(1, 0, 0), (2, 0, 0), (0, 0, 1), (2, 2, 2), (1, 0, 0), (2, 0, 0)]
)
reflections["d"] = flex.double([0.8, 2.1, 2.0, 1.4, 1.6, 2.5])
reflections["partiality"] = flex.double(6, 1.0)
reflections["xyzobs.px.value"] = flex.vec3_double(
[
(0.0, 0.0, 0.0),
(0.0, 0.0, 5.0),
(0.0, 0.0, 8.0),
(0.0, 0.0, 10.0),
(0.0, 0.0, 12.0),
(0.0, 0.0, 15.0),
]
)
reflections["s1"] = flex.vec3_double(
[
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
]
)
reflections.set_flags(flex.bool(6, True), reflections.flags.integrated)
reflections.set_flags(flex.bool(6, False), reflections.flags.bad_for_scaling)
return reflections
def import_integrated(integrate_hkl, min_ios=3):
reader = integrate_hkl_as_flex.reader(integrate_hkl, "IOBS,SIGMA,XCAL,YCAL,ZCAL,XOBS,YOBS,ZOBS,ISEG".split(","))
# reference: dials/command_line/import_xds.py
table = flex.reflection_table()
table["id"] = flex.int(len(reader.hkl), 0)
table["panel"] = flex.size_t(len(reader.hkl), 0) # only assuming single panel
table["miller_index"] = reader.hkl
table["xyzcal.px.value"] = flex.vec3_double(reader.data["XCAL"], reader.data["YCAL"], reader.data["ZCAL"])
table["xyzobs.px.value"] = flex.vec3_double(reader.data["XOBS"], reader.data["YOBS"], reader.data["ZOBS"])
table["intensity.cor.value"] = reader.data["IOBS"]
table["intensity.cor.sigma"] = reader.data["SIGMA"] # not valid name, just for making selection
table["flags"] = flex.size_t(len(table), table.flags.indexed | table.flags.strong)
table = table.select(table["intensity.cor.sigma"] > 0)
table = table.select(table["intensity.cor.value"]/table["intensity.cor.sigma"] >= min_ios)
table = table.select(table["xyzobs.px.value"].norms() > 0) # remove invalid xyzobs
# dummy
table["xyzobs.px.variance"] = flex.vec3_double(len(table), (1,1,1)) # TODO appropriate variance value
table["s1"] = flex.vec3_double(len(table), (0,0,0)) # will be updated by set_obs_s1()
del table["intensity.cor.value"]
del table["intensity.cor.sigma"]
return table
def get_pix_coords(wavelength, A, mill_arr, detector, delta_i=0.02):
""" Code copied from sim.py courtesy of Aaron and Tara """
s0 = col((0, 0, -1 / wavelength))
q = flex.vec3_double(
[A * col(idx) for idx in mill_arr.indices().as_vec3_double()])
s0_hat = flex.vec3_double([s0.normalize()] * len(q))
q_hat = q.each_normalize()
#q_hat.cross(flex.vec3_double([s0_hat]*len(q_hat)))
e1_hat = q_hat.cross(s0_hat)
c0_hat = s0_hat.cross(e1_hat)
q_len_sq = flex.double([col(v).length_sq() for v in q])
a_side = q_len_sq * wavelength / 2
b_side = flex.sqrt(q_len_sq) - a_side**2
#flex.vec3_double([sqrt(q.length_sq()-a_side**2 for idx in mill_arr)])
r_vec = flex.vec3_double(-a_side * s0_hat + b_side * c0_hat)
s1 = r_vec + s0
EQ = q + s0
len_EQ = flex.double([col(v).length() for v in EQ])
ratio = len_EQ * wavelength
indices = flex.miller_index()
coords = flex.vec2_double()
for i in range(len(s1)):
if ratio[i] > 1 - delta_i and ratio[i] < 1 + delta_i:
indices.append(mill_arr.indices()[i])
pix = detector[0].get_ray_intersection_px(s1[i])
if detector[0].is_coord_valid(pix):
coords.append(pix)
return coords, indices
def prepare_reflection_list(self,detector):
spots = self.triclinic.get_observations_with_outlier_removal()
ordinary_python_list_of_indexed_observations = [
{
"id":0,
"panel":0,
"miller_index":item["pred"],
"xyzobs.px.value":(spots[item["spot"]].ctr_mass_x(),spots[item["spot"]].ctr_mass_y(),0.0),
"xyzobs.px.variance":(0.25,0.25,0.25),
"spotfinder_lookup":item["spot"]
}
for item in self.triclinic_pairs
]
self.length = len(ordinary_python_list_of_indexed_observations)
R= flex.reflection_table.empty_standard(self.length)
R['miller_index'] = flex.miller_index([item["miller_index"] for item in ordinary_python_list_of_indexed_observations])
R['xyzobs.px.value'] = flex.vec3_double([item["xyzobs.px.value"] for item in ordinary_python_list_of_indexed_observations])
R['xyzobs.px.variance'] = flex.vec3_double([item["xyzobs.px.variance"] for item in ordinary_python_list_of_indexed_observations])
R['spotfinder_lookup'] = flex.int([item["spotfinder_lookup"] for item in ordinary_python_list_of_indexed_observations])
R['xyzobs.mm.value'] = flex.vec3_double(self.length)
R['xyzobs.mm.variance'] = flex.vec3_double(self.length)
pxlsz = detector[0].get_pixel_size()
for idx in xrange(self.length):
R['xyzobs.mm.value'][idx] = (R['xyzobs.px.value'][idx][0]*pxlsz[0], R['xyzobs.px.value'][idx][1]*pxlsz[1], R['xyzobs.px.value'][idx][2])
R['xyzobs.mm.variance'][idx] = (R['xyzobs.px.variance'][idx][0]*pxlsz[0], R['xyzobs.px.variance'][idx][1]*pxlsz[1], R['xyzobs.px.variance'][idx][2])
return R
def _make_input_for_exclude_tests(exclude_images=True):
"""Generate input data, that upon exclusion should leave only the first
reflection."""
if exclude_images:
exclude_images = [["0:360:720"], ["1:360:720"]]
expt1 = Experiment(scan=Scan(image_range=(0, 720), oscillation=(0.0, 1.0)))
expt2 = Experiment(scan=Scan(image_range=(0, 720), oscillation=(0.0,
-1.0)))
refls1 = flex.reflection_table()
refls2 = flex.reflection_table()
refls1["xyzobs.mm.value"] = flex.vec3_double([
(0.0, 0.0, 10.0 * math.pi / 180.0), (0.0, 0.0, 370.0 * math.pi / 180.0)
])
refls1["xyzobs.px.value"] = flex.vec3_double([(0.0, 0.0, 10.0),
(0.0, 0.0, 370.0)])
refls1["i"] = flex.int([0, 1])
refls2["xyzobs.mm.value"] = flex.vec3_double([
(0.0, 0.0, -10.0 * math.pi / 180.0),
(0.0, 0.0, -370.0 * math.pi / 180.0)
])
refls2["xyzobs.px.value"] = flex.vec3_double([(0.0, 0.0, 10.0),
(0.0, 0.0, 370.0)])
refls2["i"] = flex.int([0, 1])
expts = ExperimentList([expt1, expt2])
tables = [refls1, refls2]
expts, tables = assign_unique_identifiers(expts, tables)
return exclude_images, expts, tables
def _xyzcal_mm_to_px(self, experiments, reflections):
# set xyzcal.px field in reflections
reflections["xyzcal.px"] = flex.vec3_double(len(reflections))
for i, expt in enumerate(experiments):
imgset_sel = reflections["imageset_id"] == i
refined_reflections = reflections.select(imgset_sel)
panel_numbers = flex.size_t(refined_reflections["panel"])
xyzcal_mm = refined_reflections["xyzcal.mm"]
x_mm, y_mm, z_rad = xyzcal_mm.parts()
xy_cal_mm = flex.vec2_double(x_mm, y_mm)
xy_cal_px = flex.vec2_double(len(xy_cal_mm))
for i_panel in range(len(expt.detector)):
panel = expt.detector[i_panel]
sel = panel_numbers == i_panel
xy_cal_px.set_selected(
sel, panel.millimeter_to_pixel(xy_cal_mm.select(sel))
)
x_px, y_px = xy_cal_px.parts()
if expt.scan is not None:
z_px = expt.scan.get_array_index_from_angle(z_rad, deg=False)
else:
# must be a still image, z centroid not meaningful
z_px = z_rad
xyzcal_px = flex.vec3_double(x_px, y_px, z_px)
reflections["xyzcal.px"].set_selected(imgset_sel, xyzcal_px)
def fix_xy(reflections_in, reflections_out):
reflections = pickle.load(open(reflections_in, "r"))
# validate that input is from 60 panel detector, assumed to be
# 5 x 12 configuration
assert flex.max(reflections["panel"]) == 59
delta_x = 487 + 7
delta_y = 195 + 17
panel_x = reflections["panel"].as_int() % 5
panel_y = reflections["panel"].as_int() / 5
x_offset = delta_x * panel_x
y_offset = delta_y * panel_y
# apply fixes
x, y, z = reflections["xyzobs.px.value"].parts()
x += x_offset.as_double()
y += y_offset.as_double()
reflections["xyzobs.px.value"] = flex.vec3_double(x, y, z)
x, y, z = reflections["xyzcal.px"].parts()
x += x_offset.as_double()
y += y_offset.as_double()
reflections["xyzcal.px"] = flex.vec3_double(x, y, z)
# save - should probably do this "properly"
pickle.dump(reflections, open(reflections_out, "wb"))
return
def abs_correction_flex(self, s1_flex):
s1_flex_length = len(s1_flex)
surface_normal = flex.vec3_double(s1_flex_length, self.surface_normal)
edge_of_tape_normal = flex.vec3_double(s1_flex_length, self.edge_of_tape_normal)
dsurf1 = flex.double(len(s1_flex), 0)
dsurf2 = flex.double(len(s1_flex), 0)
dot_product = s1_flex.dot(surface_normal)
sel = dot_product != 0
dsurf1.set_selected(sel, self.sn1 / dot_product.select(sel))
dsurf2.set_selected(sel, self.sn2 / dot_product.select(sel))
dsurf3 = self.sn3 / (s1_flex.dot(edge_of_tape_normal))
# determine path length through kapton tape
kapton_path_mm = flex.double(s1_flex_length, 0)
unshadowed_sel = (dsurf3 < dsurf1) | (dsurf1 < 0)
nearsel = ~unshadowed_sel & (dsurf3 < dsurf2)
kapton_path_mm.set_selected(nearsel, (dsurf3 - dsurf1).select(nearsel))
farsel = ~unshadowed_sel & (dsurf3 >= dsurf2)
kapton_path_mm.set_selected(farsel, (dsurf2 - dsurf1).select(farsel))
# determine absorption correction
absorption_correction = 1 / flex.exp(
-self.abs_coeff * kapton_path_mm
) # unitless, >=1
return absorption_correction
def _predict(self):
"""
Predict the position of the spots
"""
# Get some stuff from experiment
A = matrix.sqr(self.experiments[0].crystal.get_A())
s0 = matrix.col(self.experiments[0].beam.get_s0())
# Compute the vector to the reciprocal lattice point
# since this is not on the ewald sphere, lets call it s2
h = self.reflections["miller_index"]
s1 = flex.vec3_double(len(h))
s2 = flex.vec3_double(len(h))
for i in range(len(self.reflections)):
r = A * matrix.col(h[i])
s2[i] = s0 + r
s1[i] = matrix.col(s2[i]).normalize() * s0.length()
self.reflections["s1"] = s1
self.reflections["s2"] = s2
self.reflections["entering"] = flex.bool(len(h), False)
# Compute the ray intersections
xyzpx = flex.vec3_double()
xyzmm = flex.vec3_double()
for i in range(len(s2)):
ss = s1[i]
mm = self.experiments[0].detector[0].get_ray_intersection(ss)
px = self.experiments[0].detector[0].millimeter_to_pixel(mm)
xyzpx.append(px + (0, ))
xyzmm.append(mm + (0, ))
self.reflections["xyzcal.mm"] = xyzmm
self.reflections["xyzcal.px"] = xyzpx
logger.info("Do prediction for %d reflections" % len(self.reflections))
def show_experiments(experiments, show_scan_varying=False):
text = []
for i_expt, expt in enumerate(experiments):
text.append("Experiment %i:" % i_expt)
format_class = expt.imageset.get_format_class()
if not format_class.is_abstract():
text.append(f"Format class: {format_class.__name__}")
if expt.identifier != "":
text.append(f"Experiment identifier: {expt.identifier}")
try:
template = expt.imageset.get_template()
except AttributeError:
template = None
if template:
text.append(f"Image template: {template}")
text.append(str(expt.detector))
text.append("Max resolution (at corners): %f" %
(expt.detector.get_max_resolution(expt.beam.get_s0())))
text.append(
"Max resolution (inscribed): %f" %
(expt.detector.get_max_inscribed_resolution(expt.beam.get_s0())))
text.append("")
text.append(show_beam(expt.detector, expt.beam))
if expt.scan is not None:
text.append(str(expt.scan))
if expt.goniometer is not None:
text.append(show_goniometer(expt.goniometer))
if expt.crystal is not None:
text.append(
expt.crystal.as_str(show_scan_varying=show_scan_varying))
if expt.crystal.num_scan_points:
abc = flex.vec3_double()
angles = flex.vec3_double()
for n in range(expt.crystal.num_scan_points):
(
a,
b,
c,
alpha,
beta,
gamma,
) = expt.crystal.get_unit_cell_at_scan_point(
n).parameters()
abc.append((a, b, c))
angles.append((alpha, beta, gamma))
a, b, c = abc.mean()
alpha, beta, gamma = angles.mean()
mean_unit_cell = uctbx.unit_cell((a, b, c, alpha, beta, gamma))
text.append(f" Average unit cell: {mean_unit_cell}")
if expt.profile is not None:
text.append(str(expt.profile))
if expt.scaling_model is not None:
text.append(str(expt.scaling_model))
return "\n".join(text)
def __call__(self, params, options):
''' Import the integrate.hkl file. '''
from iotbx.xds import integrate_hkl
from dials.array_family import flex
from dials.util.command_line import Command
from cctbx import sgtbx
# Get the unit cell to calculate the resolution
uc = self._experiment.crystal.get_unit_cell()
# Read the INTEGRATE.HKL file
Command.start('Reading INTEGRATE.HKL')
handle = integrate_hkl.reader()
handle.read_file(self._integrate_hkl)
hkl = flex.miller_index(handle.hkl)
xyzcal = flex.vec3_double(handle.xyzcal)
xyzobs = flex.vec3_double(handle.xyzobs)
iobs = flex.double(handle.iobs)
sigma = flex.double(handle.sigma)
rlp = flex.double(handle.rlp)
peak = flex.double(handle.peak) * 0.01
Command.end('Read %d reflections from INTEGRATE.HKL file.' % len(hkl))
# Derive the reindex matrix
rdx = self.derive_reindex_matrix(handle)
print 'Reindex matrix:\n%d %d %d\n%d %d %d\n%d %d %d' % (rdx.elems)
# Reindex the reflections
Command.start('Reindexing reflections')
cb_op = sgtbx.change_of_basis_op(sgtbx.rt_mx(sgtbx.rot_mx(rdx.elems)))
hkl = cb_op.apply(hkl)
Command.end('Reindexed %d reflections' % len(hkl))
# Create the reflection list
Command.start('Creating reflection table')
table = flex.reflection_table()
table['id'] = flex.int(len(hkl), 0)
table['panel'] = flex.size_t(len(hkl), 0)
table['miller_index'] = hkl
table['xyzcal.px'] = xyzcal
table['xyzobs.px.value'] = xyzobs
table['intensity.cor.value'] = iobs
table['intensity.cor.variance'] = sigma**2
table['intensity.prf.value'] = iobs * peak / rlp
table['intensity.prf.variance'] = (sigma * peak / rlp)**2
table['lp'] = 1.0 / rlp
table['d'] = flex.double(uc.d(h) for h in hkl)
Command.end('Created table with {0} reflections'.format(len(table)))
# Output the table to pickle file
if params.output.filename is None:
params.output.filename = 'integrate_hkl.pickle'
Command.start('Saving reflection table to %s' % params.output.filename)
table.as_pickle(params.output.filename)
Command.end('Saved reflection table to %s' % params.output.filename)
def generate_reflection_table():
"""Create a reflection table with s1 and phi."""
rt = flex.reflection_table()
s1_vec = (1.0 / sqrt(2.0), 0.0, 1.0 / sqrt(2.0))
rt["s1"] = flex.vec3_double([s1_vec, s1_vec, s1_vec])
rt["phi"] = flex.double([0.0, 45.0, 90.0])
rt["xyzobs.px.value"] = flex.vec3_double([(0.0, 0.0, 0.0),
(0.0, 0.0, 45.0),
(0.0, 0.0, 90.0)])
return rt
def align_rotation_axis_along_z(exp_rot_axis, vectors):
"""Rotate the coordinate system such that the exp_rot_axis is along z."""
if list(exp_rot_axis) == [(0.0, 0.0, 1.0)]:
return vectors
(ux, uy, uz) = exp_rot_axis[0][0], exp_rot_axis[0][1], exp_rot_axis[0][2]
cross_prod_uz = flex.vec3_double([(uy, -1.0 * ux, 0.0)])
angle_between_u_z = +1.0 * acos(uz / ((ux**2 + uy**2 + uz**2)**0.5))
phi = flex.double(vectors.size(), angle_between_u_z)
new_vectors = rotate_vectors_about_axis(cross_prod_uz, vectors, phi)
return flex.vec3_double(new_vectors)
def generated_refl():
"""Create a reflection table."""
rt = flex.reflection_table()
rt["xyzobs.px.value"] = flex.vec3_double([(0.1, 0.1, 0.1), (0.1, 0.1, 0.2)])
rt["s1"] = flex.vec3_double([(0.1, 0.1, 0.1), (0.1, 0.1, 1.1)])
rt["s1c"] = flex.vec3_double([(0.1, 0.1, 0.1), (0.1, 0.1, 1.1)])
rt["s0c"] = flex.vec3_double([(0.0, 0.0, 1.0), (0.0, 0.0, 1.0)])
rt["d"] = flex.double([1.0, 1.0])
rt["batch"] = flex.int([0, 1])
return rt
def __call__(self, params, options):
''' Import the spot.xds file. '''
from iotbx.xds import spot_xds
from dials.util.command_line import Command
from dials.array_family import flex
# Read the SPOT.XDS file
Command.start('Reading SPOT.XDS')
handle = spot_xds.reader()
handle.read_file(self._spot_xds)
centroid = handle.centroid
intensity = handle.intensity
try:
miller_index = handle.miller_index
except AttributeError:
miller_index = None
Command.end('Read {0} spots from SPOT.XDS file.'.format(len(centroid)))
# Create the reflection list
Command.start('Creating reflection list')
table = flex.reflection_table()
table['id'] = flex.int(len(centroid), 0)
table['panel'] = flex.size_t(len(centroid), 0)
if miller_index:
table['miller_index'] = flex.miller_index(miller_index)
table['xyzobs.px.value'] = flex.vec3_double(centroid)
table['intensity.sum.value'] = flex.double(intensity)
Command.end('Created reflection list')
# Remove invalid reflections
Command.start('Removing invalid reflections')
if miller_index and params.remove_invalid:
flags = flex.bool([h != (0, 0, 0) for h in table['miller_index']])
table = table.select(flags)
Command.end('Removed invalid reflections, %d remaining' % len(table))
# Fill empty standard columns
if params.add_standard_columns:
Command.start('Adding standard columns')
rt = flex.reflection_table.empty_standard(len(table))
rt.update(table)
table = rt
# set variances to unity
table['xyzobs.mm.variance'] = flex.vec3_double(
len(table), (1, 1, 1))
table['xyzobs.px.variance'] = flex.vec3_double(
len(table), (1, 1, 1))
Command.end('Standard columns added')
# Output the table to pickle file
if params.output.filename is None:
params.output.filename = 'spot_xds.pickle'
Command.start('Saving reflection table to %s' % params.output.filename)
table.as_pickle(params.output.filename)
Command.end('Saved reflection table to %s' % params.output.filename)
def get_fd_gradients(self, pred_param, ref_predictor):
# get finite difference gradients
p_vals = pred_param.get_param_vals()
deltas = [1.0e-7] * len(p_vals)
fd_grads = []
p_names = pred_param.get_param_names()
for i in range(len(deltas)):
# save parameter value
val = p_vals[i]
# calc reverse state
p_vals[i] -= deltas[i] / 2.0
pred_param.set_param_vals(p_vals)
ref_predictor(self.reflections)
x, y, _ = self.reflections["xyzcal.mm"].deep_copy().parts()
delpsi = self.reflections["delpsical.rad"].deep_copy()
rev_state = flex.vec3_double(x, y, delpsi)
# calc forward state
p_vals[i] += deltas[i]
pred_param.set_param_vals(p_vals)
ref_predictor(self.reflections)
x, y, _ = self.reflections["xyzcal.mm"].deep_copy().parts()
delpsi = self.reflections["delpsical.rad"].deep_copy()
fwd_state = flex.vec3_double(x, y, delpsi)
# reset parameter to saved value
p_vals[i] = val
# finite difference
fd = fwd_state - rev_state
x_grads, y_grads, delpsi_grads = fd.parts()
x_grads /= deltas[i]
y_grads /= deltas[i]
delpsi_grads /= deltas[i]
fd_grads.append({
"name": p_names[i],
"dX_dp": x_grads,
"dY_dp": y_grads,
"dDeltaPsi_dp": delpsi_grads,
})
# return to the initial state
pred_param.set_param_vals(p_vals)
return fd_grads
def generated_refl_for_comb():
"""Create a reflection table suitable for splitting into blocks."""
reflections = flex.reflection_table()
reflections["intensity"] = flex.double(
[1.0, 2.0, 3.0, 4.0, 500.0, 6.0, 2.0, 2.0])
reflections["variance"] = flex.double(8, 1.0)
reflections["intensity.prf.value"] = flex.double(
[1.0, 3.0, 3.0, 4.0, 50.0, 6.0, 3.0, 2.0])
reflections["intensity.prf.variance"] = flex.double(
[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0])
reflections["intensity.sum.value"] = flex.double(
[1.0, 4.0, 3.0, 4.0, 500.0, 6.0, 6.0, 2.0])
reflections["intensity.sum.variance"] = flex.double(8, 1.0)
reflections["miller_index"] = flex.miller_index([
(1, 0, 0),
(2, 0, 0),
(0, 0, 1),
(2, 2, 2),
(1, 0, 0),
(2, 0, 0),
(1, 0, 0),
(1, 0, 0),
])
reflections["d"] = flex.double([0.8, 2.1, 2.0, 1.4, 1.6, 2.5, 2.5, 2.5])
reflections["partiality"] = flex.double(8, 1.0)
reflections["Esq"] = flex.double(8, 1.0)
reflections["inverse_scale_factor"] = flex.double(8, 1.0)
reflections["xyzobs.px.value"] = flex.vec3_double([
(0.0, 0.0, 0.0),
(0.0, 0.0, 5.0),
(0.0, 0.0, 8.0),
(0.0, 0.0, 10.0),
(0.0, 0.0, 12.0),
(0.0, 0.0, 15.0),
(0.0, 0.0, 15.0),
(0.0, 0.0, 15.0),
])
reflections["s1"] = flex.vec3_double([
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
])
reflections.set_flags(flex.bool(8, True), reflections.flags.integrated)
reflections.set_flags(flex.bool([False] * 5 + [True] + [False] * 2),
reflections.flags.bad_for_scaling)
reflections["id"] = flex.int(8, 0)
reflections.experiment_identifiers()[0] = "0"
reflections = calculate_prescaling_correction(reflections)
return reflections
def __call__(self, params, options):
""" Import the spot.xds file. """
from iotbx.xds import spot_xds
from dials.util.command_line import Command
# Read the SPOT.XDS file
Command.start("Reading SPOT.XDS")
handle = spot_xds.reader()
handle.read_file(self._spot_xds)
centroid = handle.centroid
intensity = handle.intensity
try:
miller_index = handle.miller_index
except AttributeError:
miller_index = None
Command.end("Read {} spots from SPOT.XDS file.".format(len(centroid)))
# Create the reflection list
Command.start("Creating reflection list")
table = flex.reflection_table()
table["id"] = flex.int(len(centroid), 0)
table["panel"] = flex.size_t(len(centroid), 0)
if miller_index:
table["miller_index"] = flex.miller_index(miller_index)
table["xyzobs.px.value"] = flex.vec3_double(centroid)
table["intensity.sum.value"] = flex.double(intensity)
Command.end("Created reflection list")
# Remove invalid reflections
Command.start("Removing invalid reflections")
if miller_index and params.remove_invalid:
flags = flex.bool([h != (0, 0, 0) for h in table["miller_index"]])
table = table.select(flags)
Command.end("Removed invalid reflections, %d remaining" % len(table))
# Fill empty standard columns
if params.add_standard_columns:
Command.start("Adding standard columns")
rt = flex.reflection_table.empty_standard(len(table))
rt.update(table)
table = rt
# set variances to unity
table["xyzobs.mm.variance"] = flex.vec3_double(
len(table), (1, 1, 1))
table["xyzobs.px.variance"] = flex.vec3_double(
len(table), (1, 1, 1))
Command.end("Standard columns added")
# Output the table to pickle file
if params.output.filename is None:
params.output.filename = "spot_xds.refl"
Command.start("Saving reflection table to %s" % params.output.filename)
table.as_file(params.output.filename)
Command.end("Saved reflection table to %s" % params.output.filename)
def get_fd_gradients(self, pred_param, ref_predictor):
# get finite difference gradients
p_vals = pred_param.get_param_vals()
deltas = [1.e-7] * len(p_vals)
fd_grads = []
p_names = pred_param.get_param_names()
for i in range(len(deltas)):
# save parameter value
val = p_vals[i]
# calc reverse state
p_vals[i] -= deltas[i] / 2.
pred_param.set_param_vals(p_vals)
ref_predictor.update()
ref_predictor.predict(self.reflections)
x, y, _ = self.reflections['xyzcal.mm'].deep_copy().parts()
delpsi = self.reflections['delpsical.rad'].deep_copy()
rev_state = flex.vec3_double(x, y, delpsi)
# calc forward state
p_vals[i] += deltas[i]
pred_param.set_param_vals(p_vals)
ref_predictor.update()
ref_predictor.predict(self.reflections)
x, y, _ = self.reflections['xyzcal.mm'].deep_copy().parts()
delpsi = self.reflections['delpsical.rad'].deep_copy()
fwd_state = flex.vec3_double(x, y, delpsi)
# reset parameter to saved value
p_vals[i] = val
# finite difference
fd = (fwd_state - rev_state)
x_grads, y_grads, delpsi_grads = fd.parts()
x_grads /= deltas[i]
y_grads /= deltas[i]
delpsi_grads /= deltas[i]
fd_grads.append({'name':p_names[i],
'dX_dp':x_grads,
'dY_dp':y_grads,
'dDeltaPsi_dp':delpsi_grads})
# return to the initial state
pred_param.set_param_vals(p_vals)
return fd_grads
def test_RefinerData(testdata):
experiment = testdata.experiment
reflections = testdata.reflections
panel = experiment.detector[0]
s0_length = matrix.col(experiment.beam.get_s0()).length()
reflections["bbox"] = flex.int6(len(reflections))
reflections["xyzobs.px.value"] = flex.vec3_double(len(reflections))
reflections["s2"] = reflections["s1"].each_normalize() * s0_length
reflections["sp"] = flex.vec3_double(len(reflections))
for i, (x, y, z) in enumerate(reflections["xyzcal.px"]):
x0 = int(x) - 5
x1 = int(x) + 5 + 1
y0 = int(y) - 5
y1 = int(y) + 5 + 1
z0 = int(z)
z1 = z0 + 1
reflections["bbox"][i] = x0, x1, y0, y1, z0, z1
reflections["xyzobs.px.value"][i] = (int(x) + 0.5, int(y) + 0.5,
int(z) + 0.5)
reflections["sp"][i] = (matrix.col(
panel.get_pixel_lab_coord(
reflections["xyzobs.px.value"][i][0:2])).normalize() *
s0_length)
reflections["shoebox"] = flex.shoebox(reflections["panel"],
reflections["bbox"],
allocate=True)
shoebox_data = flex.float(flex.grid(1, 11, 11))
shoebox_mask = flex.int(flex.grid(1, 11, 11))
for j in range(11):
for i in range(11):
shoebox_data[0, j, i] = (100 * exp(-0.5 * (j - 5)**2 / 1**2) *
exp(-0.5 * (i - 5)**2 / 1**2))
shoebox_mask[0, j, i] = 5
for sbox in reflections["shoebox"]:
sbox.data = shoebox_data
sbox.mask = shoebox_mask
data = RefinerData.from_reflections(experiment, reflections)
assert tuple(data.s0) == pytest.approx(experiment.beam.get_s0())
assert data.h_list == reflections["miller_index"]
for i, sp in enumerate(reflections["sp"]):
assert data.sp_list[:, i] == pytest.approx(sp)
assert data.ctot_list[0] == sum(shoebox_data)
mobs1 = np.abs(data.mobs_list[0, :])
mobs2 = np.abs(data.mobs_list[1, :])
assert np.max(mobs1) < 1e-6
assert np.max(mobs2) < 1e-6
def __init__(self):
from dials.array_family import flex
self.reflections = flex.reflection_table()
self.reflections['panel'] = flex.size_t()
self.reflections['bbox'] = flex.int6()
self.reflections['miller_index'] = flex.miller_index()
self.reflections['s1'] = flex.vec3_double()
self.reflections['xyzcal.px'] = flex.vec3_double()
self.reflections['xyzcal.mm'] = flex.vec3_double()
self.reflections['entering'] = flex.bool()
self.reflections['id'] = flex.int()
self.reflections["flags"] = flex.size_t()
self.npanels = 2
self.width = 1000
self.height = 1000
self.nrefl = 10000
self.array_range = (0, 130)
self.block_size = 20
from random import randint, seed, choice
seed(0)
self.processed = [[] for i in range(12)]
for i in range(self.nrefl):
x0 = randint(0, self.width - 10)
y0 = randint(0, self.height - 10)
zs = randint(2, 9)
x1 = x0 + randint(2, 10)
y1 = y0 + randint(2, 10)
for k, j in enumerate(
[10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]):
m = k + i * 12
pos = choice(["left", "right", "centre"])
if pos == 'left':
z0 = j - zs
z1 = j
elif pos == 'right':
z0 = j
z1 = j + zs
else:
z0 = j - zs // 2
z1 = j + zs // 2
bbox = (x0, x1, y0, y1, z0, z1)
self.reflections.append({
"panel":
randint(0, 1),
"bbox":
bbox,
"flags":
flex.reflection_table.flags.reference_spot
})
self.processed[k].append(m)
def test_align_rotation_axis_along_z():
"""Test the function to rotate the coordinate system such that the rotation
axis is along z. In the test, the rotation axis is x, so we expect the
transformation to be: x > z, y > y, z > -x, x+z > -x+z."""
rot_axis = flex.vec3_double([(1.0, 0.0, 0.0)])
vectors = flex.vec3_double(
[(1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, 1.0), (1.0, 0.0, 1.0)]
)
rotated_vectors = align_rotation_axis_along_z(rot_axis, vectors)
expected = [(0.0, 0.0, 1.0), (0.0, 1.0, 0.0), (-1.0, 0.0, 0.0), (-1.0, 0.0, 1.0)]
for v1, v2 in zip(rotated_vectors, expected):
assert v1 == pytest.approx(v2)
def __call__(self, params, options):
''' Import the spot.xds file. '''
from iotbx.xds import spot_xds
from dials.util.command_line import Command
from dials.array_family import flex
# Read the SPOT.XDS file
Command.start('Reading SPOT.XDS')
handle = spot_xds.reader()
handle.read_file(self._spot_xds)
centroid = handle.centroid
intensity = handle.intensity
try:
miller_index = handle.miller_index
except AttributeError:
miller_index = None
Command.end('Read {0} spots from SPOT.XDS file.'.format(len(centroid)))
# Create the reflection list
Command.start('Creating reflection list')
table = flex.reflection_table()
table['id'] = flex.int(len(centroid), 0)
table['panel'] = flex.size_t(len(centroid), 0)
if miller_index:
table['miller_index'] = flex.miller_index(miller_index)
table['xyzobs.px.value'] = flex.vec3_double(centroid)
table['intensity.sum.value'] = flex.double(intensity)
Command.end('Created reflection list')
# Remove invalid reflections
Command.start('Removing invalid reflections')
if miller_index and params.remove_invalid:
flags = flex.bool([h != (0, 0, 0) for h in table['miller_index']])
table = table.select(flags)
Command.end('Removed invalid reflections, %d remaining' % len(table))
# Fill empty standard columns
if params.add_standard_columns:
Command.start('Adding standard columns')
rt = flex.reflection_table.empty_standard(len(table))
rt.update(table)
table = rt
# set variances to unity
table['xyzobs.mm.variance'] = flex.vec3_double(len(table), (1,1,1))
table['xyzobs.px.variance'] = flex.vec3_double(len(table), (1,1,1))
Command.end('Standard columns added')
# Output the table to pickle file
if params.output.filename is None:
params.output.filename = 'spot_xds.pickle'
Command.start('Saving reflection table to %s' % params.output.filename)
table.as_pickle(params.output.filename)
Command.end('Saved reflection table to %s' % params.output.filename)
def _prepare_for_compose(self, reflections, skip_derivatives=False):
"""Add columns to the reflection table to hold the varying state matrices
or vectors for the experimental models, if required. Also add columns for
the derivatives of states that are scan-varying"""
nref = len(reflections)
# set columns if needed
if 'u_matrix' not in reflections:
reflections['u_matrix'] = flex.mat3_double(nref)
if 'b_matrix' not in reflections:
reflections['b_matrix'] = flex.mat3_double(nref)
if 's0_vector' not in reflections:
reflections['s0_vector'] = flex.vec3_double(nref)
if 'd_matrix' not in reflections:
reflections['d_matrix'] = flex.mat3_double(nref)
if 'D_matrix' not in reflections:
reflections['D_matrix'] = flex.mat3_double(nref)
# set columns in the reflection table to store the derivative of state for
# each reflection, if needed
if not skip_derivatives:
null9 = (0., 0., 0., 0., 0., 0., 0., 0., 0.)
null3 = (0., 0., 0.)
if self._varying_xl_orientations and "dU_dp0" not in reflections:
max_free_params = max([
e.num_free()
for e in self._xl_orientation_parameterisations
])
for i in range(max_free_params):
colname = "dU_dp{0}".format(i)
reflections[colname] = flex.mat3_double(nref, null9)
if self._varying_xl_unit_cells and "dB_dp0" not in reflections:
max_free_params = max([
e.num_free() for e in self._xl_unit_cell_parameterisations
])
for i in range(max_free_params):
colname = "dB_dp{0}".format(i)
reflections[colname] = flex.mat3_double(nref, null9)
if self._varying_detectors and "dd_dp0" not in reflections:
max_free_params = max(
[e.num_free() for e in self._detector_parameterisations])
for i in range(max_free_params):
colname = "dd_dp{0}".format(i)
reflections[colname] = flex.mat3_double(nref, null9)
if self._varying_beams and "ds0_dp0" not in reflections:
max_free_params = max(
[e.num_free() for e in self._beam_parameterisations])
for i in range(max_free_params):
colname = "ds0_dp{0}".format(i)
reflections[colname] = flex.vec3_double(nref, null3)
return
def generated_10_refl():
"""Generate reflection table to test the basis and target function."""
# these miller_idx/d_values don't make physical sense, but I didn't want to
# have to write the tests for lots of reflections.
reflections = flex.reflection_table()
reflections["intensity.prf.value"] = flex.double(
[75.0, 10.0, 100.0, 25.0, 50.0, 100.0, 25.0, 20.0, 300.0, 10.0])
reflections["intensity.prf.variance"] = flex.double(
[50.0, 10.0, 100.0, 50.0, 10.0, 100.0, 50.0, 10.0, 100.0, 10.0])
reflections["miller_index"] = flex.miller_index([
(1, 0, 0),
(0, 0, 1),
(1, 0, 0),
(1, 0, 0),
(0, 0, 1),
(1, 0, 0),
(0, 4, 0),
(0, 0, 1),
(1, 0, 0),
(0, 4, 0),
]) # don't change
reflections["d"] = flex.double(
[2.0, 0.8, 2.0, 2.0, 0.8, 2.0, 2.0, 0.8, 2.0, 1.0]) # don't change
reflections["partiality"] = flex.double(10, 1.0)
reflections["xyzobs.px.value"] = flex.vec3_double([
(0.0, 0.0, 0.0),
(0.0, 0.0, 5.0),
(0.0, 0.0, 10.0),
(0.0, 0.0, 15.0),
(0.0, 0.0, 20.0),
(0.0, 0.0, 25.0),
(0.0, 0.0, 30.0),
(0.0, 0.0, 35.0),
(0.0, 0.0, 40.0),
(0.0, 0.0, 59.0),
])
reflections["s1"] = flex.vec3_double([
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
(0.0, 0.1, 20.0),
(0.0, 0.1, 20.0),
(0.0, 0.1, 20.0),
(0.0, 0.1, 20.0),
(0.0, 0.1, 20.0),
(0.0, 0.1, 20.0),
(0.0, 0.1, 20.0),
(0.0, 0.1, 20.0),
])
reflections.set_flags(flex.bool(10, True), reflections.flags.integrated)
reflections["id"] = flex.int(10, 0)
reflections.experiment_identifiers()[0] = str(0)
return [reflections]
def main():
parser = OptionParser(read_experiments=True, read_reflections=True)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
data = reflections[0]
expt = experiments[0]
data = data.select(data.get_flags(data.flags.integrated))
# data = data.select(data['d'] > 2.0)
z = data["xyzcal.px"].parts()[2]
shoebox = data["shoebox"]
# make a new reflection table, this one empty - then add columns for
# every event in this set (with some shuffle) - flags = 32 apparently,
# id == id above, intensity.sum.value=1 variance=1 n_signal=1 panel=panel
# xyzobs.px.value = pixel + random.random() - 0.5 variance = 1/12 in each
# direction - then map to reciprocal space
events = flex.vec3_double()
for s in shoebox:
d = s.data
k0, j0, i0 = s.bbox[0], s.bbox[2], s.bbox[4]
k1, j1, i1 = d.focus()
for k in range(k1):
for j in range(j1):
for i in range(i1):
for n in range(int(d[k, j, i])):
if random.random() > 0.1:
continue
z = k + k0 + random.random()
y = j + j0 + random.random()
x = i + i0 + random.random()
events.append((z, y, x))
rt = flex.reflection_table()
rt["xyzobs.px.value"] = events
variance = flex.double(events.size(), 1.0 / 12.0)
rt["xyzobs.px.variance"] = flex.vec3_double(variance, variance, variance)
rt["flags"] = flex.size_t(events.size(), 32)
rt["id"] = flex.int(events.size(), 0)
rt["panel"] = flex.size_t(events.size(), 0)
rt["intensity.sum.value"] = flex.double(events.size(), 1)
rt["intensity.sum.variance"] = flex.double(events.size(), 1)
rt.as_file("events.refl")
def __init__(self):
from dials.array_family import flex
self.reflections = flex.reflection_table()
self.reflections['panel'] = flex.size_t()
self.reflections['bbox'] = flex.int6()
self.reflections['miller_index'] = flex.miller_index()
self.reflections['s1'] = flex.vec3_double()
self.reflections['xyzcal.px'] = flex.vec3_double()
self.reflections['xyzcal.mm'] = flex.vec3_double()
self.reflections['entering'] = flex.bool()
self.reflections['id'] = flex.int()
self.reflections["flags"] = flex.size_t()
self.npanels = 2
self.width = 1000
self.height = 1000
self.nrefl = 10000
self.array_range = (0, 130)
self.block_size = 20
from random import randint, seed, choice
seed(0)
self.processed = [[] for i in range(12)]
for i in range(self.nrefl):
x0 = randint(0, self.width-10)
y0 = randint(0, self.height-10)
zs = randint(2, 9)
x1 = x0 + randint(2, 10)
y1 = y0 + randint(2, 10)
for k, j in enumerate([10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]):
m = k + i * 12
pos = choice(["left", "right", "centre"])
if pos == 'left':
z0 = j - zs
z1 = j
elif pos == 'right':
z0 = j
z1 = j + zs
else:
z0 = j - zs // 2
z1 = j + zs // 2
bbox = (x0, x1, y0, y1, z0, z1)
self.reflections.append({
"panel" : randint(0,1),
"bbox" : bbox,
"flags" : flex.reflection_table.flags.reference_spot
})
self.processed[k].append(m)
def get_gradients(self, reflections, callback=None):
"""
Calculate gradients of the prediction formula with respect to each
of the parameters of the contained models, for all of the reflections.
To be implemented by a derived class, which determines the space of the
prediction formula (e.g. we calculate dX/dp, dY/dp, dphi/dp for the
prediction formula for a rotation scan expressed in detector space, but
components of d\vec{r}/dp for the prediction formula in reciprocal space
"""
### Calculate various quantities of interest for the reflections
# Set up arrays of values for each reflection
n = len(reflections)
D = flex.mat3_double(n)
s0 = flex.vec3_double(n)
U = flex.mat3_double(n)
B = flex.mat3_double(n)
axis = flex.vec3_double(n)
fixed_rotation = flex.mat3_double(n)
for iexp, exp in enumerate(self._experiments):
sel = reflections['id'] == iexp
isel = sel.iselection()
# D matrix array
panels = reflections['panel'].select(isel)
for ipanel, D_mat in enumerate([p.get_D_matrix() for p in exp.detector]):
subsel = isel.select(panels == ipanel)
D.set_selected(subsel, D_mat)
# s0 array
s0.set_selected(isel, exp.beam.get_s0())
# U and B arrays
exp_U, exp_B = self._get_U_B_for_experiment(exp.crystal, reflections, isel)
U.set_selected(isel, exp_U)
B.set_selected(isel, exp_B)
# axis array
if exp.goniometer:
axis.set_selected(isel, exp.goniometer.get_rotation_axis())
fixed_rotation.set_selected(isel, exp.goniometer.get_fixed_rotation())
return self._get_gradients_core(reflections, D, s0, U, B, axis, fixed_rotation, callback)
def refine(self, experiments, reflections):
sel = ((reflections['id'] >= -1))
refl = reflections.select(sel)
acceptance_flags = self.identify_outliers(self.all_params, experiments, refl)
#create a new "indexed" list with outliers thrown out:
refl = refl.select(acceptance_flags)
print "$$$ stills_indexer::refine"
R = e_refine(params = self.all_params, experiments=experiments, reflections=refl, graph_verbose=False)
ref_experiments = R.get_experiments()
# try to improve the outcome with a second round of outlier rejection post-initial refinement:
acceptance_flags = self.identify_outliers(self.all_params, ref_experiments, refl)
# insert a round of Nave-outlier rejection on top of the r.m.s.d. rejection
nv0 = nave_parameters(params = self.all_params, experiments=ref_experiments, reflections=refl, refinery=R, graph_verbose=False)
crystal_model_nv0 = nv0()
acceptance_flags_nv0 = nv0.nv_acceptance_flags
refl = refl.select(acceptance_flags & acceptance_flags_nv0)
print "$$$ stills_indexer::refine after positional and delta-psi outlier rejection"
refiner = e_refine(params = self.all_params, experiments=ref_experiments, reflections=refl, graph_verbose=False)
matches = refiner.get_matches()
xyzcal_mm = flex.vec3_double(len(refl))
xyzcal_mm.set_selected(matches['iobs'], matches['xyzcal.mm'])
refl['xyzcal.mm'] = xyzcal_mm
refl.set_flags(matches['iobs'], refl.flags.used_in_refinement)
refl['entering'] = flex.bool(len(refl), False)
return refiner.get_experiments(), refl
def calculate_weights(self, reflections):
"""Set weights to constant terms. If stills, the z weights are
the 'delpsical.weights' attribute of the reflection table. Otherwise, use
the usual 'xyzobs.mm.weights'"""
wx = flex.double(len(reflections), self._wx)
wy = flex.double(len(reflections), self._wy)
wz = flex.double(len(reflections), self._wz)
if self._stills:
null = flex.double(len(reflections), 0)
reflections['xyzobs.mm.weights'] = flex.vec3_double(wx, wy, null)
reflections['delpsical.weights'] = wz
else:
reflections['xyzobs.mm.weights'] = flex.vec3_double(wx, wy, wz)
return reflections
def generate_spots(crystal_model, detector, beam, goniometer=None, scan=None,
sel_fraction=1.0):
import math
experiment = Experiment(beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
crystal=crystal_model)
# if we don't set the imageset then from_predictions uses the StillsReflectionPredictor :-(
from dxtbx.imageset import NullReader, ImageSweep
imageset = ImageSweep(NullReader, indices=range(len(scan.get_epochs())), beam=beam, goniometer=goniometer,
detector=detector, scan=scan)
experiment.imageset = imageset
predicted = flex.reflection_table.from_predictions(experiment)
sel = flex.random_selection(len(predicted),
int(math.floor(sel_fraction*len(predicted))))
predicted = predicted.select(sel)
predicted['imageset_id'] = flex.size_t(len(predicted), 0)
predicted['xyzobs.px.value'] = predicted['xyzcal.px']
predicted['xyzobs.px.variance'] = flex.vec3_double(
len(predicted), (0.5,0.5,0.5))
return predicted
def add_random_noise_xyz(datablock, strong_spots, rmsds):
import scitbx.random
errors = flex.vec3_double(*list(
scitbx.random.variate(
scitbx.random.normal_distribution(0, rmsds[i]))(len(strong_spots))
for i in range(3)))
strong_spots['xyzobs.px.value'] += errors
def get_s1_array(self, values): miller_vec = self.MILLER.as_vec3_double() ref_ori = matrix.sqr(self.ORI.reciprocal_matrix()) Rx = matrix.col((1,0,0)).axis_and_angle_as_r3_rotation_matrix(values.thetax) Ry = matrix.col((0,1,0)).axis_and_angle_as_r3_rotation_matrix(values.thetay) s_array = flex.mat3_double(len(self.MILLER),Ry * Rx * ref_ori) * miller_vec s1_array = s_array + flex.vec3_double(len(self.MILLER), self.BEAM) return s1_array
def _beam_derivatives(self, isel, parameterisation=None, reflections=None):
"""helper function to extend the derivatives lists by derivatives of the
beam parameterisations"""
# Get required data
s0 = self._s0.select(isel)
s0u = self._s0u.select(isel)
wl = self._wavelength.select(isel)
r = self._r.select(isel)
e1 = self._e1.select(isel)
q = self._q.select(isel)
c0 = self._c0.select(isel)
DeltaPsi = self._DeltaPsi.select(isel)
D = self._D.select(isel)
# get the derivatives of the beam vector wrt the parameters
ds0_dbeam_p = parameterisation.get_ds_dp(use_none_as_null=True)
dDeltaPsi_dp = []
dpv_dp = []
# loop through the parameters
for der in ds0_dbeam_p:
if der is None:
dpv_dp.append(None)
dDeltaPsi_dp.append(None)
continue
# repeat the derivative in an array
ds0 = flex.vec3_double(len(s0u), der.elems)
# we need the derivative of the unit beam direction too. This requires
# scaling by the wavelength and projection onto the Ewald sphere
scaled = ds0 * wl
ds0u = scaled.dot(c0) * c0 + scaled.dot(e1) * e1
# calculate the derivative of DeltaPsi for this parameter
dDeltaPsi = -1.0 * (r.dot(ds0)) / (e1.cross(r).dot(s0))
dDeltaPsi_dp.append(dDeltaPsi)
# calculate (d[r]/d[e1])(d[e1]/dp)
de1_dp = c0.cross(ds0u)
dr_de1 = dRq_de(DeltaPsi, e1, q)
drde_dedp = dr_de1 * de1_dp
#dp = 1.e-8 # finite step size for the parameter
#del_e1 = de1_dp * dp
#e1f = e1 + del_e1 * 0.5
#rfwd = q.rotate_around_origin(e1f, DeltaPsi)
#e1r = e1 - del_e1 * 0.5
#rrev = q.rotate_around_origin(e1r, DeltaPsi)
#drde_dedp = (rfwd - rrev) * (1 / dp)
# calculate the derivative of pv for this parameter
dpv_dp.append(D * (ds0 + e1.cross(r) * dDeltaPsi + drde_dedp))
return dpv_dp, dDeltaPsi_dp
def _prepare_for_compose(self, reflections, skip_derivatives=False):
"""Add columns to the reflection table to hold the varying state matrices
or vectors for the experimental models, if required. Also add columns for
the derivatives of states that are scan-varying"""
nref = len(reflections)
# set columns if needed
if 'u_matrix' not in reflections:
reflections['u_matrix'] = flex.mat3_double(nref)
if 'b_matrix' not in reflections:
reflections['b_matrix'] = flex.mat3_double(nref)
if 's0_vector' not in reflections:
reflections['s0_vector'] = flex.vec3_double(nref)
if 'd_matrix' not in reflections:
reflections['d_matrix'] = flex.mat3_double(nref)
if 'D_matrix' not in reflections:
reflections['D_matrix'] = flex.mat3_double(nref)
# set columns in the reflection table to store the derivative of state for
# each reflection, if needed
if not skip_derivatives:
null9 = (0., 0., 0., 0., 0., 0., 0., 0., 0.)
null3 = (0., 0., 0.)
if self._varying_xl_orientations and "dU_dp0" not in reflections:
max_free_params = max([e.num_free() for e in self._xl_orientation_parameterisations])
for i in range(max_free_params):
colname = "dU_dp{0}".format(i)
reflections[colname] = flex.mat3_double(nref, null9)
if self._varying_xl_unit_cells and "dB_dp0" not in reflections:
max_free_params = max([e.num_free() for e in self._xl_unit_cell_parameterisations])
for i in range(max_free_params):
colname = "dB_dp{0}".format(i)
reflections[colname] = flex.mat3_double(nref, null9)
if self._varying_detectors and "dd_dp0" not in reflections:
max_free_params = max([e.num_free() for e in self._detector_parameterisations])
for i in range(max_free_params):
colname = "dd_dp{0}".format(i)
reflections[colname] = flex.mat3_double(nref, null9)
if self._varying_beams and "ds0_dp0" not in reflections:
max_free_params = max([e.num_free() for e in self._beam_parameterisations])
for i in range(max_free_params):
colname = "ds0_dp{0}".format(i)
reflections[colname] = flex.vec3_double(nref, null3)
return
def calculate_weights(reflections):
"""set 'statistical weights', that is w(x) = 1/var(x)"""
weights = (reflections['xyzobs.mm.variance']).deep_copy()
parts = weights.parts()
for w in parts:
sel = w > 0.
w.set_selected(sel, 1./w.select(sel))
reflections['xyzobs.mm.weights'] = flex.vec3_double(*parts)
return reflections
def _beam_derivatives(self, isel, parameterisation=None, reflections=None):
"""helper function to extend the derivatives lists by derivatives of the
beam parameterisations"""
# Get required data
s0 = self._s0.select(isel)
s0u = self._s0u.select(isel)
nu = self._nu.select(isel)
r = self._r.select(isel)
e1 = self._e1.select(isel)
q_s0 = self._q_s0.select(isel)
inv_s = self._inv_s.select(isel)
inv_sss = self._inv_sss.select(isel)
DeltaPsi = self._DeltaPsi.select(isel)
D = self._D.select(isel)
# get the derivatives of the beam vector wrt the parameters
ds0_dbeam_p = parameterisation.get_ds_dp(use_none_as_null=True)
dDeltaPsi_dp = []
dpv_dp = []
# loop through the parameters
for der in ds0_dbeam_p:
if der is None:
dpv_dp.append(None)
dDeltaPsi_dp.append(None)
continue
# repeat the derivative in an array
ds0 = flex.vec3_double(len(s0u), der.elems)
# we need the derivative of the unit beam direction too. This requires
# scaling by the wavelength and projection onto the Ewald sphere
#scaled = ds0 * wl
#ds0u = scaled.dot(c0) * c0 + scaled.dot(e1) * e1
# calculate the derivative of DeltaPsi for this parameter
dDeltaPsi = -1.0 * (r.dot(ds0)) / (e1.cross(r).dot(s0))
dDeltaPsi_dp.append(dDeltaPsi)
# term 1
term1 = (s0u.dot(ds0) * q_s0 + nu * ds0) * inv_s
# term 2
term2 = (nu * q_s0 * q_s0.dot(ds0)) * inv_sss
# calculate the derivative of pv for this parameter
dpv_dp.append(D * (term1 - term2))
return dpv_dp, dDeltaPsi_dp
def _id_refs_to_keep(self, obs_data):
"""Create a selection of observations that pass certain conditions.
This step includes rejection of reflections too close to the spindle,
reflections measured outside the scan range, rejection of the (0,0,0)
Miller index and rejection of reflections with the overload flag set.
Outlier rejection is done later."""
# first exclude reflections with miller index set to 0,0,0
sel1 = obs_data['miller_index'] != (0,0,0)
# exclude reflections with overloads, as these have worse centroids
sel2 = ~obs_data.get_flags(obs_data.flags.overloaded)
# combine selections
sel = sel1 & sel2
inc = flex.size_t_range(len(obs_data)).select(sel)
obs_data = obs_data.select(sel)
# Default to True to pass the following test if there is no rotation axis
# for a particular experiment
to_keep = flex.bool(len(inc), True)
for iexp, exp in enumerate(self._experiments):
axis = self._axes[iexp]
if not axis or exp.scan is None: continue
if exp.scan.get_oscillation()[1] == 0.0: continue
sel = obs_data['id'] == iexp
s0 = self._s0vecs[iexp]
s1 = obs_data['s1'].select(sel)
phi = obs_data['xyzobs.mm.value'].parts()[2].select(sel)
# first test: reject reflections for which the parallelepiped formed
# between the gonio axis, s0 and s1 has a volume of less than the cutoff.
# Those reflections are by definition closer to the spindle-beam
# plane and for low values of the cutoff are troublesome to
# integrate anyway.
p_vol = flex.abs(s1.cross(flex.vec3_double(s1.size(), s0)).dot(axis))
passed1 = p_vol > self._close_to_spindle_cutoff
# second test: reject reflections that lie outside the scan range
phi_min, phi_max = exp.scan.get_oscillation_range(deg=False)
passed2 = (phi >= phi_min) & (phi <= phi_max)
# combine tests
to_update = passed1 & passed2
to_keep.set_selected(sel, to_update)
inc = inc.select(to_keep)
return inc
def set_general_variables(self):
self.frame = load(self.raw)
self.detector = self.frame.get_detector()
self.beam = self.frame.get_beam()
self.s0 = self.beam.get_s0()
self.gonio = self.frame.get_goniometer()
self.scan = self.frame.get_scan()
self.lab_coordinates = flex.vec3_double()
for panel in self.detector:
self.beam_center_mm_x, self.beam_center_mm_y = col(panel.get_beam_centre(self.s0))
pixels = flex.vec2_double(panel.get_image_size())
mms = panel.pixel_to_millimeter(pixels)
self.lab_coordinates.extend(panel.get_lab_coord(mms))
self.Isizex, self.Isizey = panel.get_image_size()
self.beam_center_x, self.beam_center_y = col(panel.get_beam_centre_px(self.s0))
self.detector_distance = panel.get_distance()
thrshim_min, thrshim_max = panel.get_trusted_range()
self.pixel_size = panel.get_pixel_size()[0]
self.raw_data = self.frame.get_raw_data()
if thrshim_min < 0 :
self.thrshim_min = int(0)
else:
self.thrshim_min = thrshim_min
if thrshim_max > 32767:
self.thrshim_max = int(32767)
else:
self.thrshim_max = int(thrshim_max)
self.polarization_fraction = self.beam.get_polarization_fraction()
self.polarization_offset = 0.0
self.cassette_x = 0.0
self.cassette_y = 0.0
self.windim_xmax = int(self.Isizex)-100 # right border for processed image (pixels)
self.windim_xmin = 100 # left border for processed image (pixels)
self.windim_ymax = int(self.Isizey)-100 # top border for processed image (pixels)
self.windim_ymin = 100 # bottom border for processed image (pixels)
### beamstop borders
self.punchim_xmax = int(self.Isizex) # right border of beam stop shadow (pixels)
self.punchim_xmin = int(self.beam_center_x)-80 # left border of beam stop shadow (pixels)
self.punchim_ymax = int(self.beam_center_y)+100 # top border of beam stop shadow (pixels)
self.punchim_ymin = int(self.beam_center_y)-40 # bottom border of beam stop shadow (pixels)
self.mode_filter_footprint = int(20)
return
def generate_single_central_non_negative_profiles(self):
from dials.array_family import flex
from tst_profile_helpers import gaussian
rlist = flex.reflection_table(1)
profile = gaussian(self.grid_size, 1000, (4, 4, 4), (1.5, 1.5, 1.5))
x = 500
y = 500
z = 5
xyz = flex.vec3_double(1)
xyz[0] = (x, y, z)
profiles = [profile.deep_copy()]
rlist["xyzcal.px"] = xyz
return rlist, profiles, profile
def generate_identical_non_negative_profiles(self):
from dials.array_family import flex
from random import uniform
from tst_profile_helpers import gaussian
rlist = flex.reflection_table(1000)
profile = gaussian(self.grid_size, 1000, (4, 4, 4), (1.5, 1.5, 1.5))
xyz = flex.vec3_double(1000)
profiles = []
for i in range(1000):
x = uniform(0, 1000)
y = uniform(0, 1000)
z = uniform(0, 10)
xyz[i] = (x, y, z)
profiles.append(profile.deep_copy())
rlist["xyzcal.px"] = xyz
return rlist, profiles, profile
def generate_systematically_offset_profiles(self):
from dials.array_family import flex
from random import uniform
from tst_profile_helpers import gaussian
rlist = flex.reflection_table(1000)
xyz = flex.vec3_double(1000)
profiles = []
for i in range(1000):
x = uniform(0, 1000)
y = uniform(0, 1000)
z = uniform(0, 10)
offset = -4.5 + 9 * x / 1000.0
profile = gaussian(self.grid_size, 1000, (4 + offset, 4, 4), (1.5, 1.5, 1.5))
xyz[i] = (x, y, z)
profiles.append(profile)
rlist["xyzcal.px"] = xyz
return rlist, profiles
def _beam_derivatives(self, isel, parameterisation=None,
ds0_dbeam_p=None, reflections=None):
"""helper function to extend the derivatives lists by derivatives of the
beam parameterisations."""
# Get required data
r = self._r.select(isel)
e_X_r = self._e_X_r.select(isel)
e_r_s0 = self._e_r_s0.select(isel)
D = self._D.select(isel)
if ds0_dbeam_p is None:
# get the derivatives of the beam vector wrt the parameters
ds0_dbeam_p = parameterisation.get_ds_dp(use_none_as_null=True)
ds0_dbeam_p = [None if e is None else flex.vec3_double(len(r), e.elems) \
for e in ds0_dbeam_p]
dphi_dp = []
dpv_dp = []
# loop through the parameters
for der in ds0_dbeam_p:
if der is None:
dphi_dp.append(None)
dpv_dp.append(None)
continue
# calculate the derivative of phi for this parameter
dphi = (r.dot(der) / e_r_s0) * -1.0
dphi_dp.append(dphi)
# calculate the derivative of pv for this parameter
dpv_dp.append(D * (e_X_r * dphi + der))
return dpv_dp, dphi_dp
def __init__(self):
from dials.algorithms.integration.profile import GridSampler2D
from dials.array_family import flex
from random import randint, uniform
# Number of reflections
nrefl = 1000
# Size of the images
width = 1000
height = 1000
# Create the grid
self.grid = GridSampler2D((width, height), (5,5))
# Create the list of xyz and bboxes
self.xyz = flex.vec3_double(nrefl)
self.bbox = flex.int6(nrefl)
self.panel = flex.size_t(nrefl, 0)
for i in range(nrefl):
x0 = randint(0,width-10)
x1 = x0 + randint(3,10)
y0 = randint(0,height-10)
y1 = y0 + randint(3,10)
z0 = randint(0,10)
z1 = z0 + randint(1,10)
b = x0, x1, y0, y1, z0, z1
c = (x1 + x0) / 2, (y1 + y0) / 2, (z1 + z0) / 2
self.xyz[i] = c
self.bbox[i] = b
# Create the array of shoeboxes
self.sbox = flex.shoebox(self.panel, self.bbox)
self.sbox.allocate()
for i in range(len(self.sbox)):
data = self.sbox[i].data
for j in range(len(data)):
data[j] = uniform(0, 100)
def tst_for_reflection_table(self):
from libtbx.test_utils import approx_equal
from dials.algorithms.spot_prediction import \
ScanVaryingReflectionPredictor, ScanStaticReflectionPredictor
from dials.array_family import flex
predict = ScanStaticReflectionPredictor(self.experiments[0])
preds = predict.for_ub(self.experiments[0].crystal.get_A())
preds['ub_matrix'] = flex.mat3_double(len(preds), self.experiments[0].crystal.get_A())
preds['s0'] = flex.vec3_double(len(preds), self.experiments[0].beam.get_s0())
preds['d_matrix'] = flex.mat3_double(len(preds))
for ipanel, panel in enumerate(self.experiments[0].detector):
sel = preds['panel'] == ipanel
D = panel.get_d_matrix()
preds['d_matrix'].set_selected(sel, D)
predict = ScanVaryingReflectionPredictor(self.experiments[0])
from copy import deepcopy
old_preds = deepcopy(preds)
predict.for_reflection_table(preds,
preds['ub_matrix'],
preds['s0'],
preds['d_matrix'])
# Because UB, s0 and d values are the same for all reflections, the new
# reflections should be approx equal to those produced by the scan static
# predictor
old_x, old_y, old_z = old_preds['xyzcal.px'].parts()
new_x, new_y, new_z = preds['xyzcal.px'].parts()
assert old_x.all_approx_equal(new_x)
assert old_y.all_approx_equal(new_y)
assert old_z.all_approx_equal(new_z)
print "OK"
return
def main(reflections, experiment, params):
nref0 = len(reflections)
method = params.method
ids = params.id
#if 'intensity.prf.variance' in reflections:
# selection = reflections.get_flags(
# reflections.flags.integrated,
# all=True)
#else:
# selection = reflections.get_flags(
# reflections.flags.integrated_sum)
#reflections = reflections.select(selection)
# filter according to rules
if params.min_isum:
selection = reflections['intensity.sum.value'] > params.min_isum
reflections = reflections.select(selection)
if params.num > len(reflections):
raise RuntimeError, 'you asked for too many reflections sorry'
from dials.array_family import flex
if params.seed > 0 and params.num > 0:
import random
random.seed(params.seed)
selected = flex.bool(len(reflections), False)
while len(selected.iselection()) < params.num:
selected[random.randint(0, len(reflections))] = True
reflections = reflections.select(selected)
nref1 = len(reflections)
print 'Removed %d reflections, %d remain' % (nref0 - nref1, nref1)
results = []
simmed = reflections.copy()[0:0]
simmed['dqe'] = flex.double()
simmed['xyzsim.mm'] = flex.vec3_double()
simmed['xyzpred.mm'] = flex.vec3_double()
for j, reflection in enumerate(reflections):
result = None
if ids:
if j in ids:
result = globals()['model_reflection_%s' % method](reflection, experiment, params)
elif params.id_start and params.id_end:
if j < params.id_start or j >= params.id_end:
continue
print j
result = globals()['model_reflection_%s' % method](reflection, experiment, params)
else:
print j
result = globals()['model_reflection_%s' % method](reflection, experiment, params)
if not result is None:
results.append(result[0])
simmed.append(result[1])
import math
if params.debug:
print "RMSD obs - cal:", math.sqrt((simmed['xyzobs.mm.value'] - simmed['xyzcal.mm']).sum_sq()/len(simmed)) * 1000
print "RMSD obs - sim:", math.sqrt((simmed['xyzobs.mm.value'] - simmed['xyzsim.mm']).sum_sq()/len(simmed)) * 1000
print "Functional:", (simmed['xyzobs.mm.value'] - simmed['xyzsim.mm']).sum_sq()
return results
if len(self.refined_experiments) > 1:
from dials.algorithms.indexing.compare_orientation_matrices \
import show_rotation_matrix_differences
show_rotation_matrix_differences(
self.refined_experiments.crystals(), out=info_handle)
info("Final refined crystal models:")
for i, crystal_model in enumerate(self.refined_experiments.crystals()):
n_indexed = 0
for i_expt in experiments.where(crystal=crystal_model):
n_indexed += (self.reflections['id'] == i).count(True)
info("model %i (%i reflections):" %(i+1, n_indexed))
info(crystal_model)
self.refined_reflections['xyzcal.px'] = flex.vec3_double(
len(self.refined_reflections))
for i, imageset in enumerate(self.imagesets):
imgset_sel = self.refined_reflections['imageset_id'] == i
# set xyzcal.px field in self.refined_reflections
refined_reflections = self.refined_reflections.select(imgset_sel)
panel_numbers = flex.size_t(refined_reflections['panel'])
xyzcal_mm = refined_reflections['xyzcal.mm']
x_mm, y_mm, z_rad = xyzcal_mm.parts()
xy_cal_mm = flex.vec2_double(x_mm, y_mm)
xy_cal_px = flex.vec2_double(len(xy_cal_mm))
for i_panel in range(len(imageset.get_detector())):
panel = imageset.get_detector()[i_panel]
sel = (panel_numbers == i_panel)
isel = sel.iselection()
ref_panel = refined_reflections.select(panel_numbers == i_panel)
xy_cal_px.set_selected(
def get_gradients(self, reflections, callback=None):
"""Calculate gradients of the prediction formula with respect to each
of the parameters of the contained models, for all of the reflections.
This method sets up required quantities relevant to the current step of
refinement and then loops over the parameterisations of each type extending
a results list each time."""
# Set up arrays of quantities of interest for each reflection
self._nref = len(reflections)
self._D = flex.mat3_double(self._nref)
self._s0 = flex.vec3_double(self._nref)
self._U = flex.mat3_double(self._nref)
self._B = flex.mat3_double(self._nref)
self._axis = flex.vec3_double(self._nref)
self._fixed_rotation = flex.mat3_double(self._nref)
self._setting_rotation = flex.mat3_double(self._nref)
# Set up experiment to index mapping
self._experiment_to_idx = []
# Populate values in these arrays
for iexp, exp in enumerate(self._experiments):
sel = reflections['id'] == iexp
isel = sel.iselection()
self._experiment_to_idx.append(isel)
subref = reflections.select(sel)
states = self._get_model_data_for_experiment(exp, subref)
self._D.set_selected(sel, states['D'])
self._s0.set_selected(sel, states['s0'])
self._U.set_selected(sel, states['U'])
self._B.set_selected(sel, states['B'])
if exp.goniometer:
self._axis.set_selected(sel, exp.goniometer.get_rotation_axis_datum())
self._fixed_rotation.set_selected(sel, exp.goniometer.get_fixed_rotation())
self._setting_rotation.set_selected(sel, exp.goniometer.get_setting_rotation())
# Other derived values
self._h = reflections['miller_index'].as_vec3_double()
self._UB = self._U * self._B
self._s1 = reflections['s1']
self._pv = self._D * self._s1 # 'projection vector' for the ray along s1.
# Quantities derived from pv, precalculated for efficiency
u, v, w = self._pv.parts()
self._w_inv = 1./w
self._u_w_inv = u * self._w_inv
self._v_w_inv = v * self._w_inv
# Reset a pointer to the parameter number
self._iparam = 0
# Do additional setup specified by derived classes
self._local_setup(reflections)
# Set up empty list in which to store gradients
results = []
# loop over detector parameterisations and extend results
results = self._grads_detector_loop(reflections, results, callback)
# loop over the beam parameterisations and extend results
results = self._grads_beam_loop(reflections, results, callback)
# loop over the crystal orientation parameterisations and extend results
results = self._grads_xl_orientation_loop(reflections, results, callback)
# loop over the crystal unit cell parameterisations and extend results
results = self._grads_xl_unit_cell_loop(reflections, results, callback)
return results
print "You really should supply datasize, but ok"
# raise ValueError,"Output datasze file must be specified."
else:
datasize = args.pop(datasizeidx).split("=")[1]
import copy, os
import dxtbx
from dxtbx.model.experiment_list import ExperimentListFactory
from dials.array_family import flex
experiments = ExperimentListFactory.from_json_file(json, check_format=False)
beam = experiments[0].beam
detector = experiments[0].detector
lab_coordinates = flex.vec3_double()
for panel in detector:
pixels = flex.vec2_double(panel.get_image_size())
mms = panel.pixel_to_millimeter(pixels)
lab_coordinates.extend(panel.get_lab_coord(mms))
# generate s1 vectors
s1 = lab_coordinates.each_normalize() * (1/beam.get_wavelength())
# Generate x vectors
x = np.asarray(s1 - beam.get_s0())
# DATAsize = np.asarray(detector[0].get_image_size())
# np.save(xvectors,x)
# np.save(datasize,DATAsize)
x.astype('float32').tofile(xvectors)
def integration_concept_detail(self, experiments, reflections, spots,image_number,cb_op_to_primitive,**kwargs):
detector = experiments[0].detector
crystal = experiments[0].crystal
from cctbx.crystal import symmetry
c_symmetry = symmetry(space_group = crystal.get_space_group(), unit_cell = crystal.get_unit_cell())
self.image_number = image_number
NEAR = 10
pxlsz = detector[0].get_pixel_size()
Predicted = self.get_predictions_accounting_for_centering(experiments,reflections,cb_op_to_primitive,**kwargs)
FWMOSAICITY = self.inputai.getMosaicity()
self.DOMAIN_SZ_ANG = kwargs.get("domain_size_ang", self.__dict__.get("actual",0) )
refineflag = {True:0,False:1}[kwargs.get("domain_size_ang",0)==0]
c_symmetry.show_summary(prefix="EXCURSION%1d REPORT FWMOS= %6.4f DOMAIN= %6.1f "%(refineflag,FWMOSAICITY,self.DOMAIN_SZ_ANG))
from annlib_ext import AnnAdaptor
self.cell = c_symmetry.unit_cell()
query = flex.double()
print len(self.predicted)
for pred in self.predicted: # predicted spot coord in pixels
query.append(pred[0]/pxlsz[0])
query.append(pred[1]/pxlsz[1])
self.reserve_hkllist_for_signal_search = self.hkllist
reference = flex.double()
assert self.length>NEAR# Can't do spot/pred matching with too few spots
for spot in spots:
reference.append(spot.ctr_mass_x())
reference.append(spot.ctr_mass_y())
IS_adapt = AnnAdaptor(data=reference,dim=2,k=NEAR)
IS_adapt.query(query)
idx_cutoff = float(min(self.mask_focus[image_number]))
from rstbx.apps.slip_helpers import slip_callbacks
cache_refinement_spots = getattr(slip_callbacks.slip_callback,"requires_refinement_spots",False)
indexed_pairs_provisional = []
correction_vectors_provisional = []
c_v_p_flex = flex.vec3_double()
this_setting_matched_indices = reflections["miller_index"]
for j,item in enumerate(this_setting_matched_indices):
this_setting_index = self.hkllist.first_index(item)
if this_setting_index:
Match = dict(spot=j,pred=this_setting_index)
indexed_pairs_provisional.append(Match)
vector = matrix.col(
[reflections["xyzobs.px.value"][j][0] - self.predicted[Match["pred"]][0]/pxlsz[0],
reflections["xyzobs.px.value"][j][1] - self.predicted[Match["pred"]][1]/pxlsz[1]])
correction_vectors_provisional.append(vector)
c_v_p_flex.append((vector[0],vector[1],0.))
self.N_correction_vectors = len(correction_vectors_provisional)
self.rmsd_px = math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex)))
print "... %d provisional matches"%self.N_correction_vectors,
print "r.m.s.d. in pixels: %6.3f"%(self.rmsd_px)
if self.horizons_phil.integration.enable_residual_scatter:
from matplotlib import pyplot as plt
fig = plt.figure()
for cv in correction_vectors_provisional:
plt.plot([cv[1]],[-cv[0]],"r.")
plt.title(" %d matches, r.m.s.d. %5.2f pixels"%(len(correction_vectors_provisional),math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex)))))
plt.axes().set_aspect("equal")
self.show_figure(plt,fig,"res")
plt.close()
if self.horizons_phil.integration.enable_residual_map:
from matplotlib import pyplot as plt
PX = reflections["xyzobs.px.value"]
fig = plt.figure()
for match,cv in zip(indexed_pairs_provisional,correction_vectors_provisional):
plt.plot([PX[match["spot"]][1]],[-PX[match["spot"]][0]],"r.")
plt.plot([self.predicted[match["pred"]][1]/pxlsz[1]],[-self.predicted[match["pred"]][0]/pxlsz[0]],"g.")
plt.plot([PX[match["spot"]][1], PX[match["spot"]][1] + 10.*cv[1]],
[-PX[match["spot"]][0], -PX[match["spot"]][0] - 10.*cv[0]],'r-')
if kwargs.get("user-reentrant") != None and self.horizons_phil.integration.spot_prediction == "dials" \
and self.horizons_phil.integration.enable_residual_map_deltapsi:
from rstbx.apps.stills.util import residual_map_special_deltapsi_add_on
residual_map_special_deltapsi_add_on(
reflections = self.dials_spot_prediction,
matches = indexed_pairs_provisional, experiments=experiments,
hkllist = self.hkllist,
predicted = self.predicted, plot=plt, eta_deg=FWMOSAICITY, deff=self.DOMAIN_SZ_ANG
)
plt.xlim([0,detector[0].get_image_size()[1]])
plt.ylim([-detector[0].get_image_size()[0],0])
plt.title(" %d matches, r.m.s.d. %5.2f pixels"%(len(correction_vectors_provisional),math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex)))))
plt.axes().set_aspect("equal")
self.show_figure(plt,fig,"map")
plt.close()
indexed_pairs = indexed_pairs_provisional
correction_vectors = correction_vectors_provisional
########### skip outlier rejection for this derived class
### However must retain the ability to write out correction vectiors.
if True: # at Aaron's request; test later
correction_lengths = flex.double([v.length() for v in correction_vectors_provisional])
clorder = flex.sort_permutation(correction_lengths)
sorted_cl = correction_lengths.select(clorder)
indexed_pairs = []
correction_vectors = []
self.correction_vectors = []
for icand in xrange(len(sorted_cl)):
# somewhat arbitrary sigma = 1.0 cutoff for outliers
indexed_pairs.append(indexed_pairs_provisional[clorder[icand]])
correction_vectors.append(correction_vectors_provisional[clorder[icand]])
if cache_refinement_spots:
self.spotfinder.images[self.frame_numbers[self.image_number]]["refinement_spots"].append(
spots[reflections[indexed_pairs[-1]["spot"]]['spotfinder_lookup']])
if kwargs.get("verbose_cv")==True:
print "CV OBSCENTER %7.2f %7.2f REFINEDCENTER %7.2f %7.2f"%(
float(self.inputpd["size1"])/2.,float(self.inputpd["size2"])/2.,
self.inputai.xbeam()/pxlsz[0], self.inputai.ybeam()/pxlsz[1]),
print "OBSSPOT %7.2f %7.2f PREDSPOT %7.2f %7.2f"%(
reflections[indexed_pairs[-1]["spot"]]['xyzobs.px.value'][0],
reflections[indexed_pairs[-1]["spot"]]['xyzobs.px.value'][1],
self.predicted[indexed_pairs[-1]["pred"]][0]/pxlsz[0],
self.predicted[indexed_pairs[-1]["pred"]][1]/pxlsz[1]),
the_hkl = self.hkllist[indexed_pairs[-1]["pred"]]
print "HKL %4d %4d %4d"%the_hkl,"%2d"%self.setting_id,
radial, azimuthal = spots[indexed_pairs[-1]["spot"]].get_radial_and_azimuthal_size(
self.inputai.xbeam()/pxlsz[0], self.inputai.ybeam()/pxlsz[1])
print "RADIALpx %5.3f AZIMUTpx %5.3f"%(radial,azimuthal)
# Store a list of correction vectors in self.
radial, azimuthal = spots[indexed_pairs[-1]['spot']].get_radial_and_azimuthal_size(
self.inputai.xbeam()/pxlsz[0], self.inputai.ybeam()/pxlsz[1])
self.correction_vectors.append(
dict(obscenter=(float(self.inputpd['size1']) / 2,
float(self.inputpd['size2']) / 2),
refinedcenter=(self.inputai.xbeam() / pxlsz[0],
self.inputai.ybeam() / pxlsz[1]),
obsspot=(reflections[indexed_pairs[-1]['spot']]['xyzobs.px.value'][0],
reflections[indexed_pairs[-1]['spot']]['xyzobs.px.value'][1]),
predspot=(self.predicted[indexed_pairs[-1]['pred']][0] / pxlsz[0],
self.predicted[indexed_pairs[-1]['pred']][1] / pxlsz[1]),
hkl=(self.hkllist[indexed_pairs[-1]['pred']][0],
self.hkllist[indexed_pairs[-1]['pred']][1],
self.hkllist[indexed_pairs[-1]['pred']][2]),
setting_id=self.setting_id,
radial=radial,
azimuthal=azimuthal))
self.inputpd["symmetry"] = c_symmetry
self.inputpd["symmetry"].show_summary(prefix="SETTING ")
if self.horizons_phil.integration.model == "user_supplied":
# Not certain of whether the reentrant_* dictionary keys create a memory leak
if kwargs.get("user-reentrant",None)==None:
kwargs["reentrant_experiments"] = experiments
kwargs["reentrant_reflections"] = reflections
from cxi_user import post_outlier_rejection
self.indexed_pairs = indexed_pairs
self.spots = spots
post_outlier_rejection(self,image_number,cb_op_to_primitive,self.horizons_phil,kwargs)
return
########### finished with user-supplied code
correction_lengths=flex.double([v.length() for v in correction_vectors])
self.r_residual = pxlsz[0]*flex.mean(correction_lengths)
#assert len(indexed_pairs)>NEAR # must have enough indexed spots
if (len(indexed_pairs) <= NEAR):
raise Sorry("Not enough indexed spots, only found %d, need %d" % (len(indexed_pairs), NEAR))
reference = flex.double()
for item in indexed_pairs:
reference.append(spots[item["spot"]].ctr_mass_x())
reference.append(spots[item["spot"]].ctr_mass_y())
PS_adapt = AnnAdaptor(data=reference,dim=2,k=NEAR)
PS_adapt.query(query)
self.BSmasks = []
# do not use null: self.null_correction_mapping( predicted=self.predicted,
self.positional_correction_mapping( predicted=self.predicted,
correction_vectors = correction_vectors,
PS_adapt = PS_adapt,
IS_adapt = IS_adapt,
spots = spots)
# which spots are close enough to interfere with background?
MAXOVER=6
OS_adapt = AnnAdaptor(data=query,dim=2,k=MAXOVER) #six near nbrs
OS_adapt.query(query)
if self.mask_focus[image_number] is None:
raise Sorry("No observed/predicted spot agreement; no Spotfinder masks; skip integration")
nbr_cutoff = 2.0* max(self.mask_focus[image_number])
FRAME = int(nbr_cutoff/2)
#print "The overlap cutoff is %d pixels"%nbr_cutoff
nbr_cutoff_sq = nbr_cutoff * nbr_cutoff
#print "Optimized C++ section...",
self.set_frame(FRAME)
self.set_background_factor(kwargs["background_factor"])
self.set_nbr_cutoff_sq(nbr_cutoff_sq)
self.set_guard_width_sq(self.horizons_phil.integration.guard_width_sq)
self.set_detector_gain(self.horizons_phil.integration.detector_gain)
flex_sorted = flex.int()
for item in self.sorted:
flex_sorted.append(item[0]);flex_sorted.append(item[1]);
if self.horizons_phil.integration.mask_pixel_value is not None:
self.set_mask_pixel_val(self.horizons_phil.integration.mask_pixel_value)
image_obj = self.imagefiles.imageindex(self.frame_numbers[self.image_number])
image_obj.read()
rawdata = image_obj.linearintdata # assume image #1
if self.inputai.active_areas != None:
self.detector_xy_draft = self.safe_background( rawdata=rawdata,
predicted=self.predicted,
OS_adapt=OS_adapt,
sorted=flex_sorted,
tiles=self.inputai.active_areas.IT,
tile_id=self.inputai.active_areas.tile_id);
else:
self.detector_xy_draft = self.safe_background( rawdata=rawdata,
predicted=self.predicted,
OS_adapt=OS_adapt,
sorted=flex_sorted);
for i in xrange(len(self.predicted)): # loop over predicteds
B_S_mask = {}
keys = self.get_bsmask(i)
for k in xrange(0,len(keys),2):
B_S_mask[(keys[k],keys[k+1])]=True
self.BSmasks.append(B_S_mask)
#print "Done"
return
