def get_pix_coords(wavelength, A, mill_arr, detector, delta_i=0.02): """ Code copied from sim.py courtesy of Aaron and Tara """ s0=col((0,0,-1/wavelength)) q=flex.vec3_double([A*col(idx) for idx in mill_arr.indices().as_vec3_double()]) s0_hat=flex.vec3_double([s0.normalize()]*len(q)) q_hat=q.each_normalize() #q_hat.cross(flex.vec3_double([s0_hat]*len(q_hat))) e1_hat = q_hat.cross(s0_hat) c0_hat = s0_hat.cross(e1_hat) q_len_sq = flex.double([col(v).length_sq() for v in q]) a_side=q_len_sq*wavelength/2 b_side=flex.sqrt(q_len_sq)-a_side**2 #flex.vec3_double([sqrt(q.length_sq()-a_side**2 for idx in mill_arr)]) r_vec=flex.vec3_double(-a_side*s0_hat+b_side*c0_hat) s1=r_vec+s0 EQ=q+s0 len_EQ=flex.double([col(v).length() for v in EQ]) ratio=len_EQ*wavelength indices = flex.miller_index() coords =flex.vec2_double() for i in range(len(s1)): if ratio[i] > 1 - delta_i and ratio[i] < 1 + delta_i: indices.append(mill_arr.indices()[i]) pix = detector[0].get_ray_intersection_px(s1[i]) if detector[0].is_coord_valid(pix): coords.append(pix) return coords, indices
def large_reflection_table(): """Generate reflection table to test the basis and target function.""" # these miller_idx/d_values don't make physical sense, but I didn't want to # have to write the tests for lots of reflections. reflections = flex.reflection_table() reflections["intensity"] = flex.double( [75.0, 10.0, 100.0, 25.0, 50.0, 100.0, 25.0, 20.0, 300.0, 10.0]) reflections["variance"] = flex.double( [50.0, 10.0, 100.0, 50.0, 10.0, 100.0, 50.0, 10.0, 100.0, 10.0]) reflections["inverse_scale_factor"] = flex.double(10, 1.0) reflections["miller_index"] = flex.miller_index([ (1, 0, 0), (0, 0, 1), (1, 0, 0), (1, 0, 0), (0, 0, 1), (1, 0, 0), (0, 4, 0), (0, 0, 1), (1, 0, 0), (0, 4, 0), ]) # don't change reflections["d"] = flex.double( [2.0, 0.8, 2.0, 2.0, 0.8, 2.0, 2.0, 0.8, 2.0, 1.0]) # don't change reflections["partiality"] = flex.double(10, 1.0) reflections["xyzobs.px.value"] = flex.vec3_double( [(0.0, 0.0, i) for i in range(0, 45, 5)] + [(0.0, 0.0, 59.0)]) reflections["s1"] = flex.vec3_double([(0.0, 0.1, 1.0)] * 2 + [(0.0, 0.1, 20.0)] * 8) return reflections
def generate_reflections(self): """Use reeke_model to generate indices of reflections near to the Ewald sphere that might be observed on a still image. Build a reflection_table of these.""" from cctbx.sgtbx import space_group_info space_group_type = space_group_info("P 1").group().type() # create a ReekeIndexGenerator UB = self.crystal.get_A() axis = self.goniometer.get_rotation_axis() s0 = self.beam.get_s0() dmin = 1.5 # use the same UB at the beginning and end - the margin parameter ensures # we still have indices close to the Ewald sphere generated from dials.algorithms.spot_prediction import ReekeIndexGenerator r = ReekeIndexGenerator(UB, UB, space_group_type, axis, s0, dmin=1.5, margin=1) # generate indices hkl = r.to_array() nref = len(hkl) # create a reflection table from dials.array_family import flex table = flex.reflection_table() table['flags'] = flex.size_t(nref, 0) table['id'] = flex.int(nref, 0) table['panel'] = flex.size_t(nref, 0) table['miller_index'] = flex.miller_index(hkl) table['entering'] = flex.bool(nref, True) table['s1'] = flex.vec3_double(nref) table['xyzcal.mm'] = flex.vec3_double(nref) table['xyzcal.px'] = flex.vec3_double(nref) return table
def __call__(self, experiments, reflections, add_correction_column=False): result = flex.reflection_table() for expt_id, experiment in enumerate(experiments): refls = reflections.select(reflections['id'] == expt_id) beam = experiment.beam # Remove the need for pixel size within cxi.merge. Allows multipanel detector with dissimilar panels. # Relies on new frame extractor code called by dials.stills_process that writes s0, s1 and polarization normal # vectors all to the integration pickle. Future path (IE THIS CODE): use dials json and reflection file. s0_vec = matrix.col(beam.get_s0()).normalize() s0_polar_norm = beam.get_polarization_normal() s1_vec = refls['s1'] Ns1 = len(s1_vec) # project the s1_vector onto the plane normal to s0. Get result by subtracting the # projection of s1 onto s0, which is (s1.dot.s0_norm)s0_norm s0_norm = flex.vec3_double(Ns1,s0_vec) s1_proj = (s1_vec.dot(s0_norm))*s0_norm s1_in_normal_plane = s1_vec - s1_proj # Now want the polar angle between the projected s1 and the polarization normal s0_polar_norms = flex.vec3_double(Ns1,s0_polar_norm) dotprod = (s1_in_normal_plane.dot(s0_polar_norms)) costheta = dotprod/(s1_in_normal_plane.norms()) theta = flex.acos(costheta) cos_two_polar_angle = flex.cos(2.0*theta) # gives same as old answer to ~1% but not exact. Not sure why, should not matter. tt_vec = experiment.crystal.get_unit_cell().two_theta(miller_indices = refls['miller_index'], wavelength = beam.get_wavelength()) cos_tt_vec = flex.cos(tt_vec) sin_tt_vec = flex.sin(tt_vec) cos_sq_tt_vec = cos_tt_vec * cos_tt_vec sin_sq_tt_vec = sin_tt_vec * sin_tt_vec P_nought_vec = 0.5 * (1. + cos_sq_tt_vec) F_prime = -1.0 # Hard-coded value defines the incident polarization axis P_prime = 0.5 * F_prime * cos_two_polar_angle * sin_sq_tt_vec # added as a diagnostic #prange=P_nought_vec - P_prime #other_F_prime = 1.0 #otherP_prime = 0.5 * other_F_prime * cos_two_polar_angle * sin_sq_tt_vec #otherprange=P_nought_vec - otherP_prime #diff2 = flex.abs(prange - otherprange) #print >> out, "mean diff is",flex.mean(diff2), "range",flex.min(diff2), flex.max(diff2) # done correction = 1 / ( P_nought_vec - P_prime ) refls['intensity.sum.value'] = refls['intensity.sum.value'] * correction refls['intensity.sum.variance'] = refls['intensity.sum.variance'] * correction**2 # propagated error # This corrects observations for polarization assuming 100% polarization on # one axis (thus the F_prime = -1.0 rather than the perpendicular axis, 1.0) # Polarization model as described by Kahn, Fourme, Gadet, Janin, Dumas & Andre # (1982) J. Appl. Cryst. 15, 330-337, equations 13 - 15. if add_correction_column: refls['polarization_correction'] = correction result.extend(refls) return result
def test_calc_crystal_frame_vectors(test_reflection_table, mock_exp): """Test the namesake function, to check that the vectors are correctly rotated into the crystal frame.""" rt, exp = test_reflection_table, mock_exp s0_vec = (1.0, 0.0, 0.0) s1_vec = (1.0 / sqrt(2.0), 0.0, 1.0 / sqrt(2.0)) reflection_table = calc_crystal_frame_vectors(rt, exp) assert list(reflection_table["s0"]) == list( flex.vec3_double([s0_vec, s0_vec, s0_vec]) ) assert approx_equal( list(reflection_table["s0c"]), list( flex.vec3_double( [s0_vec, (1.0 / sqrt(2.0), -1.0 / sqrt(2.0), 0.0), (0.0, -1.0, 0.0)] ) ), ) assert approx_equal( list(reflection_table["s1c"]), list( flex.vec3_double( [ s1_vec, (1.0 / 2.0, -1.0 / 2.0, 1.0 / sqrt(2.0)), (0.0, -1.0 / sqrt(2.0), 1.0 / sqrt(2.0)), ] ) ), )
def generated_refl(not_integrated=False, idval=0): """Generate a test reflection table.""" reflections = flex.reflection_table() reflections["intensity.prf.value"] = flex.double([1.0, 10.0, 100.0, 1.0]) reflections["intensity.prf.variance"] = flex.double( [1.0, 10.0, 100.0, 1.0]) reflections["intensity.sum.value"] = flex.double( [12.0, 120.0, 1200.0, 21.0]) reflections["intensity.sum.variance"] = flex.double( [12.0, 120.0, 2100.0, 1.0]) # reflections['inverse_scale_factor'] = flex.double(4, 1.0) reflections["miller_index"] = flex.miller_index([(1, 0, 0), (0, 0, 1), (2, 0, 0), (2, 2, 2) ]) # don't change reflections["d"] = flex.double([0.8, 2.0, 2.0, 0.0]) # don't change reflections["partiality"] = flex.double([1.0, 1.0, 1.0, 1.0]) reflections["xyzobs.px.value"] = flex.vec3_double([(0.0, 0.0, 0.0), (0.0, 0.0, 5.0), (0.0, 0.0, 10.0), (0.0, 0.0, 10.0)]) reflections["s1"] = flex.vec3_double([(0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0)]) if not_integrated: reflections.set_flags(flex.bool([False, False, False, False]), reflections.flags.integrated) else: reflections.set_flags(flex.bool([True, True, False, False]), reflections.flags.integrated) reflections.set_flags(flex.bool([False, False, True, True]), reflections.flags.bad_for_scaling) reflections["id"] = flex.int(4, idval) reflections.experiment_identifiers()[idval] = str(idval) return reflections
def generate_reflections(self): """Use reeke_model to generate indices of reflections near to the Ewald sphere that might be observed on a still image. Build a reflection_table of these.""" from cctbx.sgtbx import space_group_info space_group_type = space_group_info("P 1").group().type() # create a ReekeIndexGenerator UB = self.crystal.get_U() * self.crystal.get_B() axis = self.goniometer.get_rotation_axis() s0 = self.beam.get_s0() dmin = 1.5 # use the same UB at the beginning and end - the margin parameter ensures # we still have indices close to the Ewald sphere generated from dials.algorithms.spot_prediction import ReekeIndexGenerator r = ReekeIndexGenerator(UB, UB, space_group_type, axis, s0, dmin=1.5, margin=1) # generate indices hkl = r.to_array() nref = len(hkl) # create a reflection table from dials.array_family import flex table = flex.reflection_table() table['flags'] = flex.size_t(nref, 0) table['id'] = flex.int(nref, 0) table['panel'] = flex.size_t(nref, 0) table['miller_index'] = flex.miller_index(hkl) table['entering'] = flex.bool(nref, True) table['s1'] = flex.vec3_double(nref) table['xyzcal.mm'] = flex.vec3_double(nref) table['xyzcal.px'] = flex.vec3_double(nref) return table
def generated_refl_2(exclude_refl=True): """Generate a reflection table.""" # these miller_idx/d_values don't make physical sense, but I didn't want to # have to write the tests for lots of reflections. reflections = flex.reflection_table() reflections["intensity"] = flex.double([1.0, 10.0, 100.0, 1.0]) reflections["variance"] = flex.double([1.0, 10.0, 100.0, 1.0]) reflections["miller_index"] = flex.miller_index([(1, 0, 0), (0, 0, 1), (2, 0, 0), (2, 2, 2) ]) # don't change reflections["d"] = flex.double([0.8, 2.0, 2.0, 0.0]) # don't change reflections["d"] = flex.double([0.8, 2.0, 2.1, 0.1]) reflections["Esq"] = flex.double([1.0, 1.0, 1.0, 1.0]) reflections["inverse_scale_factor"] = flex.double([1.0, 1.0, 1.0, 1.0]) reflections["id"] = flex.int(4, 0) reflections["xyzobs.px.value"] = flex.vec3_double([(0.0, 0.0, 0.0), (0.0, 0.0, 5.0), (0.0, 0.0, 10.0), (0.0, 0.0, 10.0)]) reflections["s1"] = flex.vec3_double([(0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0)]) if exclude_refl: integrated_list = flex.bool([True, True, False, False]) bad_list = flex.bool([False, False, True, True]) else: integrated_list = flex.bool(4, True) bad_list = flex.bool(4, False) reflections.set_flags(integrated_list, reflections.flags.integrated) reflections.set_flags(bad_list, reflections.flags.bad_for_scaling) return reflections
def refls_to_q(refls, detector, beam, update_table=False): orig_vecs = {} fs_vecs = {} ss_vecs = {} u_pids = set([r['panel'] for r in refls]) for pid in u_pids: orig_vecs[pid] = np.array(detector[pid].get_origin()) fs_vecs[pid] = np.array(detector[pid].get_fast_axis()) ss_vecs[pid] = np.array(detector[pid].get_slow_axis()) s1_vecs = [] q_vecs = [] for r in refls: pid = r['panel'] i_fs, i_ss, _ = r['xyzobs.px.value'] panel = detector[pid] orig = orig_vecs[pid] #panel.get_origin() fs = fs_vecs[pid] #panel.get_fast_axis() ss = ss_vecs[pid] #panel.get_slow_axis() fs_pixsize, ss_pixsize = panel.get_pixel_size() s1 = orig + i_fs * fs * fs_pixsize + i_ss * ss * ss_pixsize # scattering vector s1 = s1 / np.linalg.norm(s1) / beam.get_wavelength() s1_vecs.append(s1) q_vecs.append(s1 - beam.get_s0()) if update_table: refls['s1'] = flex.vec3_double(tuple(map(tuple, s1_vecs))) refls['rlp'] = flex.vec3_double(tuple(map(tuple, q_vecs))) return np.vstack(q_vecs)
def generated_refl_for_splitting_1(): """"Make a reflection table.""" reflections = flex.reflection_table() reflections["intensity"] = flex.double([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]) reflections["variance"] = flex.double(6, 1.0) reflections["inverse_scale_factor"] = flex.double(6, 1.0) reflections["miller_index"] = flex.miller_index( [(1, 0, 0), (2, 0, 0), (0, 0, 1), (2, 2, 2), (1, 0, 0), (2, 0, 0)] ) reflections["d"] = flex.double([0.8, 2.1, 2.0, 1.4, 1.6, 2.5]) reflections["partiality"] = flex.double(6, 1.0) reflections["xyzobs.px.value"] = flex.vec3_double( [ (0.0, 0.0, 0.0), (0.0, 0.0, 5.0), (0.0, 0.0, 8.0), (0.0, 0.0, 10.0), (0.0, 0.0, 12.0), (0.0, 0.0, 15.0), ] ) reflections["s1"] = flex.vec3_double( [ (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), ] ) reflections.set_flags(flex.bool(6, True), reflections.flags.integrated) reflections.set_flags(flex.bool(6, False), reflections.flags.bad_for_scaling) return reflections
def import_integrated(integrate_hkl, min_ios=3): reader = integrate_hkl_as_flex.reader(integrate_hkl, "IOBS,SIGMA,XCAL,YCAL,ZCAL,XOBS,YOBS,ZOBS,ISEG".split(",")) # reference: dials/command_line/import_xds.py table = flex.reflection_table() table["id"] = flex.int(len(reader.hkl), 0) table["panel"] = flex.size_t(len(reader.hkl), 0) # only assuming single panel table["miller_index"] = reader.hkl table["xyzcal.px.value"] = flex.vec3_double(reader.data["XCAL"], reader.data["YCAL"], reader.data["ZCAL"]) table["xyzobs.px.value"] = flex.vec3_double(reader.data["XOBS"], reader.data["YOBS"], reader.data["ZOBS"]) table["intensity.cor.value"] = reader.data["IOBS"] table["intensity.cor.sigma"] = reader.data["SIGMA"] # not valid name, just for making selection table["flags"] = flex.size_t(len(table), table.flags.indexed | table.flags.strong) table = table.select(table["intensity.cor.sigma"] > 0) table = table.select(table["intensity.cor.value"]/table["intensity.cor.sigma"] >= min_ios) table = table.select(table["xyzobs.px.value"].norms() > 0) # remove invalid xyzobs # dummy table["xyzobs.px.variance"] = flex.vec3_double(len(table), (1,1,1)) # TODO appropriate variance value table["s1"] = flex.vec3_double(len(table), (0,0,0)) # will be updated by set_obs_s1() del table["intensity.cor.value"] del table["intensity.cor.sigma"] return table
def get_pix_coords(wavelength, A, mill_arr, detector, delta_i=0.02): """ Code copied from sim.py courtesy of Aaron and Tara """ s0 = col((0, 0, -1 / wavelength)) q = flex.vec3_double( [A * col(idx) for idx in mill_arr.indices().as_vec3_double()]) s0_hat = flex.vec3_double([s0.normalize()] * len(q)) q_hat = q.each_normalize() #q_hat.cross(flex.vec3_double([s0_hat]*len(q_hat))) e1_hat = q_hat.cross(s0_hat) c0_hat = s0_hat.cross(e1_hat) q_len_sq = flex.double([col(v).length_sq() for v in q]) a_side = q_len_sq * wavelength / 2 b_side = flex.sqrt(q_len_sq) - a_side**2 #flex.vec3_double([sqrt(q.length_sq()-a_side**2 for idx in mill_arr)]) r_vec = flex.vec3_double(-a_side * s0_hat + b_side * c0_hat) s1 = r_vec + s0 EQ = q + s0 len_EQ = flex.double([col(v).length() for v in EQ]) ratio = len_EQ * wavelength indices = flex.miller_index() coords = flex.vec2_double() for i in range(len(s1)): if ratio[i] > 1 - delta_i and ratio[i] < 1 + delta_i: indices.append(mill_arr.indices()[i]) pix = detector[0].get_ray_intersection_px(s1[i]) if detector[0].is_coord_valid(pix): coords.append(pix) return coords, indices
def prepare_reflection_list(self,detector): spots = self.triclinic.get_observations_with_outlier_removal() ordinary_python_list_of_indexed_observations = [ { "id":0, "panel":0, "miller_index":item["pred"], "xyzobs.px.value":(spots[item["spot"]].ctr_mass_x(),spots[item["spot"]].ctr_mass_y(),0.0), "xyzobs.px.variance":(0.25,0.25,0.25), "spotfinder_lookup":item["spot"] } for item in self.triclinic_pairs ] self.length = len(ordinary_python_list_of_indexed_observations) R= flex.reflection_table.empty_standard(self.length) R['miller_index'] = flex.miller_index([item["miller_index"] for item in ordinary_python_list_of_indexed_observations]) R['xyzobs.px.value'] = flex.vec3_double([item["xyzobs.px.value"] for item in ordinary_python_list_of_indexed_observations]) R['xyzobs.px.variance'] = flex.vec3_double([item["xyzobs.px.variance"] for item in ordinary_python_list_of_indexed_observations]) R['spotfinder_lookup'] = flex.int([item["spotfinder_lookup"] for item in ordinary_python_list_of_indexed_observations]) R['xyzobs.mm.value'] = flex.vec3_double(self.length) R['xyzobs.mm.variance'] = flex.vec3_double(self.length) pxlsz = detector[0].get_pixel_size() for idx in xrange(self.length): R['xyzobs.mm.value'][idx] = (R['xyzobs.px.value'][idx][0]*pxlsz[0], R['xyzobs.px.value'][idx][1]*pxlsz[1], R['xyzobs.px.value'][idx][2]) R['xyzobs.mm.variance'][idx] = (R['xyzobs.px.variance'][idx][0]*pxlsz[0], R['xyzobs.px.variance'][idx][1]*pxlsz[1], R['xyzobs.px.variance'][idx][2]) return R
def _make_input_for_exclude_tests(exclude_images=True): """Generate input data, that upon exclusion should leave only the first reflection.""" if exclude_images: exclude_images = [["0:360:720"], ["1:360:720"]] expt1 = Experiment(scan=Scan(image_range=(0, 720), oscillation=(0.0, 1.0))) expt2 = Experiment(scan=Scan(image_range=(0, 720), oscillation=(0.0, -1.0))) refls1 = flex.reflection_table() refls2 = flex.reflection_table() refls1["xyzobs.mm.value"] = flex.vec3_double([ (0.0, 0.0, 10.0 * math.pi / 180.0), (0.0, 0.0, 370.0 * math.pi / 180.0) ]) refls1["xyzobs.px.value"] = flex.vec3_double([(0.0, 0.0, 10.0), (0.0, 0.0, 370.0)]) refls1["i"] = flex.int([0, 1]) refls2["xyzobs.mm.value"] = flex.vec3_double([ (0.0, 0.0, -10.0 * math.pi / 180.0), (0.0, 0.0, -370.0 * math.pi / 180.0) ]) refls2["xyzobs.px.value"] = flex.vec3_double([(0.0, 0.0, 10.0), (0.0, 0.0, 370.0)]) refls2["i"] = flex.int([0, 1]) expts = ExperimentList([expt1, expt2]) tables = [refls1, refls2] expts, tables = assign_unique_identifiers(expts, tables) return exclude_images, expts, tables
def _xyzcal_mm_to_px(self, experiments, reflections): # set xyzcal.px field in reflections reflections["xyzcal.px"] = flex.vec3_double(len(reflections)) for i, expt in enumerate(experiments): imgset_sel = reflections["imageset_id"] == i refined_reflections = reflections.select(imgset_sel) panel_numbers = flex.size_t(refined_reflections["panel"]) xyzcal_mm = refined_reflections["xyzcal.mm"] x_mm, y_mm, z_rad = xyzcal_mm.parts() xy_cal_mm = flex.vec2_double(x_mm, y_mm) xy_cal_px = flex.vec2_double(len(xy_cal_mm)) for i_panel in range(len(expt.detector)): panel = expt.detector[i_panel] sel = panel_numbers == i_panel xy_cal_px.set_selected( sel, panel.millimeter_to_pixel(xy_cal_mm.select(sel)) ) x_px, y_px = xy_cal_px.parts() if expt.scan is not None: z_px = expt.scan.get_array_index_from_angle(z_rad, deg=False) else: # must be a still image, z centroid not meaningful z_px = z_rad xyzcal_px = flex.vec3_double(x_px, y_px, z_px) reflections["xyzcal.px"].set_selected(imgset_sel, xyzcal_px)
def fix_xy(reflections_in, reflections_out): reflections = pickle.load(open(reflections_in, "r")) # validate that input is from 60 panel detector, assumed to be # 5 x 12 configuration assert flex.max(reflections["panel"]) == 59 delta_x = 487 + 7 delta_y = 195 + 17 panel_x = reflections["panel"].as_int() % 5 panel_y = reflections["panel"].as_int() / 5 x_offset = delta_x * panel_x y_offset = delta_y * panel_y # apply fixes x, y, z = reflections["xyzobs.px.value"].parts() x += x_offset.as_double() y += y_offset.as_double() reflections["xyzobs.px.value"] = flex.vec3_double(x, y, z) x, y, z = reflections["xyzcal.px"].parts() x += x_offset.as_double() y += y_offset.as_double() reflections["xyzcal.px"] = flex.vec3_double(x, y, z) # save - should probably do this "properly" pickle.dump(reflections, open(reflections_out, "wb")) return
def abs_correction_flex(self, s1_flex): s1_flex_length = len(s1_flex) surface_normal = flex.vec3_double(s1_flex_length, self.surface_normal) edge_of_tape_normal = flex.vec3_double(s1_flex_length, self.edge_of_tape_normal) dsurf1 = flex.double(len(s1_flex), 0) dsurf2 = flex.double(len(s1_flex), 0) dot_product = s1_flex.dot(surface_normal) sel = dot_product != 0 dsurf1.set_selected(sel, self.sn1 / dot_product.select(sel)) dsurf2.set_selected(sel, self.sn2 / dot_product.select(sel)) dsurf3 = self.sn3 / (s1_flex.dot(edge_of_tape_normal)) # determine path length through kapton tape kapton_path_mm = flex.double(s1_flex_length, 0) unshadowed_sel = (dsurf3 < dsurf1) | (dsurf1 < 0) nearsel = ~unshadowed_sel & (dsurf3 < dsurf2) kapton_path_mm.set_selected(nearsel, (dsurf3 - dsurf1).select(nearsel)) farsel = ~unshadowed_sel & (dsurf3 >= dsurf2) kapton_path_mm.set_selected(farsel, (dsurf2 - dsurf1).select(farsel)) # determine absorption correction absorption_correction = 1 / flex.exp( -self.abs_coeff * kapton_path_mm ) # unitless, >=1 return absorption_correction
def _predict(self): """ Predict the position of the spots """ # Get some stuff from experiment A = matrix.sqr(self.experiments[0].crystal.get_A()) s0 = matrix.col(self.experiments[0].beam.get_s0()) # Compute the vector to the reciprocal lattice point # since this is not on the ewald sphere, lets call it s2 h = self.reflections["miller_index"] s1 = flex.vec3_double(len(h)) s2 = flex.vec3_double(len(h)) for i in range(len(self.reflections)): r = A * matrix.col(h[i]) s2[i] = s0 + r s1[i] = matrix.col(s2[i]).normalize() * s0.length() self.reflections["s1"] = s1 self.reflections["s2"] = s2 self.reflections["entering"] = flex.bool(len(h), False) # Compute the ray intersections xyzpx = flex.vec3_double() xyzmm = flex.vec3_double() for i in range(len(s2)): ss = s1[i] mm = self.experiments[0].detector[0].get_ray_intersection(ss) px = self.experiments[0].detector[0].millimeter_to_pixel(mm) xyzpx.append(px + (0, )) xyzmm.append(mm + (0, )) self.reflections["xyzcal.mm"] = xyzmm self.reflections["xyzcal.px"] = xyzpx logger.info("Do prediction for %d reflections" % len(self.reflections))
def show_experiments(experiments, show_scan_varying=False): text = [] for i_expt, expt in enumerate(experiments): text.append("Experiment %i:" % i_expt) format_class = expt.imageset.get_format_class() if not format_class.is_abstract(): text.append(f"Format class: {format_class.__name__}") if expt.identifier != "": text.append(f"Experiment identifier: {expt.identifier}") try: template = expt.imageset.get_template() except AttributeError: template = None if template: text.append(f"Image template: {template}") text.append(str(expt.detector)) text.append("Max resolution (at corners): %f" % (expt.detector.get_max_resolution(expt.beam.get_s0()))) text.append( "Max resolution (inscribed): %f" % (expt.detector.get_max_inscribed_resolution(expt.beam.get_s0()))) text.append("") text.append(show_beam(expt.detector, expt.beam)) if expt.scan is not None: text.append(str(expt.scan)) if expt.goniometer is not None: text.append(show_goniometer(expt.goniometer)) if expt.crystal is not None: text.append( expt.crystal.as_str(show_scan_varying=show_scan_varying)) if expt.crystal.num_scan_points: abc = flex.vec3_double() angles = flex.vec3_double() for n in range(expt.crystal.num_scan_points): ( a, b, c, alpha, beta, gamma, ) = expt.crystal.get_unit_cell_at_scan_point( n).parameters() abc.append((a, b, c)) angles.append((alpha, beta, gamma)) a, b, c = abc.mean() alpha, beta, gamma = angles.mean() mean_unit_cell = uctbx.unit_cell((a, b, c, alpha, beta, gamma)) text.append(f" Average unit cell: {mean_unit_cell}") if expt.profile is not None: text.append(str(expt.profile)) if expt.scaling_model is not None: text.append(str(expt.scaling_model)) return "\n".join(text)
def __call__(self, params, options): ''' Import the integrate.hkl file. ''' from iotbx.xds import integrate_hkl from dials.array_family import flex from dials.util.command_line import Command from cctbx import sgtbx # Get the unit cell to calculate the resolution uc = self._experiment.crystal.get_unit_cell() # Read the INTEGRATE.HKL file Command.start('Reading INTEGRATE.HKL') handle = integrate_hkl.reader() handle.read_file(self._integrate_hkl) hkl = flex.miller_index(handle.hkl) xyzcal = flex.vec3_double(handle.xyzcal) xyzobs = flex.vec3_double(handle.xyzobs) iobs = flex.double(handle.iobs) sigma = flex.double(handle.sigma) rlp = flex.double(handle.rlp) peak = flex.double(handle.peak) * 0.01 Command.end('Read %d reflections from INTEGRATE.HKL file.' % len(hkl)) # Derive the reindex matrix rdx = self.derive_reindex_matrix(handle) print 'Reindex matrix:\n%d %d %d\n%d %d %d\n%d %d %d' % (rdx.elems) # Reindex the reflections Command.start('Reindexing reflections') cb_op = sgtbx.change_of_basis_op(sgtbx.rt_mx(sgtbx.rot_mx(rdx.elems))) hkl = cb_op.apply(hkl) Command.end('Reindexed %d reflections' % len(hkl)) # Create the reflection list Command.start('Creating reflection table') table = flex.reflection_table() table['id'] = flex.int(len(hkl), 0) table['panel'] = flex.size_t(len(hkl), 0) table['miller_index'] = hkl table['xyzcal.px'] = xyzcal table['xyzobs.px.value'] = xyzobs table['intensity.cor.value'] = iobs table['intensity.cor.variance'] = sigma**2 table['intensity.prf.value'] = iobs * peak / rlp table['intensity.prf.variance'] = (sigma * peak / rlp)**2 table['lp'] = 1.0 / rlp table['d'] = flex.double(uc.d(h) for h in hkl) Command.end('Created table with {0} reflections'.format(len(table))) # Output the table to pickle file if params.output.filename is None: params.output.filename = 'integrate_hkl.pickle' Command.start('Saving reflection table to %s' % params.output.filename) table.as_pickle(params.output.filename) Command.end('Saved reflection table to %s' % params.output.filename)
def generate_reflection_table(): """Create a reflection table with s1 and phi.""" rt = flex.reflection_table() s1_vec = (1.0 / sqrt(2.0), 0.0, 1.0 / sqrt(2.0)) rt["s1"] = flex.vec3_double([s1_vec, s1_vec, s1_vec]) rt["phi"] = flex.double([0.0, 45.0, 90.0]) rt["xyzobs.px.value"] = flex.vec3_double([(0.0, 0.0, 0.0), (0.0, 0.0, 45.0), (0.0, 0.0, 90.0)]) return rt
def align_rotation_axis_along_z(exp_rot_axis, vectors): """Rotate the coordinate system such that the exp_rot_axis is along z.""" if list(exp_rot_axis) == [(0.0, 0.0, 1.0)]: return vectors (ux, uy, uz) = exp_rot_axis[0][0], exp_rot_axis[0][1], exp_rot_axis[0][2] cross_prod_uz = flex.vec3_double([(uy, -1.0 * ux, 0.0)]) angle_between_u_z = +1.0 * acos(uz / ((ux**2 + uy**2 + uz**2)**0.5)) phi = flex.double(vectors.size(), angle_between_u_z) new_vectors = rotate_vectors_about_axis(cross_prod_uz, vectors, phi) return flex.vec3_double(new_vectors)
def generated_refl(): """Create a reflection table.""" rt = flex.reflection_table() rt["xyzobs.px.value"] = flex.vec3_double([(0.1, 0.1, 0.1), (0.1, 0.1, 0.2)]) rt["s1"] = flex.vec3_double([(0.1, 0.1, 0.1), (0.1, 0.1, 1.1)]) rt["s1c"] = flex.vec3_double([(0.1, 0.1, 0.1), (0.1, 0.1, 1.1)]) rt["s0c"] = flex.vec3_double([(0.0, 0.0, 1.0), (0.0, 0.0, 1.0)]) rt["d"] = flex.double([1.0, 1.0]) rt["batch"] = flex.int([0, 1]) return rt
def __call__(self, params, options): ''' Import the spot.xds file. ''' from iotbx.xds import spot_xds from dials.util.command_line import Command from dials.array_family import flex # Read the SPOT.XDS file Command.start('Reading SPOT.XDS') handle = spot_xds.reader() handle.read_file(self._spot_xds) centroid = handle.centroid intensity = handle.intensity try: miller_index = handle.miller_index except AttributeError: miller_index = None Command.end('Read {0} spots from SPOT.XDS file.'.format(len(centroid))) # Create the reflection list Command.start('Creating reflection list') table = flex.reflection_table() table['id'] = flex.int(len(centroid), 0) table['panel'] = flex.size_t(len(centroid), 0) if miller_index: table['miller_index'] = flex.miller_index(miller_index) table['xyzobs.px.value'] = flex.vec3_double(centroid) table['intensity.sum.value'] = flex.double(intensity) Command.end('Created reflection list') # Remove invalid reflections Command.start('Removing invalid reflections') if miller_index and params.remove_invalid: flags = flex.bool([h != (0, 0, 0) for h in table['miller_index']]) table = table.select(flags) Command.end('Removed invalid reflections, %d remaining' % len(table)) # Fill empty standard columns if params.add_standard_columns: Command.start('Adding standard columns') rt = flex.reflection_table.empty_standard(len(table)) rt.update(table) table = rt # set variances to unity table['xyzobs.mm.variance'] = flex.vec3_double( len(table), (1, 1, 1)) table['xyzobs.px.variance'] = flex.vec3_double( len(table), (1, 1, 1)) Command.end('Standard columns added') # Output the table to pickle file if params.output.filename is None: params.output.filename = 'spot_xds.pickle' Command.start('Saving reflection table to %s' % params.output.filename) table.as_pickle(params.output.filename) Command.end('Saved reflection table to %s' % params.output.filename)
def get_fd_gradients(self, pred_param, ref_predictor): # get finite difference gradients p_vals = pred_param.get_param_vals() deltas = [1.0e-7] * len(p_vals) fd_grads = [] p_names = pred_param.get_param_names() for i in range(len(deltas)): # save parameter value val = p_vals[i] # calc reverse state p_vals[i] -= deltas[i] / 2.0 pred_param.set_param_vals(p_vals) ref_predictor(self.reflections) x, y, _ = self.reflections["xyzcal.mm"].deep_copy().parts() delpsi = self.reflections["delpsical.rad"].deep_copy() rev_state = flex.vec3_double(x, y, delpsi) # calc forward state p_vals[i] += deltas[i] pred_param.set_param_vals(p_vals) ref_predictor(self.reflections) x, y, _ = self.reflections["xyzcal.mm"].deep_copy().parts() delpsi = self.reflections["delpsical.rad"].deep_copy() fwd_state = flex.vec3_double(x, y, delpsi) # reset parameter to saved value p_vals[i] = val # finite difference fd = fwd_state - rev_state x_grads, y_grads, delpsi_grads = fd.parts() x_grads /= deltas[i] y_grads /= deltas[i] delpsi_grads /= deltas[i] fd_grads.append({ "name": p_names[i], "dX_dp": x_grads, "dY_dp": y_grads, "dDeltaPsi_dp": delpsi_grads, }) # return to the initial state pred_param.set_param_vals(p_vals) return fd_grads
def generated_refl_for_comb(): """Create a reflection table suitable for splitting into blocks.""" reflections = flex.reflection_table() reflections["intensity"] = flex.double( [1.0, 2.0, 3.0, 4.0, 500.0, 6.0, 2.0, 2.0]) reflections["variance"] = flex.double(8, 1.0) reflections["intensity.prf.value"] = flex.double( [1.0, 3.0, 3.0, 4.0, 50.0, 6.0, 3.0, 2.0]) reflections["intensity.prf.variance"] = flex.double( [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0]) reflections["intensity.sum.value"] = flex.double( [1.0, 4.0, 3.0, 4.0, 500.0, 6.0, 6.0, 2.0]) reflections["intensity.sum.variance"] = flex.double(8, 1.0) reflections["miller_index"] = flex.miller_index([ (1, 0, 0), (2, 0, 0), (0, 0, 1), (2, 2, 2), (1, 0, 0), (2, 0, 0), (1, 0, 0), (1, 0, 0), ]) reflections["d"] = flex.double([0.8, 2.1, 2.0, 1.4, 1.6, 2.5, 2.5, 2.5]) reflections["partiality"] = flex.double(8, 1.0) reflections["Esq"] = flex.double(8, 1.0) reflections["inverse_scale_factor"] = flex.double(8, 1.0) reflections["xyzobs.px.value"] = flex.vec3_double([ (0.0, 0.0, 0.0), (0.0, 0.0, 5.0), (0.0, 0.0, 8.0), (0.0, 0.0, 10.0), (0.0, 0.0, 12.0), (0.0, 0.0, 15.0), (0.0, 0.0, 15.0), (0.0, 0.0, 15.0), ]) reflections["s1"] = flex.vec3_double([ (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), ]) reflections.set_flags(flex.bool(8, True), reflections.flags.integrated) reflections.set_flags(flex.bool([False] * 5 + [True] + [False] * 2), reflections.flags.bad_for_scaling) reflections["id"] = flex.int(8, 0) reflections.experiment_identifiers()[0] = "0" reflections = calculate_prescaling_correction(reflections) return reflections
def __call__(self, params, options): """ Import the spot.xds file. """ from iotbx.xds import spot_xds from dials.util.command_line import Command # Read the SPOT.XDS file Command.start("Reading SPOT.XDS") handle = spot_xds.reader() handle.read_file(self._spot_xds) centroid = handle.centroid intensity = handle.intensity try: miller_index = handle.miller_index except AttributeError: miller_index = None Command.end("Read {} spots from SPOT.XDS file.".format(len(centroid))) # Create the reflection list Command.start("Creating reflection list") table = flex.reflection_table() table["id"] = flex.int(len(centroid), 0) table["panel"] = flex.size_t(len(centroid), 0) if miller_index: table["miller_index"] = flex.miller_index(miller_index) table["xyzobs.px.value"] = flex.vec3_double(centroid) table["intensity.sum.value"] = flex.double(intensity) Command.end("Created reflection list") # Remove invalid reflections Command.start("Removing invalid reflections") if miller_index and params.remove_invalid: flags = flex.bool([h != (0, 0, 0) for h in table["miller_index"]]) table = table.select(flags) Command.end("Removed invalid reflections, %d remaining" % len(table)) # Fill empty standard columns if params.add_standard_columns: Command.start("Adding standard columns") rt = flex.reflection_table.empty_standard(len(table)) rt.update(table) table = rt # set variances to unity table["xyzobs.mm.variance"] = flex.vec3_double( len(table), (1, 1, 1)) table["xyzobs.px.variance"] = flex.vec3_double( len(table), (1, 1, 1)) Command.end("Standard columns added") # Output the table to pickle file if params.output.filename is None: params.output.filename = "spot_xds.refl" Command.start("Saving reflection table to %s" % params.output.filename) table.as_file(params.output.filename) Command.end("Saved reflection table to %s" % params.output.filename)
def get_fd_gradients(self, pred_param, ref_predictor): # get finite difference gradients p_vals = pred_param.get_param_vals() deltas = [1.e-7] * len(p_vals) fd_grads = [] p_names = pred_param.get_param_names() for i in range(len(deltas)): # save parameter value val = p_vals[i] # calc reverse state p_vals[i] -= deltas[i] / 2. pred_param.set_param_vals(p_vals) ref_predictor.update() ref_predictor.predict(self.reflections) x, y, _ = self.reflections['xyzcal.mm'].deep_copy().parts() delpsi = self.reflections['delpsical.rad'].deep_copy() rev_state = flex.vec3_double(x, y, delpsi) # calc forward state p_vals[i] += deltas[i] pred_param.set_param_vals(p_vals) ref_predictor.update() ref_predictor.predict(self.reflections) x, y, _ = self.reflections['xyzcal.mm'].deep_copy().parts() delpsi = self.reflections['delpsical.rad'].deep_copy() fwd_state = flex.vec3_double(x, y, delpsi) # reset parameter to saved value p_vals[i] = val # finite difference fd = (fwd_state - rev_state) x_grads, y_grads, delpsi_grads = fd.parts() x_grads /= deltas[i] y_grads /= deltas[i] delpsi_grads /= deltas[i] fd_grads.append({'name':p_names[i], 'dX_dp':x_grads, 'dY_dp':y_grads, 'dDeltaPsi_dp':delpsi_grads}) # return to the initial state pred_param.set_param_vals(p_vals) return fd_grads
def test_RefinerData(testdata): experiment = testdata.experiment reflections = testdata.reflections panel = experiment.detector[0] s0_length = matrix.col(experiment.beam.get_s0()).length() reflections["bbox"] = flex.int6(len(reflections)) reflections["xyzobs.px.value"] = flex.vec3_double(len(reflections)) reflections["s2"] = reflections["s1"].each_normalize() * s0_length reflections["sp"] = flex.vec3_double(len(reflections)) for i, (x, y, z) in enumerate(reflections["xyzcal.px"]): x0 = int(x) - 5 x1 = int(x) + 5 + 1 y0 = int(y) - 5 y1 = int(y) + 5 + 1 z0 = int(z) z1 = z0 + 1 reflections["bbox"][i] = x0, x1, y0, y1, z0, z1 reflections["xyzobs.px.value"][i] = (int(x) + 0.5, int(y) + 0.5, int(z) + 0.5) reflections["sp"][i] = (matrix.col( panel.get_pixel_lab_coord( reflections["xyzobs.px.value"][i][0:2])).normalize() * s0_length) reflections["shoebox"] = flex.shoebox(reflections["panel"], reflections["bbox"], allocate=True) shoebox_data = flex.float(flex.grid(1, 11, 11)) shoebox_mask = flex.int(flex.grid(1, 11, 11)) for j in range(11): for i in range(11): shoebox_data[0, j, i] = (100 * exp(-0.5 * (j - 5)**2 / 1**2) * exp(-0.5 * (i - 5)**2 / 1**2)) shoebox_mask[0, j, i] = 5 for sbox in reflections["shoebox"]: sbox.data = shoebox_data sbox.mask = shoebox_mask data = RefinerData.from_reflections(experiment, reflections) assert tuple(data.s0) == pytest.approx(experiment.beam.get_s0()) assert data.h_list == reflections["miller_index"] for i, sp in enumerate(reflections["sp"]): assert data.sp_list[:, i] == pytest.approx(sp) assert data.ctot_list[0] == sum(shoebox_data) mobs1 = np.abs(data.mobs_list[0, :]) mobs2 = np.abs(data.mobs_list[1, :]) assert np.max(mobs1) < 1e-6 assert np.max(mobs2) < 1e-6
def __init__(self): from dials.array_family import flex self.reflections = flex.reflection_table() self.reflections['panel'] = flex.size_t() self.reflections['bbox'] = flex.int6() self.reflections['miller_index'] = flex.miller_index() self.reflections['s1'] = flex.vec3_double() self.reflections['xyzcal.px'] = flex.vec3_double() self.reflections['xyzcal.mm'] = flex.vec3_double() self.reflections['entering'] = flex.bool() self.reflections['id'] = flex.int() self.reflections["flags"] = flex.size_t() self.npanels = 2 self.width = 1000 self.height = 1000 self.nrefl = 10000 self.array_range = (0, 130) self.block_size = 20 from random import randint, seed, choice seed(0) self.processed = [[] for i in range(12)] for i in range(self.nrefl): x0 = randint(0, self.width - 10) y0 = randint(0, self.height - 10) zs = randint(2, 9) x1 = x0 + randint(2, 10) y1 = y0 + randint(2, 10) for k, j in enumerate( [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]): m = k + i * 12 pos = choice(["left", "right", "centre"]) if pos == 'left': z0 = j - zs z1 = j elif pos == 'right': z0 = j z1 = j + zs else: z0 = j - zs // 2 z1 = j + zs // 2 bbox = (x0, x1, y0, y1, z0, z1) self.reflections.append({ "panel": randint(0, 1), "bbox": bbox, "flags": flex.reflection_table.flags.reference_spot }) self.processed[k].append(m)
def test_align_rotation_axis_along_z(): """Test the function to rotate the coordinate system such that the rotation axis is along z. In the test, the rotation axis is x, so we expect the transformation to be: x > z, y > y, z > -x, x+z > -x+z.""" rot_axis = flex.vec3_double([(1.0, 0.0, 0.0)]) vectors = flex.vec3_double( [(1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, 1.0), (1.0, 0.0, 1.0)] ) rotated_vectors = align_rotation_axis_along_z(rot_axis, vectors) expected = [(0.0, 0.0, 1.0), (0.0, 1.0, 0.0), (-1.0, 0.0, 0.0), (-1.0, 0.0, 1.0)] for v1, v2 in zip(rotated_vectors, expected): assert v1 == pytest.approx(v2)
def __call__(self, params, options): ''' Import the spot.xds file. ''' from iotbx.xds import spot_xds from dials.util.command_line import Command from dials.array_family import flex # Read the SPOT.XDS file Command.start('Reading SPOT.XDS') handle = spot_xds.reader() handle.read_file(self._spot_xds) centroid = handle.centroid intensity = handle.intensity try: miller_index = handle.miller_index except AttributeError: miller_index = None Command.end('Read {0} spots from SPOT.XDS file.'.format(len(centroid))) # Create the reflection list Command.start('Creating reflection list') table = flex.reflection_table() table['id'] = flex.int(len(centroid), 0) table['panel'] = flex.size_t(len(centroid), 0) if miller_index: table['miller_index'] = flex.miller_index(miller_index) table['xyzobs.px.value'] = flex.vec3_double(centroid) table['intensity.sum.value'] = flex.double(intensity) Command.end('Created reflection list') # Remove invalid reflections Command.start('Removing invalid reflections') if miller_index and params.remove_invalid: flags = flex.bool([h != (0, 0, 0) for h in table['miller_index']]) table = table.select(flags) Command.end('Removed invalid reflections, %d remaining' % len(table)) # Fill empty standard columns if params.add_standard_columns: Command.start('Adding standard columns') rt = flex.reflection_table.empty_standard(len(table)) rt.update(table) table = rt # set variances to unity table['xyzobs.mm.variance'] = flex.vec3_double(len(table), (1,1,1)) table['xyzobs.px.variance'] = flex.vec3_double(len(table), (1,1,1)) Command.end('Standard columns added') # Output the table to pickle file if params.output.filename is None: params.output.filename = 'spot_xds.pickle' Command.start('Saving reflection table to %s' % params.output.filename) table.as_pickle(params.output.filename) Command.end('Saved reflection table to %s' % params.output.filename)
def _prepare_for_compose(self, reflections, skip_derivatives=False): """Add columns to the reflection table to hold the varying state matrices or vectors for the experimental models, if required. Also add columns for the derivatives of states that are scan-varying""" nref = len(reflections) # set columns if needed if 'u_matrix' not in reflections: reflections['u_matrix'] = flex.mat3_double(nref) if 'b_matrix' not in reflections: reflections['b_matrix'] = flex.mat3_double(nref) if 's0_vector' not in reflections: reflections['s0_vector'] = flex.vec3_double(nref) if 'd_matrix' not in reflections: reflections['d_matrix'] = flex.mat3_double(nref) if 'D_matrix' not in reflections: reflections['D_matrix'] = flex.mat3_double(nref) # set columns in the reflection table to store the derivative of state for # each reflection, if needed if not skip_derivatives: null9 = (0., 0., 0., 0., 0., 0., 0., 0., 0.) null3 = (0., 0., 0.) if self._varying_xl_orientations and "dU_dp0" not in reflections: max_free_params = max([ e.num_free() for e in self._xl_orientation_parameterisations ]) for i in range(max_free_params): colname = "dU_dp{0}".format(i) reflections[colname] = flex.mat3_double(nref, null9) if self._varying_xl_unit_cells and "dB_dp0" not in reflections: max_free_params = max([ e.num_free() for e in self._xl_unit_cell_parameterisations ]) for i in range(max_free_params): colname = "dB_dp{0}".format(i) reflections[colname] = flex.mat3_double(nref, null9) if self._varying_detectors and "dd_dp0" not in reflections: max_free_params = max( [e.num_free() for e in self._detector_parameterisations]) for i in range(max_free_params): colname = "dd_dp{0}".format(i) reflections[colname] = flex.mat3_double(nref, null9) if self._varying_beams and "ds0_dp0" not in reflections: max_free_params = max( [e.num_free() for e in self._beam_parameterisations]) for i in range(max_free_params): colname = "ds0_dp{0}".format(i) reflections[colname] = flex.vec3_double(nref, null3) return
def generated_10_refl(): """Generate reflection table to test the basis and target function.""" # these miller_idx/d_values don't make physical sense, but I didn't want to # have to write the tests for lots of reflections. reflections = flex.reflection_table() reflections["intensity.prf.value"] = flex.double( [75.0, 10.0, 100.0, 25.0, 50.0, 100.0, 25.0, 20.0, 300.0, 10.0]) reflections["intensity.prf.variance"] = flex.double( [50.0, 10.0, 100.0, 50.0, 10.0, 100.0, 50.0, 10.0, 100.0, 10.0]) reflections["miller_index"] = flex.miller_index([ (1, 0, 0), (0, 0, 1), (1, 0, 0), (1, 0, 0), (0, 0, 1), (1, 0, 0), (0, 4, 0), (0, 0, 1), (1, 0, 0), (0, 4, 0), ]) # don't change reflections["d"] = flex.double( [2.0, 0.8, 2.0, 2.0, 0.8, 2.0, 2.0, 0.8, 2.0, 1.0]) # don't change reflections["partiality"] = flex.double(10, 1.0) reflections["xyzobs.px.value"] = flex.vec3_double([ (0.0, 0.0, 0.0), (0.0, 0.0, 5.0), (0.0, 0.0, 10.0), (0.0, 0.0, 15.0), (0.0, 0.0, 20.0), (0.0, 0.0, 25.0), (0.0, 0.0, 30.0), (0.0, 0.0, 35.0), (0.0, 0.0, 40.0), (0.0, 0.0, 59.0), ]) reflections["s1"] = flex.vec3_double([ (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 20.0), (0.0, 0.1, 20.0), (0.0, 0.1, 20.0), (0.0, 0.1, 20.0), (0.0, 0.1, 20.0), (0.0, 0.1, 20.0), (0.0, 0.1, 20.0), (0.0, 0.1, 20.0), ]) reflections.set_flags(flex.bool(10, True), reflections.flags.integrated) reflections["id"] = flex.int(10, 0) reflections.experiment_identifiers()[0] = str(0) return [reflections]
def main(): parser = OptionParser(read_experiments=True, read_reflections=True) params, options = parser.parse_args(show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) data = reflections[0] expt = experiments[0] data = data.select(data.get_flags(data.flags.integrated)) # data = data.select(data['d'] > 2.0) z = data["xyzcal.px"].parts()[2] shoebox = data["shoebox"] # make a new reflection table, this one empty - then add columns for # every event in this set (with some shuffle) - flags = 32 apparently, # id == id above, intensity.sum.value=1 variance=1 n_signal=1 panel=panel # xyzobs.px.value = pixel + random.random() - 0.5 variance = 1/12 in each # direction - then map to reciprocal space events = flex.vec3_double() for s in shoebox: d = s.data k0, j0, i0 = s.bbox[0], s.bbox[2], s.bbox[4] k1, j1, i1 = d.focus() for k in range(k1): for j in range(j1): for i in range(i1): for n in range(int(d[k, j, i])): if random.random() > 0.1: continue z = k + k0 + random.random() y = j + j0 + random.random() x = i + i0 + random.random() events.append((z, y, x)) rt = flex.reflection_table() rt["xyzobs.px.value"] = events variance = flex.double(events.size(), 1.0 / 12.0) rt["xyzobs.px.variance"] = flex.vec3_double(variance, variance, variance) rt["flags"] = flex.size_t(events.size(), 32) rt["id"] = flex.int(events.size(), 0) rt["panel"] = flex.size_t(events.size(), 0) rt["intensity.sum.value"] = flex.double(events.size(), 1) rt["intensity.sum.variance"] = flex.double(events.size(), 1) rt.as_file("events.refl")
def __init__(self): from dials.array_family import flex self.reflections = flex.reflection_table() self.reflections['panel'] = flex.size_t() self.reflections['bbox'] = flex.int6() self.reflections['miller_index'] = flex.miller_index() self.reflections['s1'] = flex.vec3_double() self.reflections['xyzcal.px'] = flex.vec3_double() self.reflections['xyzcal.mm'] = flex.vec3_double() self.reflections['entering'] = flex.bool() self.reflections['id'] = flex.int() self.reflections["flags"] = flex.size_t() self.npanels = 2 self.width = 1000 self.height = 1000 self.nrefl = 10000 self.array_range = (0, 130) self.block_size = 20 from random import randint, seed, choice seed(0) self.processed = [[] for i in range(12)] for i in range(self.nrefl): x0 = randint(0, self.width-10) y0 = randint(0, self.height-10) zs = randint(2, 9) x1 = x0 + randint(2, 10) y1 = y0 + randint(2, 10) for k, j in enumerate([10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]): m = k + i * 12 pos = choice(["left", "right", "centre"]) if pos == 'left': z0 = j - zs z1 = j elif pos == 'right': z0 = j z1 = j + zs else: z0 = j - zs // 2 z1 = j + zs // 2 bbox = (x0, x1, y0, y1, z0, z1) self.reflections.append({ "panel" : randint(0,1), "bbox" : bbox, "flags" : flex.reflection_table.flags.reference_spot }) self.processed[k].append(m)
def get_gradients(self, reflections, callback=None): """ Calculate gradients of the prediction formula with respect to each of the parameters of the contained models, for all of the reflections. To be implemented by a derived class, which determines the space of the prediction formula (e.g. we calculate dX/dp, dY/dp, dphi/dp for the prediction formula for a rotation scan expressed in detector space, but components of d\vec{r}/dp for the prediction formula in reciprocal space """ ### Calculate various quantities of interest for the reflections # Set up arrays of values for each reflection n = len(reflections) D = flex.mat3_double(n) s0 = flex.vec3_double(n) U = flex.mat3_double(n) B = flex.mat3_double(n) axis = flex.vec3_double(n) fixed_rotation = flex.mat3_double(n) for iexp, exp in enumerate(self._experiments): sel = reflections['id'] == iexp isel = sel.iselection() # D matrix array panels = reflections['panel'].select(isel) for ipanel, D_mat in enumerate([p.get_D_matrix() for p in exp.detector]): subsel = isel.select(panels == ipanel) D.set_selected(subsel, D_mat) # s0 array s0.set_selected(isel, exp.beam.get_s0()) # U and B arrays exp_U, exp_B = self._get_U_B_for_experiment(exp.crystal, reflections, isel) U.set_selected(isel, exp_U) B.set_selected(isel, exp_B) # axis array if exp.goniometer: axis.set_selected(isel, exp.goniometer.get_rotation_axis()) fixed_rotation.set_selected(isel, exp.goniometer.get_fixed_rotation()) return self._get_gradients_core(reflections, D, s0, U, B, axis, fixed_rotation, callback)
def refine(self, experiments, reflections): sel = ((reflections['id'] >= -1)) refl = reflections.select(sel) acceptance_flags = self.identify_outliers(self.all_params, experiments, refl) #create a new "indexed" list with outliers thrown out: refl = refl.select(acceptance_flags) print "$$$ stills_indexer::refine" R = e_refine(params = self.all_params, experiments=experiments, reflections=refl, graph_verbose=False) ref_experiments = R.get_experiments() # try to improve the outcome with a second round of outlier rejection post-initial refinement: acceptance_flags = self.identify_outliers(self.all_params, ref_experiments, refl) # insert a round of Nave-outlier rejection on top of the r.m.s.d. rejection nv0 = nave_parameters(params = self.all_params, experiments=ref_experiments, reflections=refl, refinery=R, graph_verbose=False) crystal_model_nv0 = nv0() acceptance_flags_nv0 = nv0.nv_acceptance_flags refl = refl.select(acceptance_flags & acceptance_flags_nv0) print "$$$ stills_indexer::refine after positional and delta-psi outlier rejection" refiner = e_refine(params = self.all_params, experiments=ref_experiments, reflections=refl, graph_verbose=False) matches = refiner.get_matches() xyzcal_mm = flex.vec3_double(len(refl)) xyzcal_mm.set_selected(matches['iobs'], matches['xyzcal.mm']) refl['xyzcal.mm'] = xyzcal_mm refl.set_flags(matches['iobs'], refl.flags.used_in_refinement) refl['entering'] = flex.bool(len(refl), False) return refiner.get_experiments(), refl
def calculate_weights(self, reflections): """Set weights to constant terms. If stills, the z weights are the 'delpsical.weights' attribute of the reflection table. Otherwise, use the usual 'xyzobs.mm.weights'""" wx = flex.double(len(reflections), self._wx) wy = flex.double(len(reflections), self._wy) wz = flex.double(len(reflections), self._wz) if self._stills: null = flex.double(len(reflections), 0) reflections['xyzobs.mm.weights'] = flex.vec3_double(wx, wy, null) reflections['delpsical.weights'] = wz else: reflections['xyzobs.mm.weights'] = flex.vec3_double(wx, wy, wz) return reflections
def generate_spots(crystal_model, detector, beam, goniometer=None, scan=None, sel_fraction=1.0): import math experiment = Experiment(beam=beam, detector=detector, goniometer=goniometer, scan=scan, crystal=crystal_model) # if we don't set the imageset then from_predictions uses the StillsReflectionPredictor :-( from dxtbx.imageset import NullReader, ImageSweep imageset = ImageSweep(NullReader, indices=range(len(scan.get_epochs())), beam=beam, goniometer=goniometer, detector=detector, scan=scan) experiment.imageset = imageset predicted = flex.reflection_table.from_predictions(experiment) sel = flex.random_selection(len(predicted), int(math.floor(sel_fraction*len(predicted)))) predicted = predicted.select(sel) predicted['imageset_id'] = flex.size_t(len(predicted), 0) predicted['xyzobs.px.value'] = predicted['xyzcal.px'] predicted['xyzobs.px.variance'] = flex.vec3_double( len(predicted), (0.5,0.5,0.5)) return predicted
def add_random_noise_xyz(datablock, strong_spots, rmsds): import scitbx.random errors = flex.vec3_double(*list( scitbx.random.variate( scitbx.random.normal_distribution(0, rmsds[i]))(len(strong_spots)) for i in range(3))) strong_spots['xyzobs.px.value'] += errors
def get_s1_array(self, values): miller_vec = self.MILLER.as_vec3_double() ref_ori = matrix.sqr(self.ORI.reciprocal_matrix()) Rx = matrix.col((1,0,0)).axis_and_angle_as_r3_rotation_matrix(values.thetax) Ry = matrix.col((0,1,0)).axis_and_angle_as_r3_rotation_matrix(values.thetay) s_array = flex.mat3_double(len(self.MILLER),Ry * Rx * ref_ori) * miller_vec s1_array = s_array + flex.vec3_double(len(self.MILLER), self.BEAM) return s1_array
def _beam_derivatives(self, isel, parameterisation=None, reflections=None): """helper function to extend the derivatives lists by derivatives of the beam parameterisations""" # Get required data s0 = self._s0.select(isel) s0u = self._s0u.select(isel) wl = self._wavelength.select(isel) r = self._r.select(isel) e1 = self._e1.select(isel) q = self._q.select(isel) c0 = self._c0.select(isel) DeltaPsi = self._DeltaPsi.select(isel) D = self._D.select(isel) # get the derivatives of the beam vector wrt the parameters ds0_dbeam_p = parameterisation.get_ds_dp(use_none_as_null=True) dDeltaPsi_dp = [] dpv_dp = [] # loop through the parameters for der in ds0_dbeam_p: if der is None: dpv_dp.append(None) dDeltaPsi_dp.append(None) continue # repeat the derivative in an array ds0 = flex.vec3_double(len(s0u), der.elems) # we need the derivative of the unit beam direction too. This requires # scaling by the wavelength and projection onto the Ewald sphere scaled = ds0 * wl ds0u = scaled.dot(c0) * c0 + scaled.dot(e1) * e1 # calculate the derivative of DeltaPsi for this parameter dDeltaPsi = -1.0 * (r.dot(ds0)) / (e1.cross(r).dot(s0)) dDeltaPsi_dp.append(dDeltaPsi) # calculate (d[r]/d[e1])(d[e1]/dp) de1_dp = c0.cross(ds0u) dr_de1 = dRq_de(DeltaPsi, e1, q) drde_dedp = dr_de1 * de1_dp #dp = 1.e-8 # finite step size for the parameter #del_e1 = de1_dp * dp #e1f = e1 + del_e1 * 0.5 #rfwd = q.rotate_around_origin(e1f, DeltaPsi) #e1r = e1 - del_e1 * 0.5 #rrev = q.rotate_around_origin(e1r, DeltaPsi) #drde_dedp = (rfwd - rrev) * (1 / dp) # calculate the derivative of pv for this parameter dpv_dp.append(D * (ds0 + e1.cross(r) * dDeltaPsi + drde_dedp)) return dpv_dp, dDeltaPsi_dp
def _prepare_for_compose(self, reflections, skip_derivatives=False): """Add columns to the reflection table to hold the varying state matrices or vectors for the experimental models, if required. Also add columns for the derivatives of states that are scan-varying""" nref = len(reflections) # set columns if needed if 'u_matrix' not in reflections: reflections['u_matrix'] = flex.mat3_double(nref) if 'b_matrix' not in reflections: reflections['b_matrix'] = flex.mat3_double(nref) if 's0_vector' not in reflections: reflections['s0_vector'] = flex.vec3_double(nref) if 'd_matrix' not in reflections: reflections['d_matrix'] = flex.mat3_double(nref) if 'D_matrix' not in reflections: reflections['D_matrix'] = flex.mat3_double(nref) # set columns in the reflection table to store the derivative of state for # each reflection, if needed if not skip_derivatives: null9 = (0., 0., 0., 0., 0., 0., 0., 0., 0.) null3 = (0., 0., 0.) if self._varying_xl_orientations and "dU_dp0" not in reflections: max_free_params = max([e.num_free() for e in self._xl_orientation_parameterisations]) for i in range(max_free_params): colname = "dU_dp{0}".format(i) reflections[colname] = flex.mat3_double(nref, null9) if self._varying_xl_unit_cells and "dB_dp0" not in reflections: max_free_params = max([e.num_free() for e in self._xl_unit_cell_parameterisations]) for i in range(max_free_params): colname = "dB_dp{0}".format(i) reflections[colname] = flex.mat3_double(nref, null9) if self._varying_detectors and "dd_dp0" not in reflections: max_free_params = max([e.num_free() for e in self._detector_parameterisations]) for i in range(max_free_params): colname = "dd_dp{0}".format(i) reflections[colname] = flex.mat3_double(nref, null9) if self._varying_beams and "ds0_dp0" not in reflections: max_free_params = max([e.num_free() for e in self._beam_parameterisations]) for i in range(max_free_params): colname = "ds0_dp{0}".format(i) reflections[colname] = flex.vec3_double(nref, null3) return
def calculate_weights(reflections): """set 'statistical weights', that is w(x) = 1/var(x)""" weights = (reflections['xyzobs.mm.variance']).deep_copy() parts = weights.parts() for w in parts: sel = w > 0. w.set_selected(sel, 1./w.select(sel)) reflections['xyzobs.mm.weights'] = flex.vec3_double(*parts) return reflections
def _beam_derivatives(self, isel, parameterisation=None, reflections=None): """helper function to extend the derivatives lists by derivatives of the beam parameterisations""" # Get required data s0 = self._s0.select(isel) s0u = self._s0u.select(isel) nu = self._nu.select(isel) r = self._r.select(isel) e1 = self._e1.select(isel) q_s0 = self._q_s0.select(isel) inv_s = self._inv_s.select(isel) inv_sss = self._inv_sss.select(isel) DeltaPsi = self._DeltaPsi.select(isel) D = self._D.select(isel) # get the derivatives of the beam vector wrt the parameters ds0_dbeam_p = parameterisation.get_ds_dp(use_none_as_null=True) dDeltaPsi_dp = [] dpv_dp = [] # loop through the parameters for der in ds0_dbeam_p: if der is None: dpv_dp.append(None) dDeltaPsi_dp.append(None) continue # repeat the derivative in an array ds0 = flex.vec3_double(len(s0u), der.elems) # we need the derivative of the unit beam direction too. This requires # scaling by the wavelength and projection onto the Ewald sphere #scaled = ds0 * wl #ds0u = scaled.dot(c0) * c0 + scaled.dot(e1) * e1 # calculate the derivative of DeltaPsi for this parameter dDeltaPsi = -1.0 * (r.dot(ds0)) / (e1.cross(r).dot(s0)) dDeltaPsi_dp.append(dDeltaPsi) # term 1 term1 = (s0u.dot(ds0) * q_s0 + nu * ds0) * inv_s # term 2 term2 = (nu * q_s0 * q_s0.dot(ds0)) * inv_sss # calculate the derivative of pv for this parameter dpv_dp.append(D * (term1 - term2)) return dpv_dp, dDeltaPsi_dp
def _id_refs_to_keep(self, obs_data): """Create a selection of observations that pass certain conditions. This step includes rejection of reflections too close to the spindle, reflections measured outside the scan range, rejection of the (0,0,0) Miller index and rejection of reflections with the overload flag set. Outlier rejection is done later.""" # first exclude reflections with miller index set to 0,0,0 sel1 = obs_data['miller_index'] != (0,0,0) # exclude reflections with overloads, as these have worse centroids sel2 = ~obs_data.get_flags(obs_data.flags.overloaded) # combine selections sel = sel1 & sel2 inc = flex.size_t_range(len(obs_data)).select(sel) obs_data = obs_data.select(sel) # Default to True to pass the following test if there is no rotation axis # for a particular experiment to_keep = flex.bool(len(inc), True) for iexp, exp in enumerate(self._experiments): axis = self._axes[iexp] if not axis or exp.scan is None: continue if exp.scan.get_oscillation()[1] == 0.0: continue sel = obs_data['id'] == iexp s0 = self._s0vecs[iexp] s1 = obs_data['s1'].select(sel) phi = obs_data['xyzobs.mm.value'].parts()[2].select(sel) # first test: reject reflections for which the parallelepiped formed # between the gonio axis, s0 and s1 has a volume of less than the cutoff. # Those reflections are by definition closer to the spindle-beam # plane and for low values of the cutoff are troublesome to # integrate anyway. p_vol = flex.abs(s1.cross(flex.vec3_double(s1.size(), s0)).dot(axis)) passed1 = p_vol > self._close_to_spindle_cutoff # second test: reject reflections that lie outside the scan range phi_min, phi_max = exp.scan.get_oscillation_range(deg=False) passed2 = (phi >= phi_min) & (phi <= phi_max) # combine tests to_update = passed1 & passed2 to_keep.set_selected(sel, to_update) inc = inc.select(to_keep) return inc
def set_general_variables(self): self.frame = load(self.raw) self.detector = self.frame.get_detector() self.beam = self.frame.get_beam() self.s0 = self.beam.get_s0() self.gonio = self.frame.get_goniometer() self.scan = self.frame.get_scan() self.lab_coordinates = flex.vec3_double() for panel in self.detector: self.beam_center_mm_x, self.beam_center_mm_y = col(panel.get_beam_centre(self.s0)) pixels = flex.vec2_double(panel.get_image_size()) mms = panel.pixel_to_millimeter(pixels) self.lab_coordinates.extend(panel.get_lab_coord(mms)) self.Isizex, self.Isizey = panel.get_image_size() self.beam_center_x, self.beam_center_y = col(panel.get_beam_centre_px(self.s0)) self.detector_distance = panel.get_distance() thrshim_min, thrshim_max = panel.get_trusted_range() self.pixel_size = panel.get_pixel_size()[0] self.raw_data = self.frame.get_raw_data() if thrshim_min < 0 : self.thrshim_min = int(0) else: self.thrshim_min = thrshim_min if thrshim_max > 32767: self.thrshim_max = int(32767) else: self.thrshim_max = int(thrshim_max) self.polarization_fraction = self.beam.get_polarization_fraction() self.polarization_offset = 0.0 self.cassette_x = 0.0 self.cassette_y = 0.0 self.windim_xmax = int(self.Isizex)-100 # right border for processed image (pixels) self.windim_xmin = 100 # left border for processed image (pixels) self.windim_ymax = int(self.Isizey)-100 # top border for processed image (pixels) self.windim_ymin = 100 # bottom border for processed image (pixels) ### beamstop borders self.punchim_xmax = int(self.Isizex) # right border of beam stop shadow (pixels) self.punchim_xmin = int(self.beam_center_x)-80 # left border of beam stop shadow (pixels) self.punchim_ymax = int(self.beam_center_y)+100 # top border of beam stop shadow (pixels) self.punchim_ymin = int(self.beam_center_y)-40 # bottom border of beam stop shadow (pixels) self.mode_filter_footprint = int(20) return
def generate_single_central_non_negative_profiles(self): from dials.array_family import flex from tst_profile_helpers import gaussian rlist = flex.reflection_table(1) profile = gaussian(self.grid_size, 1000, (4, 4, 4), (1.5, 1.5, 1.5)) x = 500 y = 500 z = 5 xyz = flex.vec3_double(1) xyz[0] = (x, y, z) profiles = [profile.deep_copy()] rlist["xyzcal.px"] = xyz return rlist, profiles, profile
def generate_identical_non_negative_profiles(self): from dials.array_family import flex from random import uniform from tst_profile_helpers import gaussian rlist = flex.reflection_table(1000) profile = gaussian(self.grid_size, 1000, (4, 4, 4), (1.5, 1.5, 1.5)) xyz = flex.vec3_double(1000) profiles = [] for i in range(1000): x = uniform(0, 1000) y = uniform(0, 1000) z = uniform(0, 10) xyz[i] = (x, y, z) profiles.append(profile.deep_copy()) rlist["xyzcal.px"] = xyz return rlist, profiles, profile
def generate_systematically_offset_profiles(self): from dials.array_family import flex from random import uniform from tst_profile_helpers import gaussian rlist = flex.reflection_table(1000) xyz = flex.vec3_double(1000) profiles = [] for i in range(1000): x = uniform(0, 1000) y = uniform(0, 1000) z = uniform(0, 10) offset = -4.5 + 9 * x / 1000.0 profile = gaussian(self.grid_size, 1000, (4 + offset, 4, 4), (1.5, 1.5, 1.5)) xyz[i] = (x, y, z) profiles.append(profile) rlist["xyzcal.px"] = xyz return rlist, profiles
def _beam_derivatives(self, isel, parameterisation=None, ds0_dbeam_p=None, reflections=None): """helper function to extend the derivatives lists by derivatives of the beam parameterisations.""" # Get required data r = self._r.select(isel) e_X_r = self._e_X_r.select(isel) e_r_s0 = self._e_r_s0.select(isel) D = self._D.select(isel) if ds0_dbeam_p is None: # get the derivatives of the beam vector wrt the parameters ds0_dbeam_p = parameterisation.get_ds_dp(use_none_as_null=True) ds0_dbeam_p = [None if e is None else flex.vec3_double(len(r), e.elems) \ for e in ds0_dbeam_p] dphi_dp = [] dpv_dp = [] # loop through the parameters for der in ds0_dbeam_p: if der is None: dphi_dp.append(None) dpv_dp.append(None) continue # calculate the derivative of phi for this parameter dphi = (r.dot(der) / e_r_s0) * -1.0 dphi_dp.append(dphi) # calculate the derivative of pv for this parameter dpv_dp.append(D * (e_X_r * dphi + der)) return dpv_dp, dphi_dp
def __init__(self): from dials.algorithms.integration.profile import GridSampler2D from dials.array_family import flex from random import randint, uniform # Number of reflections nrefl = 1000 # Size of the images width = 1000 height = 1000 # Create the grid self.grid = GridSampler2D((width, height), (5,5)) # Create the list of xyz and bboxes self.xyz = flex.vec3_double(nrefl) self.bbox = flex.int6(nrefl) self.panel = flex.size_t(nrefl, 0) for i in range(nrefl): x0 = randint(0,width-10) x1 = x0 + randint(3,10) y0 = randint(0,height-10) y1 = y0 + randint(3,10) z0 = randint(0,10) z1 = z0 + randint(1,10) b = x0, x1, y0, y1, z0, z1 c = (x1 + x0) / 2, (y1 + y0) / 2, (z1 + z0) / 2 self.xyz[i] = c self.bbox[i] = b # Create the array of shoeboxes self.sbox = flex.shoebox(self.panel, self.bbox) self.sbox.allocate() for i in range(len(self.sbox)): data = self.sbox[i].data for j in range(len(data)): data[j] = uniform(0, 100)
def tst_for_reflection_table(self): from libtbx.test_utils import approx_equal from dials.algorithms.spot_prediction import \ ScanVaryingReflectionPredictor, ScanStaticReflectionPredictor from dials.array_family import flex predict = ScanStaticReflectionPredictor(self.experiments[0]) preds = predict.for_ub(self.experiments[0].crystal.get_A()) preds['ub_matrix'] = flex.mat3_double(len(preds), self.experiments[0].crystal.get_A()) preds['s0'] = flex.vec3_double(len(preds), self.experiments[0].beam.get_s0()) preds['d_matrix'] = flex.mat3_double(len(preds)) for ipanel, panel in enumerate(self.experiments[0].detector): sel = preds['panel'] == ipanel D = panel.get_d_matrix() preds['d_matrix'].set_selected(sel, D) predict = ScanVaryingReflectionPredictor(self.experiments[0]) from copy import deepcopy old_preds = deepcopy(preds) predict.for_reflection_table(preds, preds['ub_matrix'], preds['s0'], preds['d_matrix']) # Because UB, s0 and d values are the same for all reflections, the new # reflections should be approx equal to those produced by the scan static # predictor old_x, old_y, old_z = old_preds['xyzcal.px'].parts() new_x, new_y, new_z = preds['xyzcal.px'].parts() assert old_x.all_approx_equal(new_x) assert old_y.all_approx_equal(new_y) assert old_z.all_approx_equal(new_z) print "OK" return
def main(reflections, experiment, params): nref0 = len(reflections) method = params.method ids = params.id #if 'intensity.prf.variance' in reflections: # selection = reflections.get_flags( # reflections.flags.integrated, # all=True) #else: # selection = reflections.get_flags( # reflections.flags.integrated_sum) #reflections = reflections.select(selection) # filter according to rules if params.min_isum: selection = reflections['intensity.sum.value'] > params.min_isum reflections = reflections.select(selection) if params.num > len(reflections): raise RuntimeError, 'you asked for too many reflections sorry' from dials.array_family import flex if params.seed > 0 and params.num > 0: import random random.seed(params.seed) selected = flex.bool(len(reflections), False) while len(selected.iselection()) < params.num: selected[random.randint(0, len(reflections))] = True reflections = reflections.select(selected) nref1 = len(reflections) print 'Removed %d reflections, %d remain' % (nref0 - nref1, nref1) results = [] simmed = reflections.copy()[0:0] simmed['dqe'] = flex.double() simmed['xyzsim.mm'] = flex.vec3_double() simmed['xyzpred.mm'] = flex.vec3_double() for j, reflection in enumerate(reflections): result = None if ids: if j in ids: result = globals()['model_reflection_%s' % method](reflection, experiment, params) elif params.id_start and params.id_end: if j < params.id_start or j >= params.id_end: continue print j result = globals()['model_reflection_%s' % method](reflection, experiment, params) else: print j result = globals()['model_reflection_%s' % method](reflection, experiment, params) if not result is None: results.append(result[0]) simmed.append(result[1]) import math if params.debug: print "RMSD obs - cal:", math.sqrt((simmed['xyzobs.mm.value'] - simmed['xyzcal.mm']).sum_sq()/len(simmed)) * 1000 print "RMSD obs - sim:", math.sqrt((simmed['xyzobs.mm.value'] - simmed['xyzsim.mm']).sum_sq()/len(simmed)) * 1000 print "Functional:", (simmed['xyzobs.mm.value'] - simmed['xyzsim.mm']).sum_sq() return results
if len(self.refined_experiments) > 1: from dials.algorithms.indexing.compare_orientation_matrices \ import show_rotation_matrix_differences show_rotation_matrix_differences( self.refined_experiments.crystals(), out=info_handle) info("Final refined crystal models:") for i, crystal_model in enumerate(self.refined_experiments.crystals()): n_indexed = 0 for i_expt in experiments.where(crystal=crystal_model): n_indexed += (self.reflections['id'] == i).count(True) info("model %i (%i reflections):" %(i+1, n_indexed)) info(crystal_model) self.refined_reflections['xyzcal.px'] = flex.vec3_double( len(self.refined_reflections)) for i, imageset in enumerate(self.imagesets): imgset_sel = self.refined_reflections['imageset_id'] == i # set xyzcal.px field in self.refined_reflections refined_reflections = self.refined_reflections.select(imgset_sel) panel_numbers = flex.size_t(refined_reflections['panel']) xyzcal_mm = refined_reflections['xyzcal.mm'] x_mm, y_mm, z_rad = xyzcal_mm.parts() xy_cal_mm = flex.vec2_double(x_mm, y_mm) xy_cal_px = flex.vec2_double(len(xy_cal_mm)) for i_panel in range(len(imageset.get_detector())): panel = imageset.get_detector()[i_panel] sel = (panel_numbers == i_panel) isel = sel.iselection() ref_panel = refined_reflections.select(panel_numbers == i_panel) xy_cal_px.set_selected(
def get_gradients(self, reflections, callback=None): """Calculate gradients of the prediction formula with respect to each of the parameters of the contained models, for all of the reflections. This method sets up required quantities relevant to the current step of refinement and then loops over the parameterisations of each type extending a results list each time.""" # Set up arrays of quantities of interest for each reflection self._nref = len(reflections) self._D = flex.mat3_double(self._nref) self._s0 = flex.vec3_double(self._nref) self._U = flex.mat3_double(self._nref) self._B = flex.mat3_double(self._nref) self._axis = flex.vec3_double(self._nref) self._fixed_rotation = flex.mat3_double(self._nref) self._setting_rotation = flex.mat3_double(self._nref) # Set up experiment to index mapping self._experiment_to_idx = [] # Populate values in these arrays for iexp, exp in enumerate(self._experiments): sel = reflections['id'] == iexp isel = sel.iselection() self._experiment_to_idx.append(isel) subref = reflections.select(sel) states = self._get_model_data_for_experiment(exp, subref) self._D.set_selected(sel, states['D']) self._s0.set_selected(sel, states['s0']) self._U.set_selected(sel, states['U']) self._B.set_selected(sel, states['B']) if exp.goniometer: self._axis.set_selected(sel, exp.goniometer.get_rotation_axis_datum()) self._fixed_rotation.set_selected(sel, exp.goniometer.get_fixed_rotation()) self._setting_rotation.set_selected(sel, exp.goniometer.get_setting_rotation()) # Other derived values self._h = reflections['miller_index'].as_vec3_double() self._UB = self._U * self._B self._s1 = reflections['s1'] self._pv = self._D * self._s1 # 'projection vector' for the ray along s1. # Quantities derived from pv, precalculated for efficiency u, v, w = self._pv.parts() self._w_inv = 1./w self._u_w_inv = u * self._w_inv self._v_w_inv = v * self._w_inv # Reset a pointer to the parameter number self._iparam = 0 # Do additional setup specified by derived classes self._local_setup(reflections) # Set up empty list in which to store gradients results = [] # loop over detector parameterisations and extend results results = self._grads_detector_loop(reflections, results, callback) # loop over the beam parameterisations and extend results results = self._grads_beam_loop(reflections, results, callback) # loop over the crystal orientation parameterisations and extend results results = self._grads_xl_orientation_loop(reflections, results, callback) # loop over the crystal unit cell parameterisations and extend results results = self._grads_xl_unit_cell_loop(reflections, results, callback) return results
print "You really should supply datasize, but ok" # raise ValueError,"Output datasze file must be specified." else: datasize = args.pop(datasizeidx).split("=")[1] import copy, os import dxtbx from dxtbx.model.experiment_list import ExperimentListFactory from dials.array_family import flex experiments = ExperimentListFactory.from_json_file(json, check_format=False) beam = experiments[0].beam detector = experiments[0].detector lab_coordinates = flex.vec3_double() for panel in detector: pixels = flex.vec2_double(panel.get_image_size()) mms = panel.pixel_to_millimeter(pixels) lab_coordinates.extend(panel.get_lab_coord(mms)) # generate s1 vectors s1 = lab_coordinates.each_normalize() * (1/beam.get_wavelength()) # Generate x vectors x = np.asarray(s1 - beam.get_s0()) # DATAsize = np.asarray(detector[0].get_image_size()) # np.save(xvectors,x) # np.save(datasize,DATAsize) x.astype('float32').tofile(xvectors)
def integration_concept_detail(self, experiments, reflections, spots,image_number,cb_op_to_primitive,**kwargs): detector = experiments[0].detector crystal = experiments[0].crystal from cctbx.crystal import symmetry c_symmetry = symmetry(space_group = crystal.get_space_group(), unit_cell = crystal.get_unit_cell()) self.image_number = image_number NEAR = 10 pxlsz = detector[0].get_pixel_size() Predicted = self.get_predictions_accounting_for_centering(experiments,reflections,cb_op_to_primitive,**kwargs) FWMOSAICITY = self.inputai.getMosaicity() self.DOMAIN_SZ_ANG = kwargs.get("domain_size_ang", self.__dict__.get("actual",0) ) refineflag = {True:0,False:1}[kwargs.get("domain_size_ang",0)==0] c_symmetry.show_summary(prefix="EXCURSION%1d REPORT FWMOS= %6.4f DOMAIN= %6.1f "%(refineflag,FWMOSAICITY,self.DOMAIN_SZ_ANG)) from annlib_ext import AnnAdaptor self.cell = c_symmetry.unit_cell() query = flex.double() print len(self.predicted) for pred in self.predicted: # predicted spot coord in pixels query.append(pred[0]/pxlsz[0]) query.append(pred[1]/pxlsz[1]) self.reserve_hkllist_for_signal_search = self.hkllist reference = flex.double() assert self.length>NEAR# Can't do spot/pred matching with too few spots for spot in spots: reference.append(spot.ctr_mass_x()) reference.append(spot.ctr_mass_y()) IS_adapt = AnnAdaptor(data=reference,dim=2,k=NEAR) IS_adapt.query(query) idx_cutoff = float(min(self.mask_focus[image_number])) from rstbx.apps.slip_helpers import slip_callbacks cache_refinement_spots = getattr(slip_callbacks.slip_callback,"requires_refinement_spots",False) indexed_pairs_provisional = [] correction_vectors_provisional = [] c_v_p_flex = flex.vec3_double() this_setting_matched_indices = reflections["miller_index"] for j,item in enumerate(this_setting_matched_indices): this_setting_index = self.hkllist.first_index(item) if this_setting_index: Match = dict(spot=j,pred=this_setting_index) indexed_pairs_provisional.append(Match) vector = matrix.col( [reflections["xyzobs.px.value"][j][0] - self.predicted[Match["pred"]][0]/pxlsz[0], reflections["xyzobs.px.value"][j][1] - self.predicted[Match["pred"]][1]/pxlsz[1]]) correction_vectors_provisional.append(vector) c_v_p_flex.append((vector[0],vector[1],0.)) self.N_correction_vectors = len(correction_vectors_provisional) self.rmsd_px = math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex))) print "... %d provisional matches"%self.N_correction_vectors, print "r.m.s.d. in pixels: %6.3f"%(self.rmsd_px) if self.horizons_phil.integration.enable_residual_scatter: from matplotlib import pyplot as plt fig = plt.figure() for cv in correction_vectors_provisional: plt.plot([cv[1]],[-cv[0]],"r.") plt.title(" %d matches, r.m.s.d. %5.2f pixels"%(len(correction_vectors_provisional),math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex))))) plt.axes().set_aspect("equal") self.show_figure(plt,fig,"res") plt.close() if self.horizons_phil.integration.enable_residual_map: from matplotlib import pyplot as plt PX = reflections["xyzobs.px.value"] fig = plt.figure() for match,cv in zip(indexed_pairs_provisional,correction_vectors_provisional): plt.plot([PX[match["spot"]][1]],[-PX[match["spot"]][0]],"r.") plt.plot([self.predicted[match["pred"]][1]/pxlsz[1]],[-self.predicted[match["pred"]][0]/pxlsz[0]],"g.") plt.plot([PX[match["spot"]][1], PX[match["spot"]][1] + 10.*cv[1]], [-PX[match["spot"]][0], -PX[match["spot"]][0] - 10.*cv[0]],'r-') if kwargs.get("user-reentrant") != None and self.horizons_phil.integration.spot_prediction == "dials" \ and self.horizons_phil.integration.enable_residual_map_deltapsi: from rstbx.apps.stills.util import residual_map_special_deltapsi_add_on residual_map_special_deltapsi_add_on( reflections = self.dials_spot_prediction, matches = indexed_pairs_provisional, experiments=experiments, hkllist = self.hkllist, predicted = self.predicted, plot=plt, eta_deg=FWMOSAICITY, deff=self.DOMAIN_SZ_ANG ) plt.xlim([0,detector[0].get_image_size()[1]]) plt.ylim([-detector[0].get_image_size()[0],0]) plt.title(" %d matches, r.m.s.d. %5.2f pixels"%(len(correction_vectors_provisional),math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex))))) plt.axes().set_aspect("equal") self.show_figure(plt,fig,"map") plt.close() indexed_pairs = indexed_pairs_provisional correction_vectors = correction_vectors_provisional ########### skip outlier rejection for this derived class ### However must retain the ability to write out correction vectiors. if True: # at Aaron's request; test later correction_lengths = flex.double([v.length() for v in correction_vectors_provisional]) clorder = flex.sort_permutation(correction_lengths) sorted_cl = correction_lengths.select(clorder) indexed_pairs = [] correction_vectors = [] self.correction_vectors = [] for icand in xrange(len(sorted_cl)): # somewhat arbitrary sigma = 1.0 cutoff for outliers indexed_pairs.append(indexed_pairs_provisional[clorder[icand]]) correction_vectors.append(correction_vectors_provisional[clorder[icand]]) if cache_refinement_spots: self.spotfinder.images[self.frame_numbers[self.image_number]]["refinement_spots"].append( spots[reflections[indexed_pairs[-1]["spot"]]['spotfinder_lookup']]) if kwargs.get("verbose_cv")==True: print "CV OBSCENTER %7.2f %7.2f REFINEDCENTER %7.2f %7.2f"%( float(self.inputpd["size1"])/2.,float(self.inputpd["size2"])/2., self.inputai.xbeam()/pxlsz[0], self.inputai.ybeam()/pxlsz[1]), print "OBSSPOT %7.2f %7.2f PREDSPOT %7.2f %7.2f"%( reflections[indexed_pairs[-1]["spot"]]['xyzobs.px.value'][0], reflections[indexed_pairs[-1]["spot"]]['xyzobs.px.value'][1], self.predicted[indexed_pairs[-1]["pred"]][0]/pxlsz[0], self.predicted[indexed_pairs[-1]["pred"]][1]/pxlsz[1]), the_hkl = self.hkllist[indexed_pairs[-1]["pred"]] print "HKL %4d %4d %4d"%the_hkl,"%2d"%self.setting_id, radial, azimuthal = spots[indexed_pairs[-1]["spot"]].get_radial_and_azimuthal_size( self.inputai.xbeam()/pxlsz[0], self.inputai.ybeam()/pxlsz[1]) print "RADIALpx %5.3f AZIMUTpx %5.3f"%(radial,azimuthal) # Store a list of correction vectors in self. radial, azimuthal = spots[indexed_pairs[-1]['spot']].get_radial_and_azimuthal_size( self.inputai.xbeam()/pxlsz[0], self.inputai.ybeam()/pxlsz[1]) self.correction_vectors.append( dict(obscenter=(float(self.inputpd['size1']) / 2, float(self.inputpd['size2']) / 2), refinedcenter=(self.inputai.xbeam() / pxlsz[0], self.inputai.ybeam() / pxlsz[1]), obsspot=(reflections[indexed_pairs[-1]['spot']]['xyzobs.px.value'][0], reflections[indexed_pairs[-1]['spot']]['xyzobs.px.value'][1]), predspot=(self.predicted[indexed_pairs[-1]['pred']][0] / pxlsz[0], self.predicted[indexed_pairs[-1]['pred']][1] / pxlsz[1]), hkl=(self.hkllist[indexed_pairs[-1]['pred']][0], self.hkllist[indexed_pairs[-1]['pred']][1], self.hkllist[indexed_pairs[-1]['pred']][2]), setting_id=self.setting_id, radial=radial, azimuthal=azimuthal)) self.inputpd["symmetry"] = c_symmetry self.inputpd["symmetry"].show_summary(prefix="SETTING ") if self.horizons_phil.integration.model == "user_supplied": # Not certain of whether the reentrant_* dictionary keys create a memory leak if kwargs.get("user-reentrant",None)==None: kwargs["reentrant_experiments"] = experiments kwargs["reentrant_reflections"] = reflections from cxi_user import post_outlier_rejection self.indexed_pairs = indexed_pairs self.spots = spots post_outlier_rejection(self,image_number,cb_op_to_primitive,self.horizons_phil,kwargs) return ########### finished with user-supplied code correction_lengths=flex.double([v.length() for v in correction_vectors]) self.r_residual = pxlsz[0]*flex.mean(correction_lengths) #assert len(indexed_pairs)>NEAR # must have enough indexed spots if (len(indexed_pairs) <= NEAR): raise Sorry("Not enough indexed spots, only found %d, need %d" % (len(indexed_pairs), NEAR)) reference = flex.double() for item in indexed_pairs: reference.append(spots[item["spot"]].ctr_mass_x()) reference.append(spots[item["spot"]].ctr_mass_y()) PS_adapt = AnnAdaptor(data=reference,dim=2,k=NEAR) PS_adapt.query(query) self.BSmasks = [] # do not use null: self.null_correction_mapping( predicted=self.predicted, self.positional_correction_mapping( predicted=self.predicted, correction_vectors = correction_vectors, PS_adapt = PS_adapt, IS_adapt = IS_adapt, spots = spots) # which spots are close enough to interfere with background? MAXOVER=6 OS_adapt = AnnAdaptor(data=query,dim=2,k=MAXOVER) #six near nbrs OS_adapt.query(query) if self.mask_focus[image_number] is None: raise Sorry("No observed/predicted spot agreement; no Spotfinder masks; skip integration") nbr_cutoff = 2.0* max(self.mask_focus[image_number]) FRAME = int(nbr_cutoff/2) #print "The overlap cutoff is %d pixels"%nbr_cutoff nbr_cutoff_sq = nbr_cutoff * nbr_cutoff #print "Optimized C++ section...", self.set_frame(FRAME) self.set_background_factor(kwargs["background_factor"]) self.set_nbr_cutoff_sq(nbr_cutoff_sq) self.set_guard_width_sq(self.horizons_phil.integration.guard_width_sq) self.set_detector_gain(self.horizons_phil.integration.detector_gain) flex_sorted = flex.int() for item in self.sorted: flex_sorted.append(item[0]);flex_sorted.append(item[1]); if self.horizons_phil.integration.mask_pixel_value is not None: self.set_mask_pixel_val(self.horizons_phil.integration.mask_pixel_value) image_obj = self.imagefiles.imageindex(self.frame_numbers[self.image_number]) image_obj.read() rawdata = image_obj.linearintdata # assume image #1 if self.inputai.active_areas != None: self.detector_xy_draft = self.safe_background( rawdata=rawdata, predicted=self.predicted, OS_adapt=OS_adapt, sorted=flex_sorted, tiles=self.inputai.active_areas.IT, tile_id=self.inputai.active_areas.tile_id); else: self.detector_xy_draft = self.safe_background( rawdata=rawdata, predicted=self.predicted, OS_adapt=OS_adapt, sorted=flex_sorted); for i in xrange(len(self.predicted)): # loop over predicteds B_S_mask = {} keys = self.get_bsmask(i) for k in xrange(0,len(keys),2): B_S_mask[(keys[k],keys[k+1])]=True self.BSmasks.append(B_S_mask) #print "Done" return