def test(dials_regression): from iotbx.xds import xparm, integrate_hkl from dials.util import ioutil from dials.algorithms.spot_prediction import IndexGenerator import numpy from rstbx.cftbx.coordinate_frame_converter import \ coordinate_frame_converter from scitbx import matrix # The XDS files to read from integrate_filename = os.path.join(dials_regression, 'data', 'sim_mx', 'INTEGRATE.HKL') gxparm_filename = os.path.join(dials_regression, 'data', 'sim_mx', 'GXPARM.XDS') # Read the XDS files integrate_handle = integrate_hkl.reader() integrate_handle.read_file(integrate_filename) gxparm_handle = xparm.reader() gxparm_handle.read_file(gxparm_filename) # Get the parameters we need from the GXPARM file d_min = 1.6 space_group_type = ioutil.get_space_group_type_from_xparm(gxparm_handle) cfc = coordinate_frame_converter(gxparm_filename) a_vec = cfc.get('real_space_a') b_vec = cfc.get('real_space_b') c_vec = cfc.get('real_space_c') unit_cell = cfc.get_unit_cell() UB = matrix.sqr(a_vec + b_vec + c_vec).inverse() ub_matrix = UB # Generate the indices index_generator = IndexGenerator(unit_cell, space_group_type, d_min) miller_indices = index_generator.to_array() # Get individual generated hkl gen_h = [hkl[0] for hkl in miller_indices] gen_k = [hkl[1] for hkl in miller_indices] gen_l = [hkl[2] for hkl in miller_indices] # Get individual xds generated hkl xds_h = [hkl[0] for hkl in integrate_handle.hkl] xds_k = [hkl[1] for hkl in integrate_handle.hkl] xds_l = [hkl[2] for hkl in integrate_handle.hkl] # Get min/max generated hkl min_gen_h, max_gen_h = numpy.min(gen_h), numpy.max(gen_h) min_gen_k, max_gen_k = numpy.min(gen_k), numpy.max(gen_k) min_gen_l, max_gen_l = numpy.min(gen_l), numpy.max(gen_l) # Get min/max xds generated hkl min_xds_h, max_xds_h = numpy.min(xds_h), numpy.max(xds_h) min_xds_k, max_xds_k = numpy.min(xds_k), numpy.max(xds_k) min_xds_l, max_xds_l = numpy.min(xds_l), numpy.max(xds_l) # Ensure we have the whole xds range in the generated set assert min_gen_h <= min_xds_h and max_gen_h >= max_xds_h assert min_gen_k <= min_xds_k and max_gen_k >= max_xds_k assert min_gen_l <= min_xds_l and max_gen_l >= max_xds_l
def test_Simple1ProfileModel_predict_reflections( simple1_profile_model, test_experiment, ): # Create the index generator index_generator = IndexGenerator( test_experiment.crystal.get_unit_cell(), test_experiment.crystal.get_space_group().type(), 2.0, ) # Get an array of miller indices miller_indices = index_generator.to_array() reflections = simple1_profile_model.predict_reflections([test_experiment], miller_indices, probability=0.9973) s0 = matrix.col(test_experiment.beam.get_s0()) quantile = chisq_quantile(3, 0.9973) sigma_inv = matrix.sqr(flumpy.from_numpy( simple1_profile_model.sigma())).inverse() for s2 in reflections["s2"]: s2_ = matrix.col(s2) x = s2_.normalize() * s0.length() - s2_ d = (x.transpose() * sigma_inv * x)[0] assert d < quantile
def ref_gen_static(experiments): """Generate some reflections using the static predictor""" beam = experiments[0].beam crystal = experiments[0].crystal detector = experiments[0].detector scan = experiments[0].scan # All indices to the detector max resolution dmin = detector.get_max_resolution(beam.get_s0()) index_generator = IndexGenerator( crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), dmin ) indices = index_generator.to_array() # Predict rays within the sequence range sequence_range = scan.get_oscillation_range(deg=False) ray_predictor = ScansRayPredictor(experiments, sequence_range) refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(detector, refs) refs = refs.select(intersects) # Make a reflection predictor and re-predict for these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ScansExperimentsPredictor(experiments) refs["id"] = flex.int(len(refs), 0) refs = ref_predictor(refs) return refs
def test_Simple6ProfileModel_compute_mask(simple6_profile_model, test_experiment): experiments = [test_experiment] # Create the index generator index_generator = IndexGenerator( experiments[0].crystal.get_unit_cell(), experiments[0].crystal.get_space_group().type(), 2.0, ) # Get an array of miller indices miller_indices = index_generator.to_array() reflections = simple6_profile_model.predict_reflections(experiments, miller_indices, probability=0.9973) s2 = reflections["s2"] s0 = matrix.col(experiments[0].beam.get_s0()) quantile = chisq_quantile(3, 0.9973) sigma_inv = matrix.sqr(flumpy.from_numpy( simple6_profile_model.sigma())).inverse() for s2 in map(matrix.col, reflections["s2"]): x = s2.normalize() * s0.length() - s2 d = (x.transpose() * sigma_inv * x)[0] assert d < quantile simple6_profile_model.compute_bbox(experiments, reflections) reflections["shoebox"] = flex.shoebox(reflections["panel"], reflections["bbox"], allocate=True) simple6_profile_model.compute_mask(experiments, reflections)
def ref_gen_static(experiments): """Generate some reflections using the static predictor""" beam = experiments[0].beam crystal = experiments[0].crystal goniometer = experiments[0].goniometer detector = experiments[0].detector scan = experiments[0].scan # All indices to the detector max resolution dmin = detector.get_max_resolution(beam.get_s0()) index_generator = IndexGenerator(crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), dmin) indices = index_generator.to_array() # Predict rays within the sweep range sweep_range = scan.get_oscillation_range(deg=False) ray_predictor = ScansRayPredictor(experiments, sweep_range) refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(detector, refs) refs = refs.select(intersects) # Make a reflection predictor and re-predict for these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) refs['id'] = flex.int(len(refs), 0) refs = ref_predictor(refs) return refs
def generate_reflections(self): # Build a mock scan for a 3 degree sweep from dxtbx.model import ScanFactory sf = ScanFactory() self.scan = sf.make_scan(image_range=(1, 1), exposure_times=0.1, oscillation=(0, 3.0), epochs=range(1), deg=True) sweep_range = self.scan.get_oscillation_range(deg=False) # Create a scans ExperimentList, only for generating reflections experiments = ExperimentList() experiments.append( Experiment(beam=self.beam, detector=self.detector, goniometer=self.gonio, scan=self.scan, crystal=self.crystal, imageset=None)) # Create a ScansRayPredictor ray_predictor = ScansRayPredictor(experiments, sweep_range) # Generate rays - only to work out which hkls are predicted resolution = 2.0 index_generator = IndexGenerator( self.crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices = index_generator.to_array() rays = ray_predictor(indices) # Make a standard reflection_table and copy in the ray data self.reflections = flex.reflection_table.empty_standard(len(rays)) self.reflections.update(rays)
def predict(experiments, d_min=None, prediction_probability=0.9973): """Predict the reflections""" logger.info("\n" + "=" * 80 + "\nPredicting reflections") # Set a resolution range if d_min is None: s0 = experiments[0].beam.get_s0() d_min = experiments[0].detector.get_max_resolution(s0) # Create the index generator index_generator = IndexGenerator( experiments[0].crystal.get_unit_cell(), experiments[0].crystal.get_space_group().type(), d_min, ) # Get an array of miller indices miller_indices_to_test = index_generator.to_array() logger.info("Generated %d miller indices" % len(miller_indices_to_test)) # Get the covariance matrix profile = experiments[0].crystal.mosaicity reflection_table = profile.parameterisation.predict_reflections( experiments, miller_indices_to_test, prediction_probability) # Do the prediction reflection_table.compute_d(experiments) logger.info("Predicted %d reflections" % len(reflection_table)) return reflection_table
def list_possible_reflections(self, spacegroup, unit_cell, dmin, dmax): # List the possible miller indices in the minimum resolution range # and subtract those outside the maximum resolution range from dials.algorithms.spot_prediction import IndexGenerator # TODO: Does this make sense? Pass spacegroup.type() and then filter (again?)? possible_indices_p1 = set( IndexGenerator(unit_cell, spacegroup.type(), dmin).to_array()) lowres_indices_p1 = set( IndexGenerator(unit_cell, spacegroup.type(), dmax + 1e-8).to_array()) # [dmin, dmax] are inclusive ranges, thus reduce inner sphere radius by a small value (1e-8) possible_indices_p1 = possible_indices_p1 - lowres_indices_p1 print("%5d unique reflections possible in P 1" % len(possible_indices_p1)) # filter systematic absent miller indices systematic_present_indices = [ n for n in possible_indices_p1 if not spacegroup.is_sys_absent(n) ] print("%5d of these reflections are not systematically absent in %s" % (len(systematic_present_indices), spacegroup.type().lookup_symbol())) return systematic_present_indices
def test_new_from_array(self): from dials.algorithms.spot_prediction import ScanStaticRayPredictor from dials.algorithms.spot_prediction import IndexGenerator # Create the index generator self.generate_indices = IndexGenerator(self.unit_cell, self.space_group_type, self.d_min) s0 = self.beam.get_s0() m2 = self.gonio.get_rotation_axis() fixed_rotation = self.gonio.get_fixed_rotation() setting_rotation = self.gonio.get_setting_rotation() UB = self.ub_matrix dphi = self.scan.get_oscillation_range(deg=False) # Create the ray predictor self.predict_rays = ScanStaticRayPredictor(s0, m2, fixed_rotation, setting_rotation, dphi) # Predict the spot locations h = self.generate_indices.to_array() reflections = self.predict_rays(h, UB) self.reflections3 = [] for r in reflections: if r['phi'] >= dphi[0] and r['phi'] <= dphi[1]: self.reflections3.append(r) eps = 1e-7 assert (len(self.reflections) == len(self.reflections3)) for r1, r2 in zip(self.reflections, self.reflections3): assert (all(abs(a - b) < eps for a, b in zip(r1['s1'], r2['s1']))) assert (abs(r1['phi'] - r2['phi']) < eps) assert (r1['entering'] == r2['entering']) print 'OK'
def generate_reflections(self): # Build a mock scan for a 3 degree sweep from dxtbx.model.scan import scan_factory sf = scan_factory() self.scan = sf.make_scan(image_range = (1,1), exposure_times = 0.1, oscillation = (0, 3.0), epochs = range(1), deg = True) sweep_range = self.scan.get_oscillation_range(deg=False) # Create a scans ExperimentList, only for generating reflections experiments = ExperimentList() experiments.append(Experiment( beam=self.beam, detector=self.detector, goniometer=self.gonio, scan=self.scan, crystal=self.crystal, imageset=None)) # Create a ScansRayPredictor ray_predictor = ScansRayPredictor(experiments, sweep_range) # Generate rays - only to work out which hkls are predicted resolution = 2.0 index_generator = IndexGenerator(self.crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices = index_generator.to_array() rays = ray_predictor.predict(indices) # Make a standard reflection_table and copy in the ray data self.reflections = flex.reflection_table.empty_standard(len(rays)) self.reflections.update(rays) return
def test(dials_regression): import numpy as np from iotbx.xds import integrate_hkl, xparm from rstbx.cftbx.coordinate_frame_converter import coordinate_frame_converter from dials.algorithms.spot_prediction import IndexGenerator from dials.util import ioutil # The XDS files to read from integrate_filename = os.path.join(dials_regression, "data", "sim_mx", "INTEGRATE.HKL") gxparm_filename = os.path.join(dials_regression, "data", "sim_mx", "GXPARM.XDS") # Read the XDS files integrate_handle = integrate_hkl.reader() integrate_handle.read_file(integrate_filename) gxparm_handle = xparm.reader() gxparm_handle.read_file(gxparm_filename) # Get the parameters we need from the GXPARM file d_min = 1.6 space_group_type = ioutil.get_space_group_type_from_xparm(gxparm_handle) cfc = coordinate_frame_converter(gxparm_filename) unit_cell = cfc.get_unit_cell() # Generate the indices index_generator = IndexGenerator(unit_cell, space_group_type, d_min) miller_indices = index_generator.to_array() # Get individual generated hkl gen_h = [hkl[0] for hkl in miller_indices] gen_k = [hkl[1] for hkl in miller_indices] gen_l = [hkl[2] for hkl in miller_indices] # Get individual xds generated hkl xds_h = [hkl[0] for hkl in integrate_handle.hkl] xds_k = [hkl[1] for hkl in integrate_handle.hkl] xds_l = [hkl[2] for hkl in integrate_handle.hkl] # Get min/max generated hkl min_gen_h, max_gen_h = np.min(gen_h), np.max(gen_h) min_gen_k, max_gen_k = np.min(gen_k), np.max(gen_k) min_gen_l, max_gen_l = np.min(gen_l), np.max(gen_l) # Get min/max xds generated hkl min_xds_h, max_xds_h = np.min(xds_h), np.max(xds_h) min_xds_k, max_xds_k = np.min(xds_k), np.max(xds_k) min_xds_l, max_xds_l = np.min(xds_l), np.max(xds_l) # Ensure we have the whole xds range in the generated set assert min_gen_h <= min_xds_h and max_gen_h >= max_xds_h assert min_gen_k <= min_xds_k and max_gen_k >= max_xds_k assert min_gen_l <= min_xds_l and max_gen_l >= max_xds_l
def generate_reflections(self): from cctbx.sgtbx import space_group, space_group_symbols from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection sequence_range = self.scan.get_oscillation_range(deg=False) resolution = 2.0 index_generator = IndexGenerator( self.crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() # Predict rays within the sequence range ray_predictor = ScansRayPredictor(self.experiments, sequence_range) obs_refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(self.detector, obs_refs) obs_refs = obs_refs.select(intersects) # Re-predict using the Experiments predictor for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns obs_refs["id"] = flex.int(len(obs_refs), 0) obs_refs = self.ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180.0 px_size = self.detector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2) obs_refs["xyzobs.mm.variance"] = flex.vec3_double( var_x, var_y, var_phi) # set the flex random seed to an 'uninteresting' number flex.set_random_seed(12407) # take 10 random reflections for speed reflections = obs_refs.select(flex.random_selection(len(obs_refs), 10)) # use a BlockCalculator to calculate the blocks per image from dials.algorithms.refinement.reflection_manager import BlockCalculator block_calculator = BlockCalculator(self.experiments, reflections) reflections = block_calculator.per_image() return reflections
def generate_reflections(experiments): from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.refinement.prediction.managed_predictors import ( ScansRayPredictor, ScansExperimentsPredictor, ) from dials.algorithms.spot_prediction import ray_intersection from cctbx.sgtbx import space_group, space_group_symbols from scitbx.array_family import flex detector = experiments[0].detector crystal = experiments[0].crystal # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator( crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() # Predict rays within the sequence range scan = experiments[0].scan sequence_range = scan.get_oscillation_range(deg=False) ray_predictor = ScansRayPredictor(experiments, sequence_range) obs_refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(detector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ScansExperimentsPredictor(experiments) obs_refs["id"] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180.0 px_size = detector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2) obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) return obs_refs, ref_predictor
def test_new_from_array(raypredictor): from dials.algorithms.spot_prediction import IndexGenerator, ScanStaticRayPredictor # Create the index generator raypredictor.generate_indices = IndexGenerator( raypredictor.unit_cell, raypredictor.space_group_type, raypredictor.d_min) s0 = raypredictor.beam.get_s0() m2 = raypredictor.gonio.get_rotation_axis() fixed_rotation = raypredictor.gonio.get_fixed_rotation() setting_rotation = raypredictor.gonio.get_setting_rotation() UB = raypredictor.ub_matrix dphi = raypredictor.scan.get_oscillation_range(deg=False) # Create the ray predictor raypredictor.predict_rays = ScanStaticRayPredictor(s0, m2, fixed_rotation, setting_rotation, dphi) # Predict the spot locations h = raypredictor.generate_indices.to_array() reflections = raypredictor.predict_rays(h, UB) raypredictor.reflections3 = [] for r in reflections.rows(): if r["phi"] >= dphi[0] and r["phi"] <= dphi[1]: raypredictor.reflections3.append(r) assert len(raypredictor.reflections) == len(raypredictor.reflections3) for r1, r2 in zip(raypredictor.reflections.rows(), raypredictor.reflections3): assert all(a == pytest.approx(b, abs=1e-7) for a, b in zip(r1["s1"], r2["s1"])) assert r1["phi"] == pytest.approx(r2["phi"], abs=1e-7) assert r1["entering"] == r2["entering"]
def test_new(raypredictor): from dials.algorithms.spot_prediction import ScanStaticRayPredictor from dials.algorithms.spot_prediction import IndexGenerator # Create the index generator raypredictor.generate_indices = IndexGenerator(raypredictor.unit_cell, raypredictor.space_group_type, raypredictor.d_min) s0 = raypredictor.beam.get_s0() m2 = raypredictor.gonio.get_rotation_axis() fixed_rotation = raypredictor.gonio.get_fixed_rotation() setting_rotation = raypredictor.gonio.get_setting_rotation() UB = raypredictor.ub_matrix dphi = raypredictor.scan.get_oscillation_range(deg=False) # Create the ray predictor raypredictor.predict_rays = ScanStaticRayPredictor(s0, m2, fixed_rotation, setting_rotation, dphi) # Predict the spot locations raypredictor.reflections2 = [] for h in raypredictor.generate_indices.to_array(): rays = raypredictor.predict_rays(h, UB) for ray in rays: if ray.angle >= dphi[0] and ray.angle <= dphi[1]: raypredictor.reflections2.append(ray) assert len(raypredictor.reflections) == len(raypredictor.reflections2) for r1, r2 in zip(raypredictor.reflections, raypredictor.reflections2): assert all(a == pytest.approx(b, abs=1e-7) for a, b in zip(r1['s1'], r2.s1)) assert r1['phi'] == pytest.approx(r2.angle, abs=1e-7) assert r1['entering'] == r2.entering
def test_new_from_array(self): from dials.algorithms.spot_prediction import ScanStaticRayPredictor from dials.algorithms.spot_prediction import IndexGenerator # Create the index generator self.generate_indices = IndexGenerator(self.unit_cell, self.space_group_type, self.d_min) s0 = self.beam.get_s0() m2 = self.gonio.get_rotation_axis() fixed_rotation = self.gonio.get_fixed_rotation() setting_rotation = self.gonio.get_setting_rotation() UB = self.ub_matrix dphi = self.scan.get_oscillation_range(deg=False) # Create the ray predictor self.predict_rays = ScanStaticRayPredictor(s0, m2, fixed_rotation, setting_rotation, dphi) # Predict the spot locations h = self.generate_indices.to_array() reflections = self.predict_rays(h, UB) self.reflections3 = [] for r in reflections: if r['phi'] >= dphi[0] and r['phi'] <= dphi[1]: self.reflections3.append(r) eps = 1e-7 assert(len(self.reflections) == len(self.reflections3)) for r1, r2 in zip(self.reflections, self.reflections3): assert(all(abs(a - b) < eps for a, b in zip(r1['s1'], r2['s1']))) assert(abs(r1['phi'] - r2['phi']) < eps) assert(r1['entering'] == r2['entering']) print 'OK'
def generate_reflections(experiments): from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.refinement.prediction import \ ScansRayPredictor, ExperimentsPredictor from dials.algorithms.spot_prediction import ray_intersection from cctbx.sgtbx import space_group, space_group_symbols from scitbx.array_family import flex detector = experiments[0].detector crystal = experiments[0].crystal # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator(crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices = index_generator.to_array() # Predict rays within the sweep range scan = experiments[0].scan sweep_range = scan.get_oscillation_range(deg=False) ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(detector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs['id'] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = detector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.)**2) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) return obs_refs, ref_predictor
def generate_reflections(self): sweep_range = self.scan.get_oscillation_range(deg=False) resolution = 2.0 index_generator = IndexGenerator(self.crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices = index_generator.to_array() # Predict rays within the sweep range ray_predictor = ScansRayPredictor(self.experiments, sweep_range) obs_refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(self.detector, obs_refs) obs_refs = obs_refs.select(intersects) # Re-predict using the Experiments predictor for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns obs_refs['id'] = flex.int(len(obs_refs), 0) obs_refs = self.ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = self.detector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.)**2) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) # set the flex random seed to an 'uninteresting' number flex.set_random_seed(12407) # take 5 random reflections for speed reflections = obs_refs.select(flex.random_selection(len(obs_refs), 5)) # use a BlockCalculator to calculate the blocks per image from dials.algorithms.refinement.reflection_manager import BlockCalculator block_calculator = BlockCalculator(self.experiments, reflections) reflections = block_calculator.per_image() return reflections
def test_Simple1ProfileModel_compute_bbox(simple1_profile_model, test_experiment): experiments = [test_experiment] # Create the index generator index_generator = IndexGenerator( experiments[0].crystal.get_unit_cell(), experiments[0].crystal.get_space_group().type(), 2.0, ) # Get an array of miller indices miller_indices = index_generator.to_array() reflections = simple1_profile_model.predict_reflections(experiments, miller_indices, probability=0.9973) simple1_profile_model.compute_bbox(experiments, reflections)
def generate_reflections(self): # Build a mock scan for a 3 degree sequence sf = ScanFactory() self.scan = sf.make_scan( image_range=(1, 1), exposure_times=0.1, oscillation=(0, 3.0), epochs=list(range(1)), deg=True, ) sequence_range = self.scan.get_oscillation_range(deg=False) # Create a scans ExperimentList, only for generating reflections experiments = ExperimentList() experiments.append( Experiment( beam=self.beam, detector=self.detector, goniometer=self.gonio, scan=self.scan, crystal=self.crystal, imageset=None, )) # Create a ScansRayPredictor ray_predictor = ScansRayPredictor(experiments, sequence_range) # Generate rays - only to work out which hkls are predicted resolution = 2.0 index_generator = IndexGenerator( self.crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() rays = ray_predictor(indices) # Make a standard reflection_table and copy in the ray data self.reflections = flex.reflection_table.empty_standard(len(rays)) self.reflections.update(rays) # Set dummy observed variances to allow statistical weights to be set self.reflections["xyzobs.mm.variance"] += (1e-3, 1e-3, 1e-6)
def generate_reflections(experiments): from cctbx.sgtbx import space_group, space_group_symbols from dials.algorithms.refinement.prediction.managed_predictors import ( ScansExperimentsPredictor, ScansRayPredictor, ) from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection detector = experiments[0].detector crystal = experiments[0].crystal # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator( crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() # Predict rays within the sequence range scan = experiments[0].scan sequence_range = scan.get_oscillation_range(deg=False) ray_predictor = ScansRayPredictor(experiments, sequence_range) obs_refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(detector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ScansExperimentsPredictor(experiments) obs_refs["id"] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"] return obs_refs
def predict(self): """ Predict the reflections """ logger.info("") logger.info("Predicting reflections") # Set a resolution range if self.params.prediction.d_min is None: s0 = self.experiments[0].beam.get_s0() d_min = self.experiments[0].detector.get_max_resolution(s0) else: d_min = self.params.predictions.d_min # Create the index generator index_generator = IndexGenerator( self.experiments[0].crystal.get_unit_cell(), self.experiments[0].crystal.get_space_group().type(), d_min, ) # Get an array of miller indices miller_indices_to_test = index_generator.to_array() logger.info("Generated %d miller indices" % len(miller_indices_to_test)) # Get the covariance matrix profile = self.experiments[0].crystal.mosaicity self.reflections = profile.predict_reflections( self.experiments, miller_indices_to_test, self.params.prediction.probability) # Do the prediction self.reference = self.reference self.reflections.compute_d(self.experiments) logger.info("Predicted %d reflections" % len(self.reflections)) # Match with the reference reflections _, _, unmatched = self.reflections.match_with_reference(self.reference)
X = tuple([e * 1.e5 for e in S.forward_independent_parameters()]) xluc.set_param_vals(X) ############################# # Generate some reflections # ############################# #print "Reflections will be generated with the following geometry:" #print mybeam #print mydetector #print crystal1 #print crystal2 # All indices in a 2.0 Angstrom sphere for crystal1 resolution = 2.0 index_generator = IndexGenerator(crystal1.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices1 = index_generator.to_array() # All indices in a 2.0 Angstrom sphere for crystal2 resolution = 2.0 index_generator = IndexGenerator(crystal2.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices2 = index_generator.to_array() # Predict rays within the sweep range. Set experiment IDs ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs1 = ray_predictor.predict(indices1, experiment_id=0) obs_refs1['id'] = flex.int(len(obs_refs1), 0) obs_refs2 = ray_predictor.predict(indices1, experiment_id=1) obs_refs2['id'] = flex.int(len(obs_refs2), 1)
] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(mycrystal.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.e5 for e in S.forward_independent_parameters()]) xluc_param.set_param_vals(X) ############################# # Generate some reflections # ############################# # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator( mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices = index_generator.to_array() sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 0.1 * pi / 180.) # Predict rays within the sweep range ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects)
def test(): # Build models, with a larger crystal than default in order to get plenty of # reflections on the 'still' image overrides = """ geometry.parameters.crystal.a.length.range=40 50; geometry.parameters.crystal.b.length.range=40 50; geometry.parameters.crystal.c.length.range=40 50; geometry.parameters.random_seed = 42""" master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True, ) models = Extract(master_phil, overrides) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam # Build a mock scan for a 3 degree sweep from dxtbx.model import ScanFactory sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 1), exposure_times=0.1, oscillation=(0, 3.0), epochs=list(range(1)), deg=True, ) sweep_range = myscan.get_oscillation_range(deg=False) # Create parameterisations of these models det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) # Create a scans ExperimentList, only for generating reflections experiments = ExperimentList() experiments.append( Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None, )) # Create a stills ExperimentList stills_experiments = ExperimentList() stills_experiments.append( Experiment(beam=mybeam, detector=mydetector, crystal=mycrystal, imageset=None)) # Generate rays - only to work out which hkls are predicted ray_predictor = ScansRayPredictor(experiments, sweep_range) resolution = 2.0 index_generator = IndexGenerator( mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() rays = ray_predictor(indices) # Make a standard reflection_table and copy in the ray data reflections = flex.reflection_table.empty_standard(len(rays)) reflections.update(rays) # Build a standard prediction parameterisation for the stills experiment to do # FD calculation (not used for its analytical gradients) pred_param = StillsPredictionParameterisation( stills_experiments, detector_parameterisations=[det_param], beam_parameterisations=[s0_param], xl_orientation_parameterisations=[xlo_param], xl_unit_cell_parameterisations=[xluc_param], ) # Make a managed SphericalRelpStillsReflectionPredictor reflection predictor # for the first (only) experiment ref_predictor = Predictor(stills_experiments) # Predict these reflections in place. Must do this ahead of calculating # the analytical gradients so quantities like s1 are correct ref_predictor.update() ref_predictor.predict(reflections) # calculate analytical gradients ag = AnalyticalGradients( stills_experiments, detector_parameterisation=det_param, beam_parameterisation=s0_param, xl_orientation_parameterisation=xlo_param, xl_unit_cell_parameterisation=xluc_param, ) an_grads = ag.get_beam_gradients(reflections) an_grads.update(ag.get_crystal_orientation_gradients(reflections)) an_grads.update(ag.get_crystal_unit_cell_gradients(reflections)) # get finite difference gradients p_vals = pred_param.get_param_vals() deltas = [1.0e-7] * len(p_vals) fd_grads = [] p_names = pred_param.get_param_names() for i, delta in enumerate(deltas): # save parameter value val = p_vals[i] # calc reverse state p_vals[i] -= delta / 2.0 pred_param.set_param_vals(p_vals) ref_predictor.update() ref_predictor.predict(reflections) x, y, _ = reflections["xyzcal.mm"].deep_copy().parts() s1 = reflections["s1"].deep_copy() rev_state = s1 # calc forward state p_vals[i] += delta pred_param.set_param_vals(p_vals) ref_predictor.update() ref_predictor.predict(reflections) x, y, _ = reflections["xyzcal.mm"].deep_copy().parts() s1 = reflections["s1"].deep_copy() fwd_state = s1 # reset parameter to saved value p_vals[i] = val # finite difference - currently for s1 only fd = fwd_state - rev_state inv_delta = 1.0 / delta s1_grads = fd * inv_delta # store gradients fd_grads.append({"name": p_names[i], "ds1": s1_grads}) # return to the initial state pred_param.set_param_vals(p_vals) for i, fd_grad in enumerate(fd_grads): ## compare FD with analytical calculations print("\n\nParameter {0}: {1}".format(i, fd_grad["name"])) print("d[s1]/dp for the first reflection") print("finite diff", fd_grad["ds1"][0]) try: an_grad = an_grads[fd_grad["name"]] except KeyError: continue print("checking analytical vs finite difference gradients for s1") for a, b in zip(fd_grad["ds1"], an_grad["ds1"]): assert a == pytest.approx(b, abs=1e-7)
def __init__(self, dials_regression): import dxtbx from iotbx.xds import integrate_hkl, xparm from rstbx.cftbx.coordinate_frame_converter import coordinate_frame_converter from dials.algorithms.spot_prediction import ( IndexGenerator, ScanStaticRayPredictor, ) from dials.util import ioutil # The XDS files to read from integrate_filename = os.path.join(dials_regression, "data", "sim_mx", "INTEGRATE.HKL") gxparm_filename = os.path.join(dials_regression, "data", "sim_mx", "GXPARM.XDS") # Read the XDS files self.integrate_handle = integrate_hkl.reader() self.integrate_handle.read_file(integrate_filename) self.gxparm_handle = xparm.reader() self.gxparm_handle.read_file(gxparm_filename) # Get the parameters we need from the GXPARM file models = dxtbx.load(gxparm_filename) self.beam = models.get_beam() self.gonio = models.get_goniometer() self.detector = models.get_detector() self.scan = models.get_scan() # Get crystal parameters self.space_group_type = ioutil.get_space_group_type_from_xparm( self.gxparm_handle) cfc = coordinate_frame_converter(gxparm_filename) a_vec = cfc.get("real_space_a") b_vec = cfc.get("real_space_b") c_vec = cfc.get("real_space_c") self.unit_cell = cfc.get_unit_cell() self.ub_matrix = matrix.sqr(a_vec + b_vec + c_vec).inverse() # Get the minimum resolution in the integrate file d = [self.unit_cell.d(h) for h in self.integrate_handle.hkl] self.d_min = min(d) # extend the resolution shell by epsilon>0 # to account for rounding artifacts on 32-bit platforms self.d_min = self.d_min - 1e-15 # Get the number of frames from the max z value xcal, ycal, zcal = zip(*self.integrate_handle.xyzcal) self.scan.set_image_range(( self.scan.get_image_range()[0], self.scan.get_image_range()[0] + int(math.ceil(max(zcal))), )) # Print stuff # print self.beam # print self.gonio # print self.detector # print self.scan # Create the index generator self.generate_indices = IndexGenerator(self.unit_cell, self.space_group_type, self.d_min) s0 = self.beam.get_s0() m2 = self.gonio.get_rotation_axis() fixed_rotation = self.gonio.get_fixed_rotation() setting_rotation = self.gonio.get_setting_rotation() UB = self.ub_matrix dphi = self.scan.get_oscillation_range(deg=False) # Create the ray predictor self.predict_rays = ScanStaticRayPredictor(s0, m2, fixed_rotation, setting_rotation, dphi) # Predict the spot locations self.reflections = self.predict_rays(self.generate_indices.to_array(), UB)
class TestRayPredictor: def __init__(self): from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.spot_prediction import ScanStaticRayPredictor from iotbx.xds import xparm, integrate_hkl from dials.util import ioutil from math import ceil import dxtbx from rstbx.cftbx.coordinate_frame_converter import \ coordinate_frame_converter from scitbx import matrix # The XDS files to read from integrate_filename = join(dials_regression, 'data/sim_mx/INTEGRATE.HKL') gxparm_filename = join(dials_regression, 'data/sim_mx/GXPARM.XDS') # Read the XDS files self.integrate_handle = integrate_hkl.reader() self.integrate_handle.read_file(integrate_filename) self.gxparm_handle = xparm.reader() self.gxparm_handle.read_file(gxparm_filename) # Get the parameters we need from the GXPARM file models = dxtbx.load(gxparm_filename) self.beam = models.get_beam() self.gonio = models.get_goniometer() self.detector = models.get_detector() self.scan = models.get_scan() # Get crystal parameters self.space_group_type = ioutil.get_space_group_type_from_xparm( self.gxparm_handle) cfc = coordinate_frame_converter(gxparm_filename) a_vec = cfc.get('real_space_a') b_vec = cfc.get('real_space_b') c_vec = cfc.get('real_space_c') self.unit_cell = cfc.get_unit_cell() self.ub_matrix = matrix.sqr(a_vec + b_vec + c_vec).inverse() # Get the minimum resolution in the integrate file d = [self.unit_cell.d(h) for h in self.integrate_handle.hkl] self.d_min = min(d) # extend the resolution shell by epsilon>0 # to account for rounding artifacts on 32-bit platforms self.d_min = self.d_min - 1e-15 # Get the number of frames from the max z value xcal, ycal, zcal = zip(*self.integrate_handle.xyzcal) self.scan.set_image_range( (self.scan.get_image_range()[0], self.scan.get_image_range()[0] + int(ceil(max(zcal))))) # Print stuff # print self.beam # print self.gonio # print self.detector # print self.scan # Create the index generator self.generate_indices = IndexGenerator(self.unit_cell, self.space_group_type, self.d_min) s0 = self.beam.get_s0() m2 = self.gonio.get_rotation_axis() fixed_rotation = self.gonio.get_fixed_rotation() setting_rotation = self.gonio.get_setting_rotation() UB = self.ub_matrix dphi = self.scan.get_oscillation_range(deg=False) # Create the ray predictor self.predict_rays = ScanStaticRayPredictor(s0, m2, fixed_rotation, setting_rotation, dphi) # Predict the spot locations self.reflections = self.predict_rays(self.generate_indices.to_array(), UB) def test_miller_index_set(self): """Ensure we have the whole set of miller indices""" gen_hkl = {r['miller_index'] for r in self.reflections} missing = [] for hkl in self.integrate_handle.hkl: if hkl not in gen_hkl: missing.append(hkl) assert len( missing ) == 0, "%d out of %d reflections not in set, including %s" % ( len(missing), len(self.integrate_handle.hkl), str(missing[0])) def test_rotation_angles(self): """Ensure the rotation angles agree with XDS""" # Create a dict of lists of xy for each hkl gen_phi = {} for r in self.reflections: hkl = r['miller_index'] phi = r['phi'] try: a = gen_phi[hkl] a.append(phi) gen_phi[hkl] = a except KeyError: gen_phi[hkl] = [phi] # For each hkl in the xds file for hkl, xyz in zip(self.integrate_handle.hkl, self.integrate_handle.xyzcal): # Calculate the XDS phi value xds_phi = self.scan.get_oscillation(deg=False)[0] + \ xyz[2]*self.scan.get_oscillation(deg=False)[1] # Select the nearest xy to use if there are 2 my_phi = gen_phi[hkl] if len(my_phi) == 2: my_phi0 = my_phi[0] my_phi1 = my_phi[1] diff0 = abs(xds_phi - my_phi0) diff1 = abs(xds_phi - my_phi1) if diff0 < diff1: my_phi = my_phi0 else: my_phi = my_phi1 else: my_phi = my_phi[0] # Check the Phi values are the same assert (abs(xds_phi - my_phi) < 0.1) print "OK" def test_beam_vectors(self): """Ensure |s1| == |s0|""" from scitbx import matrix s0_length = matrix.col(self.beam.get_s0()).length() for r in self.reflections: s1 = r['s1'] s1_length = matrix.col(s1).length() assert (abs(s0_length - s1_length) < 1e-7) print "OK" def test_new(self): from dials.algorithms.spot_prediction import ScanStaticRayPredictor from dials.algorithms.spot_prediction import IndexGenerator # Create the index generator self.generate_indices = IndexGenerator(self.unit_cell, self.space_group_type, self.d_min) s0 = self.beam.get_s0() m2 = self.gonio.get_rotation_axis() fixed_rotation = self.gonio.get_fixed_rotation() setting_rotation = self.gonio.get_setting_rotation() UB = self.ub_matrix dphi = self.scan.get_oscillation_range(deg=False) # Create the ray predictor self.predict_rays = ScanStaticRayPredictor(s0, m2, fixed_rotation, setting_rotation, dphi) # Predict the spot locations self.reflections2 = [] for h in self.generate_indices.to_array(): rays = self.predict_rays(h, UB) for ray in rays: if ray.angle >= dphi[0] and ray.angle <= dphi[1]: self.reflections2.append(ray) eps = 1e-7 assert (len(self.reflections) == len(self.reflections2)) for r1, r2 in zip(self.reflections, self.reflections2): assert (all(abs(a - b) < eps for a, b in zip(r1['s1'], r2.s1))) assert (abs(r1['phi'] - r2.angle) < eps) assert (r1['entering'] == r2.entering) print 'OK' def test_new_from_array(self): from dials.algorithms.spot_prediction import ScanStaticRayPredictor from dials.algorithms.spot_prediction import IndexGenerator # Create the index generator self.generate_indices = IndexGenerator(self.unit_cell, self.space_group_type, self.d_min) s0 = self.beam.get_s0() m2 = self.gonio.get_rotation_axis() fixed_rotation = self.gonio.get_fixed_rotation() setting_rotation = self.gonio.get_setting_rotation() UB = self.ub_matrix dphi = self.scan.get_oscillation_range(deg=False) # Create the ray predictor self.predict_rays = ScanStaticRayPredictor(s0, m2, fixed_rotation, setting_rotation, dphi) # Predict the spot locations h = self.generate_indices.to_array() reflections = self.predict_rays(h, UB) self.reflections3 = [] for r in reflections: if r['phi'] >= dphi[0] and r['phi'] <= dphi[1]: self.reflections3.append(r) eps = 1e-7 assert (len(self.reflections) == len(self.reflections3)) for r1, r2 in zip(self.reflections, self.reflections3): assert (all(abs(a - b) < eps for a, b in zip(r1['s1'], r2['s1']))) assert (abs(r1['phi'] - r2['phi']) < eps) assert (r1['entering'] == r2['entering']) print 'OK' def run(self): self.test_miller_index_set() self.test_rotation_angles() self.test_beam_vectors() self.test_new() self.test_new_from_array()
def __init__(self): from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.spot_prediction import ScanStaticRayPredictor from dials.algorithms.spot_prediction import ray_intersection from iotbx.xds import xparm, integrate_hkl from dials.util import ioutil from math import ceil from os.path import realpath, dirname, join import dxtbx from rstbx.cftbx.coordinate_frame_converter import \ coordinate_frame_converter from scitbx import matrix # The XDS files to read from test_path = dirname(dirname(dirname(realpath(__file__)))) integrate_filename = join(test_path, 'data/sim_mx/INTEGRATE.HKL') gxparm_filename = join(test_path, 'data/sim_mx/GXPARM.XDS') # Read the XDS files self.integrate_handle = integrate_hkl.reader() self.integrate_handle.read_file(integrate_filename) self.gxparm_handle = xparm.reader() self.gxparm_handle.read_file(gxparm_filename) # Get the parameters we need from the GXPARM file models = dxtbx.load(gxparm_filename) self.beam = models.get_beam() self.gonio = models.get_goniometer() self.detector = models.get_detector() self.scan = models.get_scan() assert(len(self.detector) == 1) #print self.detector # Get crystal parameters self.space_group_type = ioutil.get_space_group_type_from_xparm( self.gxparm_handle) cfc = coordinate_frame_converter(gxparm_filename) a_vec = cfc.get('real_space_a') b_vec = cfc.get('real_space_b') c_vec = cfc.get('real_space_c') self.unit_cell = cfc.get_unit_cell() self.ub_matrix = matrix.sqr(a_vec + b_vec + c_vec).inverse() # Get the minimum resolution in the integrate file self.d_min = self.detector[0].get_max_resolution_at_corners( self.beam.get_s0()) # Get the number of frames from the max z value xcal, ycal, zcal = zip(*self.integrate_handle.xyzcal) self.scan.set_image_range((self.scan.get_image_range()[0], self.scan.get_image_range()[0] + int(ceil(max(zcal))))) # Create the index generator generate_indices = IndexGenerator(self.unit_cell, self.space_group_type, self.d_min) s0 = self.beam.get_s0() m2 = self.gonio.get_rotation_axis() fixed_rotation = self.gonio.get_fixed_rotation() UB = self.ub_matrix dphi = self.scan.get_oscillation_range(deg=False) # Create the ray predictor self.predict_rays = ScanStaticRayPredictor(s0, m2, fixed_rotation, dphi) # Predict the spot locations self.reflections = self.predict_rays( generate_indices.to_array(), UB) # Calculate the intersection of the detector and reflection frames success = ray_intersection(self.detector, self.reflections) self.reflections.select(success)
def __init__(self): from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.spot_prediction import ScanStaticRayPredictor from dials.algorithms.spot_prediction import ray_intersection from iotbx.xds import xparm, integrate_hkl from dials.util import ioutil from math import ceil import dxtbx from rstbx.cftbx.coordinate_frame_converter import \ coordinate_frame_converter from scitbx import matrix # The XDS files to read from integrate_filename = join(dials_regression, 'data/sim_mx/INTEGRATE.HKL') gxparm_filename = join(dials_regression, 'data/sim_mx/GXPARM.XDS') # Read the XDS files self.integrate_handle = integrate_hkl.reader() self.integrate_handle.read_file(integrate_filename) self.gxparm_handle = xparm.reader() self.gxparm_handle.read_file(gxparm_filename) # Get the parameters we need from the GXPARM file models = dxtbx.load(gxparm_filename) self.beam = models.get_beam() self.gonio = models.get_goniometer() self.detector = models.get_detector() self.scan = models.get_scan() assert (len(self.detector) == 1) #print self.detector # Get crystal parameters self.space_group_type = ioutil.get_space_group_type_from_xparm( self.gxparm_handle) cfc = coordinate_frame_converter(gxparm_filename) a_vec = cfc.get('real_space_a') b_vec = cfc.get('real_space_b') c_vec = cfc.get('real_space_c') self.unit_cell = cfc.get_unit_cell() self.ub_matrix = matrix.sqr(a_vec + b_vec + c_vec).inverse() # Get the minimum resolution in the integrate file self.d_min = self.detector[0].get_max_resolution_at_corners( self.beam.get_s0()) # Get the number of frames from the max z value xcal, ycal, zcal = zip(*self.integrate_handle.xyzcal) self.scan.set_image_range( (self.scan.get_image_range()[0], self.scan.get_image_range()[0] + int(ceil(max(zcal))))) # Create the index generator generate_indices = IndexGenerator(self.unit_cell, self.space_group_type, self.d_min) s0 = self.beam.get_s0() m2 = self.gonio.get_rotation_axis() fixed_rotation = self.gonio.get_fixed_rotation() setting_rotation = self.gonio.get_setting_rotation() UB = self.ub_matrix dphi = self.scan.get_oscillation_range(deg=False) # Create the ray predictor self.predict_rays = ScanStaticRayPredictor(s0, m2, fixed_rotation, setting_rotation, dphi) # Predict the spot locations self.reflections = self.predict_rays(generate_indices.to_array(), UB) # Calculate the intersection of the detector and reflection frames success = ray_intersection(self.detector, self.reflections) self.reflections.select(success)
def run(verbose = False): # Build models, with a larger crystal than default in order to get plenty of # reflections on the 'still' image overrides = """ geometry.parameters.crystal.a.length.range=40 50; geometry.parameters.crystal.b.length.range=40 50; geometry.parameters.crystal.c.length.range=40 50; geometry.parameters.random_seed = 42""" master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True) models = Extract(master_phil, overrides) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam # Build a mock scan for a 3 degree sweep from dxtbx.model.scan import scan_factory sf = scan_factory() myscan = sf.make_scan(image_range = (1,1), exposure_times = 0.1, oscillation = (0, 3.0), epochs = range(1), deg = True) sweep_range = myscan.get_oscillation_range(deg=False) # Create parameterisations of these models det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) # Create a scans ExperimentList, only for generating reflections experiments = ExperimentList() experiments.append(Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None)) # Create a stills ExperimentList stills_experiments = ExperimentList() stills_experiments.append(Experiment( beam=mybeam, detector=mydetector, crystal=mycrystal, imageset=None)) # Generate rays - only to work out which hkls are predicted ray_predictor = ScansRayPredictor(experiments, sweep_range) resolution = 2.0 index_generator = IndexGenerator(mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices = index_generator.to_array() rays = ray_predictor.predict(indices) # Make a standard reflection_table and copy in the ray data reflections = flex.reflection_table.empty_standard(len(rays)) reflections.update(rays) # Build a standard prediction parameterisation for the stills experiment to do # FD calculation (not used for its analytical gradients) pred_param = StillsPredictionParameterisation(stills_experiments, detector_parameterisations = [det_param], beam_parameterisations = [s0_param], xl_orientation_parameterisations = [xlo_param], xl_unit_cell_parameterisations = [xluc_param]) # Make a managed SphericalRelpStillsReflectionPredictor reflection predictor # for the first (only) experiment ref_predictor = Predictor(stills_experiments) # Predict these reflections in place. Must do this ahead of calculating # the analytical gradients so quantities like s1 are correct ref_predictor.update() ref_predictor.predict(reflections) # calculate analytical gradients ag = AnalyticalGradients(stills_experiments, detector_parameterisation=det_param, beam_parameterisation=s0_param, xl_orientation_parameterisation=xlo_param, xl_unit_cell_parameterisation=xluc_param) an_grads = ag.get_beam_gradients(reflections) an_grads.update(ag.get_crystal_orientation_gradients(reflections)) an_grads.update(ag.get_crystal_unit_cell_gradients(reflections)) # get finite difference gradients p_vals = pred_param.get_param_vals() deltas = [1.e-7] * len(p_vals) fd_grads = [] p_names = pred_param.get_param_names() for i in range(len(deltas)): # save parameter value val = p_vals[i] # calc reverse state p_vals[i] -= deltas[i] / 2. pred_param.set_param_vals(p_vals) ref_predictor.update() ref_predictor.predict(reflections) x, y, _ = reflections['xyzcal.mm'].deep_copy().parts() delpsi = reflections['delpsical.rad'].deep_copy() s1 = reflections['s1'].deep_copy() rev_state = s1 # calc forward state p_vals[i] += deltas[i] pred_param.set_param_vals(p_vals) ref_predictor.update() ref_predictor.predict(reflections) x, y, _ = reflections['xyzcal.mm'].deep_copy().parts() delpsi = reflections['delpsical.rad'].deep_copy() s1 = reflections['s1'].deep_copy() fwd_state = s1 # reset parameter to saved value p_vals[i] = val # finite difference - currently for s1 only fd = (fwd_state - rev_state) inv_delta = 1. / deltas[i] s1_grads = fd * inv_delta # store gradients fd_grads.append({'name':p_names[i], 'ds1':s1_grads}) # return to the initial state pred_param.set_param_vals(p_vals) for i, fd_grad in enumerate(fd_grads): ## compare FD with analytical calculations if verbose: print "\n\nParameter {0}: {1}". format(i, fd_grad['name']) print "d[s1]/dp for the first reflection" print 'finite diff', fd_grad['ds1'][0] try: an_grad = an_grads[fd_grad['name']] except KeyError: continue print 'checking analytical vs finite difference gradients for s1' for a, b in zip(fd_grad['ds1'], an_grad['ds1']): assert approx_equal(a, b) print 'OK'
class TestRayPredictor: def __init__(self): from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.spot_prediction import ScanStaticRayPredictor from iotbx.xds import xparm, integrate_hkl from dials.util import ioutil from math import ceil import dxtbx from rstbx.cftbx.coordinate_frame_converter import \ coordinate_frame_converter from scitbx import matrix # The XDS files to read from integrate_filename = join(dials_regression, 'data/sim_mx/INTEGRATE.HKL') gxparm_filename = join(dials_regression, 'data/sim_mx/GXPARM.XDS') # Read the XDS files self.integrate_handle = integrate_hkl.reader() self.integrate_handle.read_file(integrate_filename) self.gxparm_handle = xparm.reader() self.gxparm_handle.read_file(gxparm_filename) # Get the parameters we need from the GXPARM file models = dxtbx.load(gxparm_filename) self.beam = models.get_beam() self.gonio = models.get_goniometer() self.detector = models.get_detector() self.scan = models.get_scan() # Get crystal parameters self.space_group_type = ioutil.get_space_group_type_from_xparm( self.gxparm_handle) cfc = coordinate_frame_converter(gxparm_filename) a_vec = cfc.get('real_space_a') b_vec = cfc.get('real_space_b') c_vec = cfc.get('real_space_c') self.unit_cell = cfc.get_unit_cell() self.ub_matrix = matrix.sqr(a_vec + b_vec + c_vec).inverse() # Get the minimum resolution in the integrate file d = [self.unit_cell.d(h) for h in self.integrate_handle.hkl] self.d_min = min(d) # extend the resolution shell by epsilon>0 # to account for rounding artifacts on 32-bit platforms self.d_min = self.d_min - 1e-15 # Get the number of frames from the max z value xcal, ycal, zcal = zip(*self.integrate_handle.xyzcal) self.scan.set_image_range((self.scan.get_image_range()[0], self.scan.get_image_range()[0] + int(ceil(max(zcal))))) # Print stuff # print self.beam # print self.gonio # print self.detector # print self.scan # Create the index generator self.generate_indices = IndexGenerator(self.unit_cell, self.space_group_type, self.d_min) s0 = self.beam.get_s0() m2 = self.gonio.get_rotation_axis() fixed_rotation = self.gonio.get_fixed_rotation() setting_rotation = self.gonio.get_setting_rotation() UB = self.ub_matrix dphi = self.scan.get_oscillation_range(deg=False) # Create the ray predictor self.predict_rays = ScanStaticRayPredictor(s0, m2, fixed_rotation, setting_rotation, dphi) # Predict the spot locations self.reflections = self.predict_rays( self.generate_indices.to_array(), UB) def test_miller_index_set(self): """Ensure we have the whole set of miller indices""" gen_hkl = { r['miller_index'] for r in self.reflections } missing = [] for hkl in self.integrate_handle.hkl: if hkl not in gen_hkl: missing.append(hkl) assert len(missing) == 0, "%d out of %d reflections not in set, including %s" % (len(missing), len(self.integrate_handle.hkl), str(missing[0])) def test_rotation_angles(self): """Ensure the rotation angles agree with XDS""" # Create a dict of lists of xy for each hkl gen_phi = {} for r in self.reflections: hkl = r['miller_index'] phi = r['phi'] try: a = gen_phi[hkl] a.append(phi) gen_phi[hkl] = a except KeyError: gen_phi[hkl] = [phi] # For each hkl in the xds file for hkl, xyz in zip(self.integrate_handle.hkl, self.integrate_handle.xyzcal): # Calculate the XDS phi value xds_phi = self.scan.get_oscillation(deg=False)[0] + \ xyz[2]*self.scan.get_oscillation(deg=False)[1] # Select the nearest xy to use if there are 2 my_phi = gen_phi[hkl] if len(my_phi) == 2: my_phi0 = my_phi[0] my_phi1 = my_phi[1] diff0 = abs(xds_phi - my_phi0) diff1 = abs(xds_phi - my_phi1) if diff0 < diff1: my_phi = my_phi0 else: my_phi = my_phi1 else: my_phi = my_phi[0] # Check the Phi values are the same assert(abs(xds_phi - my_phi) < 0.1) print "OK" def test_beam_vectors(self): """Ensure |s1| == |s0|""" from scitbx import matrix s0_length = matrix.col(self.beam.get_s0()).length() for r in self.reflections: s1 = r['s1'] s1_length = matrix.col(s1).length() assert(abs(s0_length - s1_length) < 1e-7) print "OK" def test_new(self): from dials.algorithms.spot_prediction import ScanStaticRayPredictor from dials.algorithms.spot_prediction import IndexGenerator # Create the index generator self.generate_indices = IndexGenerator(self.unit_cell, self.space_group_type, self.d_min) s0 = self.beam.get_s0() m2 = self.gonio.get_rotation_axis() fixed_rotation = self.gonio.get_fixed_rotation() setting_rotation = self.gonio.get_setting_rotation() UB = self.ub_matrix dphi = self.scan.get_oscillation_range(deg=False) # Create the ray predictor self.predict_rays = ScanStaticRayPredictor(s0, m2, fixed_rotation, setting_rotation, dphi) # Predict the spot locations self.reflections2 = [] for h in self.generate_indices.to_array(): rays = self.predict_rays(h, UB) for ray in rays: if ray.angle >= dphi[0] and ray.angle <= dphi[1]: self.reflections2.append(ray) eps = 1e-7 assert(len(self.reflections) == len(self.reflections2)) for r1, r2 in zip(self.reflections, self.reflections2): assert(all(abs(a - b) < eps for a, b in zip(r1['s1'], r2.s1))) assert(abs(r1['phi'] - r2.angle) < eps) assert(r1['entering'] == r2.entering) print 'OK' def test_new_from_array(self): from dials.algorithms.spot_prediction import ScanStaticRayPredictor from dials.algorithms.spot_prediction import IndexGenerator # Create the index generator self.generate_indices = IndexGenerator(self.unit_cell, self.space_group_type, self.d_min) s0 = self.beam.get_s0() m2 = self.gonio.get_rotation_axis() fixed_rotation = self.gonio.get_fixed_rotation() setting_rotation = self.gonio.get_setting_rotation() UB = self.ub_matrix dphi = self.scan.get_oscillation_range(deg=False) # Create the ray predictor self.predict_rays = ScanStaticRayPredictor(s0, m2, fixed_rotation, setting_rotation, dphi) # Predict the spot locations h = self.generate_indices.to_array() reflections = self.predict_rays(h, UB) self.reflections3 = [] for r in reflections: if r['phi'] >= dphi[0] and r['phi'] <= dphi[1]: self.reflections3.append(r) eps = 1e-7 assert(len(self.reflections) == len(self.reflections3)) for r1, r2 in zip(self.reflections, self.reflections3): assert(all(abs(a - b) < eps for a, b in zip(r1['s1'], r2['s1']))) assert(abs(r1['phi'] - r2['phi']) < eps) assert(r1['entering'] == r2['entering']) print 'OK' def run(self): self.test_miller_index_set() self.test_rotation_angles() self.test_beam_vectors() self.test_new() self.test_new_from_array()
def test(args=[]): # Python and cctbx imports from math import pi from scitbx import matrix from libtbx.phil import parse from libtbx.test_utils import approx_equal # Import for surgery on reflection_tables from dials.array_family import flex # Get module to build models using PHIL import dials.test.algorithms.refinement.setup_geometry as setup_geometry # We will set up a mock scan and a mock experiment list from dxtbx.model import ScanFactory from dxtbx.model.experiment_list import ExperimentList, Experiment # Crystal parameterisations from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalOrientationParameterisation, CrystalUnitCellParameterisation, ) # Symmetry constrained parameterisation for the unit cell from cctbx.uctbx import unit_cell from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.refinement.prediction.managed_predictors import ( ScansRayPredictor, StillsExperimentsPredictor, ) from dials.algorithms.spot_prediction import ray_intersection from cctbx.sgtbx import space_group, space_group_symbols ############################# # Setup experimental models # ############################# master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True, ) # build models, with a larger crystal than default in order to get enough # reflections on the 'still' image param = """ geometry.parameters.crystal.a.length.range=40 50; geometry.parameters.crystal.b.length.range=40 50; geometry.parameters.crystal.c.length.range=40 50; geometry.parameters.random_seed = 42""" models = setup_geometry.Extract(master_phil, cmdline_args=args, local_overrides=param) crystal = models.crystal mydetector = models.detector mygonio = models.goniometer mybeam = models.beam # Build a mock scan for a 1.5 degree wedge. Only used for generating indices near # the Ewald sphere sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 1), exposure_times=0.1, oscillation=(0, 1.5), epochs=list(range(1)), deg=True, ) sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert approx_equal(im_width, 1.5 * pi / 180.0) # Build experiment lists stills_experiments = ExperimentList() stills_experiments.append( Experiment(beam=mybeam, detector=mydetector, crystal=crystal, imageset=None)) scans_experiments = ExperimentList() scans_experiments.append( Experiment( beam=mybeam, detector=mydetector, crystal=crystal, goniometer=mygonio, scan=myscan, imageset=None, )) ########################################################## # Parameterise the models (only for perturbing geometry) # ########################################################## xlo_param = CrystalOrientationParameterisation(crystal) xluc_param = CrystalUnitCellParameterisation(crystal) ################################ # Apply known parameter shifts # ################################ # rotate crystal (=5 mrad each rotation) xlo_p_vals = [] p_vals = xlo_param.get_param_vals() xlo_p_vals.append(p_vals) new_p_vals = [a + b for a, b in zip(p_vals, [5.0, 5.0, 5.0])] xlo_param.set_param_vals(new_p_vals) # change unit cell (=1.0 Angstrom length upsets, 0.5 degree of # gamma angle) xluc_p_vals = [] p_vals = xluc_param.get_param_vals() xluc_p_vals.append(p_vals) cell_params = crystal.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [1.0, 1.0, -1.0, 0.0, 0.0, 0.5]) ] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(crystal.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()]) xluc_param.set_param_vals(X) # keep track of the target crystal model to compare with refined from copy import deepcopy target_crystal = deepcopy(crystal) ############################# # Generate some reflections # ############################# # All indices in a 2.0 Angstrom sphere for crystal resolution = 2.0 index_generator = IndexGenerator( crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() # Build a ray predictor and predict rays close to the Ewald sphere by using # the narrow rotation scan ref_predictor = ScansRayPredictor(scans_experiments, sweep_range) obs_refs = ref_predictor(indices, experiment_id=0) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects) # Add in flags and ID columns by copying into standard reflection table tmp = flex.reflection_table.empty_standard(len(obs_refs)) tmp.update(obs_refs) obs_refs = tmp # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180.0 px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2) obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) # Re-predict using the stills reflection predictor stills_ref_predictor = StillsExperimentsPredictor(stills_experiments) obs_refs_stills = stills_ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs_stills["xyzobs.mm.value"] = obs_refs_stills["xyzcal.mm"] ############################### # Undo known parameter shifts # ############################### xlo_param.set_param_vals(xlo_p_vals[0]) xluc_param.set_param_vals(xluc_p_vals[0]) # make a refiner from dials.algorithms.refinement.refiner import phil_scope params = phil_scope.fetch(source=parse("")).extract() # Change this to get a plot do_plot = False if do_plot: params.refinement.refinery.journal.track_parameter_correlation = True from dials.algorithms.refinement.refiner import RefinerFactory # decrease bin_size_fraction to terminate on RMSD convergence params.refinement.target.bin_size_fraction = 0.01 params.refinement.parameterisation.beam.fix = "all" params.refinement.parameterisation.detector.fix = "all" refiner = RefinerFactory.from_parameters_data_experiments( params, obs_refs_stills, stills_experiments) # run refinement history = refiner.run() # regression tests assert len(history["rmsd"]) == 9 refined_crystal = refiner.get_experiments()[0].crystal uc1 = refined_crystal.get_unit_cell() uc2 = target_crystal.get_unit_cell() assert uc1.is_similar_to(uc2) if do_plot: plt = refiner.parameter_correlation_plot( len(history["parameter_correlation"]) - 1) plt.show()
def __init__(self): from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.spot_prediction import ScanStaticRayPredictor from iotbx.xds import xparm, integrate_hkl from dials.util import ioutil from math import ceil import dxtbx from rstbx.cftbx.coordinate_frame_converter import \ coordinate_frame_converter from scitbx import matrix # The XDS files to read from integrate_filename = join(dials_regression, 'data/sim_mx/INTEGRATE.HKL') gxparm_filename = join(dials_regression, 'data/sim_mx/GXPARM.XDS') # Read the XDS files self.integrate_handle = integrate_hkl.reader() self.integrate_handle.read_file(integrate_filename) self.gxparm_handle = xparm.reader() self.gxparm_handle.read_file(gxparm_filename) # Get the parameters we need from the GXPARM file models = dxtbx.load(gxparm_filename) self.beam = models.get_beam() self.gonio = models.get_goniometer() self.detector = models.get_detector() self.scan = models.get_scan() # Get crystal parameters self.space_group_type = ioutil.get_space_group_type_from_xparm( self.gxparm_handle) cfc = coordinate_frame_converter(gxparm_filename) a_vec = cfc.get('real_space_a') b_vec = cfc.get('real_space_b') c_vec = cfc.get('real_space_c') self.unit_cell = cfc.get_unit_cell() self.ub_matrix = matrix.sqr(a_vec + b_vec + c_vec).inverse() # Get the minimum resolution in the integrate file d = [self.unit_cell.d(h) for h in self.integrate_handle.hkl] self.d_min = min(d) # extend the resolution shell by epsilon>0 # to account for rounding artifacts on 32-bit platforms self.d_min = self.d_min - 1e-15 # Get the number of frames from the max z value xcal, ycal, zcal = zip(*self.integrate_handle.xyzcal) self.scan.set_image_range((self.scan.get_image_range()[0], self.scan.get_image_range()[0] + int(ceil(max(zcal))))) # Print stuff # print self.beam # print self.gonio # print self.detector # print self.scan # Create the index generator self.generate_indices = IndexGenerator(self.unit_cell, self.space_group_type, self.d_min) s0 = self.beam.get_s0() m2 = self.gonio.get_rotation_axis() fixed_rotation = self.gonio.get_fixed_rotation() setting_rotation = self.gonio.get_setting_rotation() UB = self.ub_matrix dphi = self.scan.get_oscillation_range(deg=False) # Create the ray predictor self.predict_rays = ScanStaticRayPredictor(s0, m2, fixed_rotation, setting_rotation, dphi) # Predict the spot locations self.reflections = self.predict_rays( self.generate_indices.to_array(), UB)
def test2(): """Test on simulated data""" # Get models for reflection prediction import dials.test.algorithms.refinement.setup_geometry as setup_geometry from libtbx.phil import parse overrides = """geometry.parameters.crystal.a.length.value = 77 geometry.parameters.crystal.b.length.value = 77 geometry.parameters.crystal.c.length.value = 37""" master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True, ) from dxtbx.model import Crystal models = setup_geometry.Extract(master_phil) crystal = Crystal( real_space_a=(2.62783398111729, -63.387215823567125, -45.751375737456975), real_space_b=(15.246640559660356, -44.48254330406616, 62.50501032727026), real_space_c=(-76.67246874451074, -11.01804131886244, 10.861322446352226), space_group_symbol="I 2 3", ) detector = models.detector goniometer = models.goniometer beam = models.beam # Build a mock scan for a 180 degree sweep from dxtbx.model import ScanFactory sf = ScanFactory() scan = sf.make_scan( image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=range(1800), deg=True, ) # Build an experiment list from dxtbx.model.experiment_list import ExperimentList, Experiment experiments = ExperimentList() experiments.append( Experiment( beam=beam, detector=detector, goniometer=goniometer, scan=scan, crystal=crystal, imageset=None, )) # Generate all indices in a 1.5 Angstrom sphere from dials.algorithms.spot_prediction import IndexGenerator from cctbx.sgtbx import space_group, space_group_symbols resolution = 1.5 index_generator = IndexGenerator( crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() # Predict rays within the sweep range from dials.algorithms.refinement.prediction import ScansRayPredictor sweep_range = scan.get_oscillation_range(deg=False) ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor(indices) # Take only those rays that intersect the detector from dials.algorithms.spot_prediction import ray_intersection intersects = ray_intersection(detector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns from dials.algorithms.refinement.prediction import ExperimentsPredictor ref_predictor = ExperimentsPredictor(experiments) obs_refs["id"] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor(obs_refs) # Copy 'observed' centroids from the predicted ones, applying sinusoidal # offsets obs_x, obs_y, obs_z = obs_refs["xyzcal.mm"].parts() # obs_z is in range (0, pi). Calculate offsets for phi at twice that # frequency im_width = scan.get_oscillation(deg=False)[1] z_off = flex.sin(2 * obs_z) * im_width obs_z += z_off # Calculate offsets for x pixel_size = detector[0].get_pixel_size() x_off = flex.sin(20 * obs_z) * pixel_size[0] # Calculate offsets for y with a phase-shifted sine wave from math import pi y_off = flex.sin(4 * obs_z + pi / 6) * pixel_size[1] # Incorporate the offsets into the 'observed' centroids obs_z += z_off obs_x += x_off obs_y += y_off obs_refs["xyzobs.mm.value"] = flex.vec3_double(obs_x, obs_y, obs_z) # Now do centroid analysis of the residuals results = CentroidAnalyser(obs_refs, debug=True)() # FIXME this test shows that the suggested interval width heuristic is not # yet robust. This simulation function seems a useful direction to proceed # in though raise RuntimeError("test2 failed") print("OK") return
def test(args=[]): # Python and cctbx imports from math import pi from scitbx import matrix from scitbx.array_family import flex from libtbx.phil import parse from libtbx.test_utils import approx_equal # Get module to build models using PHIL import dials.test.algorithms.refinement.setup_geometry as setup_geometry # We will set up a mock scan and a mock experiment list from dxtbx.model import ScanFactory from dxtbx.model.experiment_list import ExperimentList, Experiment # Model parameterisations from dials.algorithms.refinement.parameterisation.detector_parameters import \ DetectorParameterisationSinglePanel from dials.algorithms.refinement.parameterisation.beam_parameters import \ BeamParameterisation from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalOrientationParameterisation, CrystalUnitCellParameterisation # Symmetry constrained parameterisation for the unit cell from cctbx.uctbx import unit_cell from rstbx.symmetry.constraints.parameter_reduction import \ symmetrize_reduce_enlarge # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.refinement.prediction import ScansRayPredictor, \ ExperimentsPredictor from dials.algorithms.spot_prediction import ray_intersection from cctbx.sgtbx import space_group, space_group_symbols # Parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.prediction_parameters import \ XYPhiPredictionParameterisation # implicit import # Imports for the target function from dials.algorithms.refinement.target import \ LeastSquaresPositionalResidualWithRmsdCutoff # implicit import ############################# # Setup experimental models # ############################# master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True) models = setup_geometry.Extract( master_phil, cmdline_args=args, local_overrides="geometry.parameters.random_seed = 1") crystal1 = models.crystal models = setup_geometry.Extract( master_phil, cmdline_args=args, local_overrides="geometry.parameters.random_seed = 2") mydetector = models.detector mygonio = models.goniometer crystal2 = models.crystal mybeam = models.beam # Build a mock scan for a 180 degree sweep sf = ScanFactory() myscan = sf.make_scan(image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=range(1800), deg=True) sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 0.1 * pi / 180.) # Build an experiment list experiments = ExperimentList() experiments.append( Experiment(beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=crystal1, imageset=None)) experiments.append( Experiment(beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=crystal2, imageset=None)) assert len(experiments.detectors()) == 1 ########################################################## # Parameterise the models (only for perturbing geometry) # ########################################################## det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xl1o_param = CrystalOrientationParameterisation(crystal1) xl1uc_param = CrystalUnitCellParameterisation(crystal1) xl2o_param = CrystalOrientationParameterisation(crystal2) xl2uc_param = CrystalUnitCellParameterisation(crystal2) # Fix beam to the X-Z plane (imgCIF geometry), fix wavelength s0_param.set_fixed([True, False, True]) # Fix crystal parameters #xluc_param.set_fixed([True, True, True, True, True, True]) ######################################################################## # Link model parameterisations together into a parameterisation of the # # prediction equation # ######################################################################## #pred_param = XYPhiPredictionParameterisation(experiments, # [det_param], [s0_param], [xlo_param], [xluc_param]) ################################ # Apply known parameter shifts # ################################ # shift detector by 1.0 mm each translation and 2 mrad each rotation det_p_vals = det_param.get_param_vals() p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2., 2., 2.])] det_param.set_param_vals(p_vals) # shift beam by 2 mrad in free axis s0_p_vals = s0_param.get_param_vals() p_vals = list(s0_p_vals) p_vals[0] += 2. s0_param.set_param_vals(p_vals) # rotate crystal a bit (=2 mrad each rotation) xlo_p_vals = [] for xlo in (xl1o_param, xl2o_param): p_vals = xlo.get_param_vals() xlo_p_vals.append(p_vals) new_p_vals = [a + b for a, b in zip(p_vals, [2., 2., 2.])] xlo.set_param_vals(new_p_vals) # change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of # gamma angle) xluc_p_vals = [] for xluc, xl in ((xl1uc_param, crystal1), ((xl2uc_param, crystal2))): p_vals = xluc.get_param_vals() xluc_p_vals.append(p_vals) cell_params = xl.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1]) ] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(xl.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.e5 for e in S.forward_independent_parameters()]) xluc.set_param_vals(X) ############################# # Generate some reflections # ############################# #print "Reflections will be generated with the following geometry:" #print mybeam #print mydetector #print crystal1 #print crystal2 # All indices in a 2.0 Angstrom sphere for crystal1 resolution = 2.0 index_generator = IndexGenerator( crystal1.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices1 = index_generator.to_array() # All indices in a 2.0 Angstrom sphere for crystal2 resolution = 2.0 index_generator = IndexGenerator( crystal2.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices2 = index_generator.to_array() # Predict rays within the sweep range. Set experiment IDs ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs1 = ray_predictor(indices1, experiment_id=0) obs_refs1['id'] = flex.int(len(obs_refs1), 0) obs_refs2 = ray_predictor(indices1, experiment_id=1) obs_refs2['id'] = flex.int(len(obs_refs2), 1) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs1) obs_refs1 = obs_refs1.select(intersects) intersects = ray_intersection(mydetector, obs_refs2) obs_refs2 = obs_refs2.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs1 = ref_predictor(obs_refs1) obs_refs2 = ref_predictor(obs_refs2) # Set 'observed' centroids from the predicted ones obs_refs1['xyzobs.mm.value'] = obs_refs1['xyzcal.mm'] obs_refs2['xyzobs.mm.value'] = obs_refs2['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs1), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs1), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs1), (im_width / 2.)**2) obs_refs1['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) var_x = flex.double(len(obs_refs2), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs2), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs2), (im_width / 2.)**2) obs_refs2['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) #print "Total number of reflections excited for crystal1", len(obs_refs1) #print "Total number of reflections excited for crystal2", len(obs_refs2) # concatenate reflection lists obs_refs1.extend(obs_refs2) obs_refs = obs_refs1 ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) xl1o_param.set_param_vals(xlo_p_vals[0]) xl2o_param.set_param_vals(xlo_p_vals[1]) xl1uc_param.set_param_vals(xluc_p_vals[0]) xl2uc_param.set_param_vals(xluc_p_vals[1]) #print "Initial values of parameters are" #msg = "Parameters: " + "%.5f " * len(pred_param) #print msg % tuple(pred_param.get_param_vals()) #print # make a refiner from dials.algorithms.refinement.refiner import phil_scope params = phil_scope.fetch(source=parse('')).extract() # in case we want a plot params.refinement.refinery.journal.track_parameter_correlation = True # scan static first from dials.algorithms.refinement.refiner import RefinerFactory refiner = RefinerFactory.from_parameters_data_experiments(params, obs_refs, experiments, verbosity=0) history = refiner.run() # scan varying params.refinement.parameterisation.scan_varying = True refiner = RefinerFactory.from_parameters_data_experiments(params, obs_refs, experiments, verbosity=0) history = refiner.run()
def init_test(): models = setup_geometry.Extract(master_phil) single_panel_detector = models.detector gonio = models.goniometer crystal = models.crystal beam = models.beam # Make a 3x3 multi panel detector filling the same space as the existing # single panel detector. Each panel of the multi-panel detector has pixels # with 1/3 the length dimensions of the single panel. multi_panel_detector = Detector() for x in range(3): for y in range(3): new_panel = make_panel_in_array((x, y), single_panel_detector[0]) multi_panel_detector.add_panel(new_panel) # Build a mock scan for a 180 degree sequence sf = ScanFactory() scan = sf.make_scan( image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=list(range(1800)), deg=True, ) sequence_range = scan.get_oscillation_range(deg=False) im_width = scan.get_oscillation(deg=False)[1] assert sequence_range == (0.0, pi) assert approx_equal(im_width, 0.1 * pi / 180.0) # Build ExperimentLists experiments_single_panel = ExperimentList() experiments_multi_panel = ExperimentList() experiments_single_panel.append( Experiment( beam=beam, detector=single_panel_detector, goniometer=gonio, scan=scan, crystal=crystal, imageset=None, ) ) experiments_multi_panel.append( Experiment( beam=beam, detector=multi_panel_detector, goniometer=gonio, scan=scan, crystal=crystal, imageset=None, ) ) # Generate some reflections # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator( crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() # for the reflection predictor, it doesn't matter which experiment list is # passed, as the detector is not used ref_predictor = ScansRayPredictor( experiments_single_panel, scan.get_oscillation_range(deg=False) ) # get two sets of identical reflections obs_refs_single = ref_predictor(indices) obs_refs_multi = ref_predictor(indices) for r1, r2 in zip(obs_refs_single.rows(), obs_refs_multi.rows()): assert r1["s1"] == r2["s1"] # get the panel intersections sel = ray_intersection(single_panel_detector, obs_refs_single) obs_refs_single = obs_refs_single.select(sel) sel = ray_intersection(multi_panel_detector, obs_refs_multi) obs_refs_multi = obs_refs_multi.select(sel) assert len(obs_refs_single) == len(obs_refs_multi) # Set 'observed' centroids from the predicted ones obs_refs_single["xyzobs.mm.value"] = obs_refs_single["xyzcal.mm"] obs_refs_multi["xyzobs.mm.value"] = obs_refs_multi["xyzcal.mm"] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180.0 px_size = single_panel_detector[0].get_pixel_size() var_x = flex.double(len(obs_refs_single), (px_size[0] / 2.0) ** 2) var_y = flex.double(len(obs_refs_single), (px_size[1] / 2.0) ** 2) var_phi = flex.double(len(obs_refs_single), (im_width / 2.0) ** 2) # set the variances and frame numbers obs_refs_single["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) obs_refs_multi["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) # Add in flags and ID columns by copying into standard reflection tables tmp = flex.reflection_table.empty_standard(len(obs_refs_single)) tmp.update(obs_refs_single) obs_refs_single = tmp tmp = flex.reflection_table.empty_standard(len(obs_refs_multi)) tmp.update(obs_refs_multi) obs_refs_multi = tmp test_data = namedtuple( "test_data", [ "experiments_single_panel", "experiments_multi_panel", "observations_single_panel", "observations_multi_panel", ], ) return test_data( experiments_single_panel, experiments_multi_panel, obs_refs_single, obs_refs_multi, )
def generate_reflections(experiments, xyzvar=(0.0, 0.0, 0.0)): """Generate synthetic reflection centroids using the supplied experiments, with normally-distributed errors applied the variances in xyzvar""" # check input if [e >= 0.0 for e in xyzvar].count(False) > 0: msg = "negative variance requested in " + str(xyzvar) + "!" raise RuntimeError(msg) refs = [] for iexp, exp in enumerate(experiments): info("Generating reflections for experiment {0}".format(iexp)) # All indices in a 1.5 Angstrom sphere resolution = 1.5 index_generator = IndexGenerator( exp.crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() # Predict rays within the sweep range ray_predictor = ScansRayPredictor( experiments, exp.scan.get_oscillation_range(deg=False) ) obs_refs = ray_predictor.predict(indices, experiment_id=iexp) info("Total number of reflections excited: {0}".format(len(obs_refs))) # Take only those rays that intersect the detector intersects = ray_intersection(exp.detector, obs_refs) obs_refs = obs_refs.select(intersects) obs_refs["id"] = flex.size_t(len(obs_refs), iexp) refs.append(obs_refs) info("Total number of impacts: {0}".format(len(obs_refs))) # Concatenate reflections obs_refs = reduce(lambda x, y: x.extend(y), refs) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs = ref_predictor.predict(obs_refs) # calculate (uncorrelated) errors to offset the centroids # this is safe as elts of xyzvar are already tested to be > 0 sigX, sigY, sigZ = [sqrt(e) for e in xyzvar] shift = [ (random.gauss(0, sigX), random.gauss(0, sigY), random.gauss(0, sigZ)) for _ in xrange(len(obs_refs)) ] shift = flex.vec3_double(shift) # Set 'observed' centroids from the predicted ones obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"] + shift # Store variances for the centroid positions of the simulated data. If errors # are zero, invent some variances if tuple(xyzvar) == (0.0, 0.0, 0.0): im_width = exp.scan.get_oscillation()[1] * pi / 180.0 px_size = exp.detector[0].get_pixel_size() xyzvar = ( (px_size[0] / 2.0) ** 2, (px_size[1] / 2.0) ** 2, (im_width / 2.0) ** 2, ) var_x = flex.double(len(obs_refs), xyzvar[0]) var_y = flex.double(len(obs_refs), xyzvar[1]) var_phi = flex.double(len(obs_refs), xyzvar[2]) obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) info("Total number of observations made: {0}".format(len(obs_refs))) return obs_refs
############################# # Generate some reflections # ############################# print "Reflections will be generated with the following geometry:" print mybeam print mydetector print mycrystal print "Target values of parameters are" msg = "Parameters: " + "%.5f " * len(pred_param) print msg % tuple(pred_param.get_param_vals()) print # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator(mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices = index_generator.to_array() # Predict rays within the sweep range ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor(indices) print "Total number of reflections excited", len(obs_refs) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments)
def test(): from cctbx.sgtbx import space_group, space_group_symbols # We will set up a mock scan from dxtbx.model import ScanFactory from dxtbx.model.experiment_list import Experiment, ExperimentList from libtbx.phil import parse from scitbx import matrix from scitbx.array_family import flex from dials.algorithms.refinement.prediction.managed_predictors import ( ScansExperimentsPredictor, ScansRayPredictor, ) # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator # Building experimental models from dials.test.algorithms.refinement.setup_geometry import Extract master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True, ) overrides = """geometry.parameters.crystal.a.length.range = 10 50 geometry.parameters.crystal.b.length.range = 10 50 geometry.parameters.crystal.c.length.range = 10 50""" models = Extract(master_phil, local_overrides=overrides) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam ############################# # Generate some reflections # ############################# # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator( mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() # Build a mock scan for a 30 degree sequence sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 300), exposure_times=0.1, oscillation=(0, 0.1), epochs=list(range(300)), deg=True, ) sequence_range = myscan.get_oscillation_range(deg=False) assert sequence_range == pytest.approx((0.0, math.pi / 6.0)) im_width = myscan.get_oscillation(deg=False)[1] assert im_width == pytest.approx(0.1 * math.pi / 180.0) # Create an ExperimentList for ScansRayPredictor experiments = ExperimentList() experiments.append( Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None, )) # Select those that are excited in a 30 degree sequence and get angles ray_predictor = ScansRayPredictor(experiments, sequence_range) obs_refs = ray_predictor(indices) # Set the experiment number obs_refs["id"] = flex.int(len(obs_refs), 0) # Calculate intersections ref_predictor = ScansExperimentsPredictor(experiments) obs_refs = ref_predictor(obs_refs) print("Total number of observations made", len(obs_refs)) s0 = matrix.col(mybeam.get_s0()) spindle = matrix.col(mygonio.get_rotation_axis()) for ref in obs_refs.rows(): # get the s1 vector of this reflection s1 = matrix.col(ref["s1"]) r = s1 - s0 r_orig = r.rotate_around_origin(spindle, -1.0, deg=True) # is it outside the Ewald sphere (i.e. entering)? test = (s0 + r_orig).length() > s0.length() assert ref["entering"] == test
def test(): # Python and cctbx imports from math import pi from scitbx import matrix from scitbx.array_family import flex from libtbx.phil import parse from libtbx.test_utils import approx_equal # Get modules to build models and minimiser using PHIL import dials.test.algorithms.refinement.setup_geometry as setup_geometry import dials.test.algorithms.refinement.setup_minimiser as setup_minimiser # We will set up a mock scan and a mock experiment list from dxtbx.model import ScanFactory from dxtbx.model.experiment_list import ExperimentList, Experiment # Model parameterisations from dials.algorithms.refinement.parameterisation.detector_parameters import ( DetectorParameterisationSinglePanel, ) from dials.algorithms.refinement.parameterisation.beam_parameters import ( BeamParameterisation, ) from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalOrientationParameterisation, CrystalUnitCellParameterisation, ) # Symmetry constrained parameterisation for the unit cell from cctbx.uctbx import unit_cell from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection from dials.algorithms.refinement.prediction.managed_predictors import ( ScansRayPredictor, ScansExperimentsPredictor, ) from cctbx.sgtbx import space_group, space_group_symbols # Parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.prediction_parameters import ( XYPhiPredictionParameterisation, ) # Imports for the target function from dials.algorithms.refinement.target import ( LeastSquaresPositionalResidualWithRmsdCutoff, ) from dials.algorithms.refinement.reflection_manager import ReflectionManager ############################# # Setup experimental models # ############################# override = """geometry.parameters { beam.wavelength.random=False beam.wavelength.value=1.0 beam.direction.inclination.random=False crystal.a.length.random=False crystal.a.length.value=12.0 crystal.a.direction.method=exactly crystal.a.direction.exactly.direction=1.0 0.002 -0.004 crystal.b.length.random=False crystal.b.length.value=14.0 crystal.b.direction.method=exactly crystal.b.direction.exactly.direction=-0.002 1.0 0.002 crystal.c.length.random=False crystal.c.length.value=13.0 crystal.c.direction.method=exactly crystal.c.direction.exactly.direction=0.002 -0.004 1.0 detector.directions.method=exactly detector.directions.exactly.dir1=0.99 0.002 -0.004 detector.directions.exactly.norm=0.002 -0.001 0.99 detector.centre.method=exactly detector.centre.exactly.value=1.0 -0.5 199.0 }""" master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True, ) models = setup_geometry.Extract(master_phil, local_overrides=override, verbose=False) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam ########################### # Parameterise the models # ########################### det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) # Fix beam to the X-Z plane (imgCIF geometry), fix wavelength s0_param.set_fixed([True, False, True]) ######################################################################## # Link model parameterisations together into a parameterisation of the # # prediction equation # ######################################################################## # Build a mock scan for a 180 degree sweep sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=list(range(1800)), deg=True, ) # Build an ExperimentList experiments = ExperimentList() experiments.append( Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None, )) # Create the PredictionParameterisation pred_param = XYPhiPredictionParameterisation(experiments, [det_param], [s0_param], [xlo_param], [xluc_param]) ################################ # Apply known parameter shifts # ################################ # shift detector by 1.0 mm each translation and 4 mrad each rotation det_p_vals = det_param.get_param_vals() p_vals = [ a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 4.0, 4.0, 4.0]) ] det_param.set_param_vals(p_vals) # shift beam by 4 mrad in free axis s0_p_vals = s0_param.get_param_vals() p_vals = list(s0_p_vals) p_vals[0] += 4.0 s0_param.set_param_vals(p_vals) # rotate crystal a bit (=3 mrad each rotation) xlo_p_vals = xlo_param.get_param_vals() p_vals = [a + b for a, b in zip(xlo_p_vals, [3.0, 3.0, 3.0])] xlo_param.set_param_vals(p_vals) # change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of # alpha and beta angles) xluc_p_vals = xluc_param.get_param_vals() cell_params = mycrystal.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [0.1, -0.1, 0.1, 0.1, -0.1, 0.0]) ] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(mycrystal.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()]) xluc_param.set_param_vals(X) ############################# # Generate some reflections # ############################# # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator( mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sweep_range == (0.0, pi) assert approx_equal(im_width, 0.1 * pi / 180.0) # Predict rays within the sweep range ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ScansExperimentsPredictor(experiments) obs_refs["id"] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180.0 px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2) obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) # The total number of observations should be 1128 assert len(obs_refs) == 1128 ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) xlo_param.set_param_vals(xlo_p_vals) xluc_param.set_param_vals(xluc_p_vals) ##################################### # Select reflections for refinement # ##################################### refman = ReflectionManager(obs_refs, experiments, outlier_detector=None, close_to_spindle_cutoff=0.1) ############################## # Set up the target function # ############################## # The current 'achieved' criterion compares RMSD against 1/3 the pixel size and # 1/3 the image width in radians. For the simulated data, these are just made up mytarget = LeastSquaresPositionalResidualWithRmsdCutoff( experiments, ref_predictor, refman, pred_param, restraints_parameterisation=None) ###################################### # Set up the LSTBX refinement engine # ###################################### overrides = """minimiser.parameters.engine=GaussNewton minimiser.parameters.logfile=None""" refiner = setup_minimiser.Extract(master_phil, mytarget, pred_param, local_overrides=overrides).refiner refiner.run() assert mytarget.achieved() assert refiner.get_num_steps() == 1 assert approx_equal( mytarget.rmsds(), (0.00508252354876, 0.00420954552156, 8.97303428289e-05)) ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) xlo_param.set_param_vals(xlo_p_vals) xluc_param.set_param_vals(xluc_p_vals) ###################################################### # Set up the LBFGS with curvatures refinement engine # ###################################################### overrides = """minimiser.parameters.engine=LBFGScurvs minimiser.parameters.logfile=None""" refiner = setup_minimiser.Extract(master_phil, mytarget, pred_param, local_overrides=overrides).refiner refiner.run() assert mytarget.achieved() assert refiner.get_num_steps() == 9 assert approx_equal(mytarget.rmsds(), (0.0558857700305, 0.0333446685335, 0.000347402754278))
def test(args=[]): ############################# # Setup experimental models # ############################# master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True, ) models = setup_geometry.Extract(master_phil, cmdline_args=args) single_panel_detector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam # Make a 3x3 multi panel detector filling the same space as the existing # single panel detector. Each panel of the multi-panel detector has pixels with # 1/3 the length dimensions of the single panel. multi_panel_detector = Detector() for x in range(3): for y in range(3): new_panel = make_panel_in_array((x, y), single_panel_detector[0]) multi_panel_detector.add_panel(new_panel) # Build a mock scan for a 180 degree sweep sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=list(range(1800)), deg=True, ) sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sweep_range == (0.0, pi) assert approx_equal(im_width, 0.1 * pi / 180.0) # Build ExperimentLists experiments_single_panel = ExperimentList() experiments_multi_panel = ExperimentList() experiments_single_panel.append( Experiment( beam=mybeam, detector=single_panel_detector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None, )) experiments_multi_panel.append( Experiment( beam=mybeam, detector=multi_panel_detector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None, )) ########################### # Parameterise the models # ########################### det_param = DetectorParameterisationSinglePanel(single_panel_detector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) multi_det_param = DetectorParameterisationMultiPanel( multi_panel_detector, mybeam) # Fix beam to the X-Z plane (imgCIF geometry), fix wavelength s0_param.set_fixed([True, False, True]) # Fix crystal parameters # xluc_param.set_fixed([True, True, True, True, True, True]) ######################################################################## # Link model parameterisations together into a parameterisation of the # # prediction equation # ######################################################################## pred_param = XYPhiPredictionParameterisation(experiments_single_panel, [det_param], [s0_param], [xlo_param], [xluc_param]) pred_param2 = XYPhiPredictionParameterisation( experiments_multi_panel, [multi_det_param], [s0_param], [xlo_param], [xluc_param], ) ################################ # Apply known parameter shifts # ################################ # shift detectors by 1.0 mm each translation and 2 mrad each rotation det_p_vals = det_param.get_param_vals() p_vals = [ a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0]) ] det_param.set_param_vals(p_vals) multi_det_p_vals = multi_det_param.get_param_vals() p_vals = [ a + b for a, b in zip(multi_det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0]) ] multi_det_param.set_param_vals(p_vals) # shift beam by 2 mrad in free axis s0_p_vals = s0_param.get_param_vals() p_vals = list(s0_p_vals) p_vals[0] += 2.0 s0_param.set_param_vals(p_vals) # rotate crystal a bit (=2 mrad each rotation) xlo_p_vals = xlo_param.get_param_vals() p_vals = [a + b for a, b in zip(xlo_p_vals, [2.0, 2.0, 2.0])] xlo_param.set_param_vals(p_vals) # change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of # gamma angle) xluc_p_vals = xluc_param.get_param_vals() cell_params = mycrystal.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1]) ] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(mycrystal.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()]) xluc_param.set_param_vals(X) ############################# # Generate some reflections # ############################# # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator( mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() # for the reflection predictor, it doesn't matter which experiment list is # passed, as the detector is not used ref_predictor = ScansRayPredictor(experiments_single_panel, sweep_range) # get two sets of identical reflections obs_refs = ref_predictor(indices) obs_refs2 = ref_predictor(indices) for r1, r2 in zip(obs_refs, obs_refs2): assert r1["s1"] == r2["s1"] # get the panel intersections sel = ray_intersection(single_panel_detector, obs_refs) obs_refs = obs_refs.select(sel) sel = ray_intersection(multi_panel_detector, obs_refs2) obs_refs2 = obs_refs2.select(sel) assert len(obs_refs) == len(obs_refs2) # Set 'observed' centroids from the predicted ones obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"] obs_refs2["xyzobs.mm.value"] = obs_refs2["xyzcal.mm"] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180.0 px_size = single_panel_detector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2) # set the variances and frame numbers obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) obs_refs2["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) # Add in flags and ID columns by copying into standard reflection tables tmp = flex.reflection_table.empty_standard(len(obs_refs)) tmp.update(obs_refs) obs_refs = tmp tmp = flex.reflection_table.empty_standard(len(obs_refs2)) tmp.update(obs_refs2) obs_refs2 = tmp ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) multi_det_param.set_param_vals(det_p_vals) xlo_param.set_param_vals(xlo_p_vals) xluc_param.set_param_vals(xluc_p_vals) ##################################### # Select reflections for refinement # ##################################### refman = ReflectionManager(obs_refs, experiments_single_panel) refman2 = ReflectionManager(obs_refs, experiments_multi_panel) ############################### # Set up the target functions # ############################### mytarget = LeastSquaresPositionalResidualWithRmsdCutoff( experiments_single_panel, ScansExperimentsPredictor(experiments_single_panel), refman, pred_param, restraints_parameterisation=None, ) mytarget2 = LeastSquaresPositionalResidualWithRmsdCutoff( experiments_multi_panel, ScansExperimentsPredictor(experiments_multi_panel), refman2, pred_param2, restraints_parameterisation=None, ) ################################# # Set up the refinement engines # ################################# refiner = setup_minimiser.Extract(master_phil, mytarget, pred_param, cmdline_args=args).refiner refiner2 = setup_minimiser.Extract(master_phil, mytarget2, pred_param2, cmdline_args=args).refiner refiner.run() # reset parameters and run refinement with the multi panel detector s0_param.set_param_vals(s0_p_vals) multi_det_param.set_param_vals(det_p_vals) xlo_param.set_param_vals(xlo_p_vals) xluc_param.set_param_vals(xluc_p_vals) refiner2.run() # same number of steps assert refiner.get_num_steps() == refiner2.get_num_steps() # same rmsds for rmsd, rmsd2 in zip(refiner.history["rmsd"], refiner2.history["rmsd"]): assert approx_equal(rmsd, rmsd2) # same parameter values each step for params, params2 in zip(refiner.history["parameter_vector"], refiner.history["parameter_vector"]): assert approx_equal(params, params2)
def run(): from iotbx.xds import xparm, integrate_hkl from dials.util import ioutil from dials.algorithms.spot_prediction import IndexGenerator from os.path import realpath, dirname, join import numpy from rstbx.cftbx.coordinate_frame_converter import \ coordinate_frame_converter from scitbx import matrix # The XDS files to read from test_path = dirname(dirname(dirname(realpath(__file__)))) integrate_filename = join(test_path, 'data/sim_mx/INTEGRATE.HKL') gxparm_filename = join(test_path, 'data/sim_mx/GXPARM.XDS') # Read the XDS files integrate_handle = integrate_hkl.reader() integrate_handle.read_file(integrate_filename) gxparm_handle = xparm.reader() gxparm_handle.read_file(gxparm_filename) # Get the parameters we need from the GXPARM file d_min = 1.6 space_group_type = ioutil.get_space_group_type_from_xparm(gxparm_handle) cfc = coordinate_frame_converter(gxparm_filename) a_vec = cfc.get('real_space_a') b_vec = cfc.get('real_space_b') c_vec = cfc.get('real_space_c') unit_cell = cfc.get_unit_cell() UB = matrix.sqr(a_vec + b_vec + c_vec).inverse() ub_matrix = UB # Generate the indices index_generator = IndexGenerator(unit_cell, space_group_type, d_min) miller_indices = index_generator.to_array() # Get individual generated hkl gen_h = [hkl[0] for hkl in miller_indices] gen_k = [hkl[1] for hkl in miller_indices] gen_l = [hkl[2] for hkl in miller_indices] # Get individual xds generated hkl xds_h = [hkl[0] for hkl in integrate_handle.hkl] xds_k = [hkl[0] for hkl in integrate_handle.hkl] xds_l = [hkl[0] for hkl in integrate_handle.hkl] # Get min/max generated hkl min_gen_h, max_gen_h = numpy.min(gen_h), numpy.max(gen_h) min_gen_k, max_gen_k = numpy.min(gen_k), numpy.max(gen_k) min_gen_l, max_gen_l = numpy.min(gen_l), numpy.max(gen_l) # Get min/max xds generated hkl min_xds_h, max_xds_h = numpy.min(xds_h), numpy.max(xds_h) min_xds_k, max_xds_k = numpy.min(xds_k), numpy.max(xds_k) min_xds_l, max_xds_l = numpy.min(xds_l), numpy.max(xds_l) # Ensure we have the whole xds range in the generated set assert(min_gen_h <= min_xds_h and max_gen_h >= max_xds_h) assert(min_gen_k <= min_xds_k and max_gen_k >= max_xds_k) assert(min_gen_l <= min_xds_l and max_gen_l >= max_xds_l) # Test Passed print "OK"
def generate_reflections(experiments, xyzvar=(0., 0., 0.)): '''Generate synthetic reflection centroids using the supplied experiments, with normally-distributed errors applied the variances in xyzvar''' # check input if [e >= 0. for e in xyzvar].count(False) > 0: msg = "negative variance requested in " + str(xyzvar) + "!" raise RuntimeError(msg) refs = [] for iexp, exp in enumerate(experiments): info("Generating reflections for experiment {0}".format(iexp)) # All indices in a 1.5 Angstrom sphere resolution = 1.5 index_generator = IndexGenerator(exp.crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices = index_generator.to_array() # Predict rays within the sweep range ray_predictor = ScansRayPredictor(experiments, exp.scan.get_oscillation_range(deg=False)) obs_refs = ray_predictor.predict(indices, experiment_id=iexp) info("Total number of reflections excited: {0}".format(len(obs_refs))) # Take only those rays that intersect the detector intersects = ray_intersection(exp.detector, obs_refs) obs_refs = obs_refs.select(intersects) obs_refs['id'] = flex.size_t(len(obs_refs), iexp) refs.append(obs_refs) info("Total number of impacts: {0}".format(len(obs_refs))) # Concatenate reflections obs_refs = reduce(lambda x, y: x.extend(y), refs) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs = ref_predictor.predict(obs_refs) # calculate (uncorrelated) errors to offset the centroids # this is safe as elts of xyzvar are already tested to be > 0 sigX, sigY, sigZ = [sqrt(e) for e in xyzvar] shift = [(random.gauss(0, sigX), random.gauss(0, sigY), random.gauss(0, sigZ)) for _ in xrange(len(obs_refs))] shift = flex.vec3_double(shift) # Set 'observed' centroids from the predicted ones obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm'] + shift # Store variances for the centroid positions of the simulated data. If errors # are zero, invent some variances if tuple(xyzvar) == (0., 0., 0.): im_width = exp.scan.get_oscillation()[1] * pi / 180. px_size = exp.detector[0].get_pixel_size() xyzvar = ((px_size[0] / 2.)**2, (px_size[1] / 2.)**2, (im_width / 2.)**2) var_x = flex.double(len(obs_refs), xyzvar[0]) var_y = flex.double(len(obs_refs), xyzvar[1]) var_phi = flex.double(len(obs_refs), xyzvar[2]) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) info("Total number of observations made: {0}".format(len(obs_refs))) return obs_refs