def refine(self, experiments, reflections): acceptance_flags = self.identify_outliers(self.all_params, experiments, reflections) #create a new "reflections" list with outliers thrown out: reflections = reflections.select(acceptance_flags) R = e_refine(params = self.all_params, experiments=experiments, reflections=reflections, graph_verbose=False) ref_experiments = R.get_experiments() # try to improve the outcome with a second round of outlier rejection post-initial refinement: acceptance_flags = self.identify_outliers(self.all_params, ref_experiments, reflections) # insert a round of Nave-outlier rejection on top of the r.m.s.d. rejection nv0 = nave_parameters(params = self.all_params, experiments=ref_experiments, reflections=reflections, refinery=R, graph_verbose=False) crystal_model_nv0 = nv0() acceptance_flags_nv0 = nv0.nv_acceptance_flags reflections = reflections.select(acceptance_flags & acceptance_flags_nv0) R = e_refine(params = self.all_params, experiments=ref_experiments, reflections=reflections, graph_verbose=False) ref_experiments = R.get_experiments() nv = nave_parameters(params = self.all_params, experiments=ref_experiments, reflections=reflections, refinery=R, graph_verbose=False) nv() rmsd, _ = calc_2D_rmsd_and_displacements(R.predict_for_reflection_table(reflections)) matches = R.get_matches() xyzcal_mm = flex.vec3_double(len(reflections)) xyzcal_mm.set_selected(matches['iobs'], matches['xyzcal.mm']) reflections['xyzcal.mm'] = xyzcal_mm reflections.set_flags(matches['iobs'], reflections.flags.used_in_refinement) reflections['entering'] = flex.bool(len(reflections), False) return ref_experiments, reflections
def run(self): ''' Parse the options. ''' from dials.util.options import flatten_experiments, flatten_reflections # Parse the command line arguments params, options = self.parser.parse_args(show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) assert len(reflections) == 1 reflections = reflections[0] domain_size = flex.double() mosaic_angle = flex.double() filtered_reflections = flex.reflection_table() for i in xrange(len(experiments)): refls = reflections.select(reflections['id'] == i) try: nv = nave_parameters(params=None, experiments=experiments[i:i + 1], reflections=refls, refinery=None, graph_verbose=False) crystal_model_nv = nv() except Exception, e: continue domain_size.append(experiments[i].crystal.get_domain_size_ang() - crystal_model_nv.get_domain_size_ang()) mosaic_angle.append( experiments[i].crystal.get_half_mosaicity_deg() - crystal_model_nv.get_half_mosaicity_deg()) experiments[i].crystal = crystal_model_nv refls = refls.select(nv.nv_acceptance_flags) filtered_reflections.extend(refls)
def refine(self, experiments, reflections): sel = ((reflections['id'] >= -1)) refl = reflections.select(sel) acceptance_flags = self.identify_outliers(self.all_params, experiments, refl) #create a new "indexed" list with outliers thrown out: refl = refl.select(acceptance_flags) print "$$$ stills_indexer::refine" R = e_refine(params = self.all_params, experiments=experiments, reflections=refl, graph_verbose=False) ref_experiments = R.get_experiments() # try to improve the outcome with a second round of outlier rejection post-initial refinement: acceptance_flags = self.identify_outliers(self.all_params, ref_experiments, refl) # insert a round of Nave-outlier rejection on top of the r.m.s.d. rejection nv0 = nave_parameters(params = self.all_params, experiments=ref_experiments, reflections=refl, refinery=R, graph_verbose=False) crystal_model_nv0 = nv0() acceptance_flags_nv0 = nv0.nv_acceptance_flags refl = refl.select(acceptance_flags & acceptance_flags_nv0) print "$$$ stills_indexer::refine after positional and delta-psi outlier rejection" refiner = e_refine(params = self.all_params, experiments=ref_experiments, reflections=refl, graph_verbose=False) matches = refiner.get_matches() xyzcal_mm = flex.vec3_double(len(refl)) xyzcal_mm.set_selected(matches['iobs'], matches['xyzcal.mm']) refl['xyzcal.mm'] = xyzcal_mm refl.set_flags(matches['iobs'], refl.flags.used_in_refinement) refl['entering'] = flex.bool(len(refl), False) return refiner.get_experiments(), refl
def refine(self, experiments, centroids): if self.params.dispatch.refine: from dials.algorithms.refinement import RefinerFactory from time import time st = time() logger.info('*' * 80) logger.info('Refining Model') logger.info('*' * 80) refiner = RefinerFactory.from_parameters_data_experiments( self.params, centroids, experiments) refiner.run() experiments = refiner.get_experiments() predicted = refiner.predict_for_indexed() centroids['xyzcal.mm'] = predicted['xyzcal.mm'] centroids['entering'] = predicted['entering'] centroids = centroids.select( refiner.selection_used_for_refinement()) # Re-estimate mosaic estimates from dials.algorithms.indexing.nave_parameters import nave_parameters nv = nave_parameters(params=self.params, experiments=experiments, reflections=centroids, refinery=refiner, graph_verbose=False) nv() acceptance_flags_nv = nv.nv_acceptance_flags centroids = centroids.select(acceptance_flags_nv) if self.params.output.composite_output: if self.params.output.refined_experiments_filename or self.params.output.indexed_filename: assert self.params.output.refined_experiments_filename is not None and self.params.output.indexed_filename is not None from dials.array_family import flex n = len(self.all_indexed_experiments) self.all_indexed_experiments.extend(experiments) for i, experiment in enumerate(experiments): refls = centroids.select(centroids['id'] == i) refls['id'] = flex.int(len(refls), n) self.all_indexed_reflections.extend(refls) n += 1 else: # Dump experiments to disk if self.params.output.refined_experiments_filename: from dxtbx.model.experiment_list import ExperimentListDumper dump = ExperimentListDumper(experiments) dump.as_json(self.params.output.refined_experiments_filename) if self.params.output.indexed_filename: self.save_reflections(centroids, self.params.output.indexed_filename) if self.params.dispatch.refine: logger.info('') logger.info('Time Taken = %f seconds' % (time() - st)) return experiments, centroids
def run(self): ''' Parse the options. ''' from dials.util.options import flatten_experiments, flatten_reflections # Parse the command line arguments params, options = self.parser.parse_args(show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) assert len(reflections) == 1 reflections = reflections[0] domain_size = flex.double() mosaic_angle = flex.double() filtered_reflections = flex.reflection_table() for i in range(len(experiments)): refls = reflections.select(reflections['id'] == i) try: nv = nave_parameters(params=None, experiments=experiments[i:i + 1], reflections=refls, refinery=None, graph_verbose=False) crystal_model_nv = nv() except Exception as e: continue domain_size.append(experiments[i].crystal.get_domain_size_ang() - crystal_model_nv.get_domain_size_ang()) mosaic_angle.append( experiments[i].crystal.get_half_mosaicity_deg() - crystal_model_nv.get_half_mosaicity_deg()) experiments[i].crystal = crystal_model_nv refls = refls.select(nv.nv_acceptance_flags) filtered_reflections.extend(refls) print "Saving new experiments as %s" % params.output.experiments dump = ExperimentListDumper(experiments) dump.as_json(params.output.experiments) print "Removed %d out of %d reflections as outliers" % ( len(reflections) - len(filtered_reflections), len(reflections)) print "Saving filtered reflections as %s" % params.output.experiments filtered_reflections.as_msgpack_file(params.output.reflections) if params.plot_changes: from matplotlib import pyplot as plt domain_size = domain_size.select((domain_size >= -10) & (domain_size <= 10)) mosaic_angle = mosaic_angle.select((mosaic_angle >= -0.1) & (mosaic_angle <= 0.1)) for d in [domain_size, mosaic_angle]: f = plt.figure() plt.hist(d, bins=30) plt.show()
def refine(self, experiments, reflections): sel = ((reflections['id'] >= -1)) refl = reflections.select(sel) acceptance_flags = self.identify_outliers(self.all_params, experiments, refl) #create a new "indexed" list with outliers thrown out: refl = refl.select(acceptance_flags) print "$$$ stills_indexer::refine" R = e_refine(params=self.all_params, experiments=experiments, reflections=refl, graph_verbose=False) ref_experiments = R.get_experiments() # try to improve the outcome with a second round of outlier rejection post-initial refinement: acceptance_flags = self.identify_outliers(self.all_params, ref_experiments, refl) # insert a round of Nave-outlier rejection on top of the r.m.s.d. rejection nv0 = nave_parameters(params=self.all_params, experiments=ref_experiments, reflections=refl, refinery=R, graph_verbose=False) crystal_model_nv0 = nv0() acceptance_flags_nv0 = nv0.nv_acceptance_flags refl = refl.select(acceptance_flags & acceptance_flags_nv0) print "$$$ stills_indexer::refine after positional and delta-psi outlier rejection" refiner = e_refine(params=self.all_params, experiments=ref_experiments, reflections=refl, graph_verbose=False) matches = refiner.get_matches() xyzcal_mm = flex.vec3_double(len(refl)) xyzcal_mm.set_selected(matches['iobs'], matches['xyzcal.mm']) refl['xyzcal.mm'] = xyzcal_mm refl.set_flags(matches['iobs'], refl.flags.used_in_refinement) refl['entering'] = flex.bool(len(refl), False) return refiner.get_experiments(), refl
def refine(self, experiments, centroids): if self.params.dispatch.refine: from dials.algorithms.refinement import RefinerFactory from time import time st = time() logger.info('*' * 80) logger.info('Refining Model') logger.info('*' * 80) refiner = RefinerFactory.from_parameters_data_experiments( self.params, centroids, experiments) refiner.run() experiments = refiner.get_experiments() predicted = refiner.predict_for_indexed() centroids['xyzcal.mm'] = predicted['xyzcal.mm'] centroids['entering'] = predicted['entering'] centroids = centroids.select(refiner.selection_used_for_refinement()) # Re-estimate mosaic estimates from dials.algorithms.indexing.nave_parameters import nave_parameters nv = nave_parameters(params = self.params, experiments=experiments, reflections=centroids, refinery=refiner, graph_verbose=False) nv() acceptance_flags_nv = nv.nv_acceptance_flags centroids = centroids.select(acceptance_flags_nv) # Dump experiments to disk if self.params.output.refined_experiments_filename: from dxtbx.model.experiment_list import ExperimentListDumper dump = ExperimentListDumper(experiments) dump.as_json(self.params.output.refined_experiments_filename) if self.params.dispatch.refine: if self.params.output.indexed_filename: self.save_reflections(centroids, self.params.output.indexed_filename) logger.info('') logger.info('Time Taken = %f seconds' % (time() - st)) return experiments, centroids
def choose_best_orientation_matrix(self, candidate_orientation_matrices): from dxtbx.model.experiment_list import Experiment, ExperimentList import copy logger.info('*' * 80) logger.info('Selecting the best orientation matrix') logger.info('*' * 80) from libtbx import group_args class candidate_info(group_args): pass candidates = [] params = copy.deepcopy(self.all_params) n_cand = len(candidate_orientation_matrices) for icm, cm in enumerate(candidate_orientation_matrices): # Index reflections in P1 sel = ((self.reflections['id'] == -1)) refl = self.reflections.select(sel) experiments = self.experiment_list_for_crystal(cm) self.index_reflections(experiments, refl) indexed = refl.select(refl['id'] >= 0) indexed = indexed.select(indexed.get_flags(indexed.flags.indexed)) # If target symmetry supplied, try to apply it. Then, apply the change of basis to the reflections # indexed in P1 to the target setting if self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None: target_space_group = self.target_symmetry_primitive.space_group( ) new_crystal, cb_op_to_primitive = self.apply_symmetry( cm, target_space_group) if new_crystal is None: print( "Cannot convert to target symmetry, candidate %d/%d" % (icm, n_cand)) continue new_crystal = new_crystal.change_basis( self.cb_op_primitive_inp) cm = candidate_orientation_matrices[icm] = new_crystal experiments = self.experiment_list_for_crystal(cm) if not cb_op_to_primitive.is_identity_op(): indexed['miller_index'] = cb_op_to_primitive.apply( indexed['miller_index']) if self.cb_op_primitive_inp is not None: indexed['miller_index'] = self.cb_op_primitive_inp.apply( indexed['miller_index']) if params.indexing.stills.refine_all_candidates: try: print( "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d initial outlier identification" % (icm, n_cand)) acceptance_flags = self.identify_outliers( params, experiments, indexed) #create a new "indexed" list with outliers thrown out: indexed = indexed.select(acceptance_flags) print( "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d refinement before outlier rejection" % (icm, n_cand)) R = e_refine(params=params, experiments=experiments, reflections=indexed, graph_verbose=False) ref_experiments = R.get_experiments() # try to improve the outcome with a second round of outlier rejection post-initial refinement: acceptance_flags = self.identify_outliers( params, ref_experiments, indexed) # insert a round of Nave-outlier rejection on top of the r.m.s.d. rejection nv0 = nave_parameters(params=params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False) crystal_model_nv0 = nv0() acceptance_flags_nv0 = nv0.nv_acceptance_flags indexed = indexed.select(acceptance_flags & acceptance_flags_nv0) print( "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d after positional and delta-psi outlier rejection" % (icm, n_cand)) R = e_refine(params=params, experiments=ref_experiments, reflections=indexed, graph_verbose=False) ref_experiments = R.get_experiments() nv = nave_parameters(params=params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False) crystal_model = nv() # Drop candidates that after refinement can no longer be converted to the known target space group if not self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None: target_space_group = self.target_symmetry_primitive.space_group( ) new_crystal, cb_op_to_primitive = self.apply_symmetry( crystal_model, target_space_group) if new_crystal is None: print( "P1 refinement yielded model diverged from target, candidate %d/%d" % (icm, n_cand)) continue rmsd, _ = calc_2D_rmsd_and_displacements( R.predict_for_reflection_table(indexed)) except Exception as e: print("Couldn't refine candiate %d/%d, %s" % (icm, n_cand, str(e))) else: print( "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d done" % (icm, n_cand)) candidates.append( candidate_info( crystal=crystal_model, green_curve_area=nv.green_curve_area, ewald_proximal_volume=nv.ewald_proximal_volume(), n_indexed=len(indexed), rmsd=rmsd, indexed=indexed, experiments=ref_experiments)) else: from dials.algorithms.refinement.prediction import ExperimentsPredictor ref_predictor = ExperimentsPredictor( experiments, force_stills=True, spherical_relp=params.refinement.parameterisation. spherical_relp_model) rmsd, _ = calc_2D_rmsd_and_displacements( ref_predictor(indexed)) candidates.append( candidate_info(crystal=cm, n_indexed=len(indexed), rmsd=rmsd, indexed=indexed, experiments=experiments)) if len(candidates) == 0: raise Sorry("No suitable indexing solution found") print("**** ALL CANDIDATES:") for i, XX in enumerate(candidates): print("\n****Candidate %d" % i, XX) cc = XX.crystal if hasattr(cc, 'get_half_mosaicity_deg'): print(" half mosaicity %5.2f deg." % (cc.get_half_mosaicity_deg())) print(" domain size %.0f Ang." % (cc.get_domain_size_ang())) print("\n**** BEST CANDIDATE:") results = flex.double([c.rmsd for c in candidates]) best = candidates[flex.min_index(results)] print(best) if params.indexing.stills.refine_all_candidates: if best.rmsd > params.indexing.stills.rmsd_min_px: raise Sorry("RMSD too high, %f" % best.rmsd) if best.ewald_proximal_volume > params.indexing.stills.ewald_proximal_volume_max: raise Sorry("Ewald proximity volume too high, %f" % best.ewald_proximal_volume) if len(candidates) > 1: for i in xrange(len(candidates)): if i == flex.min_index(results): continue if best.ewald_proximal_volume > candidates[ i].ewald_proximal_volume: print( "Couldn't figure out which candidate is best; picked the one with the best RMSD." ) best.indexed['entering'] = flex.bool(best.n_indexed, False) return best.crystal, best.n_indexed
def index(self): # most of this is the same as dials.algorithms.indexing.indexer.indexer_base.index(), with some stills # specific modifications (don't re-index after choose best orientation matrix, but use the indexing from # choose best orientation matrix, also don't use macrocycles) of refinement after indexing. # 2017 update: do accept multiple lattices per shot if self.params.refinement_protocol.n_macro_cycles > 1: raise Sorry("For stills, please set refinement_protocol.n_macro_cycles = 1") experiments = ExperimentList() had_refinement_error = False have_similar_crystal_models = False # This enters into a long series of refinements and checks while True: self.d_min = self.params.refinement_protocol.d_min_start if had_refinement_error or have_similar_crystal_models: break max_lattices = self.params.multiple_lattice_search.max_lattices if max_lattices is not None and len(experiments) >= max_lattices: break if len(experiments) > 0: cutoff_fraction = \ self.params.multiple_lattice_search.recycle_unindexed_reflections_cutoff d_spacings = 1/self.reflections['rlp'].norms() d_min_indexed = flex.min(d_spacings.select(self.indexed_reflections)) min_reflections_for_indexing = \ cutoff_fraction * len(self.reflections.select(d_spacings > d_min_indexed)) crystal_ids = self.reflections.select(d_spacings > d_min_indexed)['id'] if (crystal_ids == -1).count(True) < min_reflections_for_indexing: logger.info("Finish searching for more lattices: %i unindexed reflections remaining." %( min_reflections_for_indexing)) break n_lattices_previous_cycle = len(experiments) # indexing starts around here if len(experiments) == 0: experiments.extend(self.find_lattices()) # try to find a good crystal model.. if len(experiments) == 0: raise Sorry("No suitable lattice could be found.") else: print("\n\n Do you really want to go here? Dont go here!\n\n") exit() try: new = self.find_lattices() experiments.extend(new) except Exception as e: logger.info("Indexing remaining reflections failed") logger.debug("Indexing remaining reflections failed, exception:\n" + str(e)) # reset reflection lattice flags # the lattice a given reflection belongs to: a value of -1 indicates # that a reflection doesn't belong to any lattice so far self.reflections['id'] = flex.int(len(self.reflections), -1) self.index_reflections(experiments, self.reflections) if len(experiments) == n_lattices_previous_cycle: # no more lattices found break ######################################################### if not self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None: # now apply the space group symmetry only after the first indexing # need to make sure that the symmetrized orientation is similar to the P1 model target_space_group = self.target_symmetry_primitive.space_group() for i_cryst, cryst in enumerate(experiments.crystals()): if i_cryst >= n_lattices_previous_cycle: new_cryst, cb_op_to_primitive = self.apply_symmetry( cryst, target_space_group) if self.cb_op_primitive_inp is not None: new_cryst = new_cryst.change_basis(self.cb_op_primitive_inp) logger.info(new_cryst.get_space_group().info()) cryst.update(new_cryst) cryst.set_space_group( self.params.known_symmetry.space_group.group()) for i_expt, expt in enumerate(experiments): if expt.crystal is not cryst: continue if not cb_op_to_primitive.is_identity_op(): miller_indices = self.reflections['miller_index'].select( self.reflections['id'] == i_expt) miller_indices = cb_op_to_primitive.apply(miller_indices) self.reflections['miller_index'].set_selected( self.reflections['id'] == i_expt, miller_indices) if self.cb_op_primitive_inp is not None: miller_indices = self.reflections['miller_index'].select( self.reflections['id'] == i_expt) miller_indices = self.cb_op_primitive_inp.apply(miller_indices) self.reflections['miller_index'].set_selected( self.reflections['id'] == i_expt, miller_indices) ########################################################### # discard nearly overlapping lattices on the same shot if len(experiments) > 1: from dials.algorithms.indexing.compare_orientation_matrices \ import difference_rotation_matrix_axis_angle cryst_b = experiments.crystals()[-1] have_similar_crystal_models = False for i_a, cryst_a in enumerate(experiments.crystals()[:-1]): R_ab, axis, angle, cb_op_ab = \ difference_rotation_matrix_axis_angle(cryst_a, cryst_b) min_angle = self.params.multiple_lattice_search.minimum_angular_separation if abs(angle) < min_angle: # degrees logger.info("Crystal models too similar, rejecting crystal %i:" %( len(experiments))) logger.info("Rotation matrix to transform crystal %i to crystal %i" %( i_a+1, len(experiments))) logger.info(R_ab) logger.info("Rotation of %.3f degrees" %angle + " about axis (%.3f, %.3f, %.3f)" %axis) #show_rotation_matrix_differences([cryst_a, cryst_b]) have_similar_crystal_models = True del experiments[-1] break if have_similar_crystal_models: break self.indexed_reflections = (self.reflections['id'] > -1) if self.d_min is None: sel = self.reflections['id'] <= -1 else: sel = flex.bool(len(self.reflections), False) lengths = 1/self.reflections['rlp'].norms() isel = (lengths >= self.d_min).iselection() sel.set_selected(isel, True) sel.set_selected(self.reflections['id'] > -1, False) self.unindexed_reflections = self.reflections.select(sel) reflections_for_refinement = self.reflections.select( self.indexed_reflections) if len(self.params.stills.isoforms) > 0: logger.info("") logger.info("#" * 80) logger.info("Starting refinement") logger.info("#" * 80) logger.info("") import copy isoform_experiments = ExperimentList() isoform_reflections = flex.reflection_table() # Note, changes to params after initial indexing. Cannot use tie to target when fixing the unit cell. self.all_params.refinement.reflections.outlier.algorithm = "null" self.all_params.refinement.parameterisation.crystal.fix = "cell" self.all_params.refinement.parameterisation.crystal.unit_cell.restraints.tie_to_target = [] for expt_id, experiment in enumerate(experiments): reflections = reflections_for_refinement.select(reflections_for_refinement['id'] == expt_id) reflections['id'] = flex.int(len(reflections),0) refiners = [] for isoform in self.params.stills.isoforms: iso_experiment = copy.deepcopy(experiment) crystal = iso_experiment.crystal if isoform.lookup_symbol != crystal.get_space_group().type().lookup_symbol(): logger.info("Crystal isoform lookup_symbol %s does not match isoform %s lookup_symbol %s"%(crystal.get_space_group().type().lookup_symbol(), isoform.name, isoform.lookup_symbol)) continue crystal.set_B(isoform.cell.fractionalization_matrix()) logger.info("Refining isoform %s"%isoform.name) refiners.append(e_refine(params=self.all_params, experiments=ExperimentList([iso_experiment]), reflections=reflections, graph_verbose=False)) if len(refiners) == 0: raise Sorry("No isoforms had a lookup symbol that matched") positional_rmsds = [math.sqrt(P.rmsds()[0] ** 2 + P.rmsds()[1] ** 2) for P in refiners] logger.info("Positional rmsds for all isoforms:" + str(positional_rmsds)) minrmsd_mm = min(positional_rmsds) minindex = positional_rmsds.index(minrmsd_mm) logger.info("The smallest rmsd is %5.1f um from isoform %s" % ( 1000. * minrmsd_mm, self.params.stills.isoforms[minindex].name)) if self.params.stills.isoforms[minindex].rmsd_target_mm is not None: logger.info("Asserting %f < %f"%(minrmsd_mm, self.params.stills.isoforms[minindex].rmsd_target_mm)) assert minrmsd_mm < self.params.stills.isoforms[minindex].rmsd_target_mm logger.info("Acceptable rmsd for isoform %s." % (self.params.stills.isoforms[minindex].name)) if len(self.params.stills.isoforms) == 2: logger.info("Rmsd gain over the other isoform %5.1f um." % ( 1000. * abs(positional_rmsds[0] - positional_rmsds[1]))) R = refiners[minindex] # Now one last check to see if direct beam is out of bounds if self.params.stills.isoforms[minindex].beam_restraint is not None: from scitbx import matrix refined_beam = matrix.col( R.get_experiments()[0].detector[0].get_beam_centre_lab(experiments[0].beam.get_s0())[0:2]) known_beam = matrix.col(self.params.stills.isoforms[minindex].beam_restraint) logger.info("Asserting difference in refined beam center and expected beam center %f < %f"%((refined_beam - known_beam).length(), self.params.stills.isoforms[ minindex].rmsd_target_mm)) assert (refined_beam - known_beam).length() < self.params.stills.isoforms[minindex].rmsd_target_mm # future--circle of confusion could be given as a separate length in mm instead of reusing rmsd_target experiment = R.get_experiments()[0] experiment.crystal.identified_isoform = self.params.stills.isoforms[minindex].name isoform_experiments.append(experiment) reflections['id'] = flex.int(len(reflections),expt_id) isoform_reflections.extend(reflections) experiments = isoform_experiments reflections_for_refinement = isoform_reflections if self.params.refinement_protocol.mode == 'repredict_only': from dials.algorithms.indexing.nave_parameters \ import nave_parameters from dials.algorithms.refinement.prediction.managed_predictors \ import ExperimentsPredictorFactory # TODO: need to check here if an experiment does not have any refined reflections # as nv will fail miserably if len(refined_reflections)==0 refined_experiments, refined_reflections = experiments, reflections_for_refinement ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, force_stills=True, spherical_relp=self.all_params.refinement.parameterisation.spherical_relp_model) ref_predictor(refined_reflections) refined_reflections['delpsical2'] = refined_reflections['delpsical.rad']**2 for expt_id in range(len(refined_experiments)): refls = refined_reflections.select(refined_reflections['id'] == expt_id) nv = nave_parameters(params=self.all_params, experiments=refined_experiments[expt_id:expt_id+1], reflections=refls, refinery=None, graph_verbose=False) experiments[expt_id].crystal = nv() ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, force_stills=True, spherical_relp=self.all_params.refinement.parameterisation.spherical_relp_model) ref_predictor(refined_reflections) elif self.params.refinement_protocol.mode == 'ignore': refined_experiments, refined_reflections = experiments, reflections_for_refinement else: try: refined_experiments, refined_reflections = self.refine( experiments, reflections_for_refinement) except Exception as e: s = str(e) if len(experiments) == 1: raise Sorry(e) had_refinement_error = True logger.info("Refinement failed:") logger.info(s) del experiments[-1] break # sanity check for unrealistic unit cell volume increase during refinement # usually this indicates too many parameters are being refined given the # number of observations provided. if not self.params.refinement_protocol.disable_unit_cell_volume_sanity_check: for orig_expt, refined_expt in zip(experiments, refined_experiments): uc1 = orig_expt.crystal.get_unit_cell() uc2 = refined_expt.crystal.get_unit_cell() volume_change = abs(uc1.volume()-uc2.volume())/uc1.volume() cutoff = 0.5 if volume_change > cutoff: msg = "\n".join(( "Unrealistic unit cell volume increase during refinement of %.1f%%.", "Please try refining fewer parameters, either by enforcing symmetry", "constraints (space_group=) and/or disabling experimental geometry", "refinement (detector.fix=all and beam.fix=all). To disable this", "sanity check set disable_unit_cell_volume_sanity_check=True.")) %( 100*volume_change) raise Sorry(msg) self.refined_reflections = refined_reflections.select( refined_reflections['id'] > -1) for i, imageset in enumerate(self.imagesets): ref_sel = self.refined_reflections.select( self.refined_reflections['imageset_id'] == i) ref_sel = ref_sel.select(ref_sel['id'] >= 0) for i_expt in set(ref_sel['id']): expt = refined_experiments[i_expt] imageset.set_detector(expt.detector) imageset.set_beam(expt.beam) imageset.set_goniometer(expt.goniometer) imageset.set_scan(expt.scan) expt.imageset = imageset if not (self.all_params.refinement.parameterisation.beam.fix == 'all' and self.all_params.refinement.parameterisation.detector.fix == 'all'): # Experimental geometry may have changed - re-map centroids to # reciprocal space spots_mm = self.reflections self.reflections = flex.reflection_table() for i, imageset in enumerate(self.imagesets): spots_sel = spots_mm.select(spots_mm['imageset_id'] == i) self.map_centroids_to_reciprocal_space( spots_sel, imageset.get_detector(), imageset.get_beam(), imageset.get_goniometer()) self.reflections.extend(spots_sel) # update for next cycle experiments = refined_experiments self.refined_experiments = refined_experiments ################################################### # end of the long indexing/refinement while loop... ################################################### if not 'refined_experiments' in locals(): raise Sorry("None of the experiments could refine.") # discard experiments with zero reflections after refinement id_set = set(self.refined_reflections['id']) if len(id_set) < len(self.refined_experiments): filtered_refined_reflections = flex.reflection_table() for i in xrange(len(self.refined_experiments)): if i not in id_set: del self.refined_experiments[i] for old, new in zip(sorted(id_set), range(len(id_set))): subset = self.refined_reflections.select(self.refined_reflections['id'] == old) subset['id'] = flex.int(len(subset), new) filtered_refined_reflections.extend(subset) self.refined_reflections = filtered_refined_reflections if len(self.refined_experiments) > 1: from dials.algorithms.indexing.compare_orientation_matrices \ import show_rotation_matrix_differences show_rotation_matrix_differences( self.refined_experiments.crystals(), out=info_handle) logger.info("Final refined crystal models:") for i, crystal_model in enumerate(self.refined_experiments.crystals()): n_indexed = 0 for i_expt in experiments.where(crystal=crystal_model): n_indexed += (self.reflections['id'] == i).count(True) logger.info("model %i (%i reflections):" %(i+1, n_indexed)) logger.info(crystal_model) if 'xyzcal.mm' in self.refined_reflections: # won't be there if refine_all_candidates = False and no isoforms self.refined_reflections['xyzcal.px'] = flex.vec3_double( len(self.refined_reflections)) for i, imageset in enumerate(self.imagesets): imgset_sel = self.refined_reflections['imageset_id'] == i # set xyzcal.px field in self.refined_reflections refined_reflections = self.refined_reflections.select(imgset_sel) panel_numbers = flex.size_t(refined_reflections['panel']) xyzcal_mm = refined_reflections['xyzcal.mm'] x_mm, y_mm, z_rad = xyzcal_mm.parts() xy_cal_mm = flex.vec2_double(x_mm, y_mm) xy_cal_px = flex.vec2_double(len(xy_cal_mm)) for i_panel in range(len(imageset.get_detector())): panel = imageset.get_detector()[i_panel] sel = (panel_numbers == i_panel) isel = sel.iselection() ref_panel = refined_reflections.select(panel_numbers == i_panel) xy_cal_px.set_selected( sel, panel.millimeter_to_pixel(xy_cal_mm.select(sel))) x_px, y_px = xy_cal_px.parts() scan = imageset.get_scan() if scan is not None: z_px = scan.get_array_index_from_angle(z_rad, deg=False) else: # must be a still image, z centroid not meaningful z_px = z_rad xyzcal_px = flex.vec3_double(x_px, y_px, z_px) self.refined_reflections['xyzcal.px'].set_selected(imgset_sel, xyzcal_px)
def choose_best_orientation_matrix(self, candidate_orientation_matrices): from dxtbx.model.experiment.experiment_list import Experiment, ExperimentList from logging import info import copy info('*' * 80) info('Selecting the best orientation matrix') info('*' * 80) from libtbx import group_args class candidate_info(group_args): pass candidates = [] params = copy.deepcopy(self.all_params) for icm,cm in enumerate(candidate_orientation_matrices): sel = ((self.reflections['id'] == -1)) #(1/self.reflections['rlp'].norms() > self.d_min)) refl = self.reflections.select(sel) experiments = self.experiment_list_for_crystal(cm) self.index_reflections(experiments, refl) indexed = refl.select(refl['id'] >= 0) indexed = indexed.select(indexed.get_flags(indexed.flags.indexed)) print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d initial outlier identification"%icm acceptance_flags = self.identify_outliers(params, experiments, indexed) #create a new "indexed" list with outliers thrown out: indexed = indexed.select(acceptance_flags) print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d refinement before outlier rejection"%icm R = e_refine(params = params, experiments=experiments, reflections=indexed, graph_verbose=False) ref_experiments = R.get_experiments() # try to improve the outcome with a second round of outlier rejection post-initial refinement: acceptance_flags = self.identify_outliers(params, ref_experiments, indexed) # insert a round of Nave-outlier rejection on top of the r.m.s.d. rejection nv0 = nave_parameters(params = params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False) crystal_model_nv0 = nv0() acceptance_flags_nv0 = nv0.nv_acceptance_flags indexed = indexed.select(acceptance_flags & acceptance_flags_nv0) print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d after positional and delta-psi outlier rejection"%icm R = e_refine(params = params, experiments=ref_experiments, reflections=indexed, graph_verbose=False) ref_experiments = R.get_experiments() nv = nave_parameters(params = params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False) crystal_model = nv() rmsd, _ = calc_2D_rmsd_and_displacements(R.predict_for_reflection_table(indexed)) print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d done"%icm candidates.append(candidate_info(crystal = crystal_model, green_curve_area = nv.green_curve_area, ewald_proximal_volume = nv.ewald_proximal_volume(), n_indexed = len(indexed), rmsd = rmsd, indexed = indexed, experiments = ref_experiments)) if len(candidates) == 0: raise Sorry("No suitable indexing solution found") print "**** ALL CANDIDATES:" for i,XX in enumerate(candidates): print "\n****Candidate %d"%i,XX cc = XX.crystal print " half mosaicity %5.2f deg."%(cc._ML_half_mosaicity_deg) print " domain size %.0f Ang."%(cc._ML_domain_size_ang) print "\n**** BEST CANDIDATE:" results = flex.double([c.rmsd for c in candidates]) best = candidates[flex.min_index(results)] print best if best.rmsd > 1.5: raise Sorry ("RMSD too high, %f" %rmsd) if best.ewald_proximal_volume > 0.0015: raise Sorry ("Ewald proximity volume too high, %f"%best.ewald_proximal_volume) if len(candidates) > 1: for i in xrange(len(candidates)): if i == flex.min_index(results): continue if best.ewald_proximal_volume > candidates[i].ewald_proximal_volume: print "Couldn't figure out which candidate is best; picked the one with the best RMSD." best.indexed['entering'] = flex.bool(best.n_indexed, False) self._best_indexed = best.indexed return best.crystal, best.n_indexed
def choose_best_orientation_matrix(self, candidate_orientation_matrices): from dxtbx.model.experiment_list import Experiment, ExperimentList import copy logger.info('*' * 80) logger.info('Selecting the best orientation matrix') logger.info('*' * 80) from libtbx import group_args class candidate_info(group_args): pass candidates = [] params = copy.deepcopy(self.all_params) n_cand = len(candidate_orientation_matrices) for icm,cm in enumerate(candidate_orientation_matrices): sel = ((self.reflections['id'] == -1)) #(1/self.reflections['rlp'].norms() > self.d_min)) refl = self.reflections.select(sel) experiments = self.experiment_list_for_crystal(cm) self.index_reflections(experiments, refl) indexed = refl.select(refl['id'] >= 0) indexed = indexed.select(indexed.get_flags(indexed.flags.indexed)) if params.indexing.stills.refine_all_candidates: try: print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d initial outlier identification"%(icm, n_cand) acceptance_flags = self.identify_outliers(params, experiments, indexed) #create a new "indexed" list with outliers thrown out: indexed = indexed.select(acceptance_flags) print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d refinement before outlier rejection"%(icm, n_cand) R = e_refine(params = params, experiments=experiments, reflections=indexed, graph_verbose=False) ref_experiments = R.get_experiments() # try to improve the outcome with a second round of outlier rejection post-initial refinement: acceptance_flags = self.identify_outliers(params, ref_experiments, indexed) # insert a round of Nave-outlier rejection on top of the r.m.s.d. rejection nv0 = nave_parameters(params = params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False) crystal_model_nv0 = nv0() acceptance_flags_nv0 = nv0.nv_acceptance_flags indexed = indexed.select(acceptance_flags & acceptance_flags_nv0) print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d after positional and delta-psi outlier rejection"%(icm, n_cand) R = e_refine(params = params, experiments=ref_experiments, reflections=indexed, graph_verbose=False) ref_experiments = R.get_experiments() nv = nave_parameters(params = params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False) crystal_model = nv() # Drop candidates that after refinement can no longer be converted to the known target space group if self.params.known_symmetry.space_group is not None: target_space_group = self.target_symmetry_primitive.space_group() new_crystal, cb_op_to_primitive = self.apply_symmetry(crystal_model, target_space_group) if new_crystal is None: print "P1 refinement yielded model diverged from target, candidate %d/%d"%(icm, n_cand) continue rmsd, _ = calc_2D_rmsd_and_displacements(R.predict_for_reflection_table(indexed)) except Exception, e: print "Couldn't refine candiate %d/%d, %s"%(icm, n_cand, str(e)) else: print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d done"%(icm, n_cand) candidates.append(candidate_info(crystal = crystal_model, green_curve_area = nv.green_curve_area, ewald_proximal_volume = nv.ewald_proximal_volume(), n_indexed = len(indexed), rmsd = rmsd, indexed = indexed, experiments = ref_experiments)) else: from dials.algorithms.refinement.prediction import ExperimentsPredictor ref_predictor = ExperimentsPredictor(experiments, force_stills=True, spherical_relp=params.refinement.parameterisation.spherical_relp_model) rmsd, _ = calc_2D_rmsd_and_displacements(ref_predictor(indexed)) candidates.append(candidate_info(crystal = cm, n_indexed = len(indexed), rmsd = rmsd, indexed = indexed, experiments = experiments))
def choose_best_orientation_matrix(self, candidate_orientation_matrices): from dxtbx.model.experiment.experiment_list import Experiment, ExperimentList import copy logger.info('*' * 80) logger.info('Selecting the best orientation matrix') logger.info('*' * 80) from libtbx import group_args class candidate_info(group_args): pass candidates = [] params = copy.deepcopy(self.all_params) n_cand = len(candidate_orientation_matrices) for icm,cm in enumerate(candidate_orientation_matrices): sel = ((self.reflections['id'] == -1)) #(1/self.reflections['rlp'].norms() > self.d_min)) refl = self.reflections.select(sel) experiments = self.experiment_list_for_crystal(cm) self.index_reflections(experiments, refl) indexed = refl.select(refl['id'] >= 0) indexed = indexed.select(indexed.get_flags(indexed.flags.indexed)) if params.indexing.stills.refine_all_candidates: try: print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d initial outlier identification"%(icm, n_cand) acceptance_flags = self.identify_outliers(params, experiments, indexed) #create a new "indexed" list with outliers thrown out: indexed = indexed.select(acceptance_flags) print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d refinement before outlier rejection"%(icm, n_cand) R = e_refine(params = params, experiments=experiments, reflections=indexed, graph_verbose=False) ref_experiments = R.get_experiments() # try to improve the outcome with a second round of outlier rejection post-initial refinement: acceptance_flags = self.identify_outliers(params, ref_experiments, indexed) # insert a round of Nave-outlier rejection on top of the r.m.s.d. rejection nv0 = nave_parameters(params = params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False) crystal_model_nv0 = nv0() acceptance_flags_nv0 = nv0.nv_acceptance_flags indexed = indexed.select(acceptance_flags & acceptance_flags_nv0) print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d after positional and delta-psi outlier rejection"%(icm, n_cand) R = e_refine(params = params, experiments=ref_experiments, reflections=indexed, graph_verbose=False) ref_experiments = R.get_experiments() nv = nave_parameters(params = params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False) crystal_model = nv() # Drop candidates that after refinement can no longer be converted to the known target space group if self.params.known_symmetry.space_group is not None: target_space_group = self.target_symmetry_primitive.space_group() new_crystal, cb_op_to_primitive = self.apply_symmetry(crystal_model, target_space_group) if new_crystal is None: print "P1 refinement yielded model diverged from target, candidate %d/%d"%(icm, n_cand) continue rmsd, _ = calc_2D_rmsd_and_displacements(R.predict_for_reflection_table(indexed)) except Exception, e: print "Couldn't refine candiate %d/%d, %s"%(icm, n_cand, str(e)) else: print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d done"%(icm, n_cand) candidates.append(candidate_info(crystal = crystal_model, green_curve_area = nv.green_curve_area, ewald_proximal_volume = nv.ewald_proximal_volume(), n_indexed = len(indexed), rmsd = rmsd, indexed = indexed, experiments = ref_experiments)) else: from dials.algorithms.refinement.prediction import ExperimentsPredictor ref_predictor = ExperimentsPredictor(experiments, force_stills=True, spherical_relp=params.refinement.parameterisation.spherical_relp_model) rmsd, _ = calc_2D_rmsd_and_displacements(ref_predictor(indexed)) candidates.append(candidate_info(crystal = cm, n_indexed = len(indexed), rmsd = rmsd, indexed = indexed, experiments = experiments))