def integrate(self): # Process reference reflections self.indexed, _ = self.process_reference(self.indexed) # Get integrator from input params from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory # Compute the profile model self.experiments = ProfileModelFactory.create(self.phil, self.experiments, self.indexed) # Predict the reflections predicted = flex.reflection_table.from_predictions_multi( self.experiments, dmin=self.phil.prediction.d_min, dmax=self.phil.prediction.d_max, margin=self.phil.prediction.margin, force_static=self.phil.prediction.force_static) # Match the predictions with the reference predicted.match_with_reference(self.indexed) # Create the integrator integrator = IntegratorFactory.create(self.phil, self.experiments, predicted) # Integrate the reflections self.integrated = integrator.integrate() if self.integrated.has_key('intensity.prf.value'): method = 'prf' # integration by profile fitting elif self.integrated.has_key('intensity.sum.value'): method = 'sum' # integration by simple summation self.integrated = self.integrated.select( self.integrated['intensity.' + method + '.variance'] > 0) # keep only spots with sigmas above zero # Save the reflections if selected if self.phil.output.integrated_filename: self.save_reflections(self.integrated, self.phil.output.integrated_filename) self.write_integration_pickles() from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements rmsd_indexed, _ = calc_2D_rmsd_and_displacements(self.indexed) rmsd_integrated, _ = calc_2D_rmsd_and_displacements(self.integrated) crystal_model = self.experiments.crystals()[0]
def integrate(self): # Process reference reflections self.indexed,_ = self.process_reference(self.indexed) # Get integrator from input params from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory # Compute the profile model self.experiments = ProfileModelFactory.create(self.phil, self.experiments, self.indexed) # Predict the reflections predicted = flex.reflection_table.from_predictions_multi( self.experiments, dmin=self.phil.prediction.d_min, dmax=self.phil.prediction.d_max, margin=self.phil.prediction.margin, force_static=self.phil.prediction.force_static) # Match the predictions with the reference predicted.match_with_reference(self.indexed) # Create the integrator integrator = IntegratorFactory.create(self.phil, self.experiments, predicted) # Integrate the reflections self.integrated = integrator.integrate() if self.integrated.has_key('intensity.prf.value'): method = 'prf' # integration by profile fitting elif self.integrated.has_key('intensity.sum.value'): method = 'sum' # integration by simple summation self.integrated = self.integrated.select(self.integrated['intensity.' + method + '.variance'] > 0) # keep only spots with sigmas above zero # Save the reflections if selected if self.phil.output.integrated_filename: self.save_reflections(self.integrated, self.phil.output.integrated_filename) self.write_integration_pickles() from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements rmsd_indexed, _ = calc_2D_rmsd_and_displacements(self.indexed) rmsd_integrated, _ = calc_2D_rmsd_and_displacements(self.integrated) crystal_model = self.experiments.crystals()[0]
def integrate(self, experiments, indexed): from logging import info info('*' * 80) info('Integrating Reflections') info('*' * 80) from xfel.command_line.xfel_process import Script as ProcessScript assert len(experiments) == 2 integrated = ProcessScript.integrate(self, experiments, indexed) if 'intensity.prf.value' in integrated: method = 'prf' # integration by profile fitting elif 'intensity.sum.value' in integrated: method = 'sum' # integration by simple summation integrated = integrated.select( integrated['intensity.' + method + '.variance'] > 0) # keep only spots with sigmas above zero integrated = remove_invalid_reflections(integrated) self.save_reflections(integrated, self.params.output.integrated_filename) def write_integration_pickles_callback(params, outfile, frame): from cxi_xdr_xes.two_color.two_color_dump import correction_for_metal_foil_absorption, derive_scale_and_B_to_model correction_for_metal_foil_absorption(params, frame) if params.calc_G_and_B.do_calc: derive_scale_and_B_to_model(params.calc_G_and_B, outfile, frame) self.write_integration_pickles( integrated, experiments, callback=write_integration_pickles_callback) from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed) rmsd_integrated, _ = calc_2D_rmsd_and_displacements(integrated) crystal_model = experiments.crystals()[0] print "Integrated. RMSD indexed,", rmsd_indexed, "RMSD integrated", rmsd_integrated, \ "Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg) return integrated
def integrate(self, experiments, indexed): from time import time st = time() logger.info('*' * 80) logger.info('Integrating Reflections') logger.info('*' * 80) indexed,_ = self.process_reference(indexed) # Get the integrator from the input parameters logger.info('Configuring integrator from input parameters') from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory from dials.array_family import flex # Compute the profile model # Predict the reflections # Match the predictions with the reference # Create the integrator experiments = ProfileModelFactory.create(self.params, experiments, indexed) logger.info("") logger.info("=" * 80) logger.info("") logger.info("Predicting reflections") logger.info("") predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=self.params.prediction.d_min, dmax=self.params.prediction.d_max, margin=self.params.prediction.margin, force_static=self.params.prediction.force_static) predicted.match_with_reference(indexed) logger.info("") integrator = IntegratorFactory.create(self.params, experiments, predicted) # Integrate the reflections integrated = integrator.integrate() # Select only those reflections which were integrated if 'intensity.prf.variance' in integrated: selection = integrated.get_flags( integrated.flags.integrated, all=True) else: selection = integrated.get_flags( integrated.flags.integrated_sum) integrated = integrated.select(selection) len_all = len(integrated) integrated = integrated.select(~integrated.get_flags(integrated.flags.foreground_includes_bad_pixels)) print "Filtering %d reflections with at least one bad foreground pixel out of %d"%(len_all-len(integrated), len_all) # verify sigmas are sensible if 'intensity.prf.value' in integrated: if (integrated['intensity.prf.variance'] <= 0).count(True) > 0: raise Sorry("Found negative variances") if 'intensity.sum.value' in integrated: if (integrated['intensity.sum.variance'] <= 0).count(True) > 0: raise Sorry("Found negative variances") # apply detector gain to summation variances integrated['intensity.sum.variance'] *= self.params.integration.summation.detector_gain if 'background.sum.value' in integrated: if (integrated['background.sum.variance'] < 0).count(True) > 0: raise Sorry("Found negative variances") if (integrated['background.sum.variance'] == 0).count(True) > 0: print "Filtering %d reflections with zero background variance" % ((integrated['background.sum.variance'] == 0).count(True)) integrated = integrated.select(integrated['background.sum.variance'] > 0) # apply detector gain to background summation variances integrated['background.sum.variance'] *= self.params.integration.summation.detector_gain # correct integrated intensities for absorption correction, if necessary for abs_params in self.params.integration.absorption_correction: if abs_params.apply and abs_params.algorithm == "fuller_kapton": from dials.algorithms.integration.kapton_correction import multi_kapton_correction experiments, integrated = multi_kapton_correction(experiments, integrated, abs_params.fuller_kapton, logger=logger)() if self.params.significance_filter.enable: from dials.algorithms.integration.stills_significance_filter import SignificanceFilter sig_filter = SignificanceFilter(self.params) refls = sig_filter(experiments, integrated) logger.info("Removed %d reflections out of %d when applying significance filter"%(len(integrated)-len(refls), len(integrated))) if len(refls) == 0: raise Sorry("No reflections left after applying significance filter") integrated = refls if self.params.output.integrated_filename: # Save the reflections self.save_reflections(integrated, self.params.output.integrated_filename) self.write_integration_pickles(integrated, experiments) from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed) log_str = "RMSD indexed (px): %f\n"%(rmsd_indexed) for i in xrange(6): bright_integrated = integrated.select((integrated['intensity.sum.value']/flex.sqrt(integrated['intensity.sum.variance']))>=i) if len(bright_integrated) > 0: rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated) else: rmsd_integrated = 0 log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"%(i, len(bright_integrated), rmsd_integrated) for crystal_model in experiments.crystals(): if hasattr(crystal_model, '_ML_domain_size_ang'): log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg) logger.info(log_str) logger.info('') logger.info('Time Taken = %f seconds' % (time() - st)) return integrated
def integrate_coset(self, experiments, indexed): TRANS = self.params.integration.coset.transformation # here get a deepcopy that we are not afraid to modify: experiments_local = copy.deepcopy(experiments) print("*" * 80) print("Coset Reflections for modeling or validating the background") print("*" * 80) from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import create_integrator # XXX Fixme later implement support for non-primitive lattices NKS base_set = miller_set( crystal_symmetry = symmetry( unit_cell = experiments_local[0].crystal.get_unit_cell(), space_group = experiments_local[0].crystal.get_space_group()), indices = indexed["miller_index"] ) triclinic = base_set.customized_copy( crystal_symmetry=symmetry(unit_cell = experiments_local[0].crystal.get_unit_cell(),space_group="P1")) # ================ # Compute the profile model # Predict the reflections # Create the integrator # This creates a reference to the experiment, not a copy: experiments_local = ProfileModelFactory.create(self.params, experiments_local, indexed) # for debug SLT[TRANS].show_summary() for e in experiments_local: e.crystal.set_space_group(triclinic.space_group()) Astar = e.crystal.get_A() # debug OriAstar = crystal_orientation(Astar,True) # debug OriAstar.show(legend="old ") Astarprime = sqr(Astar)* ( sqr(SLT[TRANS]._reindex_N).transpose().inverse() ) e.crystal.set_A(Astarprime) # debug OriAstarprime = crystal_orientation(Astarprime,True) # debug OriAstarprime.show(legend="new ") print("Predicting coset reflections") print("") predicted = flex.reflection_table.from_predictions_multi( experiments_local, dmin=self.params.prediction.d_min, dmax=self.params.prediction.d_max, margin=self.params.prediction.margin, force_static=self.params.prediction.force_static, ) print("sublattice total predictions %d"%len(predicted)) # filter the sublattice, keep only the coset indices miller = predicted["miller_index"] # coset of modulus 2, wherein there is a binary choice # see Sauter & Zwart, Acta D (2009) 65:553, Table 1; select the valid coset using eqn(5). coset_select_algorithm_2 = flex.bool() M_mat = SLT[TRANS].matS() # the transformation M_p = M_mat.inverse() for idx in miller: H_row = row(idx) h_orig_setting = H_row * M_p on_coset=False for icom in h_orig_setting.elems: if icom.denominator() > 1: on_coset=True; break coset_select_algorithm_2.append(on_coset) predicted = predicted.select(coset_select_algorithm_2) print("of which %d are in coset %d"%(len(predicted), TRANS)) print("") integrator = create_integrator(self.params, experiments_local, predicted) # Integrate the reflections integrated = integrator.integrate() # Delete the shoeboxes used for intermediate calculations, if requested if self.params.integration.debug.delete_shoeboxes and "shoebox" in integrated: del integrated["shoebox"] if self.params.output.composite_output: if ( self.params.output.coset_experiments_filename or self.params.output.coset_filename ): assert ( self.params.output.coset_experiments_filename is not None and self.params.output.coset_filename is not None ) n = len(self.all_coset_experiments) self.all_coset_experiments.extend(experiments_local) for i, experiment in enumerate(experiments_local): refls = integrated.select(integrated["id"] == i) refls["id"] = flex.int(len(refls), n) del refls.experiment_identifiers()[i] refls.experiment_identifiers()[n] = experiment.identifier self.all_coset_reflections.extend(refls) n += 1 else: # Dump experiments to disk if self.params.output.coset_experiments_filename: experiments_local.as_json(self.params.output.coset_experiments_filename) if self.params.output.coset_filename: # Save the reflections self.save_reflections( integrated, self.params.output.coset_filename ) rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed) log_str = "coset RMSD indexed (px): %f\n" % (rmsd_indexed) log_str += "integrated %d\n"%len(integrated) for i in range(6): bright_integrated = integrated.select( ( integrated["intensity.sum.value"] / flex.sqrt(integrated["intensity.sum.variance"]) ) >= i ) if len(bright_integrated) > 0: rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated) else: rmsd_integrated = 0 log_str += ( "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n" % (i, len(bright_integrated), rmsd_integrated) ) for crystal_model in experiments_local.crystals(): if hasattr(crystal_model, "get_domain_size_ang"): log_str += ( ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f" % ( crystal_model.get_domain_size_ang(), crystal_model.get_half_mosaicity_deg(), ) ) print(log_str) print("")
def integrate(self, experiments, indexed): from time import time st = time() logger.info('*' * 80) logger.info('Integrating Reflections') logger.info('*' * 80) indexed,_ = self.process_reference(indexed) # Get the integrator from the input parameters logger.info('Configuring integrator from input parameters') from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory from dials.array_family import flex # Compute the profile model # Predict the reflections # Match the predictions with the reference # Create the integrator experiments = ProfileModelFactory.create(self.params, experiments, indexed) logger.info("") logger.info("=" * 80) logger.info("") logger.info("Predicting reflections") logger.info("") predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=self.params.prediction.d_min, dmax=self.params.prediction.d_max, margin=self.params.prediction.margin, force_static=self.params.prediction.force_static) predicted.match_with_reference(indexed) logger.info("") integrator = IntegratorFactory.create(self.params, experiments, predicted) # Integrate the reflections integrated = integrator.integrate() # Select only those reflections which were integrated if 'intensity.prf.variance' in integrated: selection = integrated.get_flags( integrated.flags.integrated, all=True) else: selection = integrated.get_flags( integrated.flags.integrated_sum) integrated = integrated.select(selection) len_all = len(integrated) integrated = integrated.select(~integrated.get_flags(integrated.flags.foreground_includes_bad_pixels)) print "Filtering %d reflections with at least one bad foreground pixel out of %d"%(len_all-len(integrated), len_all) # verify sigmas are sensible if 'intensity.prf.value' in integrated: if (integrated['intensity.prf.variance'] <= 0).count(True) > 0: raise Sorry("Found negative variances") if 'intensity.sum.value' in integrated: if (integrated['intensity.sum.variance'] <= 0).count(True) > 0: raise Sorry("Found negative variances") # apply detector gain to summation variances integrated['intensity.sum.variance'] *= self.params.integration.summation.detector_gain if 'background.sum.value' in integrated: if (integrated['background.sum.variance'] < 0).count(True) > 0: raise Sorry("Found negative variances") if (integrated['background.sum.variance'] == 0).count(True) > 0: print "Filtering %d reflections with zero background variance" % ((integrated['background.sum.variance'] == 0).count(True)) integrated = integrated.select(integrated['background.sum.variance'] > 0) # apply detector gain to background summation variances integrated['background.sum.variance'] *= self.params.integration.summation.detector_gain if self.params.output.integrated_filename: # Save the reflections self.save_reflections(integrated, self.params.output.integrated_filename) self.write_integration_pickles(integrated, experiments) from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed) log_str = "RMSD indexed (px): %f\n"%(rmsd_indexed) for i in xrange(6): bright_integrated = integrated.select((integrated['intensity.sum.value']/flex.sqrt(integrated['intensity.sum.variance']))>=i) if len(bright_integrated) > 0: rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated) else: rmsd_integrated = 0 log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"%(i, len(bright_integrated), rmsd_integrated) crystal_model = experiments.crystals()[0] if hasattr(crystal_model, '._ML_domain_size_ang'): log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg) logger.info(log_str) logger.info('') logger.info('Time Taken = %f seconds' % (time() - st)) return integrated
def integrate(self, experiments, indexed): from time import time from logging import info st = time() info('*' * 80) info('Integrating Reflections') info('*' * 80) indexed,_ = self.process_reference(indexed) # Get the integrator from the input parameters info('Configuring integrator from input parameters') from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory from dials.array_family import flex # Compute the profile model # Predict the reflections # Match the predictions with the reference # Create the integrator experiments = ProfileModelFactory.create(self.params, experiments, indexed) info("") info("=" * 80) info("") info("Predicting reflections") info("") predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=self.params.prediction.d_min, dmax=self.params.prediction.d_max, margin=self.params.prediction.margin, force_static=self.params.prediction.force_static) predicted.match_with_reference(indexed) info("") integrator = IntegratorFactory.create(self.params, experiments, predicted) # Integrate the reflections integrated = integrator.integrate() if integrated.has_key('intensity.prf.value'): method = 'prf' # integration by profile fitting elif integrated.has_key('intensity.sum.value'): method = 'sum' # integration by simple summation integrated = integrated.select(integrated['intensity.' + method + '.variance'] > 0) # keep only spots with sigmas above zero if self.params.output.integrated_filename: # Save the reflections self.save_reflections(integrated, self.params.output.integrated_filename) self.write_integration_pickles(integrated, experiments) from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed) rmsd_integrated, _ = calc_2D_rmsd_and_displacements(integrated) crystal_model = experiments.crystals()[0] print "Integrated. RMSD indexed,", rmsd_indexed, "RMSD integrated", rmsd_integrated, \ "Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg) info('') info('Time Taken = %f seconds' % (time() - st)) return integrated
def integrate(self, experiments, indexed): if self.params.skip_hopper: return super(Hopper_Processor, self).integrate(experiments, indexed) st = time.time() logger.info("*" * 80) logger.info("Integrating Reflections") logger.info("*" * 80) indexed, _ = self.process_reference(indexed) if self.params.integration.integration_only_overrides.trusted_range: for detector in experiments.detectors(): for panel in detector: panel.set_trusted_range( self.params.integration.integration_only_overrides. trusted_range) if self.params.dispatch.coset: from xfel.util.sublattice_helper import integrate_coset integrate_coset(self, experiments, indexed) # Get the integrator from the input parameters logger.info("Configuring integrator from input parameters") from dials.algorithms.integration.integrator import create_integrator from dials.algorithms.profile_model.factory import ProfileModelFactory # Compute the profile model # Predict the reflections # Match the predictions with the reference # Create the integrator experiments = ProfileModelFactory.create(self.params, experiments, indexed) new_experiments = ExperimentList() new_reflections = flex.reflection_table() for expt_id, expt in enumerate(experiments): if (self.params.profile.gaussian_rs.parameters.sigma_b_cutoff is None or expt.profile.sigma_b() < self.params.profile.gaussian_rs.parameters.sigma_b_cutoff): refls = indexed.select(indexed["id"] == expt_id) refls["id"] = flex.int(len(refls), len(new_experiments)) # refls.reset_ids() del refls.experiment_identifiers()[expt_id] refls.experiment_identifiers()[len( new_experiments)] = expt.identifier new_reflections.extend(refls) new_experiments.append(expt) else: logger.info("Rejected expt %d with sigma_b %f" % (expt_id, expt.profile.sigma_b())) experiments = new_experiments indexed = new_reflections if len(experiments) == 0: raise RuntimeError("No experiments after filtering by sigma_b") logger.info("") logger.info("=" * 80) logger.info("") logger.info("Predicting reflections") logger.info("") # NOTE: this is the only changed needed to dials.stills_process # TODO: multi xtal # TODO: add in normal dials predictions as an option predicted, model = predictions.get_predicted_from_pandas( self.stage1_df, self.params.diffBragg, self.observed, experiments[0].identifier, self.device_id, spectrum_override=self.stage1_modeler.SIM.beam.spectrum) if self.params.refine_predictions: experiments, rnd2_refls = self.refine(experiments, predicted, refining_predictions=True, best=self.stage1_df) # TODO: match rnd2_refls with indexed.refl and re-save indexed.refl predicted, model = predictions.get_predicted_from_pandas( self.stage1_df, self.params.diffBragg, self.observed, experiments[0].identifier, self.device_id, spectrum_override=self.stage1_modeler.SIM.beam.spectrum) predicted.match_with_reference(indexed) integrator = create_integrator(self.params, experiments, predicted) # Integrate the reflections integrated = integrator.integrate() if self.params.partial_correct: integrated = predictions.normalize_by_partiality( integrated, model, default_F=self.params.diffBragg.predictions.default_Famplitude, gain=self.params.diffBragg.refiner.adu_per_photon) # correct integrated intensities for absorption correction, if necessary for abs_params in self.params.integration.absorption_correction: if abs_params.apply: if abs_params.algorithm == "fuller_kapton": from dials.algorithms.integration.kapton_correction import ( multi_kapton_correction, ) elif abs_params.algorithm == "kapton_2019": from dials.algorithms.integration.kapton_2019_correction import ( multi_kapton_correction, ) experiments, integrated = multi_kapton_correction( experiments, integrated, abs_params.fuller_kapton, logger=logger)() if self.params.significance_filter.enable: from dials.algorithms.integration.stills_significance_filter import ( SignificanceFilter, ) sig_filter = SignificanceFilter(self.params) filtered_refls = sig_filter(experiments, integrated) accepted_expts = ExperimentList() accepted_refls = flex.reflection_table() logger.info( "Removed %d reflections out of %d when applying significance filter", len(integrated) - len(filtered_refls), len(integrated), ) for expt_id, expt in enumerate(experiments): refls = filtered_refls.select(filtered_refls["id"] == expt_id) if len(refls) > 0: accepted_expts.append(expt) refls["id"] = flex.int(len(refls), len(accepted_expts) - 1) accepted_refls.extend(refls) else: logger.info( "Removed experiment %d which has no reflections left after applying significance filter", expt_id, ) if len(accepted_refls) == 0: raise Sorry( "No reflections left after applying significance filter") experiments = accepted_expts integrated = accepted_refls # Delete the shoeboxes used for intermediate calculations, if requested if self.params.integration.debug.delete_shoeboxes and "shoebox" in integrated: del integrated["shoebox"] if self.params.output.composite_output: if (self.params.output.integrated_experiments_filename or self.params.output.integrated_filename): assert (self.params.output.integrated_experiments_filename is not None and self.params.output.integrated_filename is not None) n = len(self.all_integrated_experiments) self.all_integrated_experiments.extend(experiments) for i, experiment in enumerate(experiments): refls = integrated.select(integrated["id"] == i) refls["id"] = flex.int(len(refls), n) del refls.experiment_identifiers()[i] refls.experiment_identifiers()[n] = experiment.identifier self.all_integrated_reflections.extend(refls) n += 1 else: # Dump experiments to disk if self.params.output.integrated_experiments_filename: experiments.as_json( self.params.output.integrated_experiments_filename) if self.params.output.integrated_filename: # Save the reflections self.save_reflections(integrated, self.params.output.integrated_filename) self.write_integration_pickles(integrated, experiments) from dials.algorithms.indexing.stills_indexer import ( calc_2D_rmsd_and_displacements, ) rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed) log_str = "RMSD indexed (px): %f\n" % rmsd_indexed for i in range(6): bright_integrated = integrated.select( (integrated["intensity.sum.value"] / flex.sqrt(integrated["intensity.sum.variance"])) >= i) if len(bright_integrated) > 0: rmsd_integrated, _ = calc_2D_rmsd_and_displacements( bright_integrated) else: rmsd_integrated = 0 log_str += ( "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n" % (i, len(bright_integrated), rmsd_integrated)) for crystal_model in experiments.crystals(): if hasattr(crystal_model, "get_domain_size_ang"): log_str += ". Final ML model: domain size angstroms: {:f}, half mosaicity degrees: {:f}".format( crystal_model.get_domain_size_ang(), crystal_model.get_half_mosaicity_deg(), ) logger.info(log_str) logger.info("") logger.info("Time Taken = %f seconds", time.time() - st) return integrated
def integrate(self, experiments, indexed): from time import time st = time() logger.info('*' * 80) logger.info('Integrating Reflections') logger.info('*' * 80) indexed, _ = self.process_reference(indexed) # Get the integrator from the input parameters logger.info('Configuring integrator from input parameters') from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory from dials.array_family import flex # Compute the profile model # Predict the reflections # Match the predictions with the reference # Create the integrator experiments = ProfileModelFactory.create(self.params, experiments, indexed) logger.info("") logger.info("=" * 80) logger.info("") logger.info("Predicting reflections") logger.info("") predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=self.params.prediction.d_min, dmax=self.params.prediction.d_max, margin=self.params.prediction.margin, force_static=self.params.prediction.force_static) predicted.match_with_reference(indexed) logger.info("") integrator = IntegratorFactory.create(self.params, experiments, predicted) # Integrate the reflections integrated = integrator.integrate() # correct integrated intensities for absorption correction, if necessary for abs_params in self.params.integration.absorption_correction: if abs_params.apply and abs_params.algorithm == "fuller_kapton": from dials.algorithms.integration.kapton_correction import multi_kapton_correction experiments, integrated = multi_kapton_correction( experiments, integrated, abs_params.fuller_kapton, logger=logger)() if self.params.significance_filter.enable: from dials.algorithms.integration.stills_significance_filter import SignificanceFilter sig_filter = SignificanceFilter(self.params) refls = sig_filter(experiments, integrated) logger.info( "Removed %d reflections out of %d when applying significance filter" % (len(integrated) - len(refls), len(integrated))) if len(refls) == 0: raise Sorry( "No reflections left after applying significance filter") integrated = refls # Delete the shoeboxes used for intermediate calculations, if requested if self.params.integration.debug.delete_shoeboxes and 'shoebox' in integrated: del integrated['shoebox'] if self.params.output.composite_output: if self.params.output.integrated_experiments_filename or self.params.output.integrated_filename: assert self.params.output.integrated_experiments_filename is not None and self.params.output.integrated_filename is not None from dials.array_family import flex n = len(self.all_integrated_experiments) self.all_integrated_experiments.extend(experiments) for i, experiment in enumerate(experiments): refls = integrated.select(integrated['id'] == i) refls['id'] = flex.int(len(refls), n) self.all_integrated_reflections.extend(refls) n += 1 else: # Dump experiments to disk if self.params.output.integrated_experiments_filename: from dxtbx.model.experiment_list import ExperimentListDumper dump = ExperimentListDumper(experiments) dump.as_json( self.params.output.integrated_experiments_filename) if self.params.output.integrated_filename: # Save the reflections self.save_reflections(integrated, self.params.output.integrated_filename) self.write_integration_pickles(integrated, experiments) from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed) log_str = "RMSD indexed (px): %f\n" % (rmsd_indexed) for i in xrange(6): bright_integrated = integrated.select( (integrated['intensity.sum.value'] / flex.sqrt(integrated['intensity.sum.variance'])) >= i) if len(bright_integrated) > 0: rmsd_integrated, _ = calc_2D_rmsd_and_displacements( bright_integrated) else: rmsd_integrated = 0 log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n" % ( i, len(bright_integrated), rmsd_integrated) for crystal_model in experiments.crystals(): if hasattr(crystal_model, 'get_domain_size_ang'): log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f" % ( crystal_model.get_domain_size_ang(), crystal_model.get_half_mosaicity_deg()) logger.info(log_str) logger.info('') logger.info('Time Taken = %f seconds' % (time() - st)) return integrated
def integrate(self, experiments, indexed): # TODO: Figure out if this is necessary and/or how to do this better indexed, _ = self.process_reference(indexed) if self.params.integration.integration_only_overrides.trusted_range: for detector in experiments.detectors(): for panel in detector: panel.set_trusted_range( self.params.integration.integration_only_overrides.trusted_range ) # Get the integrator from the input parameters from dials.algorithms.integration.integrator import create_integrator from dials.algorithms.profile_model.factory import ProfileModelFactory # Compute the profile model # Predict the reflections # Match the predictions with the reference # Create the integrator experiments = ProfileModelFactory.create(self.params, experiments, indexed) new_experiments = ExperimentList() new_reflections = flex.reflection_table() for expt_id, expt in enumerate(experiments): if ( self.params.profile.gaussian_rs.parameters.sigma_b_cutoff is None or expt.profile.sigma_b() < self.params.profile.gaussian_rs.parameters.sigma_b_cutoff ): refls = indexed.select(indexed["id"] == expt_id) refls["id"] = flex.int(len(refls), len(new_experiments)) # refls.reset_ids() del refls.experiment_identifiers()[expt_id] refls.experiment_identifiers()[len(new_experiments)] = expt.identifier new_reflections.extend(refls) new_experiments.append(expt) else: # TODO: this can be done better, also print( "Rejected expt %d with sigma_b %f" % (expt_id, expt.profile.sigma_b()) ) experiments = new_experiments indexed = new_reflections if len(experiments) == 0: raise RuntimeError("No experiments after filtering by sigma_b") predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=self.params.prediction.d_min, dmax=self.params.prediction.d_max, margin=self.params.prediction.margin, force_static=self.params.prediction.force_static, ) predicted.match_with_reference(indexed) integrator = create_integrator(self.params, experiments, predicted) # Integrate the reflections integrated = integrator.integrate() # correct integrated intensities for absorption correction, if necessary for abs_params in self.params.integration.absorption_correction: if abs_params.apply: if abs_params.algorithm == "fuller_kapton": from dials.algorithms.integration.kapton_correction import ( multi_kapton_correction, ) elif abs_params.algorithm == "kapton_2019": from dials.algorithms.integration.kapton_2019_correction import ( multi_kapton_correction, ) experiments, integrated = multi_kapton_correction( experiments, integrated, abs_params.fuller_kapton, logger=logger )() if self.params.significance_filter.enable: from dials.algorithms.integration.stills_significance_filter import ( SignificanceFilter, ) sig_filter = SignificanceFilter(self.params) filtered_refls = sig_filter(experiments, integrated) accepted_expts = ExperimentList() accepted_refls = flex.reflection_table() for expt_id, expt in enumerate(experiments): refls = filtered_refls.select(filtered_refls["id"] == expt_id) if len(refls) > 0: accepted_expts.append(expt) refls["id"] = flex.int(len(refls), len(accepted_expts) - 1) accepted_refls.extend(refls) else: print( "Removed experiment %d which has no reflections left after applying significance filter", expt_id, ) if len(accepted_refls) == 0: raise Sorry("No reflections left after applying significance filter") experiments = accepted_expts integrated = accepted_refls # Delete the shoeboxes used for intermediate calculations, if requested if self.params.integration.debug.delete_shoeboxes and "shoebox" in integrated: del integrated["shoebox"] # Dump experiments to disk if self.params.output.integrated_experiments_filename: experiments.as_json(self.params.output.integrated_experiments_filename) if self.params.output.integrated_filename: # Save the reflections self.save_reflections( integrated, self.params.output.integrated_filename ) self.write_integration_pickles(integrated, experiments) # TODO: Figure out what this is from dials.algorithms.indexing.stills_indexer import ( calc_2D_rmsd_and_displacements, ) rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed) log_str = f"RMSD indexed (px): {rmsd_indexed:f}\n" for i in range(6): bright_integrated = integrated.select( ( integrated["intensity.sum.value"] / flex.sqrt(integrated["intensity.sum.variance"]) ) >= i ) if len(bright_integrated) > 0: rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated) else: rmsd_integrated = 0 log_str += ( "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n" % (i, len(bright_integrated), rmsd_integrated) ) for crystal_model in experiments.crystals(): if hasattr(crystal_model, "get_domain_size_ang"): log_str += ". Final ML model: domain size angstroms: {:f}, half mosaicity degrees: {:f}".format( crystal_model.get_domain_size_ang(), crystal_model.get_half_mosaicity_deg(), ) print(log_str) return integrated