def integrate(self, integrator_type): from dials.algorithms.integration.integrator import IntegratorFactory from dials.algorithms.integration.integrator import phil_scope as master_phil_scope from libtbx.phil import parse import sys import StringIO rlist = self.rlist.copy() output = StringIO.StringIO() stdout = sys.stdout sys.stdout = output try: phil_scope = parse(''' integration.background.algorithm=null integration.intensity.algorithm=sum integration.intensity.sum.integrator=%s integration.block.size=0.5 integration.profile_fitting=False ''' % integrator_type) params = master_phil_scope.fetch(source=phil_scope).extract() integrator = IntegratorFactory.create(params, self.exlist, rlist) result = integrator.integrate() except Exception: print output raise sys.stdout = stdout return result
def integrate(self): # Process reference reflections self.indexed, _ = self.process_reference(self.indexed) # Get integrator from input params from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory # Compute the profile model self.experiments = ProfileModelFactory.create(self.phil, self.experiments, self.indexed) # Predict the reflections predicted = flex.reflection_table.from_predictions_multi( self.experiments, dmin=self.phil.prediction.d_min, dmax=self.phil.prediction.d_max, margin=self.phil.prediction.margin, force_static=self.phil.prediction.force_static) # Match the predictions with the reference predicted.match_with_reference(self.indexed) # Create the integrator integrator = IntegratorFactory.create(self.phil, self.experiments, predicted) # Integrate the reflections self.integrated = integrator.integrate() if self.integrated.has_key('intensity.prf.value'): method = 'prf' # integration by profile fitting elif self.integrated.has_key('intensity.sum.value'): method = 'sum' # integration by simple summation self.integrated = self.integrated.select( self.integrated['intensity.' + method + '.variance'] > 0) # keep only spots with sigmas above zero # Save the reflections if selected if self.phil.output.integrated_filename: self.save_reflections(self.integrated, self.phil.output.integrated_filename) self.write_integration_pickles() from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements rmsd_indexed, _ = calc_2D_rmsd_and_displacements(self.indexed) rmsd_integrated, _ = calc_2D_rmsd_and_displacements(self.integrated) crystal_model = self.experiments.crystals()[0]
def integrate(self): # Process reference reflections self.indexed,_ = self.process_reference(self.indexed) # Get integrator from input params from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory # Compute the profile model self.experiments = ProfileModelFactory.create(self.phil, self.experiments, self.indexed) # Predict the reflections predicted = flex.reflection_table.from_predictions_multi( self.experiments, dmin=self.phil.prediction.d_min, dmax=self.phil.prediction.d_max, margin=self.phil.prediction.margin, force_static=self.phil.prediction.force_static) # Match the predictions with the reference predicted.match_with_reference(self.indexed) # Create the integrator integrator = IntegratorFactory.create(self.phil, self.experiments, predicted) # Integrate the reflections self.integrated = integrator.integrate() if self.integrated.has_key('intensity.prf.value'): method = 'prf' # integration by profile fitting elif self.integrated.has_key('intensity.sum.value'): method = 'sum' # integration by simple summation self.integrated = self.integrated.select(self.integrated['intensity.' + method + '.variance'] > 0) # keep only spots with sigmas above zero # Save the reflections if selected if self.phil.output.integrated_filename: self.save_reflections(self.integrated, self.phil.output.integrated_filename) self.write_integration_pickles() from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements rmsd_indexed, _ = calc_2D_rmsd_and_displacements(self.indexed) rmsd_integrated, _ = calc_2D_rmsd_and_displacements(self.integrated) crystal_model = self.experiments.crystals()[0]
def integrate(integrator_type, rlist): from dials.algorithms.integration.integrator import IntegratorFactory from dials.algorithms.integration.integrator import phil_scope as master_phil_scope from libtbx.phil import parse rlist = rlist.copy() phil_scope = parse(''' integration.background.algorithm=null integration.intensity.algorithm=sum integration.intensity.sum.integrator=%s integration.block.size=0.5 integration.profile_fitting=False ''' % integrator_type) params = master_phil_scope.fetch(source=phil_scope).extract() integrator = IntegratorFactory.create(params, exlist, rlist) result = integrator.integrate() return result
def integrate(self, integrator_type): from dials.algorithms.integration.integrator import IntegratorFactory from dials.algorithms.integration.integrator import phil_scope as master_phil_scope from libtbx.phil import parse import sys import StringIO rlist = self.rlist.copy() output = StringIO.StringIO() stdout = sys.stdout sys.stdout = output try: phil_scope = parse(''' integration.background.algorithm=null integration.intensity.algorithm=sum integration.intensity.sum.integrator=%s integration.block.size=0.5 integration.profile_fitting=False ''' % integrator_type) params = master_phil_scope.fetch(source=phil_scope).extract() integrator = IntegratorFactory.create( params, self.exlist, rlist) result = integrator.integrate() except Exception: print output raise sys.stdout = stdout return result
def work(filename, cl=None): if cl is None: cl = [] import libtbx.phil phil_scope = libtbx.phil.parse('''\ ice_rings { filter = True .type = bool width = 0.004 .type = float(value_min=0.0) } index = False .type = bool integrate = False .type = bool indexing_min_spots = 10 .type = int(value_min=1) ''') if not os.access(filename, os.R_OK): raise RuntimeError("Server does not have read access to file %s" % filename) interp = phil_scope.command_line_argument_interpreter() params, unhandled = interp.process_and_fetch( cl, custom_processor='collect_remaining') filter_ice = params.extract().ice_rings.filter ice_rings_width = params.extract().ice_rings.width index = params.extract().index integrate = params.extract().integrate indexing_min_spots = params.extract().indexing_min_spots from dials.command_line.find_spots import phil_scope as find_spots_phil_scope from dxtbx.datablock import DataBlockFactory from dials.array_family import flex interp = find_spots_phil_scope.command_line_argument_interpreter() phil_scope, unhandled = interp.process_and_fetch( unhandled, custom_processor='collect_remaining') logger.info('The following spotfinding parameters have been modified:') logger.info(find_spots_phil_scope.fetch_diff(source=phil_scope).as_str()) params = phil_scope.extract() # no need to write the hot mask in the server/client params.spotfinder.write_hot_mask = False datablock = DataBlockFactory.from_filenames([filename])[0] t0 = time.time() reflections = flex.reflection_table.from_observations(datablock, params) t1 = time.time() logger.info('Spotfinding took %.2f seconds' % (t1 - t0)) from dials.algorithms.spot_finding import per_image_analysis imageset = datablock.extract_imagesets()[0] scan = imageset.get_scan() if scan is not None: i = scan.get_array_range()[0] else: i = 0 stats = per_image_analysis.stats_single_image( imageset, reflections, i=i, plot=False, filter_ice=filter_ice, ice_rings_width=ice_rings_width) stats = stats.__dict__ t2 = time.time() logger.info('Resolution analysis took %.2f seconds' % (t2 - t1)) if index and stats['n_spots_no_ice'] > indexing_min_spots: import logging logging.basicConfig(stream=sys.stdout, level=logging.INFO) from dials.algorithms.indexing import indexer interp = indexer.master_phil_scope.command_line_argument_interpreter() phil_scope, unhandled = interp.process_and_fetch( unhandled, custom_processor='collect_remaining') imagesets = [imageset] logger.info('The following indexing parameters have been modified:') indexer.master_phil_scope.fetch_diff(source=phil_scope).show() params = phil_scope.extract() params.indexing.scan_range = [] if (imageset.get_goniometer() is not None and imageset.get_scan() is not None and imageset.get_scan().get_oscillation()[1] == 0): imageset.set_goniometer(None) imageset.set_scan(None) try: idxr = indexer.indexer_base.from_parameters(reflections, imagesets, params=params) indexing_results = [] idxr.index() indexed_sel = idxr.refined_reflections.get_flags( idxr.refined_reflections.flags.indexed) indexed_sel &= ~(idxr.refined_reflections.get_flags( idxr.refined_reflections.flags.centroid_outlier)) for i_expt, expt in enumerate(idxr.refined_experiments): sel = idxr.refined_reflections['id'] == i_expt sel &= indexed_sel indexing_results.append({ 'crystal': expt.crystal.to_dict(), 'n_indexed': sel.count(True), 'fraction_indexed': sel.count(True) / sel.size() }) stats['lattices'] = indexing_results stats['n_indexed'] = indexed_sel.count(True) stats['fraction_indexed'] = indexed_sel.count(True) / len( reflections) except Exception as e: logger.error(e) stats['error'] = str(e) #stats.crystal = None #stats.n_indexed = None #stats.fraction_indexed = None finally: t3 = time.time() logger.info('Indexing took %.2f seconds' % (t3 - t2)) if integrate and 'lattices' in stats: from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory from dials.command_line.integrate import phil_scope as integrate_phil_scope interp = integrate_phil_scope.command_line_argument_interpreter() phil_scope, unhandled = interp.process_and_fetch( unhandled, custom_processor='collect_remaining') imagesets = [imageset] logger.error( 'The following integration parameters have been modified:') integrate_phil_scope.fetch_diff(source=phil_scope).show() params = phil_scope.extract() try: params.profile.gaussian_rs.min_spots = 0 experiments = idxr.refined_experiments reference = idxr.refined_reflections predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=params.prediction.d_min, dmax=params.prediction.d_max, margin=params.prediction.margin, force_static=params.prediction.force_static) matched, reference, unmatched = predicted.match_with_reference( reference) assert (len(matched) == len(predicted)) assert (matched.count(True) <= len(reference)) if matched.count(True) == 0: raise Sorry(''' Invalid input for reference reflections. Zero reference spots were matched to predictions ''') elif matched.count(True) != len(reference): logger.info('') logger.info('*' * 80) logger.info( 'Warning: %d reference spots were not matched to predictions' % (len(reference) - matched.count(True))) logger.info('*' * 80) logger.info('') # Compute the profile model experiments = ProfileModelFactory.create( params, experiments, reference) # Compute the bounding box predicted.compute_bbox(experiments) # Create the integrator integrator = IntegratorFactory.create(params, experiments, predicted) # Integrate the reflections reflections = integrator.integrate() #print len(reflections) stats['integrated_intensity'] = flex.sum( reflections['intensity.sum.value']) except Exception as e: logger.error(e) stats['error'] = str(e) finally: t4 = time.time() logger.info('Integration took %.2f seconds' % (t4 - t3)) return stats
def work(filename, cl=None): if cl is None: cl = [] phil_scope = libtbx.phil.parse("""\ ice_rings { filter = True .type = bool width = 0.004 .type = float(value_min=0.0) } index = False .type = bool integrate = False .type = bool indexing_min_spots = 10 .type = int(value_min=1) """) interp = phil_scope.command_line_argument_interpreter() params, unhandled = interp.process_and_fetch( cl, custom_processor="collect_remaining") filter_ice = params.extract().ice_rings.filter ice_rings_width = params.extract().ice_rings.width index = params.extract().index integrate = params.extract().integrate indexing_min_spots = params.extract().indexing_min_spots from dials.command_line.find_spots import phil_scope as find_spots_phil_scope from dxtbx.model.experiment_list import ExperimentListFactory from dials.array_family import flex interp = find_spots_phil_scope.command_line_argument_interpreter() phil_scope, unhandled = interp.process_and_fetch( unhandled, custom_processor="collect_remaining") logger.info("The following spotfinding parameters have been modified:") logger.info(find_spots_phil_scope.fetch_diff(source=phil_scope).as_str()) params = phil_scope.extract() # no need to write the hot mask in the server/client params.spotfinder.write_hot_mask = False experiments = ExperimentListFactory.from_filenames([filename]) t0 = time.time() reflections = flex.reflection_table.from_observations(experiments, params) t1 = time.time() logger.info("Spotfinding took %.2f seconds" % (t1 - t0)) from dials.algorithms.spot_finding import per_image_analysis imageset = experiments.imagesets()[0] reflections.centroid_px_to_mm(experiments) reflections.map_centroids_to_reciprocal_space(experiments) stats = per_image_analysis.stats_for_reflection_table( reflections, filter_ice=filter_ice, ice_rings_width=ice_rings_width)._asdict() t2 = time.time() logger.info("Resolution analysis took %.2f seconds" % (t2 - t1)) if index and stats["n_spots_no_ice"] > indexing_min_spots: logging.basicConfig(stream=sys.stdout, level=logging.INFO) from dials.algorithms.indexing import indexer from dials.command_line.index import phil_scope as index_phil_scope interp = index_phil_scope.command_line_argument_interpreter() phil_scope, unhandled = interp.process_and_fetch( unhandled, custom_processor="collect_remaining") logger.info("The following indexing parameters have been modified:") index_phil_scope.fetch_diff(source=phil_scope).show() params = phil_scope.extract() if (imageset.get_goniometer() is not None and imageset.get_scan() is not None and imageset.get_scan().is_still()): imageset.set_goniometer(None) imageset.set_scan(None) try: idxr = indexer.Indexer.from_parameters(reflections, experiments, params=params) indexing_results = [] idxr.index() indexed_sel = idxr.refined_reflections.get_flags( idxr.refined_reflections.flags.indexed) indexed_sel &= ~(idxr.refined_reflections.get_flags( idxr.refined_reflections.flags.centroid_outlier)) for i_expt, expt in enumerate(idxr.refined_experiments): sel = idxr.refined_reflections["id"] == i_expt sel &= indexed_sel indexing_results.append({ "crystal": expt.crystal.to_dict(), "n_indexed": sel.count(True), "fraction_indexed": sel.count(True) / sel.size(), }) stats["lattices"] = indexing_results stats["n_indexed"] = indexed_sel.count(True) stats["fraction_indexed"] = indexed_sel.count(True) / len( reflections) except Exception as e: logger.error(e) stats["error"] = str(e) finally: t3 = time.time() logger.info("Indexing took %.2f seconds" % (t3 - t2)) if integrate and "lattices" in stats: from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory from dials.command_line.integrate import phil_scope as integrate_phil_scope interp = integrate_phil_scope.command_line_argument_interpreter() phil_scope, unhandled = interp.process_and_fetch( unhandled, custom_processor="collect_remaining") logger.error( "The following integration parameters have been modified:") integrate_phil_scope.fetch_diff(source=phil_scope).show() params = phil_scope.extract() try: params.profile.gaussian_rs.min_spots = 0 experiments = idxr.refined_experiments reference = idxr.refined_reflections predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=params.prediction.d_min, dmax=params.prediction.d_max, margin=params.prediction.margin, force_static=params.prediction.force_static, ) matched, reference, unmatched = predicted.match_with_reference( reference) assert len(matched) == len(predicted) assert matched.count(True) <= len(reference) if matched.count(True) == 0: raise Sorry(""" Invalid input for reference reflections. Zero reference spots were matched to predictions """) elif matched.count(True) != len(reference): logger.info("") logger.info("*" * 80) logger.info( "Warning: %d reference spots were not matched to predictions" % (len(reference) - matched.count(True))) logger.info("*" * 80) logger.info("") # Compute the profile model experiments = ProfileModelFactory.create( params, experiments, reference) # Compute the bounding box predicted.compute_bbox(experiments) # Create the integrator integrator = IntegratorFactory.create(params, experiments, predicted) # Integrate the reflections reflections = integrator.integrate() # print len(reflections) stats["integrated_intensity"] = flex.sum( reflections["intensity.sum.value"]) except Exception as e: logger.error(e) stats["error"] = str(e) finally: t4 = time.time() logger.info("Integration took %.2f seconds" % (t4 - t3)) return stats
def integrate(self, experiments, indexed): from time import time st = time() logger.info('*' * 80) logger.info('Integrating Reflections') logger.info('*' * 80) indexed,_ = self.process_reference(indexed) # Get the integrator from the input parameters logger.info('Configuring integrator from input parameters') from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory from dials.array_family import flex # Compute the profile model # Predict the reflections # Match the predictions with the reference # Create the integrator experiments = ProfileModelFactory.create(self.params, experiments, indexed) logger.info("") logger.info("=" * 80) logger.info("") logger.info("Predicting reflections") logger.info("") predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=self.params.prediction.d_min, dmax=self.params.prediction.d_max, margin=self.params.prediction.margin, force_static=self.params.prediction.force_static) predicted.match_with_reference(indexed) logger.info("") integrator = IntegratorFactory.create(self.params, experiments, predicted) # Integrate the reflections integrated = integrator.integrate() # Select only those reflections which were integrated if 'intensity.prf.variance' in integrated: selection = integrated.get_flags( integrated.flags.integrated, all=True) else: selection = integrated.get_flags( integrated.flags.integrated_sum) integrated = integrated.select(selection) len_all = len(integrated) integrated = integrated.select(~integrated.get_flags(integrated.flags.foreground_includes_bad_pixels)) print "Filtering %d reflections with at least one bad foreground pixel out of %d"%(len_all-len(integrated), len_all) # verify sigmas are sensible if 'intensity.prf.value' in integrated: if (integrated['intensity.prf.variance'] <= 0).count(True) > 0: raise Sorry("Found negative variances") if 'intensity.sum.value' in integrated: if (integrated['intensity.sum.variance'] <= 0).count(True) > 0: raise Sorry("Found negative variances") # apply detector gain to summation variances integrated['intensity.sum.variance'] *= self.params.integration.summation.detector_gain if 'background.sum.value' in integrated: if (integrated['background.sum.variance'] < 0).count(True) > 0: raise Sorry("Found negative variances") if (integrated['background.sum.variance'] == 0).count(True) > 0: print "Filtering %d reflections with zero background variance" % ((integrated['background.sum.variance'] == 0).count(True)) integrated = integrated.select(integrated['background.sum.variance'] > 0) # apply detector gain to background summation variances integrated['background.sum.variance'] *= self.params.integration.summation.detector_gain if self.params.output.integrated_filename: # Save the reflections self.save_reflections(integrated, self.params.output.integrated_filename) self.write_integration_pickles(integrated, experiments) from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed) log_str = "RMSD indexed (px): %f\n"%(rmsd_indexed) for i in xrange(6): bright_integrated = integrated.select((integrated['intensity.sum.value']/flex.sqrt(integrated['intensity.sum.variance']))>=i) if len(bright_integrated) > 0: rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated) else: rmsd_integrated = 0 log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"%(i, len(bright_integrated), rmsd_integrated) crystal_model = experiments.crystals()[0] if hasattr(crystal_model, '._ML_domain_size_ang'): log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg) logger.info(log_str) logger.info('') logger.info('Time Taken = %f seconds' % (time() - st)) return integrated
def integrate(self, experiments, indexed): from time import time from logging import info st = time() info('*' * 80) info('Integrating Reflections') info('*' * 80) indexed,_ = self.process_reference(indexed) # Get the integrator from the input parameters info('Configuring integrator from input parameters') from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory from dials.array_family import flex # Compute the profile model # Predict the reflections # Match the predictions with the reference # Create the integrator experiments = ProfileModelFactory.create(self.params, experiments, indexed) info("") info("=" * 80) info("") info("Predicting reflections") info("") predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=self.params.prediction.d_min, dmax=self.params.prediction.d_max, margin=self.params.prediction.margin, force_static=self.params.prediction.force_static) predicted.match_with_reference(indexed) info("") integrator = IntegratorFactory.create(self.params, experiments, predicted) # Integrate the reflections integrated = integrator.integrate() if integrated.has_key('intensity.prf.value'): method = 'prf' # integration by profile fitting elif integrated.has_key('intensity.sum.value'): method = 'sum' # integration by simple summation integrated = integrated.select(integrated['intensity.' + method + '.variance'] > 0) # keep only spots with sigmas above zero if self.params.output.integrated_filename: # Save the reflections self.save_reflections(integrated, self.params.output.integrated_filename) self.write_integration_pickles(integrated, experiments) from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed) rmsd_integrated, _ = calc_2D_rmsd_and_displacements(integrated) crystal_model = experiments.crystals()[0] print "Integrated. RMSD indexed,", rmsd_indexed, "RMSD integrated", rmsd_integrated, \ "Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg) info('') info('Time Taken = %f seconds' % (time() - st)) return integrated
def run(): parser = OptionParser( phil = phil_scope) params, options = parser.parse_args(show_diff_phil=True) assert params.input.single_img is not None assert params.output_dir is not None # load the image img = dxtbx.load(params.input.single_img) imgset = MemImageSet([img]) datablock = DataBlockFactory.from_imageset(imgset)[0] spotfinder = SpotFinderFactory.from_parameters(params) reflections = spotfinder(datablock) base_name = os.path.splitext(params.input.single_img)[0] reflections.as_pickle(os.path.join(params.output_dir, base_name + "_strong.pickle")) # DGW commented out as reflections.minimum_number_of_reflections no longer exists #if len(reflections) < params.refinement.reflections.minimum_number_of_reflections: # print "Not enough spots to index" # return # create the spot finder print "Spotfinder spots found:", len(reflections) if params.indexing.method == "fft3d": from dials.algorithms.indexing.fft3d import indexer_fft3d as indexer elif params.indexing.method == "fft1d": from dials.algorithms.indexing.fft1d import indexer_fft1d as indexer elif params.method == "real_space_grid_search": from dials.algorithms.indexing.real_space_grid_search \ import indexer_real_space_grid_search as indexer try: idxr = indexer(reflections, [imgset], params=params.indexing) except (RuntimeError, Sorry) as e: print str(e) return indexed = idxr.refined_reflections experiments = idxr.refined_experiments #from dxtbx.model.experiment.experiment_list import ExperimentListDumper #dump = ExperimentListDumper(experiments) #dump.as_json(os.path.join(params.output_dir, base_name + "_experiments.json")) indexed.as_pickle(os.path.join(params.output_dir, base_name + "_indexed.pickle")) refiner = RefinerFactory.from_parameters_data_experiments( params, indexed, experiments) refiner.run() refined_experiments = refiner.get_experiments() #dump = ExperimentListDumper(refined_experiments) #dump.as_json(os.path.join(params.output_dir, base_name + "_refined.json")) # Compute the profile model # Predict the reflections # Match the predictions with the reference # Create the integrator reference = indexed reference = process_reference(reference) profile_model = ProfileModelFactory.create(params, refined_experiments, reference) predicted = flex.reflection_table.from_predictions_multi( refined_experiments, dmin=params.prediction.dmin, dmax=params.prediction.dmax, margin=params.prediction.margin, force_static=params.prediction.force_static) predicted.match_with_reference(reference) integrator = IntegratorFactory.create(params, experiments, profile_model, predicted) # Integrate the reflections integrated = integrator.integrate() integrated.as_pickle(os.path.join(params.output_dir, base_name + "_integrated.pickle"))
print "Predicting Reflections" rlist = flex.reflection_table.from_predictions(experiments[0]) rlist['id'] = flex.int(len(rlist), 0) rlist.compute_bbox(experiments, profile_model) rlist.compute_zeta_multi(experiments) rlist.compute_d(experiments) print "" print "Creating params" from dials.algorithms.integration.integrator import IntegratorFactory from dials.algorithms.integration.integrator import phil_scope from libtbx import phil user_phil = phil.parse(''' integration { mp.max_procs = %d block.size=5 filter.ice_rings.filter=False intensity.algorithm=sum3d } ''' % nproc) working_phil = phil_scope.fetch(source=user_phil) params = working_phil.extract() print "Integrating" integrator = IntegratorFactory.create(params, experiments, profile_model, rlist) result = integrator.integrate() result.as_pickle("temp.pickle")
rlist = flex.reflection_table.from_predictions(experiments[0]) rlist["id"] = flex.int(len(rlist), 0) rlist.compute_bbox(experiments, profile_model) rlist.compute_zeta_multi(experiments) rlist.compute_d(experiments) print("") print("Creating params") from dials.algorithms.integration.integrator import IntegratorFactory from dials.algorithms.integration.integrator import phil_scope from libtbx import phil user_phil = phil.parse(""" integration { mp.max_procs = %d block.size=5 filter.ice_rings.filter=False intensity.algorithm=sum3d } """ % nproc) working_phil = phil_scope.fetch(source=user_phil) params = working_phil.extract() print("Integrating") integrator = IntegratorFactory.create(params, experiments, profile_model, rlist) result = integrator.integrate() result.as_pickle("temp.pickle")
elif matched.count(True) != len(reference): logger.info('') logger.info('*' * 80) logger.info('Warning: %d reference spots were not matched to predictions' % ( len(reference) - matched.count(True))) logger.info('*' * 80) logger.info('') # Compute the profile model experiments = ProfileModelFactory.create(params, experiments, reference) # Compute the bounding box predicted.compute_bbox(experiments) # Create the integrator integrator = IntegratorFactory.create(params, experiments, predicted) # Integrate the reflections reflections = integrator.integrate() #print len(reflections) stats['integrated_intensity'] = flex.sum(reflections['intensity.sum.value']) except Exception, e: logger.error(e) stats['error'] = str(e) finally: t4 = time.time() logger.info('Integration took %.2f seconds' %(t4-t3)) return stats
def run(): parser = OptionParser(phil=phil_scope) params, options = parser.parse_args(show_diff_phil=True) assert params.input.single_img is not None assert params.output_dir is not None # load the image img = dxtbx.load(params.input.single_img) imgset = MemImageSet([img]) datablock = DataBlockFactory.from_imageset(imgset)[0] spotfinder = SpotFinderFactory.from_parameters(params) reflections = spotfinder(datablock) base_name = os.path.splitext(params.input.single_img)[0] reflections.as_pickle( os.path.join(params.output_dir, base_name + "_strong.pickle")) # DGW commented out as reflections.minimum_number_of_reflections no longer exists # if len(reflections) < params.refinement.reflections.minimum_number_of_reflections: # print "Not enough spots to index" # return # create the spot finder print("Spotfinder spots found:", len(reflections)) if params.indexing.method == "fft3d": from dials.algorithms.indexing.fft3d import indexer_fft3d as indexer elif params.indexing.method == "fft1d": from dials.algorithms.indexing.fft1d import indexer_fft1d as indexer elif params.method == "real_space_grid_search": from dials.algorithms.indexing.real_space_grid_search import ( indexer_real_space_grid_search as indexer, ) try: idxr = indexer(reflections, [imgset], params=params.indexing) except (RuntimeError, Sorry) as e: print(str(e)) return indexed = idxr.refined_reflections experiments = idxr.refined_experiments # from dxtbx.model.experiment.experiment_list import ExperimentListDumper # dump = ExperimentListDumper(experiments) # dump.as_json(os.path.join(params.output_dir, base_name + "_experiments.json")) indexed.as_pickle( os.path.join(params.output_dir, base_name + "_indexed.pickle")) refiner = RefinerFactory.from_parameters_data_experiments( params, indexed, experiments) refiner.run() refined_experiments = refiner.get_experiments() # dump = ExperimentListDumper(refined_experiments) # dump.as_json(os.path.join(params.output_dir, base_name + "_refined.json")) # Compute the profile model # Predict the reflections # Match the predictions with the reference # Create the integrator reference = indexed reference = process_reference(reference) profile_model = ProfileModelFactory.create(params, refined_experiments, reference) predicted = flex.reflection_table.from_predictions_multi( refined_experiments, dmin=params.prediction.dmin, dmax=params.prediction.dmax, margin=params.prediction.margin, force_static=params.prediction.force_static, ) predicted.match_with_reference(reference) integrator = IntegratorFactory.create(params, experiments, profile_model, predicted) # Integrate the reflections integrated = integrator.integrate() integrated.as_pickle( os.path.join(params.output_dir, base_name + "_integrated.pickle"))
elif matched.count(True) != len(reference): logger.info('') logger.info('*' * 80) logger.info('Warning: %d reference spots were not matched to predictions' % ( len(reference) - matched.count(True))) logger.info('*' * 80) logger.info('') # Compute the profile model experiments = ProfileModelFactory.create(params, experiments, reference) # Compute the bounding box predicted.compute_bbox(experiments) # Create the integrator integrator = IntegratorFactory.create(params, experiments, predicted) # Integrate the reflections reflections = integrator.integrate() #print len(reflections) stats['integrated_intensity'] = flex.sum(reflections['intensity.sum.value']) except Exception, e: logger.error(e) stats['error'] = str(e) finally: t4 = time.time() logger.info('Integration took %.2f seconds' %(t4-t3)) return stats
def run(self): """ Perform the integration. """ from dials.util.command_line import heading from dials.util.options import flatten_reflections, flatten_experiments from dials.util import log from logging import info, debug from time import time from libtbx.utils import Sorry # Check the number of arguments is correct start_time = time() # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) reference = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) if len(reference) == 0 and len(experiments) == 0: self.parser.print_help() return if len(reference) == 0: reference = None elif len(reference) != 1: raise Sorry("more than 1 reflection file was given") else: reference = reference[0] if len(experiments) == 0: raise Sorry("no experiment list was specified") # Save phil parameters if params.output.phil is not None: with open(params.output.phil, "w") as outfile: outfile.write(self.parser.diff_phil.as_str()) # Configure logging log.config(params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not "": info("The following parameters have been modified:\n") info(diff_phil) # Print if we're using a mask for i, exp in enumerate(experiments): mask = exp.imageset.external_lookup.mask if mask.filename is not None: info("Using external mask: %s" % mask.filename) info(" Mask has %d pixels masked" % mask.data.count(False)) # Print the experimental models for i, exp in enumerate(experiments): debug("Models for experiment %d" % i) debug("") debug(str(exp.beam)) debug(str(exp.detector)) if exp.goniometer: debug(str(exp.goniometer)) if exp.scan: debug(str(exp.scan)) debug(str(exp.crystal)) info("=" * 80) info("") info(heading("Initialising")) info("") # Load the data reference, rubbish = self.process_reference(reference) info("") # Initialise the integrator from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory from dials.array_family import flex # Modify experiment list if scan range is set. experiments, reference = self.split_for_scan_range(experiments, reference, params.scan_range) # Predict the reflections info("") info("=" * 80) info("") info(heading("Predicting reflections")) info("") predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=params.prediction.d_min, dmax=params.prediction.d_max, margin=params.prediction.margin, force_static=params.prediction.force_static, ) # Match reference with predicted if reference: matched, reference, unmatched = predicted.match_with_reference(reference) assert len(matched) == len(predicted) assert matched.count(True) <= len(reference) if matched.count(True) == 0: raise Sorry( """ Invalid input for reference reflections. Zero reference spots were matched to predictions """ ) elif len(unmatched) != 0: info("") info("*" * 80) info("Warning: %d reference spots were not matched to predictions" % (len(unmatched))) info("*" * 80) info("") rubbish.extend(unmatched) # Select a random sample of the predicted reflections if not params.sampling.integrate_all_reflections: predicted = self.sample_predictions(experiments, predicted, params) # Compute the profile model if reference is not None and params.create_profile_model: experiments = ProfileModelFactory.create(params, experiments, reference) else: for expr in experiments: expr.profile.params = params.profile del reference # Compute the bounding box predicted.compute_bbox(experiments) # Create the integrator info("") integrator = IntegratorFactory.create(params, experiments, predicted) # Integrate the reflections reflections = integrator.integrate() # Append rubbish data onto the end if rubbish is not None and params.output.include_bad_reference: mask = flex.bool(len(rubbish), True) rubbish.unset_flags(mask, rubbish.flags.integrated_sum) rubbish.unset_flags(mask, rubbish.flags.integrated_prf) rubbish.set_flags(mask, rubbish.flags.bad_reference) reflections.extend(rubbish) # Save the reflections self.save_reflections(reflections, params.output.reflections) self.save_experiments(experiments, params.output.experiments) # Write a report if requested if params.output.report is not None: integrator.report().as_file(params.output.report) # Print the total time taken info("\nTotal time taken: %f" % (time() - start_time))
def run(self): ''' Perform the integration. ''' from dials.util.command_line import heading from dials.util.options import flatten_reflections, flatten_experiments from dials.util import log from time import time from libtbx.utils import Sorry # Check the number of arguments is correct start_time = time() # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) reference = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) if len(reference) == 0 and len(experiments) == 0: self.parser.print_help() return if len(reference) == 0: reference = None elif len(reference) != 1: raise Sorry('more than 1 reflection file was given') else: reference = reference[0] if len(experiments) == 0: raise Sorry('no experiment list was specified') # Save phil parameters if params.output.phil is not None: with open(params.output.phil, "w") as outfile: outfile.write(self.parser.diff_phil.as_str()) # Configure logging log.config( params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version logger.info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) # Print if we're using a mask for i, exp in enumerate(experiments): mask = exp.imageset.external_lookup.mask if mask.filename is not None: if mask.data: logger.info('Using external mask: %s' % mask.filename) logger.info(' Mask has %d pixels masked' % mask.data.count(False)) # Print the experimental models for i, exp in enumerate(experiments): logger.debug("Models for experiment %d" % i) logger.debug("") logger.debug(str(exp.beam)) logger.debug(str(exp.detector)) if exp.goniometer: logger.debug(str(exp.goniometer)) if exp.scan: logger.debug(str(exp.scan)) logger.debug(str(exp.crystal)) logger.info("=" * 80) logger.info("") logger.info(heading("Initialising")) logger.info("") # Load the data reference, rubbish = self.process_reference(reference) logger.info("") # Initialise the integrator from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory from dials.array_family import flex # Modify experiment list if scan range is set. experiments, reference = self.split_for_scan_range( experiments, reference, params.scan_range) # Predict the reflections logger.info("") logger.info("=" * 80) logger.info("") logger.info(heading("Predicting reflections")) logger.info("") predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=params.prediction.d_min, dmax=params.prediction.d_max, margin=params.prediction.margin, force_static=params.prediction.force_static) # Match reference with predicted if reference: matched, reference, unmatched = predicted.match_with_reference(reference) assert(len(matched) == len(predicted)) assert(matched.count(True) <= len(reference)) if matched.count(True) == 0: raise Sorry(''' Invalid input for reference reflections. Zero reference spots were matched to predictions ''') elif len(unmatched) != 0: logger.info('') logger.info('*' * 80) logger.info('Warning: %d reference spots were not matched to predictions' % ( len(unmatched))) logger.info('*' * 80) logger.info('') rubbish.extend(unmatched) if len(experiments) > 1: # filter out any experiments without matched reference reflections # f_: filtered from dxtbx.model.experiment.experiment_list import ExperimentList f_reference = flex.reflection_table() f_predicted = flex.reflection_table() f_rubbish = flex.reflection_table() f_experiments = ExperimentList() good_expt_count = 0 def refl_extend(src, dest, eid): tmp = src.select(src['id'] == eid) tmp['id'] = flex.int(len(tmp), good_expt_count) dest.extend(tmp) for expt_id, experiment in enumerate(experiments): if len(reference.select(reference['id'] == expt_id)) != 0: refl_extend(reference, f_reference, expt_id) refl_extend(predicted, f_predicted, expt_id) refl_extend(rubbish, f_rubbish, expt_id) f_experiments.append(experiment) good_expt_count += 1 else: logger.info("Removing experiment %d: no reference reflections matched to predictions"%expt_id) reference = f_reference predicted = f_predicted experiments = f_experiments rubbish = f_rubbish # Select a random sample of the predicted reflections if not params.sampling.integrate_all_reflections: predicted = self.sample_predictions(experiments, predicted, params) # Compute the profile model if (params.create_profile_model and reference is not None and "shoebox" in reference): experiments = ProfileModelFactory.create(params, experiments, reference) else: for expr in experiments: if expr.profile is None: raise Sorry('No profile information in experiment list') expr.profile.params = params.profile del reference # Compute the bounding box predicted.compute_bbox(experiments) # Create the integrator logger.info("") integrator = IntegratorFactory.create(params, experiments, predicted) # Integrate the reflections reflections = integrator.integrate() # Append rubbish data onto the end if rubbish is not None and params.output.include_bad_reference: mask = flex.bool(len(rubbish), True) rubbish.unset_flags(mask, rubbish.flags.integrated_sum) rubbish.unset_flags(mask, rubbish.flags.integrated_prf) rubbish.set_flags(mask, rubbish.flags.bad_reference) reflections.extend(rubbish) # Save the reflections self.save_reflections(reflections, params.output.reflections) self.save_experiments(experiments, params.output.experiments) # Write a report if requested if params.output.report is not None: integrator.report().as_file(params.output.report) # Print the total time taken logger.info("\nTotal time taken: %f" % (time() - start_time))
def integrate(self, experiments, indexed): from time import time st = time() logger.info('*' * 80) logger.info('Integrating Reflections') logger.info('*' * 80) indexed,_ = self.process_reference(indexed) # Get the integrator from the input parameters logger.info('Configuring integrator from input parameters') from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory from dials.array_family import flex # Compute the profile model # Predict the reflections # Match the predictions with the reference # Create the integrator experiments = ProfileModelFactory.create(self.params, experiments, indexed) logger.info("") logger.info("=" * 80) logger.info("") logger.info("Predicting reflections") logger.info("") predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=self.params.prediction.d_min, dmax=self.params.prediction.d_max, margin=self.params.prediction.margin, force_static=self.params.prediction.force_static) predicted.match_with_reference(indexed) logger.info("") integrator = IntegratorFactory.create(self.params, experiments, predicted) # Integrate the reflections integrated = integrator.integrate() # Select only those reflections which were integrated if 'intensity.prf.variance' in integrated: selection = integrated.get_flags( integrated.flags.integrated, all=True) else: selection = integrated.get_flags( integrated.flags.integrated_sum) integrated = integrated.select(selection) len_all = len(integrated) integrated = integrated.select(~integrated.get_flags(integrated.flags.foreground_includes_bad_pixels)) print "Filtering %d reflections with at least one bad foreground pixel out of %d"%(len_all-len(integrated), len_all) # verify sigmas are sensible if 'intensity.prf.value' in integrated: if (integrated['intensity.prf.variance'] <= 0).count(True) > 0: raise Sorry("Found negative variances") if 'intensity.sum.value' in integrated: if (integrated['intensity.sum.variance'] <= 0).count(True) > 0: raise Sorry("Found negative variances") # apply detector gain to summation variances integrated['intensity.sum.variance'] *= self.params.integration.summation.detector_gain if 'background.sum.value' in integrated: if (integrated['background.sum.variance'] < 0).count(True) > 0: raise Sorry("Found negative variances") if (integrated['background.sum.variance'] == 0).count(True) > 0: print "Filtering %d reflections with zero background variance" % ((integrated['background.sum.variance'] == 0).count(True)) integrated = integrated.select(integrated['background.sum.variance'] > 0) # apply detector gain to background summation variances integrated['background.sum.variance'] *= self.params.integration.summation.detector_gain # correct integrated intensities for absorption correction, if necessary for abs_params in self.params.integration.absorption_correction: if abs_params.apply and abs_params.algorithm == "fuller_kapton": from dials.algorithms.integration.kapton_correction import multi_kapton_correction experiments, integrated = multi_kapton_correction(experiments, integrated, abs_params.fuller_kapton, logger=logger)() if self.params.significance_filter.enable: from dials.algorithms.integration.stills_significance_filter import SignificanceFilter sig_filter = SignificanceFilter(self.params) refls = sig_filter(experiments, integrated) logger.info("Removed %d reflections out of %d when applying significance filter"%(len(integrated)-len(refls), len(integrated))) if len(refls) == 0: raise Sorry("No reflections left after applying significance filter") integrated = refls if self.params.output.integrated_filename: # Save the reflections self.save_reflections(integrated, self.params.output.integrated_filename) self.write_integration_pickles(integrated, experiments) from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed) log_str = "RMSD indexed (px): %f\n"%(rmsd_indexed) for i in xrange(6): bright_integrated = integrated.select((integrated['intensity.sum.value']/flex.sqrt(integrated['intensity.sum.variance']))>=i) if len(bright_integrated) > 0: rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated) else: rmsd_integrated = 0 log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"%(i, len(bright_integrated), rmsd_integrated) for crystal_model in experiments.crystals(): if hasattr(crystal_model, '_ML_domain_size_ang'): log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg) logger.info(log_str) logger.info('') logger.info('Time Taken = %f seconds' % (time() - st)) return integrated
def integrate(self, experiments, indexed): from time import time st = time() logger.info('*' * 80) logger.info('Integrating Reflections') logger.info('*' * 80) indexed, _ = self.process_reference(indexed) # Get the integrator from the input parameters logger.info('Configuring integrator from input parameters') from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory from dials.array_family import flex # Compute the profile model # Predict the reflections # Match the predictions with the reference # Create the integrator experiments = ProfileModelFactory.create(self.params, experiments, indexed) logger.info("") logger.info("=" * 80) logger.info("") logger.info("Predicting reflections") logger.info("") predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=self.params.prediction.d_min, dmax=self.params.prediction.d_max, margin=self.params.prediction.margin, force_static=self.params.prediction.force_static) predicted.match_with_reference(indexed) logger.info("") integrator = IntegratorFactory.create(self.params, experiments, predicted) # Integrate the reflections integrated = integrator.integrate() # correct integrated intensities for absorption correction, if necessary for abs_params in self.params.integration.absorption_correction: if abs_params.apply and abs_params.algorithm == "fuller_kapton": from dials.algorithms.integration.kapton_correction import multi_kapton_correction experiments, integrated = multi_kapton_correction( experiments, integrated, abs_params.fuller_kapton, logger=logger)() if self.params.significance_filter.enable: from dials.algorithms.integration.stills_significance_filter import SignificanceFilter sig_filter = SignificanceFilter(self.params) refls = sig_filter(experiments, integrated) logger.info( "Removed %d reflections out of %d when applying significance filter" % (len(integrated) - len(refls), len(integrated))) if len(refls) == 0: raise Sorry( "No reflections left after applying significance filter") integrated = refls # Delete the shoeboxes used for intermediate calculations, if requested if self.params.integration.debug.delete_shoeboxes and 'shoebox' in integrated: del integrated['shoebox'] if self.params.output.composite_output: if self.params.output.integrated_experiments_filename or self.params.output.integrated_filename: assert self.params.output.integrated_experiments_filename is not None and self.params.output.integrated_filename is not None from dials.array_family import flex n = len(self.all_integrated_experiments) self.all_integrated_experiments.extend(experiments) for i, experiment in enumerate(experiments): refls = integrated.select(integrated['id'] == i) refls['id'] = flex.int(len(refls), n) self.all_integrated_reflections.extend(refls) n += 1 else: # Dump experiments to disk if self.params.output.integrated_experiments_filename: from dxtbx.model.experiment_list import ExperimentListDumper dump = ExperimentListDumper(experiments) dump.as_json( self.params.output.integrated_experiments_filename) if self.params.output.integrated_filename: # Save the reflections self.save_reflections(integrated, self.params.output.integrated_filename) self.write_integration_pickles(integrated, experiments) from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed) log_str = "RMSD indexed (px): %f\n" % (rmsd_indexed) for i in xrange(6): bright_integrated = integrated.select( (integrated['intensity.sum.value'] / flex.sqrt(integrated['intensity.sum.variance'])) >= i) if len(bright_integrated) > 0: rmsd_integrated, _ = calc_2D_rmsd_and_displacements( bright_integrated) else: rmsd_integrated = 0 log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n" % ( i, len(bright_integrated), rmsd_integrated) for crystal_model in experiments.crystals(): if hasattr(crystal_model, 'get_domain_size_ang'): log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f" % ( crystal_model.get_domain_size_ang(), crystal_model.get_half_mosaicity_deg()) logger.info(log_str) logger.info('') logger.info('Time Taken = %f seconds' % (time() - st)) return integrated
def run(self, args=None): """ Perform the integration. """ from dials.util.command_line import heading from dials.util.options import flatten_reflections, flatten_experiments from dials.util import log from time import time from dials.util import Sorry # Check the number of arguments is correct start_time = time() # Parse the command line params, options = self.parser.parse_args(args=args, show_diff_phil=False) reference = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) if len(reference) == 0 and len(experiments) == 0: self.parser.print_help() return if len(reference) == 0: reference = None elif len(reference) != 1: raise Sorry("more than 1 reflection file was given") else: reference = reference[0] if len(experiments) == 0: raise Sorry("no experiment list was specified") # Save phil parameters if params.output.phil is not None: with open(params.output.phil, "w") as outfile: outfile.write(self.parser.diff_phil.as_str()) if __name__ == "__main__": # Configure logging log.config(params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version logger.info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) for abs_params in params.absorption_correction: if abs_params.apply: if not (params.integration.debug.output and not params.integration.debug.separate_files): raise Sorry( "Shoeboxes must be saved to integration intermediates to apply an absorption correction. " + "Set integration.debug.output=True, integration.debug.separate_files=False and " + "integration.debug.delete_shoeboxes=True to temporarily store shoeboxes." ) # Print if we're using a mask for i, exp in enumerate(experiments): mask = exp.imageset.external_lookup.mask if mask.filename is not None: if mask.data: logger.info("Using external mask: %s" % mask.filename) for tile in mask.data: logger.info(" Mask has %d pixels masked" % tile.data().count(False)) # Print the experimental models for i, exp in enumerate(experiments): logger.info("=" * 80) logger.info("") logger.info("Experiments") logger.info("") logger.info("Models for experiment %d" % i) logger.info("") logger.info(str(exp.beam)) logger.info(str(exp.detector)) if exp.goniometer: logger.info(str(exp.goniometer)) if exp.scan: logger.info(str(exp.scan)) logger.info(str(exp.crystal)) logger.info("=" * 80) logger.info("") logger.info(heading("Initialising")) logger.info("") # Load the data reference, rubbish = self.process_reference(reference) # Check pixels don't belong to neighbours if reference is not None: if exp.goniometer is not None and exp.scan is not None: self.filter_reference_pixels(reference, experiments) logger.info("") # Initialise the integrator from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory # Modify experiment list if scan range is set. experiments, reference = self.split_for_scan_range( experiments, reference, params.scan_range) # Modify experiment list if exclude images is set experiments = self.exclude_images(experiments, params.exclude_images) # Predict the reflections logger.info("") logger.info("=" * 80) logger.info("") logger.info(heading("Predicting reflections")) logger.info("") predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=params.prediction.d_min, dmax=params.prediction.d_max, margin=params.prediction.margin, force_static=params.prediction.force_static, padding=params.prediction.padding, ) # Match reference with predicted if reference: matched, reference, unmatched = predicted.match_with_reference( reference) assert len(matched) == len(predicted) assert matched.count(True) <= len(reference) if matched.count(True) == 0: raise Sorry(""" Invalid input for reference reflections. Zero reference spots were matched to predictions """) elif len(unmatched) != 0: logger.info("") logger.info("*" * 80) logger.info( "Warning: %d reference spots were not matched to predictions" % (len(unmatched))) logger.info("*" * 80) logger.info("") rubbish.extend(unmatched) if len(experiments) > 1: # filter out any experiments without matched reference reflections # f_: filtered from dxtbx.model.experiment_list import ExperimentList f_reference = flex.reflection_table() f_predicted = flex.reflection_table() f_rubbish = flex.reflection_table() f_experiments = ExperimentList() good_expt_count = 0 def refl_extend(src, dest, eid): tmp = src.select(src["id"] == eid) tmp["id"] = flex.int(len(tmp), good_expt_count) dest.extend(tmp) for expt_id, experiment in enumerate(experiments): if len(reference.select(reference["id"] == expt_id)) != 0: refl_extend(reference, f_reference, expt_id) refl_extend(predicted, f_predicted, expt_id) refl_extend(rubbish, f_rubbish, expt_id) f_experiments.append(experiment) good_expt_count += 1 else: logger.info( "Removing experiment %d: no reference reflections matched to predictions" % expt_id) reference = f_reference predicted = f_predicted experiments = f_experiments rubbish = f_rubbish # Select a random sample of the predicted reflections if not params.sampling.integrate_all_reflections: predicted = self.sample_predictions(experiments, predicted, params) # Compute the profile model if (params.create_profile_model and reference is not None and "shoebox" in reference): experiments = ProfileModelFactory.create(params, experiments, reference) else: experiments = ProfileModelFactory.create(params, experiments) for expr in experiments: if expr.profile is None: raise Sorry("No profile information in experiment list") del reference # Compute the bounding box predicted.compute_bbox(experiments) # Create the integrator logger.info("") integrator = IntegratorFactory.create(params, experiments, predicted) # Integrate the reflections reflections = integrator.integrate() # Append rubbish data onto the end if rubbish is not None and params.output.include_bad_reference: mask = flex.bool(len(rubbish), True) rubbish.unset_flags(mask, rubbish.flags.integrated_sum) rubbish.unset_flags(mask, rubbish.flags.integrated_prf) rubbish.set_flags(mask, rubbish.flags.bad_reference) reflections.extend(rubbish) # Correct integrated intensities for absorption correction, if necessary for abs_params in params.absorption_correction: if abs_params.apply and abs_params.algorithm == "fuller_kapton": from dials.algorithms.integration.kapton_correction import ( multi_kapton_correction, ) experiments, reflections = multi_kapton_correction( experiments, reflections, abs_params.fuller_kapton, logger=logger)() if params.significance_filter.enable: from dials.algorithms.integration.stills_significance_filter import ( SignificanceFilter, ) from dxtbx.model.experiment_list import ExperimentList sig_filter = SignificanceFilter(params) filtered_refls = sig_filter(experiments, reflections) accepted_expts = ExperimentList() accepted_refls = flex.reflection_table() logger.info( "Removed %d reflections out of %d when applying significance filter" % (len(reflections) - len(filtered_refls), len(reflections))) for expt_id, expt in enumerate(experiments): refls = filtered_refls.select(filtered_refls["id"] == expt_id) if len(refls) > 0: accepted_expts.append(expt) refls["id"] = flex.int(len(refls), len(accepted_expts) - 1) accepted_refls.extend(refls) else: logger.info( "Removed experiment %d which has no reflections left after applying significance filter" % expt_id) if len(accepted_refls) == 0: raise Sorry( "No reflections left after applying significance filter") experiments = accepted_expts reflections = accepted_refls # Delete the shoeboxes used for intermediate calculations, if requested if params.integration.debug.delete_shoeboxes and "shoebox" in reflections: del reflections["shoebox"] # Save the reflections self.save_reflections(reflections, params.output.reflections) self.save_experiments(experiments, params.output.experiments) # Write a report if requested if params.output.report is not None: integrator.report().as_file(params.output.report) # Print the total time taken logger.info("\nTotal time taken: %f" % (time() - start_time)) return experiments, reflections
def run(self): ''' Perform the integration. ''' from dials.util.command_line import heading from dials.util.options import flatten_reflections, flatten_experiments from dials.util import log from time import time from libtbx.utils import Sorry # Check the number of arguments is correct start_time = time() # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) reference = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) if len(reference) == 0 and len(experiments) == 0: self.parser.print_help() return if len(reference) == 0: reference = None elif len(reference) != 1: raise Sorry('more than 1 reflection file was given') else: reference = reference[0] if len(experiments) == 0: raise Sorry('no experiment list was specified') # Save phil parameters if params.output.phil is not None: with open(params.output.phil, "w") as outfile: outfile.write(self.parser.diff_phil.as_str()) # Configure logging log.config(params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version logger.info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) # Print if we're using a mask for i, exp in enumerate(experiments): mask = exp.imageset.external_lookup.mask if mask.filename is not None: if mask.data: logger.info('Using external mask: %s' % mask.filename) logger.info(' Mask has %d pixels masked' % mask.data.count(False)) # Print the experimental models for i, exp in enumerate(experiments): logger.debug("Models for experiment %d" % i) logger.debug("") logger.debug(str(exp.beam)) logger.debug(str(exp.detector)) if exp.goniometer: logger.debug(str(exp.goniometer)) if exp.scan: logger.debug(str(exp.scan)) logger.debug(str(exp.crystal)) logger.info("=" * 80) logger.info("") logger.info(heading("Initialising")) logger.info("") # Load the data reference, rubbish = self.process_reference(reference) logger.info("") # Initialise the integrator from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory from dials.array_family import flex # Modify experiment list if scan range is set. experiments, reference = self.split_for_scan_range( experiments, reference, params.scan_range) # Predict the reflections logger.info("") logger.info("=" * 80) logger.info("") logger.info(heading("Predicting reflections")) logger.info("") predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=params.prediction.d_min, dmax=params.prediction.d_max, margin=params.prediction.margin, force_static=params.prediction.force_static) # Match reference with predicted if reference: matched, reference, unmatched = predicted.match_with_reference( reference) assert (len(matched) == len(predicted)) assert (matched.count(True) <= len(reference)) if matched.count(True) == 0: raise Sorry(''' Invalid input for reference reflections. Zero reference spots were matched to predictions ''') elif len(unmatched) != 0: logger.info('') logger.info('*' * 80) logger.info( 'Warning: %d reference spots were not matched to predictions' % (len(unmatched))) logger.info('*' * 80) logger.info('') rubbish.extend(unmatched) if len(experiments) > 1: # filter out any experiments without matched reference reflections # f_: filtered from dxtbx.model.experiment.experiment_list import ExperimentList f_reference = flex.reflection_table() f_predicted = flex.reflection_table() f_rubbish = flex.reflection_table() f_experiments = ExperimentList() good_expt_count = 0 def refl_extend(src, dest, eid): tmp = src.select(src['id'] == eid) tmp['id'] = flex.int(len(tmp), good_expt_count) dest.extend(tmp) for expt_id, experiment in enumerate(experiments): if len(reference.select(reference['id'] == expt_id)) != 0: refl_extend(reference, f_reference, expt_id) refl_extend(predicted, f_predicted, expt_id) refl_extend(rubbish, f_rubbish, expt_id) f_experiments.append(experiment) good_expt_count += 1 else: logger.info( "Removing experiment %d: no reference reflections matched to predictions" % expt_id) reference = f_reference predicted = f_predicted experiments = f_experiments rubbish = f_rubbish # Select a random sample of the predicted reflections if not params.sampling.integrate_all_reflections: predicted = self.sample_predictions(experiments, predicted, params) # Compute the profile model if (params.create_profile_model and reference is not None and "shoebox" in reference): experiments = ProfileModelFactory.create(params, experiments, reference) else: for expr in experiments: if expr.profile is None: raise Sorry('No profile information in experiment list') expr.profile.params = params.profile del reference # Compute the bounding box predicted.compute_bbox(experiments) # Create the integrator logger.info("") integrator = IntegratorFactory.create(params, experiments, predicted) # Integrate the reflections reflections = integrator.integrate() # Append rubbish data onto the end if rubbish is not None and params.output.include_bad_reference: mask = flex.bool(len(rubbish), True) rubbish.unset_flags(mask, rubbish.flags.integrated_sum) rubbish.unset_flags(mask, rubbish.flags.integrated_prf) rubbish.set_flags(mask, rubbish.flags.bad_reference) reflections.extend(rubbish) # Save the reflections self.save_reflections(reflections, params.output.reflections) self.save_experiments(experiments, params.output.experiments) # Write a report if requested if params.output.report is not None: integrator.report().as_file(params.output.report) # Print the total time taken logger.info("\nTotal time taken: %f" % (time() - start_time))