def _find_spots(self, args=None): # type: (Optional[List[str]]) -> None """ Call `dials.find_spots` on the imported experiment list. Args: args (optional): List of any additional PHIL parameters to be used by dials.import. """ info("\nFinding spots...") dials_start = timeit.default_timer() # Use some choice fillets from dials.find_spots # Ignore `args`, use `self.params` # Loop through all the imagesets and find the strong spots self.refls = flex.reflection_table.from_observations( self.expts, self.params.dials_find_spots) # Add n_signal column - before deleting shoeboxes good = MaskCode.Foreground | MaskCode.Valid self.refls["n_signal"] = self.refls["shoebox"].count_mask_values(good) # Delete the shoeboxes if not self.params.dials_find_spots.output.shoeboxes: del self.refls["shoebox"] info( 60 * "-" + "\n%s\n" + 60 * "-" + "\nSuccessfully completed (%.1f sec)", spot_counts_per_image_plot(self.refls), timeit.default_timer() - dials_start, )
def _index_prepare(self): logger.notice(banner("Spotfinding %s" % self.get_indexer_sweep_name())) super(XDSIndexerII, self)._index_prepare() reflections_file = spot_xds_to_reflection_file( self._indxr_payload["SPOT.XDS"], working_directory=self.get_working_directory(), ) refl = flex.reflection_table.from_file(reflections_file) logger.info(spot_counts_per_image_plot(refl))
def _index_prepare(self): Chatter.banner('Spotfinding %s' % self.get_indexer_sweep_name()) super(XDSIndexerII, self)._index_prepare() from dials.array_family import flex from dials.util.ascii_art import spot_counts_per_image_plot reflection_pickle = spot_xds_to_reflection_pickle( self._indxr_payload['SPOT.XDS'], working_directory=self.get_working_directory()) refl = flex.reflection_table.from_pickle(reflection_pickle) Chatter.write(spot_counts_per_image_plot(refl), strip=False)
def _index_prepare(self): Chatter.banner('Spotfinding %s' %self.get_indexer_sweep_name()) super(XDSIndexerII, self)._index_prepare() from dials.array_family import flex from dials.util.ascii_art import spot_counts_per_image_plot reflection_pickle = spot_xds_to_reflection_pickle( self._indxr_payload['SPOT.XDS'], working_directory=self.get_working_directory()) refl = flex.reflection_table.from_pickle(reflection_pickle) Chatter.write(spot_counts_per_image_plot(refl), strip=False)
def _find_spots(self, additional_parameters=None): if additional_parameters is None: additional_parameters = [] info("\nSpot finding...") command = [ "dials.find_spots", self.json_file, "nproc=%s" % self.nproc ] + additional_parameters result = run_process(command, print_stdout=False, debug=procrunner_debug) debug("result = %s" % self._prettyprint_dictionary(result)) if result['exitcode'] != 0: warn("Failed with exit code %d" % result['exitcode']) sys.exit(1) info(60 * '-') from libtbx import easy_pickle from dials.util.ascii_art import spot_counts_per_image_plot refl = easy_pickle.load('strong.pickle') info(spot_counts_per_image_plot(refl)) info(60 * '-') info("Successfully completed (%.1f sec)" % result['runtime'])
def _index_prepare(self): from xia2.Handlers.Citations import Citations Citations.cite('dials') #all_images = self.get_matching_images() #first = min(all_images) #last = max(all_images) spot_lists = [] datablocks = [] for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps): Chatter.banner('Spotfinding %s' % xsweep.get_name()) first, last = imageset.get_scan().get_image_range() # at this stage, break out to run the DIALS code: this sets itself up # now cheat and pass in some information... save re-reading all of the # image headers # FIXME need to adjust this to allow (say) three chunks of images from dxtbx.serialize import dump from dxtbx.datablock import DataBlock sweep_filename = os.path.join( self.get_working_directory(), '%s_datablock.json' % xsweep.get_name()) dump.datablock(DataBlock([imageset]), sweep_filename) genmask = self.GenerateMask() genmask.set_input_datablock(sweep_filename) genmask.set_output_datablock( os.path.join( self.get_working_directory(), '%s_%s_datablock.json' % (genmask.get_xpid(), xsweep.get_name()))) genmask.set_params(PhilIndex.params.dials.masking) sweep_filename, mask_pickle = genmask.run() Debug.write('Generated mask for %s: %s' % (xsweep.get_name(), mask_pickle)) gain = PhilIndex.params.xia2.settings.input.gain if gain is libtbx.Auto: gain_estimater = self.EstimateGain() gain_estimater.set_sweep_filename(sweep_filename) gain_estimater.run() gain = gain_estimater.get_gain() Chatter.write('Estimated gain: %.2f' % gain) PhilIndex.params.xia2.settings.input.gain = gain # FIXME this should really use the assigned spot finding regions #offset = self.get_frame_offset() spotfinder = self.Spotfinder() if last - first > 10: spotfinder.set_write_hot_mask(True) spotfinder.set_input_sweep_filename(sweep_filename) spotfinder.set_output_sweep_filename( '%s_%s_datablock.json' % (spotfinder.get_xpid(), xsweep.get_name())) spotfinder.set_input_spot_filename( '%s_%s_strong.pickle' % (spotfinder.get_xpid(), xsweep.get_name())) if PhilIndex.params.dials.fast_mode: wedges = self._index_select_images_i(imageset) spotfinder.set_scan_ranges(wedges) else: spotfinder.set_scan_ranges([(first, last)]) if PhilIndex.params.dials.find_spots.phil_file is not None: spotfinder.set_phil_file( PhilIndex.params.dials.find_spots.phil_file) min_spot_size = PhilIndex.params.dials.find_spots.min_spot_size if min_spot_size is libtbx.Auto: if imageset.get_detector()[0].get_type() == 'SENSOR_PAD': min_spot_size = 3 else: min_spot_size = None if min_spot_size is not None: spotfinder.set_min_spot_size(min_spot_size) min_local = PhilIndex.params.dials.find_spots.min_local if min_local is not None: spotfinder.set_min_local(min_local) sigma_strong = PhilIndex.params.dials.find_spots.sigma_strong if sigma_strong: spotfinder.set_sigma_strong(sigma_strong) gain = PhilIndex.params.xia2.settings.input.gain if gain: spotfinder.set_gain(gain) filter_ice_rings = PhilIndex.params.dials.find_spots.filter_ice_rings if filter_ice_rings: spotfinder.set_filter_ice_rings(filter_ice_rings) kernel_size = PhilIndex.params.dials.find_spots.kernel_size if kernel_size: spotfinder.set_kernel_size(kernel_size) global_threshold = PhilIndex.params.dials.find_spots.global_threshold if global_threshold is not None: spotfinder.set_global_threshold(global_threshold) spotfinder.run() spot_filename = spotfinder.get_spot_filename() if not os.path.exists(spot_filename): raise RuntimeError("Spotfinding failed: %s does not exist." % os.path.basename(spot_filename)) spot_lists.append(spot_filename) datablocks.append(spotfinder.get_output_sweep_filename()) from libtbx import easy_pickle from dials.util.ascii_art import spot_counts_per_image_plot refl = easy_pickle.load(spot_filename) if not len(refl): raise RuntimeError('No spots found in sweep %s' % xsweep.get_name()) Chatter.write(spot_counts_per_image_plot(refl), strip=False) if not PhilIndex.params.dials.fast_mode: detectblanks = self.DetectBlanks() detectblanks.set_sweep_filename(datablocks[-1]) detectblanks.set_reflections_filename(spot_filename) detectblanks.run() json = detectblanks.get_results() offset = imageset.get_scan().get_image_range()[0] blank_regions = json['strong']['blank_regions'] if len(blank_regions): blank_regions = [(int(s), int(e)) for s, e in blank_regions] for blank_start, blank_end in blank_regions: Chatter.write( 'WARNING: Potential blank images: %i -> %i' % (blank_start + 1, blank_end)) if PhilIndex.params.xia2.settings.remove_blanks: non_blanks = [] start, end = imageset.get_array_range() last_blank_end = start for blank_start, blank_end in blank_regions: if blank_start > start: non_blanks.append( (last_blank_end, blank_start)) last_blank_end = blank_end if last_blank_end + 1 < end: non_blanks.append((last_blank_end, end)) xsweep = self.get_indexer_sweep() xwav = xsweep.get_wavelength() xsample = xsweep.get_xsample() sweep_name = xsweep.get_name() import string for i, (nb_start, nb_end) in enumerate(non_blanks): assert i < 26 if i == 0: sub_imageset = imageset[nb_start - start:nb_end - start] xsweep._frames_to_process = (nb_start + 1, nb_end + 1) self.set_indexer_prepare_done(done=False) self._indxr_imagesets[ self._indxr_imagesets.index( imageset)] = sub_imageset xsweep._integrater._setup_from_imageset( sub_imageset) else: new_name = '_'.join( (sweep_name, string.ascii_lowercase[i])) new_sweep = xwav.add_sweep( new_name, xsample, directory=os.path.join( os.path.basename( xsweep.get_directory()), new_name), image=imageset.get_path(nb_start - start), frames_to_process=(nb_start + 1, nb_end), ) Chatter.write( "Generating new sweep: %s (%s:%i:%i)" % (new_sweep.get_name(), new_sweep.get_image(), new_sweep.get_frames_to_process()[0], new_sweep.get_frames_to_process()[1])) return if not PhilIndex.params.xia2.settings.trust_beam_centre: discovery = self.DiscoverBetterExperimentalModel() discovery.set_sweep_filename(datablocks[-1]) discovery.set_spot_filename(spot_filename) #wedges = self._index_select_images_i(imageset) #discovery.set_scan_ranges(wedges) #discovery.set_scan_ranges([(first + offset, last + offset)]) try: discovery.run() except Exception as e: Debug.write('DIALS beam centre search failed: %s' % str(e)) else: # overwrite datablock.json in datablocks list datablocks[ -1] = discovery.get_optimized_datablock_filename() self.set_indexer_payload("spot_lists", spot_lists) self.set_indexer_payload("datablocks", datablocks) return
def run(self): '''Execute the script.''' from dials.array_family import flex from dials.util.options import flatten_datablocks from time import time from dials.util import log from libtbx.utils import Sorry start_time = time() # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) # Configure the logging log.config(params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version logger.info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) # Ensure we have a data block datablocks = flatten_datablocks(params.input.datablock) if len(datablocks) == 0: self.parser.print_help() return elif len(datablocks) != 1: raise Sorry('only 1 datablock can be processed at a time') # Loop through all the imagesets and find the strong spots reflections = flex.reflection_table.from_observations( datablocks[0], params) # Delete the shoeboxes if not params.output.shoeboxes: del reflections['shoebox'] # ascii spot count per image plot from dials.util.ascii_art import spot_counts_per_image_plot for i, imageset in enumerate(datablocks[0].extract_imagesets()): ascii_plot = spot_counts_per_image_plot( reflections.select(reflections['id'] == i)) if len(ascii_plot): logger.info( '\nHistogram of per-image spot count for imageset %i:' % i) logger.info(ascii_plot) # Save the reflections to file logger.info('\n' + '-' * 80) reflections.as_pickle(params.output.reflections) logger.info('Saved {0} reflections to {1}'.format( len(reflections), params.output.reflections)) # Save the datablock if params.output.datablock: from dxtbx.datablock import DataBlockDumper logger.info('Saving datablocks to {0}'.format( params.output.datablock)) dump = DataBlockDumper(datablocks) dump.as_file(params.output.datablock) # Print some per image statistics if params.per_image_statistics: from dials.algorithms.spot_finding import per_image_analysis from cStringIO import StringIO s = StringIO() for i, imageset in enumerate(datablocks[0].extract_imagesets()): print >> s, "Number of centroids per image for imageset %i:" % i stats = per_image_analysis.stats_imageset( imageset, reflections.select(reflections['id'] == i), resolution_analysis=False) per_image_analysis.print_table(stats, out=s) logger.info(s.getvalue()) # Print the time logger.info("Time Taken: %f" % (time() - start_time))
def run(self, args=None): """Execute the script.""" from dials.array_family import flex from dials.util.options import flatten_experiments from time import time from dials.util import log start_time = time() # Parse the command line params, options = self.parser.parse_args(args=args, show_diff_phil=False) if __name__ == "__main__": # Configure the logging log.config(verbosity=options.verbose, logfile=params.output.log) from dials.util.version import dials_version logger.info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) # Ensure we have a data block experiments = flatten_experiments(params.input.experiments) if len(experiments) == 0: self.parser.print_help() return # Loop through all the imagesets and find the strong spots reflections = flex.reflection_table.from_observations( experiments, params) # Add n_signal column - before deleting shoeboxes from dials.algorithms.shoebox import MaskCode good = MaskCode.Foreground | MaskCode.Valid reflections["n_signal"] = reflections["shoebox"].count_mask_values( good) # Delete the shoeboxes if not params.output.shoeboxes: del reflections["shoebox"] # ascii spot count per image plot from dials.util.ascii_art import spot_counts_per_image_plot for i, experiment in enumerate(experiments): ascii_plot = spot_counts_per_image_plot( reflections.select(reflections["id"] == i)) if len(ascii_plot): logger.info( "\nHistogram of per-image spot count for imageset %i:" % i) logger.info(ascii_plot) # Save the reflections to file logger.info("\n" + "-" * 80) reflections.as_file(params.output.reflections) logger.info("Saved {} reflections to {}".format( len(reflections), params.output.reflections)) # Save the experiments if params.output.experiments: logger.info("Saving experiments to {}".format( params.output.experiments)) experiments.as_file(params.output.experiments) # Print some per image statistics if params.per_image_statistics: from dials.algorithms.spot_finding import per_image_analysis from six.moves import cStringIO as StringIO s = StringIO() for i, experiment in enumerate(experiments): print("Number of centroids per image for imageset %i:" % i, file=s) imageset = experiment.imageset stats = per_image_analysis.stats_imageset( imageset, reflections.select(reflections["id"] == i), resolution_analysis=False, ) per_image_analysis.print_table(stats, out=s) logger.info(s.getvalue()) # Print the time logger.info("Time Taken: %f" % (time() - start_time)) if params.output.experiments: return experiments, reflections else: return reflections
def _index_prepare(self): Citations.cite("dials") # all_images = self.get_matching_images() # first = min(all_images) # last = max(all_images) spot_lists = [] experiments_filenames = [] for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps): logger.notice(banner("Spotfinding %s" % xsweep.get_name())) first, last = imageset.get_scan().get_image_range() # at this stage, break out to run the DIALS code: this sets itself up # now cheat and pass in some information... save re-reading all of the # image headers # FIXME need to adjust this to allow (say) three chunks of images from dxtbx.model.experiment_list import ExperimentListFactory sweep_filename = os.path.join(self.get_working_directory(), "%s_import.expt" % xsweep.get_name()) ExperimentListFactory.from_imageset_and_crystal( imageset, None).as_file(sweep_filename) genmask = self.GenerateMask() genmask.set_input_experiments(sweep_filename) genmask.set_output_experiments( os.path.join( self.get_working_directory(), "%s_%s_masked.expt" % (genmask.get_xpid(), xsweep.get_name()), )) genmask.set_params(PhilIndex.params.dials.masking) sweep_filename, mask_pickle = genmask.run() logger.debug("Generated mask for %s: %s", xsweep.get_name(), mask_pickle) gain = PhilIndex.params.xia2.settings.input.gain if gain is libtbx.Auto: gain_estimater = self.EstimateGain() gain_estimater.set_sweep_filename(sweep_filename) gain_estimater.run() gain = gain_estimater.get_gain() logger.info("Estimated gain: %.2f", gain) PhilIndex.params.xia2.settings.input.gain = gain # FIXME this should really use the assigned spot finding regions # offset = self.get_frame_offset() dfs_params = PhilIndex.params.dials.find_spots spotfinder = self.Spotfinder() if last - first > 10: spotfinder.set_write_hot_mask(True) spotfinder.set_input_sweep_filename(sweep_filename) spotfinder.set_output_sweep_filename( "%s_%s_strong.expt" % (spotfinder.get_xpid(), xsweep.get_name())) spotfinder.set_input_spot_filename( "%s_%s_strong.refl" % (spotfinder.get_xpid(), xsweep.get_name())) if PhilIndex.params.dials.fast_mode: wedges = self._index_select_images_i(imageset) spotfinder.set_scan_ranges(wedges) else: spotfinder.set_scan_ranges([(first, last)]) if dfs_params.phil_file is not None: spotfinder.set_phil_file(dfs_params.phil_file) if dfs_params.min_spot_size is libtbx.Auto: if imageset.get_detector()[0].get_type() == "SENSOR_PAD": dfs_params.min_spot_size = 3 else: dfs_params.min_spot_size = None if dfs_params.min_spot_size is not None: spotfinder.set_min_spot_size(dfs_params.min_spot_size) if dfs_params.min_local is not None: spotfinder.set_min_local(dfs_params.min_local) if dfs_params.sigma_strong: spotfinder.set_sigma_strong(dfs_params.sigma_strong) gain = PhilIndex.params.xia2.settings.input.gain if gain: spotfinder.set_gain(gain) if dfs_params.filter_ice_rings: spotfinder.set_filter_ice_rings(dfs_params.filter_ice_rings) if dfs_params.kernel_size: spotfinder.set_kernel_size(dfs_params.kernel_size) if dfs_params.global_threshold is not None: spotfinder.set_global_threshold(dfs_params.global_threshold) if dfs_params.threshold.algorithm is not None: spotfinder.set_threshold_algorithm( dfs_params.threshold.algorithm) spotfinder.run() spot_filename = spotfinder.get_spot_filename() if not os.path.exists(spot_filename): raise RuntimeError("Spotfinding failed: %s does not exist." % os.path.basename(spot_filename)) spot_lists.append(spot_filename) experiments_filenames.append( spotfinder.get_output_sweep_filename()) refl = flex.reflection_table.from_file(spot_filename) if not len(refl): raise RuntimeError("No spots found in sweep %s" % xsweep.get_name()) logger.info(spot_counts_per_image_plot(refl)) if not PhilIndex.params.dials.fast_mode: detectblanks = self.DetectBlanks() detectblanks.set_sweep_filename(experiments_filenames[-1]) detectblanks.set_reflections_filename(spot_filename) detectblanks.run() json = detectblanks.get_results() blank_regions = json["strong"]["blank_regions"] if len(blank_regions): blank_regions = [(int(s), int(e)) for s, e in blank_regions] for blank_start, blank_end in blank_regions: logger.info( "WARNING: Potential blank images: %i -> %i", blank_start + 1, blank_end, ) if PhilIndex.params.xia2.settings.remove_blanks: non_blanks = [] start, end = imageset.get_array_range() last_blank_end = start for blank_start, blank_end in blank_regions: if blank_start > start: non_blanks.append( (last_blank_end, blank_start)) last_blank_end = blank_end if last_blank_end + 1 < end: non_blanks.append((last_blank_end, end)) xsweep = self.get_indexer_sweep() xwav = xsweep.get_wavelength() xsample = xsweep.sample sweep_name = xsweep.get_name() for i, (nb_start, nb_end) in enumerate(non_blanks): assert i < 26 if i == 0: sub_imageset = imageset[nb_start - start:nb_end - start] xsweep._frames_to_process = (nb_start + 1, nb_end + 1) self.set_indexer_prepare_done(done=False) self._indxr_imagesets[ self._indxr_imagesets.index( imageset)] = sub_imageset xsweep._integrater._setup_from_imageset( sub_imageset) else: min_images = (PhilIndex.params.xia2.settings. input.min_images) if (nb_end - nb_start) < min_images: continue new_name = "_".join( (sweep_name, string.ascii_lowercase[i])) new_sweep = xwav.add_sweep( new_name, xsample, directory=os.path.join( os.path.basename( xsweep.get_directory()), new_name, ), image=imageset.get_path(nb_start - start), frames_to_process=(nb_start + 1, nb_end), beam=xsweep.get_beam_centre(), reversephi=xsweep.get_reversephi(), distance=xsweep.get_distance(), gain=xsweep.get_gain(), dmin=xsweep.get_resolution_high(), dmax=xsweep.get_resolution_low(), polarization=xsweep.get_polarization(), user_lattice=xsweep.get_user_lattice(), user_cell=xsweep.get_user_cell(), epoch=xsweep._epoch, ice=xsweep._ice, excluded_regions=xsweep._excluded_regions, ) logger.info( "Generating new sweep: %s (%s:%i:%i)", new_sweep.get_name(), new_sweep.get_image(), new_sweep.get_frames_to_process()[0], new_sweep.get_frames_to_process()[1], ) return if not PhilIndex.params.xia2.settings.trust_beam_centre: discovery = self.SearchBeamPosition() discovery.set_sweep_filename(experiments_filenames[-1]) discovery.set_spot_filename(spot_filename) # set scan_range to correspond to not more than 180 degrees # if we have > 20000 reflections width = imageset.get_scan().get_oscillation()[1] if (last - first) * width > 180.0 and len(refl) > 20000: end = first + int(round(180.0 / width)) - 1 logger.debug("Using %d to %d for beam search", first, end) discovery.set_image_range((first, end)) try: discovery.run() result = discovery.get_optimized_experiments_filename() # overwrite indexed.expt in experiments list experiments_filenames[-1] = result except Exception as e: logger.debug("DIALS beam centre search failed: %s", str(e), exc_info=True) self.set_indexer_payload("spot_lists", spot_lists) self.set_indexer_payload("experiments", experiments_filenames)
def run(self, args=None): """Execute the script.""" # Parse the command line params, options = self.parser.parse_args(args=args, show_diff_phil=False) if __name__ == "__main__": # Configure the logging log.config(verbosity=options.verbose, logfile=params.output.log) logger.info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) # Ensure we have a data block experiments = flatten_experiments(params.input.experiments) # did input have identifier? had_identifiers = False if all(i != "" for i in experiments.identifiers()): had_identifiers = True else: generate_experiment_identifiers( experiments ) # add identifier e.g. if coming straight from images if len(experiments) == 0: self.parser.print_help() return # If maximum_trusted_value assigned, use this temporarily for the # spot finding if params.maximum_trusted_value is not None: logger.info("Overriding maximum trusted value to %.1f", params.maximum_trusted_value) input_trusted_ranges = {} for _d, detector in enumerate(experiments.detectors()): for _p, panel in enumerate(detector): trusted = panel.get_trusted_range() input_trusted_ranges[(_d, _p)] = trusted panel.set_trusted_range( (trusted[0], params.maximum_trusted_value)) # Loop through all the imagesets and find the strong spots reflections = flex.reflection_table.from_observations( experiments, params) # Add n_signal column - before deleting shoeboxes good = MaskCode.Foreground | MaskCode.Valid reflections["n_signal"] = reflections["shoebox"].count_mask_values( good) # Delete the shoeboxes if not params.output.shoeboxes: del reflections["shoebox"] # ascii spot count per image plot - per imageset imagesets = [] for i, experiment in enumerate(experiments): if experiment.imageset not in imagesets: imagesets.append(experiment.imageset) for imageset in imagesets: selected = flex.bool(reflections.nrows(), False) for i, experiment in enumerate(experiments): if experiment.imageset is not imageset: continue selected.set_selected(reflections["id"] == i, True) ascii_plot = spot_counts_per_image_plot( reflections.select(selected)) if len(ascii_plot): logger.info( "\nHistogram of per-image spot count for imageset %i:" % i) logger.info(ascii_plot) # Save the reflections to file logger.info("\n" + "-" * 80) # If started with images and not saving experiments, then remove id mapping # as the experiment linked to will no longer exists after exit. if not had_identifiers: if not params.output.experiments: for k in reflections.experiment_identifiers().keys(): del reflections.experiment_identifiers()[k] reflections.as_file(params.output.reflections) logger.info("Saved {} reflections to {}".format( len(reflections), params.output.reflections)) # Reset the trusted ranges if params.maximum_trusted_value is not None: for _d, detector in enumerate(experiments.detectors()): for _p, panel in enumerate(detector): trusted = input_trusted_ranges[(_d, _p)] panel.set_trusted_range(trusted) # Save the experiments if params.output.experiments: logger.info("Saving experiments to {}".format( params.output.experiments)) experiments.as_file(params.output.experiments) # Print some per image statistics if params.per_image_statistics: for i, experiment in enumerate(experiments): logger.info("Number of centroids per image for imageset %i:", i) refl = reflections.select(reflections["id"] == i) refl.centroid_px_to_mm([experiment]) refl.map_centroids_to_reciprocal_space([experiment]) stats = per_image_analysis.stats_per_image( experiment, refl, resolution_analysis=False) logger.info(str(stats)) if params.output.experiments: return experiments, reflections else: return reflections
def test_spot_counts_per_image_plot(dials_regression): from libtbx import easy_pickle from dials.util import ascii_art data_dir = os.path.join(dials_regression, "indexing_test_data", "i04_weak_data") pickle_path = os.path.join(data_dir, "full.pickle") refl = easy_pickle.load(pickle_path) output = ascii_art.spot_counts_per_image_plot(refl) expected_output = """\ 116082 spots found on 540 images (max 2075 / bin) * * * * * * * * ** * ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ 1 image 540""" output = "\n".join(line.rstrip() for line in output.splitlines()) assert output == expected_output output = ascii_art.spot_counts_per_image_plot(refl, char="o", width=80, height=15) expected_output = """\ 116082 spots found on 540 images (max 1627 / bin) o o oo o o o o o o o o o o o o o o o oo oo o o o o o o oo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo 1 image 540""" output = "\n".join(line.rstrip() for line in output.splitlines()) assert output == expected_output output = ascii_art.spot_counts_per_image_plot(refl, char="#", width=7, height=10) expected_output = """\ 116082 spots found on 540 images (max 16752 / bin) ####### ####### ####### ####### ####### ####### ####### ####### ####### ####### 1 540""" output = "\n".join(line.rstrip() for line in output.splitlines()) assert output == expected_output output = ascii_art.spot_counts_per_image_plot( refl.select(refl["xyzobs.px.value"].parts()[2] <= 9.5), char="#", width=10, height=15, ) expected_output = """\ 2065 spots found on 10 images (max 361 / bin) # # # # # # # # # # ## ## ## ########## ########## ########## ########## ########## ########## ########## 1 image 10""" output = "\n".join(line.rstrip() for line in output.splitlines()) assert output == expected_output
def run(self): '''Execute the script.''' from dials.array_family import flex from dials.util.options import flatten_datablocks from time import time from dials.util import log from libtbx.utils import Sorry start_time = time() # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) # Configure the logging log.config( params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version logger.info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) # Ensure we have a data block datablocks = flatten_datablocks(params.input.datablock) if len(datablocks) == 0: self.parser.print_help() return elif len(datablocks) != 1: raise Sorry('only 1 datablock can be processed at a time') # Loop through all the imagesets and find the strong spots reflections = flex.reflection_table.from_observations( datablocks[0], params) # Delete the shoeboxes if not params.output.shoeboxes: del reflections['shoebox'] # ascii spot count per image plot from dials.util.ascii_art import spot_counts_per_image_plot for i, imageset in enumerate(datablocks[0].extract_imagesets()): ascii_plot = spot_counts_per_image_plot( reflections.select(reflections['id'] == i)) if len(ascii_plot): logger.info('\nHistogram of per-image spot count for imageset %i:' %i) logger.info(ascii_plot) # Save the reflections to file logger.info('\n' + '-' * 80) reflections.as_pickle(params.output.reflections) logger.info('Saved {0} reflections to {1}'.format( len(reflections), params.output.reflections)) # Save the datablock if params.output.datablock: from dxtbx.datablock import DataBlockDumper logger.info('Saving datablocks to {0}'.format( params.output.datablock)) dump = DataBlockDumper(datablocks) dump.as_file(params.output.datablock) # Print some per image statistics if params.per_image_statistics: from dials.algorithms.spot_finding import per_image_analysis from cStringIO import StringIO s = StringIO() for i, imageset in enumerate(datablocks[0].extract_imagesets()): print >> s, "Number of centroids per image for imageset %i:" %i stats = per_image_analysis.stats_imageset( imageset, reflections.select(reflections['id'] == i), resolution_analysis=False) per_image_analysis.print_table(stats, out=s) logger.info(s.getvalue()) # Print the time logger.info("Time Taken: %f" % (time() - start_time))
def test_spot_counts_per_image_plot(dials_regression): from dials.util import ascii_art data_dir = os.path.join(dials_regression, "indexing_test_data", "i04_weak_data") pickle_path = os.path.join(data_dir, "full.pickle") with open(pickle_path, "rb") as fh: refl = pickle.load(fh, encoding="bytes") output = ascii_art.spot_counts_per_image_plot(refl) expected_output = """\ 116082 spots found on 540 images (max 2075 / bin) * * * * * * * * ** * ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ 1 image 540""" output = "\n".join(line.rstrip() for line in output.splitlines()) assert output == expected_output output = ascii_art.spot_counts_per_image_plot(refl, char="o", width=80, height=15) expected_output = """\ 116082 spots found on 540 images (max 1627 / bin) o o oo o o o o o o o o o o o o o o o oo oo o o o o o o oo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo 1 image 540""" output = "\n".join(line.rstrip() for line in output.splitlines()) assert output == expected_output output = ascii_art.spot_counts_per_image_plot(refl, char="#", width=7, height=10) expected_output = """\ 116082 spots found on 540 images (max 16752 / bin) ####### ####### ####### ####### ####### ####### ####### ####### ####### ####### 1 540""" output = "\n".join(line.rstrip() for line in output.splitlines()) assert output == expected_output output = ascii_art.spot_counts_per_image_plot( refl.select(refl["xyzobs.px.value"].parts()[2] <= 9.5), char="#", width=10, height=15, ) expected_output = """\ 2065 spots found on 10 images (max 361 / bin) # # # # # # # # # # ## ## ## ########## ########## ########## ########## ########## ########## ########## 1 image 10""" output = "\n".join(line.rstrip() for line in output.splitlines()) assert output == expected_output # If we do spotfinding on the first 3 images, then the spot centroids will be in the # range z=0.5 to z=2.5 output = ascii_art.spot_counts_per_image_plot( refl.select(refl["xyzobs.px.value"].parts()[2] <= 2.5), char="#", width=10, height=15, ) expected_output = """\ 624 spots found on 3 images (max 294 / bin) # # # # # # ## ### ### ### ### ### ### ### ### 1 3""" output = "\n".join(line.rstrip() for line in output.splitlines()) assert output == expected_output
def exercise_spot_counts_per_image_plot(): from libtbx import easy_pickle from dials.util import ascii_art from libtbx.test_utils import show_diff if not have_dials_regression: print 'Skipping exercise_spot_counts_per_image_plot(): dials_regression not available' return data_dir = os.path.join(dials_regression, "indexing_test_data", "i04_weak_data") pickle_path = os.path.join(data_dir, "full.pickle") refl = easy_pickle.load(pickle_path) output = ascii_art.spot_counts_per_image_plot(refl) expected_output = '''\ 116082 spots found on 540 images (max 2075 / bin) * * * * * * * * ** * ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ 1 image 540''' output = '\n'.join(line.rstrip() for line in output.splitlines()) assert not show_diff(output, expected_output) output = ascii_art.spot_counts_per_image_plot(refl, char='o', width=80, height=15) expected_output = '''\ 116082 spots found on 540 images (max 1627 / bin) o o oo o o o o o o o o o o o o o o o oo oo o o o o o o oo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo 1 image 540''' output = '\n'.join(line.rstrip() for line in output.splitlines()) assert not show_diff(output, expected_output) output = ascii_art.spot_counts_per_image_plot(refl, char='#', width=7, height=10) expected_output = '''\ 116082 spots found on 540 images (max 16752 / bin) ####### ####### ####### ####### ####### ####### ####### ####### ####### ####### 1 540''' output = '\n'.join(line.rstrip() for line in output.splitlines()) assert not show_diff(output, expected_output) output = ascii_art.spot_counts_per_image_plot( refl.select(refl['xyzobs.px.value'].parts()[2] <= 9.5), char='#', width=10, height=15) expected_output = '''\ 2065 spots found on 10 images (max 361 / bin) # # # # # # # # # # ## ## ## ########## ########## ########## ########## ########## ########## ########## 1 image 10''' output = '\n'.join(line.rstrip() for line in output.splitlines()) assert not show_diff(output, expected_output)
def exercise_spot_counts_per_image_plot(): from libtbx import easy_pickle from dials.util import ascii_art from libtbx.test_utils import show_diff if not have_dials_regression: print 'Skipping exercise_spot_counts_per_image_plot(): dials_regression not available' return data_dir = os.path.join(dials_regression, "indexing_test_data", "i04_weak_data") pickle_path = os.path.join(data_dir, "full.pickle") refl = easy_pickle.load(pickle_path) output = ascii_art.spot_counts_per_image_plot(refl) expected_output = '''\ 116082 spots found on 540 images (max 2082 / bin) * * * * * * * * ** * ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ ************************************************************ 1 image 540''' output = '\n'.join(line.rstrip() for line in output.splitlines()) assert not show_diff(output, expected_output) output = ascii_art.spot_counts_per_image_plot( refl, char='o', width=80, height=15) expected_output = '''\ 116082 spots found on 540 images (max 1568 / bin) o o o o o o o o o o o o ooo ooo o o ooo ooo ooo ooo oooo oo ooo ooo ooo ooo ooooooo ooo ooo ooo ooo oo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo 1 image 540''' output = '\n'.join(line.rstrip() for line in output.splitlines()) assert not show_diff(output, expected_output) output = ascii_art.spot_counts_per_image_plot( refl, char='#', width=7, height=10) expected_output = '''\ 116082 spots found on 540 images (max 16765 / bin) ####### ####### ####### ####### ####### ####### ####### ####### ####### ####### 1 540''' output = '\n'.join(line.rstrip() for line in output.splitlines()) assert not show_diff(output, expected_output) output = ascii_art.spot_counts_per_image_plot( refl.select(refl['xyzobs.px.value'].parts()[2] <= 9.5), char='#', width=10, height=15) expected_output = '''\ 2065 spots found on 10 images (max 259 / bin) # # ### # ##### ## ######### ######### ########## ########## ########## ########## ########## ########## ########## ########## ########## 1 image 10''' output = '\n'.join(line.rstrip() for line in output.splitlines()) assert not show_diff(output, expected_output)
def _index_prepare(self): from xia2.Handlers.Citations import Citations Citations.cite('dials') #all_images = self.get_matching_images() #first = min(all_images) #last = max(all_images) spot_lists = [] datablocks = [] for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps): Chatter.banner('Spotfinding %s' %xsweep.get_name()) first, last = imageset.get_scan().get_image_range() # at this stage, break out to run the DIALS code: this sets itself up # now cheat and pass in some information... save re-reading all of the # image headers # FIXME need to adjust this to allow (say) three chunks of images from dxtbx.serialize import dump from dxtbx.datablock import DataBlock sweep_filename = os.path.join( self.get_working_directory(), '%s_datablock.json' %xsweep.get_name()) dump.datablock(DataBlock([imageset]), sweep_filename) gain = PhilIndex.params.xia2.settings.input.gain if gain is libtbx.Auto: gain_estimater = self.EstimateGain() gain_estimater.set_sweep_filename(sweep_filename) gain_estimater.run() gain = gain_estimater.get_gain() Chatter.write('Estimated gain: %.2f' %gain) PhilIndex.params.xia2.settings.input.gain = gain # FIXME this should really use the assigned spot finding regions #offset = self.get_frame_offset() spotfinder = self.Spotfinder() if last - first > 10: spotfinder.set_write_hot_mask(True) spotfinder.set_input_sweep_filename(sweep_filename) spotfinder.set_output_sweep_filename( '%s_%s_datablock.json' %(spotfinder.get_xpid(), xsweep.get_name())) spotfinder.set_input_spot_filename( '%s_%s_strong.pickle' %(spotfinder.get_xpid(), xsweep.get_name())) if PhilIndex.params.dials.fast_mode: wedges = self._index_select_images_i(imageset) spotfinder.set_scan_ranges(wedges) else: spotfinder.set_scan_ranges([(first, last)]) if PhilIndex.params.dials.find_spots.phil_file is not None: spotfinder.set_phil_file(PhilIndex.params.dials.find_spots.phil_file) min_spot_size = PhilIndex.params.dials.find_spots.min_spot_size if min_spot_size is libtbx.Auto: if imageset.get_detector()[0].get_type() == 'SENSOR_PAD': min_spot_size = 3 else: min_spot_size = None if min_spot_size is not None: spotfinder.set_min_spot_size(min_spot_size) min_local = PhilIndex.params.dials.find_spots.min_local if min_local is not None: spotfinder.set_min_local(min_local) sigma_strong = PhilIndex.params.dials.find_spots.sigma_strong if sigma_strong: spotfinder.set_sigma_strong(sigma_strong) gain = PhilIndex.params.xia2.settings.input.gain if gain: spotfinder.set_gain(gain) filter_ice_rings = PhilIndex.params.dials.find_spots.filter_ice_rings if filter_ice_rings: spotfinder.set_filter_ice_rings(filter_ice_rings) kernel_size = PhilIndex.params.dials.find_spots.kernel_size if kernel_size: spotfinder.set_kernel_size(kernel_size) global_threshold = PhilIndex.params.dials.find_spots.global_threshold if global_threshold is not None: spotfinder.set_global_threshold(global_threshold) spotfinder.run() spot_filename = spotfinder.get_spot_filename() if not os.path.exists(spot_filename): raise RuntimeError("Spotfinding failed: %s does not exist." %os.path.basename(spot_filename)) spot_lists.append(spot_filename) datablocks.append(spotfinder.get_output_sweep_filename()) from libtbx import easy_pickle from dials.util.ascii_art import spot_counts_per_image_plot refl = easy_pickle.load(spot_filename) if not len(refl): raise RuntimeError('No spots found in sweep %s' %xsweep.get_name()) Chatter.write(spot_counts_per_image_plot(refl), strip=False) if not PhilIndex.params.dials.fast_mode: detectblanks = self.DetectBlanks() detectblanks.set_sweep_filename(datablocks[-1]) detectblanks.set_reflections_filename(spot_filename) detectblanks.run() json = detectblanks.get_results() offset = imageset.get_scan().get_image_range()[0] blank_regions = json['strong']['blank_regions'] if len(blank_regions): blank_regions = [(int(s), int(e)) for s, e in blank_regions] for blank_start, blank_end in blank_regions: Chatter.write('WARNING: Potential blank images: %i -> %i' %( blank_start+1, blank_end)) if PhilIndex.params.xia2.settings.remove_blanks: non_blanks = [] start, end = imageset.get_array_range() last_blank_end = start for blank_start, blank_end in blank_regions: if blank_start > start: non_blanks.append((last_blank_end, blank_start)) last_blank_end = blank_end if last_blank_end+1 < end: non_blanks.append((last_blank_end, end)) xsweep = self.get_indexer_sweep() xwav = xsweep.get_wavelength() xsample = xsweep.get_xsample() sweep_name = xsweep.get_name() import string for i, (nb_start, nb_end) in enumerate(non_blanks): assert i < 26 if i == 0: sub_imageset = imageset[nb_start-start:nb_end-start] xsweep._frames_to_process = (nb_start+1, nb_end+1) self.set_indexer_prepare_done(done=False) self._indxr_imagesets[self._indxr_imagesets.index(imageset)] = sub_imageset xsweep._integrater._setup_from_imageset(sub_imageset) else: new_name = '_'.join((sweep_name, string.ascii_lowercase[i])) new_sweep = xwav.add_sweep(new_name, xsample, directory=os.path.join( os.path.basename(xsweep.get_directory()), new_name), image=imageset.get_path(nb_start-start), frames_to_process=(nb_start+1, nb_end), ) Chatter.write("Generating new sweep: %s (%s:%i:%i)" %( new_sweep.get_name(), new_sweep.get_image(), new_sweep.get_frames_to_process()[0], new_sweep.get_frames_to_process()[1])) return if not PhilIndex.params.xia2.settings.trust_beam_centre: discovery = self.DiscoverBetterExperimentalModel() discovery.set_sweep_filename(datablocks[-1]) discovery.set_spot_filename(spot_filename) #wedges = self._index_select_images_i(imageset) #discovery.set_scan_ranges(wedges) #discovery.set_scan_ranges([(first + offset, last + offset)]) try: discovery.run() except Exception, e: Debug.write('DIALS beam centre search failed: %s' %str(e)) else: # overwrite datablock.json in datablocks list datablocks[-1] = discovery.get_optimized_datablock_filename()