def run_indexing(datablock, strong_spots, crystal_model, rmsds): cwd = os.path.abspath(os.curdir) tmp_dir = os.path.abspath(open_tmp_directory(suffix="test_dials_index")) os.chdir(tmp_dir) sweep_path = os.path.join(tmp_dir, "datablock.json") pickle_path = os.path.join(tmp_dir, "strong.pickle") dump.datablock(datablock, sweep_path) easy_pickle.dump(pickle_path, strong_spots) from dials.test.algorithms.indexing.tst_index import run_one_indexing space_group_info = crystal_model.get_space_group() symmetry = crystal.symmetry(unit_cell=crystal_model.get_unit_cell(), space_group=crystal_model.get_space_group()) expected_rmsds = [1.1*r for r in rmsds] imageset = datablock[0].extract_imagesets()[0] pixel_size = imageset.get_detector()[0].get_pixel_size() phi_width = imageset.get_scan().get_oscillation()[1] * math.pi/180 expected_rmsds = [1.1 * rmsds[0] * pixel_size[0], 1.1 * rmsds[1] * pixel_size[1], 1.1 * rmsds[2] * phi_width] run_one_indexing(pickle_path=pickle_path, sweep_path=sweep_path, extra_args=[], expected_unit_cell=symmetry.minimum_cell().unit_cell(), expected_rmsds=expected_rmsds, #expected_hall_symbol=crystal_model.get_space_group().type().hall_symbol(), expected_hall_symbol=' P 1', )
def run_indexing(datablock, strong_spots, crystal_model, rmsds): sweep_path = "datablock.json" pickle_path = "strong.pickle" dump.datablock(datablock, sweep_path) easy_pickle.dump(pickle_path, strong_spots) space_group_info = crystal_model.get_space_group() symmetry = crystal.symmetry(unit_cell=crystal_model.get_unit_cell(), space_group=crystal_model.get_space_group()) expected_rmsds = [1.1 * r for r in rmsds] imageset = datablock[0].extract_imagesets()[0] pixel_size = imageset.get_detector()[0].get_pixel_size() phi_width = imageset.get_scan().get_oscillation()[1] * math.pi / 180 expected_rmsds = [ 1.1 * rmsds[0] * pixel_size[0], 1.1 * rmsds[1] * pixel_size[1], 1.1 * rmsds[2] * phi_width ] run_one_indexing( pickle_path=pickle_path, sweep_path=sweep_path, extra_args=[], expected_unit_cell=symmetry.minimum_cell().unit_cell(), expected_rmsds=expected_rmsds, expected_hall_symbol=' P 1', )
def run(args): import libtbx.load_env from dials.util import log usage = "%s [options] datablock.json strong.pickle" % libtbx.env.dispatcher_name parser = OptionParser(usage=usage, phil=phil_scope, read_datablocks=True, read_reflections=True, check_format=False, epilog=help_message) params, options = parser.parse_args(show_diff_phil=False) datablocks = flatten_datablocks(params.input.datablock) reflections = flatten_reflections(params.input.reflections) if len(datablocks) == 0 or len(reflections) == 0: parser.print_help() exit(0) # Configure the logging log.config(info=params.output.log, debug=params.output.debug_log) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) imagesets = [] for datablock in datablocks: imagesets.extend(datablock.extract_imagesets()) assert len(imagesets) > 0 assert len(reflections) == len(imagesets) if params.scan_range is not None and len(params.scan_range) > 0: reflections = [ filter_reflections_by_scan_range(refl, params.scan_range) for refl in reflections ] dps_params = dps_phil_scope.extract() # for development, we want an exhaustive plot of beam probability map: dps_params.indexing.plot_search_scope = params.plot_search_scope dps_params.indexing.mm_search_scope = params.mm_search_scope new_detector, new_beam = discover_better_experimental_model( imagesets, reflections, params, dps_params, nproc=params.nproc, wide_search_binning=params.wide_search_binning) for imageset in imagesets: imageset.set_detector(new_detector) imageset.set_beam(new_beam) from dxtbx.serialize import dump dump.datablock(datablock, params.output.datablock)
def run(args): import libtbx.load_env from libtbx.utils import Sorry from dials.util import log usage = "%s [options] datablock.json strong.pickle" %libtbx.env.dispatcher_name parser = OptionParser( usage=usage, phil=phil_scope, read_datablocks=True, read_reflections=True, check_format=False, epilog=help_message) params, options = parser.parse_args(show_diff_phil=False) datablocks = flatten_datablocks(params.input.datablock) reflections = flatten_reflections(params.input.reflections) if len(datablocks) == 0 or len(reflections) == 0: parser.print_help() exit(0) # Configure the logging log.config( info=params.output.log, debug=params.output.debug_log) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil is not '': info('The following parameters have been modified:\n') info(diff_phil) imagesets = [] for datablock in datablocks: imagesets.extend(datablock.extract_imagesets()) assert len(imagesets) > 0 assert len(reflections) == len(imagesets) if params.scan_range is not None and len(params.scan_range) > 0: reflections = [ filter_reflections_by_scan_range(refl, params.scan_range) for refl in reflections] dps_params = dps_phil_scope.extract() # for development, we want an exhaustive plot of beam probability map: dps_params.indexing.plot_search_scope = params.plot_search_scope dps_params.indexing.mm_search_scope = params.mm_search_scope new_detector, new_beam = discover_better_experimental_model( imagesets, reflections, params, dps_params, nproc=params.nproc, wide_search_binning=params.wide_search_binning) for imageset in imagesets: imageset.set_detector(new_detector) imageset.set_beam(new_beam) from dxtbx.serialize import dump dump.datablock(datablock, params.output.datablock)
def save_datablock(filename): from xia2.Schema import imageset_cache from dxtbx.datablock import DataBlock from dxtbx.serialize import dump datablock = DataBlock([]) for imagesets in imageset_cache.values(): for imageset in imagesets.values(): datablock.append(imageset) dump.datablock(datablock, filename, compact=True)
def run(args): from dials.util.options import OptionParser from dials.util.options import flatten_datablocks from dials.util.options import flatten_experiments import libtbx.load_env usage = "%s [options] datablock.json | experiments.json" %( libtbx.env.dispatcher_name) parser = OptionParser( usage=usage, phil=phil_scope, read_datablocks=True, read_experiments=True, check_format=False, epilog=help_message) params, options = parser.parse_args(show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) datablocks = flatten_datablocks(params.input.datablock) if len(experiments) == 0 and len(datablocks) == 0: parser.print_help() exit(0) from dials.command_line.dials_import import ManualGeometryUpdater update_geometry = ManualGeometryUpdater(params) if len(experiments): imagesets = experiments.imagesets() elif len(datablocks): assert len(datablocks) == 1 imagesets = datablocks[0].extract_imagesets() for imageset in imagesets: imageset_new = update_geometry(imageset) imageset.set_detector(imageset_new.get_detector()) imageset.set_beam(imageset_new.get_beam()) imageset.set_goniometer(imageset_new.get_goniometer()) imageset.set_scan(imageset_new.get_scan()) from dxtbx.serialize import dump if len(experiments): print "Saving modified experiments to %s" %params.output.experiments dump.experiment_list(experiments, params.output.experiments) elif len(datablocks): print "Saving modified datablock to %s" %params.output.datablock dump.datablock(datablocks, params.output.datablock)
def run_indexing(datablock, strong_spots, crystal_model, rmsds): cwd = os.path.abspath(os.curdir) tmp_dir = os.path.abspath(open_tmp_directory(suffix="test_dials_index")) os.chdir(tmp_dir) sweep_path = os.path.join(tmp_dir, "datablock.json") pickle_path = os.path.join(tmp_dir, "strong.pickle") dump.datablock(datablock, sweep_path) easy_pickle.dump(pickle_path, strong_spots) from dials.test.algorithms.indexing.tst_index import run_one_indexing space_group_info = crystal_model.get_space_group() symmetry = crystal.symmetry(unit_cell=crystal_model.get_unit_cell(), space_group=crystal_model.get_space_group()) expected_rmsds = [1.1 * r for r in rmsds] imageset = datablock[0].extract_imagesets()[0] pixel_size = imageset.get_detector()[0].get_pixel_size() phi_width = imageset.get_scan().get_oscillation()[1] * math.pi / 180 expected_rmsds = [ 1.1 * rmsds[0] * pixel_size[0], 1.1 * rmsds[1] * pixel_size[1], 1.1 * rmsds[2] * phi_width ] run_one_indexing( pickle_path=pickle_path, sweep_path=sweep_path, extra_args=[], expected_unit_cell=symmetry.minimum_cell().unit_cell(), expected_rmsds=expected_rmsds, #expected_hall_symbol=crystal_model.get_space_group().type().hall_symbol(), expected_hall_symbol=' P 1', )
def run(args): from dials.util.options import OptionParser from dials.util.options import flatten_datablocks import libtbx.load_env usage = "%s [options] datablock.json reference=reference_datablock.json" % ( libtbx.env.dispatcher_name ) parser = OptionParser( usage=usage, phil=phil_scope, read_datablocks=True, check_format=False, epilog=help_message, ) params, options = parser.parse_args(show_diff_phil=True) datablocks = flatten_datablocks(params.input.datablock) if len(datablocks) == 0: parser.print_help() exit() # Load reference geometry reference_detector = None if params.input.reference is not None: from dxtbx.serialize import load try: reference_experiments = load.experiment_list( params.input.reference, check_format=False ) assert len(reference_experiments.detectors()) == 1 reference_detector = reference_experiments.detectors()[0] except Exception as e: reference_datablocks = load.datablock(params.input.reference) assert len(reference_datablocks) == 1 imageset = reference_datablocks[0].extract_imagesets()[0] reference_detector = imageset.get_detector() assert len(datablocks) == 1 imageset = datablocks[0].extract_imagesets()[0] detector = imageset.get_detector() h = detector.hierarchy() href = reference_detector.hierarchy() assert len(h) == len(href) assert ( abs(h.get_directed_distance() - href.get_directed_distance()) < params.max_delta_distance ) for panel, panel_ref in zip(h.children(), href.children()): panel.set_local_frame( panel_ref.get_local_fast_axis(), panel_ref.get_local_slow_axis(), panel_ref.get_local_origin(), ) print("Writing metrology-corrected datablock to %s" % params.output.datablock) from dxtbx.serialize import dump dump.datablock(datablocks, params.output.datablock) return
def _index_prepare(self): from xia2.Handlers.Citations import Citations Citations.cite('dials') #all_images = self.get_matching_images() #first = min(all_images) #last = max(all_images) spot_lists = [] datablocks = [] for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps): Chatter.banner('Spotfinding %s' % xsweep.get_name()) first, last = imageset.get_scan().get_image_range() # at this stage, break out to run the DIALS code: this sets itself up # now cheat and pass in some information... save re-reading all of the # image headers # FIXME need to adjust this to allow (say) three chunks of images from dxtbx.serialize import dump from dxtbx.datablock import DataBlock sweep_filename = os.path.join( self.get_working_directory(), '%s_datablock.json' % xsweep.get_name()) dump.datablock(DataBlock([imageset]), sweep_filename) genmask = self.GenerateMask() genmask.set_input_datablock(sweep_filename) genmask.set_output_datablock( os.path.join( self.get_working_directory(), '%s_%s_datablock.json' % (genmask.get_xpid(), xsweep.get_name()))) genmask.set_params(PhilIndex.params.dials.masking) sweep_filename, mask_pickle = genmask.run() Debug.write('Generated mask for %s: %s' % (xsweep.get_name(), mask_pickle)) gain = PhilIndex.params.xia2.settings.input.gain if gain is libtbx.Auto: gain_estimater = self.EstimateGain() gain_estimater.set_sweep_filename(sweep_filename) gain_estimater.run() gain = gain_estimater.get_gain() Chatter.write('Estimated gain: %.2f' % gain) PhilIndex.params.xia2.settings.input.gain = gain # FIXME this should really use the assigned spot finding regions #offset = self.get_frame_offset() spotfinder = self.Spotfinder() if last - first > 10: spotfinder.set_write_hot_mask(True) spotfinder.set_input_sweep_filename(sweep_filename) spotfinder.set_output_sweep_filename( '%s_%s_datablock.json' % (spotfinder.get_xpid(), xsweep.get_name())) spotfinder.set_input_spot_filename( '%s_%s_strong.pickle' % (spotfinder.get_xpid(), xsweep.get_name())) if PhilIndex.params.dials.fast_mode: wedges = self._index_select_images_i(imageset) spotfinder.set_scan_ranges(wedges) else: spotfinder.set_scan_ranges([(first, last)]) if PhilIndex.params.dials.find_spots.phil_file is not None: spotfinder.set_phil_file( PhilIndex.params.dials.find_spots.phil_file) min_spot_size = PhilIndex.params.dials.find_spots.min_spot_size if min_spot_size is libtbx.Auto: if imageset.get_detector()[0].get_type() == 'SENSOR_PAD': min_spot_size = 3 else: min_spot_size = None if min_spot_size is not None: spotfinder.set_min_spot_size(min_spot_size) min_local = PhilIndex.params.dials.find_spots.min_local if min_local is not None: spotfinder.set_min_local(min_local) sigma_strong = PhilIndex.params.dials.find_spots.sigma_strong if sigma_strong: spotfinder.set_sigma_strong(sigma_strong) gain = PhilIndex.params.xia2.settings.input.gain if gain: spotfinder.set_gain(gain) filter_ice_rings = PhilIndex.params.dials.find_spots.filter_ice_rings if filter_ice_rings: spotfinder.set_filter_ice_rings(filter_ice_rings) kernel_size = PhilIndex.params.dials.find_spots.kernel_size if kernel_size: spotfinder.set_kernel_size(kernel_size) global_threshold = PhilIndex.params.dials.find_spots.global_threshold if global_threshold is not None: spotfinder.set_global_threshold(global_threshold) spotfinder.run() spot_filename = spotfinder.get_spot_filename() if not os.path.exists(spot_filename): raise RuntimeError("Spotfinding failed: %s does not exist." % os.path.basename(spot_filename)) spot_lists.append(spot_filename) datablocks.append(spotfinder.get_output_sweep_filename()) from libtbx import easy_pickle from dials.util.ascii_art import spot_counts_per_image_plot refl = easy_pickle.load(spot_filename) if not len(refl): raise RuntimeError('No spots found in sweep %s' % xsweep.get_name()) Chatter.write(spot_counts_per_image_plot(refl), strip=False) if not PhilIndex.params.dials.fast_mode: detectblanks = self.DetectBlanks() detectblanks.set_sweep_filename(datablocks[-1]) detectblanks.set_reflections_filename(spot_filename) detectblanks.run() json = detectblanks.get_results() offset = imageset.get_scan().get_image_range()[0] blank_regions = json['strong']['blank_regions'] if len(blank_regions): blank_regions = [(int(s), int(e)) for s, e in blank_regions] for blank_start, blank_end in blank_regions: Chatter.write( 'WARNING: Potential blank images: %i -> %i' % (blank_start + 1, blank_end)) if PhilIndex.params.xia2.settings.remove_blanks: non_blanks = [] start, end = imageset.get_array_range() last_blank_end = start for blank_start, blank_end in blank_regions: if blank_start > start: non_blanks.append( (last_blank_end, blank_start)) last_blank_end = blank_end if last_blank_end + 1 < end: non_blanks.append((last_blank_end, end)) xsweep = self.get_indexer_sweep() xwav = xsweep.get_wavelength() xsample = xsweep.get_xsample() sweep_name = xsweep.get_name() import string for i, (nb_start, nb_end) in enumerate(non_blanks): assert i < 26 if i == 0: sub_imageset = imageset[nb_start - start:nb_end - start] xsweep._frames_to_process = (nb_start + 1, nb_end + 1) self.set_indexer_prepare_done(done=False) self._indxr_imagesets[ self._indxr_imagesets.index( imageset)] = sub_imageset xsweep._integrater._setup_from_imageset( sub_imageset) else: new_name = '_'.join( (sweep_name, string.ascii_lowercase[i])) new_sweep = xwav.add_sweep( new_name, xsample, directory=os.path.join( os.path.basename( xsweep.get_directory()), new_name), image=imageset.get_path(nb_start - start), frames_to_process=(nb_start + 1, nb_end), ) Chatter.write( "Generating new sweep: %s (%s:%i:%i)" % (new_sweep.get_name(), new_sweep.get_image(), new_sweep.get_frames_to_process()[0], new_sweep.get_frames_to_process()[1])) return if not PhilIndex.params.xia2.settings.trust_beam_centre: discovery = self.DiscoverBetterExperimentalModel() discovery.set_sweep_filename(datablocks[-1]) discovery.set_spot_filename(spot_filename) #wedges = self._index_select_images_i(imageset) #discovery.set_scan_ranges(wedges) #discovery.set_scan_ranges([(first + offset, last + offset)]) try: discovery.run() except Exception as e: Debug.write('DIALS beam centre search failed: %s' % str(e)) else: # overwrite datablock.json in datablocks list datablocks[ -1] = discovery.get_optimized_datablock_filename() self.set_indexer_payload("spot_lists", spot_lists) self.set_indexer_payload("datablocks", datablocks) return
def _index_prepare(self): '''Prepare to do autoindexing - in XDS terms this will mean calling xycorr, init and colspot on the input images.''' # decide on images to work with Debug.write('XDS INDEX PREPARE:') Debug.write('Wavelength: %.6f' % self.get_wavelength()) Debug.write('Distance: %.2f' % self.get_distance()) if self._indxr_images == []: _select_images_function = getattr( self, '_index_select_images_%s' % (self._index_select_images)) wedges = _select_images_function() for wedge in wedges: self.add_indexer_image_wedge(wedge) self.set_indexer_prepare_done(True) all_images = self.get_matching_images() first = min(all_images) last = max(all_images) # next start to process these - first xycorr xycorr = self.Xycorr() xycorr.set_data_range(first, last) xycorr.set_background_range(self._indxr_images[0][0], self._indxr_images[0][1]) from dxtbx.serialize.xds import to_xds converter = to_xds(self.get_imageset()) xds_beam_centre = converter.detector_origin xycorr.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1]) for block in self._indxr_images: xycorr.add_spot_range(block[0], block[1]) # FIXME need to set the origin here xycorr.run() for file in ['X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf']: self._indxr_payload[file] = xycorr.get_output_data_file(file) # next start to process these - then init if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing: # find the region of the scan with the least predicted shadow # to use for background determination in XDS INIT step from dxtbx.serialize import dump from dxtbx.datablock import DataBlock imageset = self._indxr_imagesets[0] xsweep = self._indxr_sweeps[0] sweep_filename = os.path.join( self.get_working_directory(), '%s_datablock.json' %xsweep.get_name()) dump.datablock(DataBlock([imageset]), sweep_filename) from xia2.Wrappers.Dials.ShadowPlot import ShadowPlot shadow_plot = ShadowPlot() shadow_plot.set_working_directory(self.get_working_directory()) auto_logfiler(shadow_plot) shadow_plot.set_sweep_filename(sweep_filename) shadow_plot.set_json_filename( os.path.join( self.get_working_directory(), '%s_shadow_plot.json' %shadow_plot.get_xpid())) shadow_plot.run() results = shadow_plot.get_results() from scitbx.array_family import flex fraction_shadowed = flex.double(results['fraction_shadowed']) scan_points = flex.double(results['scan_points']) phi_width = self.get_phi_width() scan = imageset.get_scan() oscillation_range = scan.get_oscillation_range() oscillation = scan.get_oscillation() bg_images = self._background_images bg_range_deg = (scan.get_angle_from_image_index(bg_images[0]), scan.get_angle_from_image_index(bg_images[1])) bg_range_width = bg_range_deg[1] - bg_range_deg[0] min_shadow = 100 best_bg_range = bg_range_deg from libtbx.utils import frange for bg_range_start in frange(flex.min(scan_points), flex.max(scan_points) - bg_range_width, step=oscillation[1]): bg_range_deg = (bg_range_start, bg_range_start + bg_range_width) sel = (scan_points >= bg_range_deg[0]) & (scan_points <= bg_range_deg[1]) mean_shadow = flex.mean(fraction_shadowed.select(sel)) if mean_shadow < min_shadow: min_shadow = mean_shadow best_bg_range = bg_range_deg self._background_images = ( scan.get_image_index_from_angle(best_bg_range[0]), scan.get_image_index_from_angle(best_bg_range[1])) Debug.write('Setting background images: %s -> %s' %self._background_images) init = self.Init() for file in ['X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf']: init.set_input_data_file(file, self._indxr_payload[file]) init.set_data_range(first, last) if self._background_images: init.set_background_range(self._background_images[0], self._background_images[1]) else: init.set_background_range(self._indxr_images[0][0], self._indxr_images[0][1]) for block in self._indxr_images: init.add_spot_range(block[0], block[1]) init.run() # at this stage, need to (perhaps) modify the BKGINIT.cbf image # to mark out the back stop if PhilIndex.params.general.backstop_mask: Debug.write('Applying mask to BKGINIT.pck') # copy the original file cbf_old = os.path.join(init.get_working_directory(), 'BKGINIT.cbf') cbf_save = os.path.join(init.get_working_directory(), 'BKGINIT.sav') shutil.copyfile(cbf_old, cbf_save) # modify the file to give the new mask from xia2.Toolkit.BackstopMask import BackstopMask mask = BackstopMask(PhilIndex.params.general.backstop_mask) mask.apply_mask_xds(self.get_header(), cbf_save, cbf_old) init.reload() for file in ['BLANK.cbf', 'BKGINIT.cbf', 'GAIN.cbf']: self._indxr_payload[file] = init.get_output_data_file(file) if PhilIndex.params.xia2.settings.developmental.use_dials_spotfinder: spotfinder = self.DialsSpotfinder() for block in self._indxr_images: spotfinder.add_spot_range(block[0], block[1]) spotfinder.run() export = self.DialsExportSpotXDS() export.set_input_data_file( 'reflections.pickle', spotfinder.get_output_data_file('reflections.pickle')) export.run() for file in ['SPOT.XDS']: self._indxr_payload[file] = export.get_output_data_file(file) else: # next start to process these - then colspot colspot = self.Colspot() for file in ['X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf', 'BLANK.cbf', 'BKGINIT.cbf', 'GAIN.cbf']: colspot.set_input_data_file(file, self._indxr_payload[file]) colspot.set_data_range(first, last) colspot.set_background_range(self._indxr_images[0][0], self._indxr_images[0][1]) for block in self._indxr_images: colspot.add_spot_range(block[0], block[1]) colspot.run() for file in ['SPOT.XDS']: self._indxr_payload[file] = colspot.get_output_data_file(file) # that should be everything prepared... all of the important # files should be loaded into memory to be able to cope with # integration happening somewhere else return
def _index_prepare(self): '''Prepare to do autoindexing - in XDS terms this will mean calling xycorr, init and colspot on the input images.''' # decide on images to work with Debug.write('XDS INDEX PREPARE:') Debug.write('Wavelength: %.6f' % self.get_wavelength()) Debug.write('Distance: %.2f' % self.get_distance()) if self._indxr_images == []: _select_images_function = getattr( self, '_index_select_images_%s' % (self._index_select_images)) wedges = _select_images_function() for wedge in wedges: self.add_indexer_image_wedge(wedge) self.set_indexer_prepare_done(True) all_images = self.get_matching_images() first = min(all_images) last = max(all_images) # next start to process these - first xycorr xycorr = self.Xycorr() xycorr.set_data_range(first, last) xycorr.set_background_range(self._indxr_images[0][0], self._indxr_images[0][1]) from dxtbx.serialize.xds import to_xds converter = to_xds(self.get_imageset()) xds_beam_centre = converter.detector_origin xycorr.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1]) for block in self._indxr_images: xycorr.add_spot_range(block[0], block[1]) # FIXME need to set the origin here xycorr.run() for file in ['X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf']: self._indxr_payload[file] = xycorr.get_output_data_file(file) # next start to process these - then init if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing: imageset = self._indxr_imagesets[0] masker = imageset.masker().format_class( imageset.paths()[0]).get_goniometer_shadow_masker() if masker is None: # disable dynamic_shadowing PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing: # find the region of the scan with the least predicted shadow # to use for background determination in XDS INIT step from dxtbx.serialize import dump from dxtbx.datablock import DataBlock imageset = self._indxr_imagesets[0] xsweep = self._indxr_sweeps[0] sweep_filename = os.path.join( self.get_working_directory(), '%s_datablock.json' % xsweep.get_name()) dump.datablock(DataBlock([imageset]), sweep_filename) from xia2.Wrappers.Dials.ShadowPlot import ShadowPlot shadow_plot = ShadowPlot() shadow_plot.set_working_directory(self.get_working_directory()) auto_logfiler(shadow_plot) shadow_plot.set_sweep_filename(sweep_filename) shadow_plot.set_json_filename( os.path.join(self.get_working_directory(), '%s_shadow_plot.json' % shadow_plot.get_xpid())) shadow_plot.run() results = shadow_plot.get_results() from scitbx.array_family import flex fraction_shadowed = flex.double(results['fraction_shadowed']) if flex.max(fraction_shadowed) == 0: PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False else: scan_points = flex.double(results['scan_points']) phi_width = self.get_phi_width() scan = imageset.get_scan() oscillation_range = scan.get_oscillation_range() oscillation = scan.get_oscillation() if self._background_images is not None: bg_images = self._background_images bg_range_deg = (scan.get_angle_from_image_index( bg_images[0]), scan.get_angle_from_image_index( bg_images[1])) bg_range_width = bg_range_deg[1] - bg_range_deg[0] min_shadow = 100 best_bg_range = bg_range_deg from libtbx.utils import frange for bg_range_start in frange(flex.min(scan_points), flex.max(scan_points) - bg_range_width, step=oscillation[1]): bg_range_deg = (bg_range_start, bg_range_start + bg_range_width) sel = (scan_points >= bg_range_deg[0]) & ( scan_points <= bg_range_deg[1]) mean_shadow = flex.mean(fraction_shadowed.select(sel)) if mean_shadow < min_shadow: min_shadow = mean_shadow best_bg_range = bg_range_deg self._background_images = (scan.get_image_index_from_angle( best_bg_range[0]), scan.get_image_index_from_angle( best_bg_range[1])) Debug.write('Setting background images: %s -> %s' % self._background_images) init = self.Init() for file in ['X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf']: init.set_input_data_file(file, self._indxr_payload[file]) init.set_data_range(first, last) if self._background_images: init.set_background_range(self._background_images[0], self._background_images[1]) else: init.set_background_range(self._indxr_images[0][0], self._indxr_images[0][1]) for block in self._indxr_images: init.add_spot_range(block[0], block[1]) init.run() # at this stage, need to (perhaps) modify the BKGINIT.cbf image # to mark out the back stop if PhilIndex.params.general.backstop_mask: Debug.write('Applying mask to BKGINIT.pck') # copy the original file cbf_old = os.path.join(init.get_working_directory(), 'BKGINIT.cbf') cbf_save = os.path.join(init.get_working_directory(), 'BKGINIT.sav') shutil.copyfile(cbf_old, cbf_save) # modify the file to give the new mask from xia2.Toolkit.BackstopMask import BackstopMask mask = BackstopMask(PhilIndex.params.general.backstop_mask) mask.apply_mask_xds(self.get_header(), cbf_save, cbf_old) init.reload() for file in ['BLANK.cbf', 'BKGINIT.cbf', 'GAIN.cbf']: self._indxr_payload[file] = init.get_output_data_file(file) if PhilIndex.params.xia2.settings.developmental.use_dials_spotfinder: spotfinder = self.DialsSpotfinder() for block in self._indxr_images: spotfinder.add_spot_range(block[0], block[1]) spotfinder.run() export = self.DialsExportSpotXDS() export.set_input_data_file( 'reflections.pickle', spotfinder.get_output_data_file('reflections.pickle')) export.run() for file in ['SPOT.XDS']: self._indxr_payload[file] = export.get_output_data_file(file) else: # next start to process these - then colspot colspot = self.Colspot() for file in [ 'X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf', 'BLANK.cbf', 'BKGINIT.cbf', 'GAIN.cbf' ]: colspot.set_input_data_file(file, self._indxr_payload[file]) colspot.set_data_range(first, last) colspot.set_background_range(self._indxr_images[0][0], self._indxr_images[0][1]) for block in self._indxr_images: colspot.add_spot_range(block[0], block[1]) colspot.run() for file in ['SPOT.XDS']: self._indxr_payload[file] = colspot.get_output_data_file(file) # that should be everything prepared... all of the important # files should be loaded into memory to be able to cope with # integration happening somewhere else return
imageset = reference_datablocks[0].extract_imagesets()[0] reference_detector = imageset.get_detector() assert len(datablocks) == 1 imageset = datablocks[0].extract_imagesets()[0] detector = imageset.get_detector() h = detector.hierarchy() href = reference_detector.hierarchy() assert len(h) == len(href) assert abs(h.get_directed_distance() - href.get_directed_distance()) < params.max_delta_distance for panel, panel_ref in zip(h.children(), href.children()): panel.set_local_frame( panel_ref.get_local_fast_axis(), panel_ref.get_local_slow_axis(), panel_ref.get_local_origin() ) print 'Writing metrology-corrected datablock to %s' %params.output.datablock from dxtbx.serialize import dump dump.datablock(datablocks, params.output.datablock) return if __name__ == '__main__': import sys run(sys.argv[1:])
def _index_prepare(self): from xia2.Handlers.Citations import Citations Citations.cite('dials') #all_images = self.get_matching_images() #first = min(all_images) #last = max(all_images) spot_lists = [] datablocks = [] for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps): Chatter.banner('Spotfinding %s' %xsweep.get_name()) first, last = imageset.get_scan().get_image_range() # at this stage, break out to run the DIALS code: this sets itself up # now cheat and pass in some information... save re-reading all of the # image headers # FIXME need to adjust this to allow (say) three chunks of images from dxtbx.serialize import dump from dxtbx.datablock import DataBlock sweep_filename = os.path.join( self.get_working_directory(), '%s_datablock.json' %xsweep.get_name()) dump.datablock(DataBlock([imageset]), sweep_filename) gain = PhilIndex.params.xia2.settings.input.gain if gain is libtbx.Auto: gain_estimater = self.EstimateGain() gain_estimater.set_sweep_filename(sweep_filename) gain_estimater.run() gain = gain_estimater.get_gain() Chatter.write('Estimated gain: %.2f' %gain) PhilIndex.params.xia2.settings.input.gain = gain # FIXME this should really use the assigned spot finding regions #offset = self.get_frame_offset() spotfinder = self.Spotfinder() if last - first > 10: spotfinder.set_write_hot_mask(True) spotfinder.set_input_sweep_filename(sweep_filename) spotfinder.set_output_sweep_filename( '%s_%s_datablock.json' %(spotfinder.get_xpid(), xsweep.get_name())) spotfinder.set_input_spot_filename( '%s_%s_strong.pickle' %(spotfinder.get_xpid(), xsweep.get_name())) if PhilIndex.params.dials.fast_mode: wedges = self._index_select_images_i(imageset) spotfinder.set_scan_ranges(wedges) else: spotfinder.set_scan_ranges([(first, last)]) if PhilIndex.params.dials.find_spots.phil_file is not None: spotfinder.set_phil_file(PhilIndex.params.dials.find_spots.phil_file) min_spot_size = PhilIndex.params.dials.find_spots.min_spot_size if min_spot_size is libtbx.Auto: if imageset.get_detector()[0].get_type() == 'SENSOR_PAD': min_spot_size = 3 else: min_spot_size = None if min_spot_size is not None: spotfinder.set_min_spot_size(min_spot_size) min_local = PhilIndex.params.dials.find_spots.min_local if min_local is not None: spotfinder.set_min_local(min_local) sigma_strong = PhilIndex.params.dials.find_spots.sigma_strong if sigma_strong: spotfinder.set_sigma_strong(sigma_strong) gain = PhilIndex.params.xia2.settings.input.gain if gain: spotfinder.set_gain(gain) filter_ice_rings = PhilIndex.params.dials.find_spots.filter_ice_rings if filter_ice_rings: spotfinder.set_filter_ice_rings(filter_ice_rings) kernel_size = PhilIndex.params.dials.find_spots.kernel_size if kernel_size: spotfinder.set_kernel_size(kernel_size) global_threshold = PhilIndex.params.dials.find_spots.global_threshold if global_threshold is not None: spotfinder.set_global_threshold(global_threshold) spotfinder.run() spot_filename = spotfinder.get_spot_filename() if not os.path.exists(spot_filename): raise RuntimeError("Spotfinding failed: %s does not exist." %os.path.basename(spot_filename)) spot_lists.append(spot_filename) datablocks.append(spotfinder.get_output_sweep_filename()) from libtbx import easy_pickle from dials.util.ascii_art import spot_counts_per_image_plot refl = easy_pickle.load(spot_filename) if not len(refl): raise RuntimeError('No spots found in sweep %s' %xsweep.get_name()) Chatter.write(spot_counts_per_image_plot(refl), strip=False) if not PhilIndex.params.dials.fast_mode: detectblanks = self.DetectBlanks() detectblanks.set_sweep_filename(datablocks[-1]) detectblanks.set_reflections_filename(spot_filename) detectblanks.run() json = detectblanks.get_results() offset = imageset.get_scan().get_image_range()[0] blank_regions = json['strong']['blank_regions'] if len(blank_regions): blank_regions = [(int(s), int(e)) for s, e in blank_regions] for blank_start, blank_end in blank_regions: Chatter.write('WARNING: Potential blank images: %i -> %i' %( blank_start+1, blank_end)) if PhilIndex.params.xia2.settings.remove_blanks: non_blanks = [] start, end = imageset.get_array_range() last_blank_end = start for blank_start, blank_end in blank_regions: if blank_start > start: non_blanks.append((last_blank_end, blank_start)) last_blank_end = blank_end if last_blank_end+1 < end: non_blanks.append((last_blank_end, end)) xsweep = self.get_indexer_sweep() xwav = xsweep.get_wavelength() xsample = xsweep.get_xsample() sweep_name = xsweep.get_name() import string for i, (nb_start, nb_end) in enumerate(non_blanks): assert i < 26 if i == 0: sub_imageset = imageset[nb_start-start:nb_end-start] xsweep._frames_to_process = (nb_start+1, nb_end+1) self.set_indexer_prepare_done(done=False) self._indxr_imagesets[self._indxr_imagesets.index(imageset)] = sub_imageset xsweep._integrater._setup_from_imageset(sub_imageset) else: new_name = '_'.join((sweep_name, string.ascii_lowercase[i])) new_sweep = xwav.add_sweep(new_name, xsample, directory=os.path.join( os.path.basename(xsweep.get_directory()), new_name), image=imageset.get_path(nb_start-start), frames_to_process=(nb_start+1, nb_end), ) Chatter.write("Generating new sweep: %s (%s:%i:%i)" %( new_sweep.get_name(), new_sweep.get_image(), new_sweep.get_frames_to_process()[0], new_sweep.get_frames_to_process()[1])) return if not PhilIndex.params.xia2.settings.trust_beam_centre: discovery = self.DiscoverBetterExperimentalModel() discovery.set_sweep_filename(datablocks[-1]) discovery.set_spot_filename(spot_filename) #wedges = self._index_select_images_i(imageset) #discovery.set_scan_ranges(wedges) #discovery.set_scan_ranges([(first + offset, last + offset)]) try: discovery.run() except Exception, e: Debug.write('DIALS beam centre search failed: %s' %str(e)) else: # overwrite datablock.json in datablocks list datablocks[-1] = discovery.get_optimized_datablock_filename()