def scale(self): '''Actually perform the scaling - this is delegated to the implementation.''' if self._scalr_integraters == {}: raise RuntimeError( \ 'no Integrater implementations assigned for scaling') xname = self._scalr_xcrystal.get_name() while not self.get_scaler_finish_done(): while not self.get_scaler_done(): while not self.get_scaler_prepare_done(): Chatter.banner('Preparing %s' % xname) self._scalr_prepare_done = True self._scale_prepare() Chatter.banner('Scaling %s' % xname) self._scalr_done = True self._scalr_result = self._scale() self._scalr_finish_done = True self._scale_finish() return self._scalr_result
def scale(self): '''Actually perform the scaling - this is delegated to the implementation.''' if self._scalr_integraters == { }: raise RuntimeError, \ 'no Integrater implementations assigned for scaling' xname = self._scalr_xcrystal.get_name() while not self.get_scaler_finish_done(): while not self.get_scaler_done(): while not self.get_scaler_prepare_done(): Chatter.banner('Preparing %s' % xname) self._scalr_prepare_done = True self._scale_prepare() Chatter.banner('Scaling %s' % xname) self._scalr_done = True self._scalr_result = self._scale() self._scalr_finish_done = True self._scale_finish() return self._scalr_result
def _scale_finish_chunk_8_raddam(self): crd = CCP4InterRadiationDamageDetector() crd.set_working_directory(self.get_working_directory()) crd.set_hklin(self._scalr_scaled_reflection_files['mtz']) if self.get_scaler_anomalous(): crd.set_anomalous(True) hklout = os.path.join(self.get_working_directory(), 'temp.mtz') FileHandler.record_temporary_file(hklout) crd.set_hklout(hklout) status = crd.detect() if status: Chatter.write('') Chatter.banner('Local Scaling %s' % self._scalr_xname) for s in status: Chatter.write('%s %s' % s) Chatter.banner('') else: Debug.write('Local scaling failed')
def integrate(self): '''Actually perform integration until we think we are done...''' while not self.get_integrater_finish_done(): while not self.get_integrater_done(): while not self.get_integrater_prepare_done(): Debug.write('Preparing to do some integration...') self.set_integrater_prepare_done(True) # if this raises an exception, perhaps the autoindexing # solution has too high symmetry. if this the case, then # perform a self._intgr_indexer.eliminate() - this should # reset the indexing system try: self._integrate_prepare() except BadLatticeError, e: Journal.banner('eliminated this lattice', size = 80) Chatter.write('Rejecting bad lattice %s' % str(e)) self._intgr_refiner.eliminate() self._integrater_reset() # FIXME x1698 - may be the case that _integrate() returns the # raw intensities, _integrate_finish() returns intensities # which may have been adjusted or corrected. See #1698 below. Debug.write('Doing some integration...') self.set_integrater_done(True) template = self.get_integrater_sweep().get_template() if self._intgr_sweep_name: if PhilIndex.params.xia2.settings.show_template: Chatter.banner('Integrating %s (%s)' % \ (self._intgr_sweep_name, template)) else: Chatter.banner('Integrating %s' % \ (self._intgr_sweep_name)) try: #1698 self._intgr_hklout_raw = self._integrate() except BadLatticeError, e: Chatter.write('Rejecting bad lattice %s' % str(e)) Journal.banner('eliminated this lattice', size = 80) self._intgr_refiner.eliminate() self._integrater_reset()
def _index_prepare(self): Chatter.banner('Spotfinding %s' % self.get_indexer_sweep_name()) super(XDSIndexerII, self)._index_prepare() from dials.array_family import flex from dials.util.ascii_art import spot_counts_per_image_plot reflection_pickle = spot_xds_to_reflection_pickle( self._indxr_payload['SPOT.XDS'], working_directory=self.get_working_directory()) refl = flex.reflection_table.from_pickle(reflection_pickle) Chatter.write(spot_counts_per_image_plot(refl), strip=False)
def _index_prepare(self): Chatter.banner('Spotfinding %s' %self.get_indexer_sweep_name()) super(XDSIndexerII, self)._index_prepare() from dials.array_family import flex from dials.util.ascii_art import spot_counts_per_image_plot reflection_pickle = spot_xds_to_reflection_pickle( self._indxr_payload['SPOT.XDS'], working_directory=self.get_working_directory()) refl = flex.reflection_table.from_pickle(reflection_pickle) Chatter.write(spot_counts_per_image_plot(refl), strip=False)
def _index_prepare(self): from xia2.Handlers.Citations import Citations Citations.cite('dials') #all_images = self.get_matching_images() #first = min(all_images) #last = max(all_images) spot_lists = [] datablocks = [] for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps): Chatter.banner('Spotfinding %s' % xsweep.get_name()) first, last = imageset.get_scan().get_image_range() # at this stage, break out to run the DIALS code: this sets itself up # now cheat and pass in some information... save re-reading all of the # image headers # FIXME need to adjust this to allow (say) three chunks of images from dxtbx.serialize import dump from dxtbx.datablock import DataBlock sweep_filename = os.path.join( self.get_working_directory(), '%s_datablock.json' % xsweep.get_name()) dump.datablock(DataBlock([imageset]), sweep_filename) genmask = self.GenerateMask() genmask.set_input_datablock(sweep_filename) genmask.set_output_datablock( os.path.join( self.get_working_directory(), '%s_%s_datablock.json' % (genmask.get_xpid(), xsweep.get_name()))) genmask.set_params(PhilIndex.params.dials.masking) sweep_filename, mask_pickle = genmask.run() Debug.write('Generated mask for %s: %s' % (xsweep.get_name(), mask_pickle)) gain = PhilIndex.params.xia2.settings.input.gain if gain is libtbx.Auto: gain_estimater = self.EstimateGain() gain_estimater.set_sweep_filename(sweep_filename) gain_estimater.run() gain = gain_estimater.get_gain() Chatter.write('Estimated gain: %.2f' % gain) PhilIndex.params.xia2.settings.input.gain = gain # FIXME this should really use the assigned spot finding regions #offset = self.get_frame_offset() spotfinder = self.Spotfinder() if last - first > 10: spotfinder.set_write_hot_mask(True) spotfinder.set_input_sweep_filename(sweep_filename) spotfinder.set_output_sweep_filename( '%s_%s_datablock.json' % (spotfinder.get_xpid(), xsweep.get_name())) spotfinder.set_input_spot_filename( '%s_%s_strong.pickle' % (spotfinder.get_xpid(), xsweep.get_name())) if PhilIndex.params.dials.fast_mode: wedges = self._index_select_images_i(imageset) spotfinder.set_scan_ranges(wedges) else: spotfinder.set_scan_ranges([(first, last)]) if PhilIndex.params.dials.find_spots.phil_file is not None: spotfinder.set_phil_file( PhilIndex.params.dials.find_spots.phil_file) min_spot_size = PhilIndex.params.dials.find_spots.min_spot_size if min_spot_size is libtbx.Auto: if imageset.get_detector()[0].get_type() == 'SENSOR_PAD': min_spot_size = 3 else: min_spot_size = None if min_spot_size is not None: spotfinder.set_min_spot_size(min_spot_size) min_local = PhilIndex.params.dials.find_spots.min_local if min_local is not None: spotfinder.set_min_local(min_local) sigma_strong = PhilIndex.params.dials.find_spots.sigma_strong if sigma_strong: spotfinder.set_sigma_strong(sigma_strong) gain = PhilIndex.params.xia2.settings.input.gain if gain: spotfinder.set_gain(gain) filter_ice_rings = PhilIndex.params.dials.find_spots.filter_ice_rings if filter_ice_rings: spotfinder.set_filter_ice_rings(filter_ice_rings) kernel_size = PhilIndex.params.dials.find_spots.kernel_size if kernel_size: spotfinder.set_kernel_size(kernel_size) global_threshold = PhilIndex.params.dials.find_spots.global_threshold if global_threshold is not None: spotfinder.set_global_threshold(global_threshold) spotfinder.run() spot_filename = spotfinder.get_spot_filename() if not os.path.exists(spot_filename): raise RuntimeError("Spotfinding failed: %s does not exist." % os.path.basename(spot_filename)) spot_lists.append(spot_filename) datablocks.append(spotfinder.get_output_sweep_filename()) from libtbx import easy_pickle from dials.util.ascii_art import spot_counts_per_image_plot refl = easy_pickle.load(spot_filename) if not len(refl): raise RuntimeError('No spots found in sweep %s' % xsweep.get_name()) Chatter.write(spot_counts_per_image_plot(refl), strip=False) if not PhilIndex.params.dials.fast_mode: detectblanks = self.DetectBlanks() detectblanks.set_sweep_filename(datablocks[-1]) detectblanks.set_reflections_filename(spot_filename) detectblanks.run() json = detectblanks.get_results() offset = imageset.get_scan().get_image_range()[0] blank_regions = json['strong']['blank_regions'] if len(blank_regions): blank_regions = [(int(s), int(e)) for s, e in blank_regions] for blank_start, blank_end in blank_regions: Chatter.write( 'WARNING: Potential blank images: %i -> %i' % (blank_start + 1, blank_end)) if PhilIndex.params.xia2.settings.remove_blanks: non_blanks = [] start, end = imageset.get_array_range() last_blank_end = start for blank_start, blank_end in blank_regions: if blank_start > start: non_blanks.append( (last_blank_end, blank_start)) last_blank_end = blank_end if last_blank_end + 1 < end: non_blanks.append((last_blank_end, end)) xsweep = self.get_indexer_sweep() xwav = xsweep.get_wavelength() xsample = xsweep.get_xsample() sweep_name = xsweep.get_name() import string for i, (nb_start, nb_end) in enumerate(non_blanks): assert i < 26 if i == 0: sub_imageset = imageset[nb_start - start:nb_end - start] xsweep._frames_to_process = (nb_start + 1, nb_end + 1) self.set_indexer_prepare_done(done=False) self._indxr_imagesets[ self._indxr_imagesets.index( imageset)] = sub_imageset xsweep._integrater._setup_from_imageset( sub_imageset) else: new_name = '_'.join( (sweep_name, string.ascii_lowercase[i])) new_sweep = xwav.add_sweep( new_name, xsample, directory=os.path.join( os.path.basename( xsweep.get_directory()), new_name), image=imageset.get_path(nb_start - start), frames_to_process=(nb_start + 1, nb_end), ) Chatter.write( "Generating new sweep: %s (%s:%i:%i)" % (new_sweep.get_name(), new_sweep.get_image(), new_sweep.get_frames_to_process()[0], new_sweep.get_frames_to_process()[1])) return if not PhilIndex.params.xia2.settings.trust_beam_centre: discovery = self.DiscoverBetterExperimentalModel() discovery.set_sweep_filename(datablocks[-1]) discovery.set_spot_filename(spot_filename) #wedges = self._index_select_images_i(imageset) #discovery.set_scan_ranges(wedges) #discovery.set_scan_ranges([(first + offset, last + offset)]) try: discovery.run() except Exception as e: Debug.write('DIALS beam centre search failed: %s' % str(e)) else: # overwrite datablock.json in datablocks list datablocks[ -1] = discovery.get_optimized_datablock_filename() self.set_indexer_payload("spot_lists", spot_lists) self.set_indexer_payload("datablocks", datablocks) return
def index(self): if not self.get_indexer_finish_done(): f = inspect.currentframe().f_back.f_back m = f.f_code.co_filename l = f.f_lineno Debug.write('Index in %s called from %s %d' % (self.__class__.__name__, m, l)) while not self.get_indexer_finish_done(): while not self.get_indexer_done(): while not self.get_indexer_prepare_done(): # -------------- # call prepare() # -------------- self.set_indexer_prepare_done(True) self._index_prepare() # -------------------------------------------- # then do the proper indexing - using the best # solution already stored if available (c/f # eliminate above) # -------------------------------------------- self.set_indexer_done(True) if self.get_indexer_sweeps(): xsweeps = [ s.get_name() for s in self.get_indexer_sweeps() ] if len(xsweeps) > 1: # find "SWEEPn, SWEEP(n+1), (..), SWEEPm" and aggregate to "SWEEPS n-m" xsweeps = map(lambda x: (int(x[5:]), int(x[5:])) if x.startswith('SWEEP') else x, xsweeps) xsweeps[0] = [xsweeps[0]] def compress(seen, nxt): if isinstance(seen[-1], tuple) and isinstance(nxt, tuple) and (seen[-1][1] + 1 == nxt[0]): seen[-1] = (seen[-1][0], nxt[1]) else: seen.append(nxt) return seen xsweeps = reduce(compress, xsweeps) xsweeps = map(lambda x: ('SWEEP%d' % x[0] if x[0] == x[1] else 'SWEEPS %d to %d' % (x[0], x[1])) if isinstance(x, tuple) else x, xsweeps) if len(xsweeps) > 1: sweep_names = ', '.join(xsweeps[:-1]) sweep_names += ' & ' + xsweeps[-1] else: sweep_names = xsweeps[0] if PhilIndex.params.xia2.settings.show_template: template = self.get_indexer_sweep().get_template() Chatter.banner( 'Autoindexing %s (%s)' %(sweep_names, template)) else: Chatter.banner('Autoindexing %s' %sweep_names) if not self._indxr_helper: result = self._index() if not self._indxr_done: Debug.write( 'Looks like indexing failed - try again!') continue solutions = { } for k in self._indxr_other_lattice_cell.keys(): solutions[k] = self._indxr_other_lattice_cell[k][ 'cell'] # create a helper for the indexer to manage solutions self._indxr_helper = _IndexerHelper(solutions) solution = self._indxr_helper.get() # compare these against the final solution, if different # reject solution and return - correct solution will # be used next cycle if self._indxr_lattice != solution[0] and \ not self._indxr_input_cell and \ not PhilIndex.params.xia2.settings.integrate_p1: Chatter.write('Rerunning indexing lattice %s to %s' % (self._indxr_lattice, solution[0])) Debug.write( 'Rerunning indexing with target lattice %s' % \ solution[0]) self.set_indexer_done(False) else: # rerun autoindexing with the best known current solution solution = self._indxr_helper.get() self._indxr_input_lattice = solution[0] self._indxr_input_cell = solution[1] result = self._index() # next finish up... self.set_indexer_finish_done(True) self._index_finish() if self._indxr_print: Chatter.write(self.show_indexer_solutions())
def integrate(self): '''Actually perform integration until we think we are done...''' while not self.get_integrater_finish_done(): while not self.get_integrater_done(): while not self.get_integrater_prepare_done(): Debug.write('Preparing to do some integration...') self.set_integrater_prepare_done(True) # if this raises an exception, perhaps the autoindexing # solution has too high symmetry. if this the case, then # perform a self._intgr_indexer.eliminate() - this should # reset the indexing system try: self._integrate_prepare() except BadLatticeError as e: Journal.banner('eliminated this lattice', size=80) Chatter.write('Rejecting bad lattice %s' % str(e)) self._intgr_refiner.eliminate() self._integrater_reset() # FIXME x1698 - may be the case that _integrate() returns the # raw intensities, _integrate_finish() returns intensities # which may have been adjusted or corrected. See #1698 below. Debug.write('Doing some integration...') self.set_integrater_done(True) template = self.get_integrater_sweep().get_template() if self._intgr_sweep_name: if PhilIndex.params.xia2.settings.show_template: Chatter.banner('Integrating %s (%s)' % \ (self._intgr_sweep_name, template)) else: Chatter.banner('Integrating %s' % \ (self._intgr_sweep_name)) try: #1698 self._intgr_hklout_raw = self._integrate() except BadLatticeError as e: Chatter.write('Rejecting bad lattice %s' % str(e)) Journal.banner('eliminated this lattice', size=80) self._intgr_refiner.eliminate() self._integrater_reset() self.set_integrater_finish_done(True) try: # allow for the fact that postrefinement may be used # to reject the lattice... self._intgr_hklout = self._integrate_finish() except BadLatticeError as e: Chatter.write('Bad Lattice Error: %s' % str(e)) self._intgr_refiner.eliminate() self._integrater_reset() return self._intgr_hklout
def index(self): if not self.get_indexer_finish_done(): f = inspect.currentframe().f_back.f_back m = f.f_code.co_filename l = f.f_lineno Debug.write( "Index in %s called from %s %d" % (self.__class__.__name__, m, l) ) while not self.get_indexer_finish_done(): while not self.get_indexer_done(): while not self.get_indexer_prepare_done(): # -------------- # call prepare() # -------------- self.set_indexer_prepare_done(True) self._index_prepare() # -------------------------------------------- # then do the proper indexing - using the best # solution already stored if available (c/f # eliminate above) # -------------------------------------------- self.set_indexer_done(True) if self.get_indexer_sweeps(): xsweeps = [s.get_name() for s in self.get_indexer_sweeps()] if len(xsweeps) > 1: # find "SWEEPn, SWEEP(n+1), (..), SWEEPm" and aggregate to "SWEEPS n-m" xsweeps = map( lambda x: (int(x[5:]), int(x[5:])) if x.startswith("SWEEP") else x, xsweeps, ) xsweeps[0] = [xsweeps[0]] def compress(seen, nxt): if ( isinstance(seen[-1], tuple) and isinstance(nxt, tuple) and (seen[-1][1] + 1 == nxt[0]) ): seen[-1] = (seen[-1][0], nxt[1]) else: seen.append(nxt) return seen xsweeps = reduce(compress, xsweeps) xsweeps = map( lambda x: ( "SWEEP%d" % x[0] if x[0] == x[1] else "SWEEPS %d to %d" % (x[0], x[1]) ) if isinstance(x, tuple) else x, xsweeps, ) if len(xsweeps) > 1: sweep_names = ", ".join(xsweeps[:-1]) sweep_names += " & " + xsweeps[-1] else: sweep_names = xsweeps[0] if PhilIndex.params.xia2.settings.show_template: template = self.get_indexer_sweep().get_template() Chatter.banner("Autoindexing %s (%s)" % (sweep_names, template)) else: Chatter.banner("Autoindexing %s" % sweep_names) if not self._indxr_helper: result = self._index() if not self._indxr_done: Debug.write("Looks like indexing failed - try again!") continue solutions = {} for k in self._indxr_other_lattice_cell.keys(): solutions[k] = self._indxr_other_lattice_cell[k]["cell"] # create a helper for the indexer to manage solutions self._indxr_helper = _IndexerHelper(solutions) solution = self._indxr_helper.get() # compare these against the final solution, if different # reject solution and return - correct solution will # be used next cycle if ( self._indxr_lattice != solution[0] and not self._indxr_input_cell and not PhilIndex.params.xia2.settings.integrate_p1 ): Chatter.write( "Rerunning indexing lattice %s to %s" % (self._indxr_lattice, solution[0]) ) Debug.write( "Rerunning indexing with target lattice %s" % solution[0] ) self.set_indexer_done(False) else: # rerun autoindexing with the best known current solution solution = self._indxr_helper.get() self._indxr_input_lattice = solution[0] self._indxr_input_cell = solution[1] result = self._index() # next finish up... self.set_indexer_finish_done(True) self._index_finish() if self._indxr_print: Chatter.write(self.show_indexer_solutions())
def index(self): if not self.get_indexer_finish_done(): f = inspect.currentframe().f_back.f_back m = f.f_code.co_filename l = f.f_lineno Debug.write('Index in %s called from %s %d' % (self.__class__.__name__, m, l)) while not self.get_indexer_finish_done(): while not self.get_indexer_done(): while not self.get_indexer_prepare_done(): # -------------- # call prepare() # -------------- self.set_indexer_prepare_done(True) self._index_prepare() # -------------------------------------------- # then do the proper indexing - using the best # solution already stored if available (c/f # eliminate above) # -------------------------------------------- self.set_indexer_done(True) if self.get_indexer_sweeps(): xsweeps = self.get_indexer_sweeps() if len(xsweeps) > 1: sweep_names = ', '.join(x.get_name() for x in xsweeps[:-1]) sweep_names += ' & ' + xsweeps[-1].get_name() else: sweep_names = xsweeps[0].get_name() if PhilIndex.params.xia2.settings.show_template: template = self.get_indexer_sweep().get_template() Chatter.banner( 'Autoindexing %s (%s)' %(sweep_names, template)) else: Chatter.banner('Autoindexing %s' %sweep_names) if not self._indxr_helper: result = self._index() if not self._indxr_done: Debug.write( 'Looks like indexing failed - try again!') continue solutions = { } for k in self._indxr_other_lattice_cell.keys(): solutions[k] = self._indxr_other_lattice_cell[k][ 'cell'] # create a helper for the indexer to manage solutions self._indxr_helper = _IndexerHelper(solutions) solution = self._indxr_helper.get() # compare these against the final solution, if different # reject solution and return - correct solution will # be used next cycle if self._indxr_lattice != solution[0] and \ not self._indxr_input_cell and \ not PhilIndex.params.xia2.settings.integrate_p1: Chatter.write('Rerunning indexing lattice %s to %s' % (self._indxr_lattice, solution[0])) Debug.write( 'Rerunning indexing with target lattice %s' % \ solution[0]) self.set_indexer_done(False) else: # rerun autoindexing with the best known current solution solution = self._indxr_helper.get() self._indxr_input_lattice = solution[0] self._indxr_input_cell = solution[1] result = self._index() # next finish up... self.set_indexer_finish_done(True) self._index_finish() if self._indxr_print: Chatter.write(self.show_indexer_solutions()) # FIXED 23/OCT/06 at this stage I need to look at the list of # reasonable solutions and try to figure out if the indexing # program has picked the highest - if not, then constrain the # unit cell (need to implement this somewhere, sure it's # around!) then rerun the autoindexing (perhaps?) with this # new target - this means that we are always working from the # top downwards with these things. Once we decide somewhere # else (e.g. in cell refinement) that this cell isn't good # then we can eliminate it from the list, select the next # lower symmetry solution and continue. This solution is a # general one, so may be implemented in the general indexer # interface rather than in specific code... return
def _update_scaled_unit_cell(self): # FIXME this could be brought in-house params = PhilIndex.params fast_mode = params.dials.fast_mode if (params.xia2.settings.integrater == 'dials' and not fast_mode and params.xia2.settings.scale.two_theta_refine): from xia2.Wrappers.Dials.TwoThetaRefine import TwoThetaRefine from xia2.lib.bits import auto_logfiler Chatter.banner('Unit cell refinement') # Collect a list of all sweeps, grouped by project, crystal, wavelength groups = {} self._scalr_cell_dict = {} tt_refine_experiments = [] tt_refine_pickles = [] tt_refine_reindex_ops = [] for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) pi = '_'.join(si.get_project_info()) intgr = si.get_integrater() groups[pi] = groups.get(pi, []) + \ [(intgr.get_integrated_experiments(), intgr.get_integrated_reflections(), intgr.get_integrater_reindex_operator())] # Two theta refine the unit cell for each group p4p_file = os.path.join(self.get_working_directory(), '%s_%s.p4p' % (self._scalr_pname, self._scalr_xname)) for pi in groups.keys(): tt_grouprefiner = TwoThetaRefine() tt_grouprefiner.set_working_directory(self.get_working_directory()) auto_logfiler(tt_grouprefiner) args = zip(*groups[pi]) tt_grouprefiner.set_experiments(args[0]) tt_grouprefiner.set_pickles(args[1]) tt_grouprefiner.set_output_p4p(p4p_file) tt_refine_experiments.extend(args[0]) tt_refine_pickles.extend(args[1]) tt_refine_reindex_ops.extend(args[2]) reindex_ops = args[2] from cctbx.sgtbx import change_of_basis_op as cb_op if self._spacegroup_reindex_operator is not None: reindex_ops = [( cb_op(str(self._spacegroup_reindex_operator)) * cb_op(str(op))).as_hkl() if op is not None else self._spacegroup_reindex_operator for op in reindex_ops] tt_grouprefiner.set_reindex_operators(reindex_ops) tt_grouprefiner.run() Chatter.write('%s: %6.2f %6.2f %6.2f %6.2f %6.2f %6.2f' % \ tuple([''.join(pi.split('_')[2:])] + list(tt_grouprefiner.get_unit_cell()))) self._scalr_cell_dict[pi] = (tt_grouprefiner.get_unit_cell(), tt_grouprefiner.get_unit_cell_esd(), tt_grouprefiner.import_cif(), tt_grouprefiner.import_mmcif()) if len(groups) > 1: cif_in = tt_grouprefiner.import_cif() cif_out = CIF.get_block(pi) for key in sorted(cif_in.keys()): cif_out[key] = cif_in[key] mmcif_in = tt_grouprefiner.import_mmcif() mmcif_out = mmCIF.get_block(pi) for key in sorted(mmcif_in.keys()): mmcif_out[key] = mmcif_in[key] # Two theta refine everything together if len(groups) > 1: tt_refiner = TwoThetaRefine() tt_refiner.set_working_directory(self.get_working_directory()) tt_refiner.set_output_p4p(p4p_file) auto_logfiler(tt_refiner) tt_refiner.set_experiments(tt_refine_experiments) tt_refiner.set_pickles(tt_refine_pickles) if self._spacegroup_reindex_operator is not None: reindex_ops = [( cb_op(str(self._spacegroup_reindex_operator)) * cb_op(str(op))).as_hkl() if op is not None else self._spacegroup_reindex_operator for op in tt_refine_reindex_ops] tt_refiner.set_reindex_operators(reindex_ops) tt_refiner.run() self._scalr_cell = tt_refiner.get_unit_cell() Chatter.write('Overall: %6.2f %6.2f %6.2f %6.2f %6.2f %6.2f' % tt_refiner.get_unit_cell()) self._scalr_cell_esd = tt_refiner.get_unit_cell_esd() cif_in = tt_refiner.import_cif() mmcif_in = tt_refiner.import_mmcif() else: self._scalr_cell, self._scalr_cell_esd, cif_in, mmcif_in = self._scalr_cell_dict.values()[0] if params.xia2.settings.small_molecule == True: FileHandler.record_data_file(p4p_file) import dials.util.version cif_out = CIF.get_block('xia2') mmcif_out = mmCIF.get_block('xia2') cif_out['_computing_cell_refinement'] = mmcif_out['_computing.cell_refinement'] = 'DIALS 2theta refinement, %s' % dials.util.version.dials_version() for key in sorted(cif_in.keys()): cif_out[key] = cif_in[key] for key in sorted(mmcif_in.keys()): mmcif_out[key] = mmcif_in[key] Debug.write('Unit cell obtained by two-theta refinement') else: ami = AnalyseMyIntensities() ami.set_working_directory(self.get_working_directory()) average_unit_cell, ignore_sg = ami.compute_average_cell( [self._scalr_scaled_refl_files[key] for key in self._scalr_scaled_refl_files]) Debug.write('Computed average unit cell (will use in all files)') self._scalr_cell = average_unit_cell self._scalr_cell_esd = None # Write average unit cell to .cif cif_out = CIF.get_block('xia2') cif_out['_computing_cell_refinement'] = 'AIMLESS averaged unit cell' for cell, cifname in zip(self._scalr_cell, ['length_a', 'length_b', 'length_c', 'angle_alpha', 'angle_beta', 'angle_gamma']): cif_out['_cell_%s' % cifname] = cell Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \ self._scalr_cell)
def get_unit_cell_errors(stop_after=None): '''Actually process something...''' wd = os.getcwd() all_miller_indices, all_two_thetas_obs, reference_cell, reference_lattice, reference_wavelength = load_sweeps_with_common_indexing() Chatter.banner('Unit cell sampling') Debug.banner('Unit cell sampling') span = miller.index_span(all_miller_indices) Chatter.write("Found %d reflections in 2theta range %.3f - %.3f deg" % (len(all_miller_indices), min(all_two_thetas_obs), max(all_two_thetas_obs))) Chatter.write("Miller index range: %s - %s" % (str(span.min()), str(span.max()))) unit_cell_info = { 'reflections': { 'count': len(all_miller_indices), 'min_2theta': min(all_two_thetas_obs), 'max_2theta': max(all_two_thetas_obs), 'min_miller': list(span.min()), 'max_miller': list(span.max()) } } # Exclude 1% of reflections to remove potential outliers # eg. indexed/integrated high angle noise two_theta_cutoff = sorted(all_two_thetas_obs)[-int(len(all_two_thetas_obs) * 0.01)-1] Chatter.write("Excluding outermost 1%% of reflections (2theta >= %.3f)" % two_theta_cutoff) two_thetas_select = all_two_thetas_obs < two_theta_cutoff all_two_thetas_obs = all_two_thetas_obs.select(two_thetas_select) all_miller_indices = all_miller_indices.select(two_thetas_select) Chatter.write("Kept %d reflections in 2theta range %.3f - %.3f deg" % (len(all_miller_indices), min(all_two_thetas_obs), max(all_two_thetas_obs))) span = miller.index_span(all_miller_indices) unit_cell_info['reflections_filtered'] = \ { 'count': len(all_miller_indices), 'min_2theta': min(all_two_thetas_obs), 'max_2theta': max(all_two_thetas_obs), 'min_miller': list(span.min()), 'max_miller': list(span.max()) } # prepare MonteCarlo sampling mc_runs = 50 sample_size = min(len(all_miller_indices) // 2, 100) unit_cell_info['sampling'] = { 'method': 'montecarlo', 'runs': mc_runs, 'used_per_run': sample_size } unit_cell_info['reference'] = { 'cell': reference_cell.parameters(), 'cell_volume': reference_cell.volume(), 'lattice': reference_lattice, 'wavelength': reference_wavelength } Chatter.write("\nRandomly sampling %d x %d reflections for Monte Carlo iterations" % (mc_runs, sample_size)) Debug.write("Refinements start with reference unit cell: %s" % reference_cell) MC = [] MCconstrained = [] used_index_range = flex.miller_index() used_two_theta_range_min = 1e300 used_two_theta_range_max = 0 used_reflections = set() for n in range(mc_runs): # MC sampling # Select sample_size reflections sample = flex.size_t(random.sample(range(len(all_miller_indices)), sample_size)) used_reflections = used_reflections.union(set(sample)) miller_indices = all_miller_indices.select(sample) two_thetas_obs = all_two_thetas_obs.select(sample) # Record span = miller.index_span(miller_indices) used_index_range.append(span.min()) used_index_range.append(span.max()) used_two_theta_range_min = min(used_two_theta_range_min, min(two_thetas_obs)) used_two_theta_range_max = max(used_two_theta_range_max, max(two_thetas_obs)) refined = _refinery(two_thetas_obs, miller_indices, reference_wavelength, reference_cell) MC.append(refined.unit_cell().parameters() + (refined.unit_cell().volume(),)) Debug.write('Run %d refined to: %s' % (n, str(refined.unit_cell()))) if reference_lattice is not None and reference_lattice is not 'aP': refined = _refinery(two_thetas_obs, miller_indices, reference_wavelength, reference_cell, reference_lattice[0]) MCconstrained.append(refined.unit_cell().parameters() + (refined.unit_cell().volume(),)) Debug.write('Run %d (constrained %s) refined to: %s' % (n, reference_lattice[0], str(refined.unit_cell()))) if (n % 50) == 0: sys.stdout.write("\n%5s ." % (str(n) if n > 0 else '')) else: sys.stdout.write(".") sys.stdout.flush() assert used_two_theta_range_min < used_two_theta_range_max def stats_summary(l): mean = sum(l) / len(l) var = 0 for y in l: var = var + ((y - mean) ** 2) popvar = var / (len(l)-1) popstddev = math.sqrt(popvar) stderr = popstddev / math.sqrt(len(l)) return { 'mean': mean, 'variance': var, 'population_variance': popvar, 'population_standard_deviation': popstddev, 'standard_error': stderr } print Chatter.write("") Chatter.write("Unit cell estimation based on %d Monte Carlo runs," % len(MC)) span = miller.index_span(used_index_range) Chatter.write("drawn from miller indices between %s and %s" % (str(span.min()), str(span.max()))) Chatter.write("with associated 2theta angles between %.3f and %.3f deg" % (used_two_theta_range_min, used_two_theta_range_max)) unit_cell_info['sampling'].update({ 'used_reflections': len(used_reflections), 'used_max_2theta': used_two_theta_range_max, 'used_min_2theta': used_two_theta_range_min, 'used_max_miller': span.max(), 'used_min_miller': span.min() }) unit_cell_info['solution_unconstrained'] = {} unit_cell_info['solution_constrained'] = { 'lattice': reference_lattice } if reference_lattice is None or reference_lattice == 'aP': Chatter.write("\n Unconstrained estimate:", strip=False) for dimension, estimate in zip(['a', 'b', 'c', 'alpha', 'beta', 'gamma', 'volume'], zip(*MC)): est_stats = stats_summary(estimate) unit_cell_info['solution_unconstrained'][dimension] = est_stats unit_cell_info['solution_constrained'][dimension] = est_stats Chatter.write(" %6s =%10.5f (SD: %.5f, SE: %.5f)" % (dimension, est_stats['mean'], est_stats['population_standard_deviation'], est_stats['standard_error']), strip=False) else: Chatter.write("\n Unconstrained estimate: | Constrained estimate (%s):" % reference_lattice, strip=False) for dimension, estimate, constrained in zip(['a', 'b', 'c', 'alpha', 'beta', 'gamma', 'volume'], zip(*MC), zip(*MCconstrained)): est_stats = stats_summary(estimate) rest_stats = stats_summary(constrained) unit_cell_info['solution_unconstrained'][dimension] = est_stats unit_cell_info['solution_constrained'][dimension] = rest_stats Chatter.write(" %6s =%10.5f (SD: %.5f, SE: %.5f) | %6s =%10.5f (SD: %.5f, SE: %.5f)" % (dimension, est_stats['mean'], est_stats['population_standard_deviation'], est_stats['standard_error'], dimension, rest_stats['mean'], rest_stats['population_standard_deviation'], rest_stats['standard_error']), strip=False) with open(os.path.join(wd, 'xia2.get_unit_cell_errors.json'), 'w') as fh: json.dump(unit_cell_info, fh, indent = 2, sort_keys=True)
def _index_prepare(self): from xia2.Handlers.Citations import Citations Citations.cite("dials") # all_images = self.get_matching_images() # first = min(all_images) # last = max(all_images) spot_lists = [] experiments_filenames = [] for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps): Chatter.banner("Spotfinding %s" % xsweep.get_name()) first, last = imageset.get_scan().get_image_range() # at this stage, break out to run the DIALS code: this sets itself up # now cheat and pass in some information... save re-reading all of the # image headers # FIXME need to adjust this to allow (say) three chunks of images from dxtbx.model.experiment_list import ExperimentListFactory sweep_filename = os.path.join(self.get_working_directory(), "%s_import.expt" % xsweep.get_name()) ExperimentListFactory.from_imageset_and_crystal( imageset, None).as_file(sweep_filename) genmask = self.GenerateMask() genmask.set_input_experiments(sweep_filename) genmask.set_output_experiments( os.path.join( self.get_working_directory(), "%s_%s_masked.expt" % (genmask.get_xpid(), xsweep.get_name()), )) genmask.set_params(PhilIndex.params.dials.masking) sweep_filename, mask_pickle = genmask.run() Debug.write("Generated mask for %s: %s" % (xsweep.get_name(), mask_pickle)) gain = PhilIndex.params.xia2.settings.input.gain if gain is libtbx.Auto: gain_estimater = self.EstimateGain() gain_estimater.set_sweep_filename(sweep_filename) gain_estimater.run() gain = gain_estimater.get_gain() Chatter.write("Estimated gain: %.2f" % gain) PhilIndex.params.xia2.settings.input.gain = gain # FIXME this should really use the assigned spot finding regions # offset = self.get_frame_offset() dfs_params = PhilIndex.params.dials.find_spots spotfinder = self.Spotfinder() if last - first > 10: spotfinder.set_write_hot_mask(True) spotfinder.set_input_sweep_filename(sweep_filename) spotfinder.set_output_sweep_filename( "%s_%s_strong.expt" % (spotfinder.get_xpid(), xsweep.get_name())) spotfinder.set_input_spot_filename( "%s_%s_strong.refl" % (spotfinder.get_xpid(), xsweep.get_name())) if PhilIndex.params.dials.fast_mode: wedges = self._index_select_images_i(imageset) spotfinder.set_scan_ranges(wedges) else: spotfinder.set_scan_ranges([(first, last)]) if dfs_params.phil_file is not None: spotfinder.set_phil_file(dfs_params.phil_file) if dfs_params.min_spot_size is libtbx.Auto: if imageset.get_detector()[0].get_type() == "SENSOR_PAD": dfs_params.min_spot_size = 3 else: dfs_params.min_spot_size = None if dfs_params.min_spot_size is not None: spotfinder.set_min_spot_size(dfs_params.min_spot_size) if dfs_params.min_local is not None: spotfinder.set_min_local(dfs_params.min_local) if dfs_params.sigma_strong: spotfinder.set_sigma_strong(dfs_params.sigma_strong) gain = PhilIndex.params.xia2.settings.input.gain if gain: spotfinder.set_gain(gain) if dfs_params.filter_ice_rings: spotfinder.set_filter_ice_rings(dfs_params.filter_ice_rings) if dfs_params.kernel_size: spotfinder.set_kernel_size(dfs_params.kernel_size) if dfs_params.global_threshold is not None: spotfinder.set_global_threshold(dfs_params.global_threshold) if dfs_params.threshold.algorithm is not None: spotfinder.set_threshold_algorithm( dfs_params.threshold.algorithm) spotfinder.run() spot_filename = spotfinder.get_spot_filename() if not os.path.exists(spot_filename): raise RuntimeError("Spotfinding failed: %s does not exist." % os.path.basename(spot_filename)) spot_lists.append(spot_filename) experiments_filenames.append( spotfinder.get_output_sweep_filename()) from dials.util.ascii_art import spot_counts_per_image_plot refl = flex.reflection_table.from_file(spot_filename) if not len(refl): raise RuntimeError("No spots found in sweep %s" % xsweep.get_name()) Chatter.write(spot_counts_per_image_plot(refl), strip=False) if not PhilIndex.params.dials.fast_mode: detectblanks = self.DetectBlanks() detectblanks.set_sweep_filename(experiments_filenames[-1]) detectblanks.set_reflections_filename(spot_filename) detectblanks.run() json = detectblanks.get_results() blank_regions = json["strong"]["blank_regions"] if len(blank_regions): blank_regions = [(int(s), int(e)) for s, e in blank_regions] for blank_start, blank_end in blank_regions: Chatter.write( "WARNING: Potential blank images: %i -> %i" % (blank_start + 1, blank_end)) if PhilIndex.params.xia2.settings.remove_blanks: non_blanks = [] start, end = imageset.get_array_range() last_blank_end = start for blank_start, blank_end in blank_regions: if blank_start > start: non_blanks.append( (last_blank_end, blank_start)) last_blank_end = blank_end if last_blank_end + 1 < end: non_blanks.append((last_blank_end, end)) xsweep = self.get_indexer_sweep() xwav = xsweep.get_wavelength() xsample = xsweep.get_xsample() sweep_name = xsweep.get_name() import string for i, (nb_start, nb_end) in enumerate(non_blanks): assert i < 26 if i == 0: sub_imageset = imageset[nb_start - start:nb_end - start] xsweep._frames_to_process = (nb_start + 1, nb_end + 1) self.set_indexer_prepare_done(done=False) self._indxr_imagesets[ self._indxr_imagesets.index( imageset)] = sub_imageset xsweep._integrater._setup_from_imageset( sub_imageset) else: min_images = (PhilIndex.params.xia2.settings. input.min_images) if (nb_end - nb_start) < min_images: continue new_name = "_".join( (sweep_name, string.ascii_lowercase[i])) new_sweep = xwav.add_sweep( new_name, xsample, directory=os.path.join( os.path.basename( xsweep.get_directory()), new_name, ), image=imageset.get_path(nb_start - start), frames_to_process=(nb_start + 1, nb_end), ) Chatter.write( "Generating new sweep: %s (%s:%i:%i)" % ( new_sweep.get_name(), new_sweep.get_image(), new_sweep.get_frames_to_process()[0], new_sweep.get_frames_to_process()[1], )) return if not PhilIndex.params.xia2.settings.trust_beam_centre: discovery = self.DiscoverBetterExperimentalModel() discovery.set_sweep_filename(experiments_filenames[-1]) discovery.set_spot_filename(spot_filename) try: discovery.run() except Exception as e: Debug.write("DIALS beam centre search failed: %s" % str(e)) else: # overwrite indexed.expt in experiments list experiments_filenames[ -1] = discovery.get_optimized_experiments_filename() self.set_indexer_payload("spot_lists", spot_lists) self.set_indexer_payload("experiments", experiments_filenames)
def _update_scaled_unit_cell(self): # FIXME this could be brought in-house fast_mode = PhilIndex.params.dials.fast_mode if PhilIndex.params.xia2.settings.integrater == 'dials' and not fast_mode: from xia2.Wrappers.Dials.TwoThetaRefine import TwoThetaRefine from xia2.lib.bits import auto_logfiler Chatter.banner('Unit cell refinement') # Collect a list of all sweeps, grouped by project, crystal, wavelength groups = {} self._scalr_cell_dict = {} tt_refine_experiments, tt_refine_pickles = [], [] for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) pi = '_'.join(si.get_project_info()) intgr = si.get_integrater() groups[pi] = groups.get(pi, []) + \ [(intgr.get_integrated_experiments(), intgr.get_integrated_reflections())] # Two theta refine the unit cell for each group for pi in groups.keys(): tt_grouprefiner = TwoThetaRefine() tt_grouprefiner.set_working_directory(self.get_working_directory()) auto_logfiler(tt_grouprefiner) files = zip(*groups[pi]) tt_grouprefiner.set_experiments(files[0]) tt_grouprefiner.set_pickles(files[1]) tt_refine_experiments.extend(files[0]) tt_refine_pickles.extend(files[1]) tt_grouprefiner.set_reindex_operator(self._spacegroup_reindex_operator) tt_grouprefiner.run() Chatter.write('%s: %6.2f %6.2f %6.2f %6.2f %6.2f %6.2f' % \ tuple([''.join(pi.split('_')[2:])] + list(tt_grouprefiner.get_unit_cell()))) self._scalr_cell_dict[pi] = (tt_grouprefiner.get_unit_cell(), tt_grouprefiner.get_unit_cell_esd(), tt_grouprefiner.import_cif(), tt_grouprefiner.import_mmcif()) if len(groups) > 1: cif_in = tt_grouprefiner.import_cif() cif_out = CIF.get_block(pi) for key in sorted(cif_in.keys()): cif_out[key] = cif_in[key] mmcif_in = tt_grouprefiner.import_mmcif() mmcif_out = mmCIF.get_block(pi) for key in sorted(mmcif_in.keys()): mmcif_out[key] = mmcif_in[key] # Two theta refine everything together if len(groups) > 1: tt_refiner = TwoThetaRefine() tt_refiner.set_working_directory(self.get_working_directory()) auto_logfiler(tt_refiner) tt_refiner.set_experiments(tt_refine_experiments) tt_refiner.set_pickles(tt_refine_pickles) tt_refiner.set_reindex_operator(self._spacegroup_reindex_operator) tt_refiner.run() self._scalr_cell = tt_refiner.get_unit_cell() Chatter.write('Overall: %6.2f %6.2f %6.2f %6.2f %6.2f %6.2f' % tt_refiner.get_unit_cell()) self._scalr_cell_esd = tt_refiner.get_unit_cell_esd() cif_in = tt_refiner.import_cif() mmcif_in = tt_refiner.import_mmcif() else: self._scalr_cell, self._scalr_cell_esd, cif_in, mmcif_in = self._scalr_cell_dict.values()[0] import dials.util.version cif_out = CIF.get_block('xia2') mmcif_out = mmCIF.get_block('xia2') cif_out['_computing_cell_refinement'] = mmcif_out['_computing.cell_refinement'] = 'DIALS 2theta refinement, %s' % dials.util.version.dials_version() for key in sorted(cif_in.keys()): cif_out[key] = cif_in[key] for key in sorted(mmcif_in.keys()): mmcif_out[key] = mmcif_in[key] Debug.write('Unit cell obtained by two-theta refinement') else: ami = AnalyseMyIntensities() ami.set_working_directory(self.get_working_directory()) average_unit_cell, ignore_sg = ami.compute_average_cell( [self._scalr_scaled_refl_files[key] for key in self._scalr_scaled_refl_files]) Debug.write('Computed average unit cell (will use in all files)') self._scalr_cell = average_unit_cell self._scalr_cell_esd = None # Write average unit cell to .cif cif_out = CIF.get_block('xia2') cif_out['_computing_cell_refinement'] = 'AIMLESS averaged unit cell' for cell, cifname in zip(self._scalr_cell, ['length_a', 'length_b', 'length_c', 'angle_alpha', 'angle_beta', 'angle_gamma']): cif_out['_cell_%s' % cifname] = cell Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \ self._scalr_cell)
def _index_prepare(self): from xia2.Handlers.Citations import Citations Citations.cite('dials') #all_images = self.get_matching_images() #first = min(all_images) #last = max(all_images) spot_lists = [] datablocks = [] for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps): Chatter.banner('Spotfinding %s' %xsweep.get_name()) first, last = imageset.get_scan().get_image_range() # at this stage, break out to run the DIALS code: this sets itself up # now cheat and pass in some information... save re-reading all of the # image headers # FIXME need to adjust this to allow (say) three chunks of images from dxtbx.serialize import dump from dxtbx.datablock import DataBlock sweep_filename = os.path.join( self.get_working_directory(), '%s_datablock.json' %xsweep.get_name()) dump.datablock(DataBlock([imageset]), sweep_filename) gain = PhilIndex.params.xia2.settings.input.gain if gain is libtbx.Auto: gain_estimater = self.EstimateGain() gain_estimater.set_sweep_filename(sweep_filename) gain_estimater.run() gain = gain_estimater.get_gain() Chatter.write('Estimated gain: %.2f' %gain) PhilIndex.params.xia2.settings.input.gain = gain # FIXME this should really use the assigned spot finding regions #offset = self.get_frame_offset() spotfinder = self.Spotfinder() if last - first > 10: spotfinder.set_write_hot_mask(True) spotfinder.set_input_sweep_filename(sweep_filename) spotfinder.set_output_sweep_filename( '%s_%s_datablock.json' %(spotfinder.get_xpid(), xsweep.get_name())) spotfinder.set_input_spot_filename( '%s_%s_strong.pickle' %(spotfinder.get_xpid(), xsweep.get_name())) if PhilIndex.params.dials.fast_mode: wedges = self._index_select_images_i(imageset) spotfinder.set_scan_ranges(wedges) else: spotfinder.set_scan_ranges([(first, last)]) if PhilIndex.params.dials.find_spots.phil_file is not None: spotfinder.set_phil_file(PhilIndex.params.dials.find_spots.phil_file) min_spot_size = PhilIndex.params.dials.find_spots.min_spot_size if min_spot_size is libtbx.Auto: if imageset.get_detector()[0].get_type() == 'SENSOR_PAD': min_spot_size = 3 else: min_spot_size = None if min_spot_size is not None: spotfinder.set_min_spot_size(min_spot_size) min_local = PhilIndex.params.dials.find_spots.min_local if min_local is not None: spotfinder.set_min_local(min_local) sigma_strong = PhilIndex.params.dials.find_spots.sigma_strong if sigma_strong: spotfinder.set_sigma_strong(sigma_strong) gain = PhilIndex.params.xia2.settings.input.gain if gain: spotfinder.set_gain(gain) filter_ice_rings = PhilIndex.params.dials.find_spots.filter_ice_rings if filter_ice_rings: spotfinder.set_filter_ice_rings(filter_ice_rings) kernel_size = PhilIndex.params.dials.find_spots.kernel_size if kernel_size: spotfinder.set_kernel_size(kernel_size) global_threshold = PhilIndex.params.dials.find_spots.global_threshold if global_threshold is not None: spotfinder.set_global_threshold(global_threshold) spotfinder.run() spot_filename = spotfinder.get_spot_filename() if not os.path.exists(spot_filename): raise RuntimeError("Spotfinding failed: %s does not exist." %os.path.basename(spot_filename)) spot_lists.append(spot_filename) datablocks.append(spotfinder.get_output_sweep_filename()) from libtbx import easy_pickle from dials.util.ascii_art import spot_counts_per_image_plot refl = easy_pickle.load(spot_filename) if not len(refl): raise RuntimeError('No spots found in sweep %s' %xsweep.get_name()) Chatter.write(spot_counts_per_image_plot(refl), strip=False) if not PhilIndex.params.dials.fast_mode: detectblanks = self.DetectBlanks() detectblanks.set_sweep_filename(datablocks[-1]) detectblanks.set_reflections_filename(spot_filename) detectblanks.run() json = detectblanks.get_results() offset = imageset.get_scan().get_image_range()[0] blank_regions = json['strong']['blank_regions'] if len(blank_regions): blank_regions = [(int(s), int(e)) for s, e in blank_regions] for blank_start, blank_end in blank_regions: Chatter.write('WARNING: Potential blank images: %i -> %i' %( blank_start+1, blank_end)) if PhilIndex.params.xia2.settings.remove_blanks: non_blanks = [] start, end = imageset.get_array_range() last_blank_end = start for blank_start, blank_end in blank_regions: if blank_start > start: non_blanks.append((last_blank_end, blank_start)) last_blank_end = blank_end if last_blank_end+1 < end: non_blanks.append((last_blank_end, end)) xsweep = self.get_indexer_sweep() xwav = xsweep.get_wavelength() xsample = xsweep.get_xsample() sweep_name = xsweep.get_name() import string for i, (nb_start, nb_end) in enumerate(non_blanks): assert i < 26 if i == 0: sub_imageset = imageset[nb_start-start:nb_end-start] xsweep._frames_to_process = (nb_start+1, nb_end+1) self.set_indexer_prepare_done(done=False) self._indxr_imagesets[self._indxr_imagesets.index(imageset)] = sub_imageset xsweep._integrater._setup_from_imageset(sub_imageset) else: new_name = '_'.join((sweep_name, string.ascii_lowercase[i])) new_sweep = xwav.add_sweep(new_name, xsample, directory=os.path.join( os.path.basename(xsweep.get_directory()), new_name), image=imageset.get_path(nb_start-start), frames_to_process=(nb_start+1, nb_end), ) Chatter.write("Generating new sweep: %s (%s:%i:%i)" %( new_sweep.get_name(), new_sweep.get_image(), new_sweep.get_frames_to_process()[0], new_sweep.get_frames_to_process()[1])) return if not PhilIndex.params.xia2.settings.trust_beam_centre: discovery = self.DiscoverBetterExperimentalModel() discovery.set_sweep_filename(datablocks[-1]) discovery.set_spot_filename(spot_filename) #wedges = self._index_select_images_i(imageset) #discovery.set_scan_ranges(wedges) #discovery.set_scan_ranges([(first + offset, last + offset)]) try: discovery.run() except Exception, e: Debug.write('DIALS beam centre search failed: %s' %str(e)) else: # overwrite datablock.json in datablocks list datablocks[-1] = discovery.get_optimized_datablock_filename()