def Cellparm(self): '''Create a Cellparm wrapper from _Cellparm - set the working directory and log file stuff as a part of this...''' cellparm = _Cellparm() cellparm.set_working_directory(self.get_working_directory()) auto_logfiler(cellparm) return cellparm
def get_real_space_primitive_matrix(lattice, matrix, wd = None): '''Get the primitive real space vectors for the unit cell and lattice type. Note that the resulting matrix will need to be scaled by a factor equal to the wavelength in Angstroms.''' # parse the orientation matrix cell, a, u = parse_matrix(matrix) # generate other possibilities o = _Othercell() if wd: o.set_working_directory(wd) auto_logfiler(o) o.set_cell(cell) o.set_lattice(lattice) o.generate() # transform the possibly centred cell to the primitive setting new_cell = o.get_cell('aP') op = symop_to_mat(o.get_reindex_op('aP')) primitive_a = matmul(invert(op), a) # then convert to real space real_a = invert(primitive_a) return real_a[0:3], real_a[3:6], real_a[6:9]
def get_reciprocal_space_primitive_matrix(lattice, matrix, wd = None): '''Get the primitive reciprocal space vectors for this matrix.''' # parse the orientation matrix cell, a, u = parse_matrix(matrix) # generate other possibilities o = _Othercell() if wd: o.set_working_directory(wd) auto_logfiler(o) o.set_cell(cell) o.set_lattice(lattice) o.generate() # transform the possibly centred cell to the primitive setting new_cell = o.get_cell('aP') op = symop_to_mat(o.get_reindex_op('aP')) primitive_a = matmul(invert(op), a) return mat2vec(primitive_a)
def Mtzdump(self): '''Create a Mtzdump wrapper from _Mtzdump - set the working directory and log file stuff as a part of this...''' mtzdump = _Mtzdump() mtzdump.set_working_directory(self.get_working_directory()) auto_logfiler(mtzdump) return mtzdump
def Correct(self): correct = _Correct(params=PhilIndex.params.xds.correct) correct.set_working_directory(self.get_working_directory()) correct.setup_from_imageset(self.get_imageset()) if self.get_distance(): correct.set_distance(self.get_distance()) if self.get_wavelength(): correct.set_wavelength(self.get_wavelength()) if self.get_integrater_ice(): correct.set_ice(self.get_integrater_ice()) if self.get_integrater_excluded_regions(): correct.set_excluded_regions(self.get_integrater_excluded_regions()) if self.get_integrater_anomalous(): correct.set_anomalous(True) if self.get_integrater_low_resolution() > 0.0: Debug.write('Using low resolution limit: %.2f' % \ self.get_integrater_low_resolution()) correct.set_resolution_high(0.0) correct.set_resolution_low( self.get_integrater_low_resolution()) auto_logfiler(correct, 'CORRECT') return correct
def RefineBravaisSettings(self): rbs = _RefineBravaisSettings() rbs.set_working_directory(self.get_working_directory()) rbs.set_close_to_spindle_cutoff( PhilIndex.params.dials.close_to_spindle_cutoff) auto_logfiler(rbs) return rbs
def Idxref(self): from xia2.Handlers.Phil import PhilIndex idxref = _Idxref(params=PhilIndex.params.xds.index) idxref.set_working_directory(self.get_working_directory()) idxref.setup_from_imageset(self.get_imageset()) if self.get_distance(): idxref.set_distance(self.get_distance()) if self.get_wavelength(): idxref.set_wavelength(self.get_wavelength()) # if we have a refined set of parameters to apply, apply these if Flags.get_xparm(): idxref.set_refined_origin(Flags.get_xparm_origin()) idxref.set_refined_beam_vector(Flags.get_xparm_beam_vector()) idxref.set_refined_rotation_axis(Flags.get_xparm_rotation_axis()) idxref.set_refined_distance(Flags.get_xparm_distance()) # hacks for Jira 493 if Flags.get_xparm_a(): idxref.set_a_axis(Flags.get_xparm_a()) if Flags.get_xparm_b(): idxref.set_b_axis(Flags.get_xparm_b()) if Flags.get_xparm_c(): idxref.set_c_axis(Flags.get_xparm_c()) auto_logfiler(idxref, 'IDXREF') return idxref
def Report(self): report = _Report() report.set_working_directory(self.get_working_directory()) report.set_experiments_filename(self._intgr_experiments_filename) report.set_reflections_filename(self._intgr_integrated_pickle) auto_logfiler(report, 'REPORT') return report
def Truncate(self): '''Create a Truncate wrapper from _Truncate - set the working directory and log file stuff as a part of this...''' truncate = _Truncate() truncate.set_working_directory(self.get_working_directory()) auto_logfiler(truncate) return truncate
def Rebatch(self): '''Create a Rebatch wrapper from _Rebatch - set the working directory and log file stuff as a part of this...''' rebatch = _Rebatch() rebatch.set_working_directory(self.get_working_directory()) auto_logfiler(rebatch) return rebatch
def _integrate_finish(self): '''Finish the integration - if necessary performing reindexing based on the pointgroup and the reindexing operator.''' if self._intgr_reindex_operator is None and \ self._intgr_spacegroup_number == lattice_to_spacegroup( self.get_integrater_refiner().get_refiner_lattice()): return self._mosflm_hklout if self._intgr_reindex_operator is None and \ self._intgr_spacegroup_number == 0: return self._mosflm_hklout Debug.write('Reindexing to spacegroup %d (%s)' % \ (self._intgr_spacegroup_number, self._intgr_reindex_operator)) hklin = self._mosflm_hklout reindex = Reindex() reindex.set_working_directory(self.get_working_directory()) auto_logfiler(reindex) reindex.set_operator(self._intgr_reindex_operator) if self._intgr_spacegroup_number: reindex.set_spacegroup(self._intgr_spacegroup_number) hklout = '%s_reindex.mtz' % hklin[:-4] reindex.set_hklin(hklin) reindex.set_hklout(hklout) reindex.reindex() return hklout
def Pointless(self): '''Create a Pointless wrapper from _Pointless - and set the working directory and log file stuff as a part of this...''' pointless = _Pointless() pointless.set_working_directory(self.get_working_directory()) auto_logfiler(pointless) return pointless
def _index_select_images(self): '''Select correct images based on image headers. This will in general use the 20 frames. N.B. only if they have good spots on them!''' phi_width = self.get_phi_width() images = self.get_matching_images() # N.B. now bodging this to use up to 20 frames which have decent # spots on, spaced from throughout the data set. spacing = max(1, int(len(images) // 20)) selected = [] for j in range(0, len(images), spacing): selected.append(images[j]) for image in selected[:20]: ld = LabelitDistl() ld.set_working_directory(self.get_working_directory()) auto_logfiler(ld) ld.add_image(self.get_image_name(image)) ld.distl() spots = ld.get_statistics( self.get_image_name(image))['spots_good'] Debug.write('Image %d good spots %d' % (image, spots)) if spots > 10: self.add_indexer_image_wedge(image) return
def Sortmtz(self): '''Create a Sortmtz wrapper from _Sortmtz - set the working directory and log file stuff as a part of this...''' sortmtz = _Sortmtz() sortmtz.set_working_directory(self.get_working_directory()) auto_logfiler(sortmtz) return sortmtz
def Reindex(self): '''Create a Reindex wrapper from _Reindex - set the working directory and log file stuff as a part of this...''' reindex = _Reindex() reindex.set_working_directory(self.get_working_directory()) auto_logfiler(reindex) return reindex
def Chef(self): '''Create a Chef wrapper from _Chef - set the working directory and log file stuff as a part of this...''' chef = _Chef() chef.set_working_directory(self.get_working_directory()) auto_logfiler(chef) return chef
def Scaleit(self): '''Create a Scaleit wrapper from _Scaleit - set the working directory and log file stuff as a part of this...''' scaleit = _Scaleit() scaleit.set_working_directory(self.get_working_directory()) auto_logfiler(scaleit) return scaleit
def Matthews_coef(self): '''Create a Matthews_coef wrapper from _Matthews_coef - set the working directory and log file stuff as a part of this...''' matthews_coef = _Matthews_coef() matthews_coef.set_working_directory(self.get_working_directory()) auto_logfiler(matthews_coef) return matthews_coef
def Freerflag(self): '''Create a Freerflag wrapper from _Freerflag - set the working directory and log file stuff as a part of this...''' freerflag = _Freerflag() freerflag.set_working_directory(self.get_working_directory()) auto_logfiler(freerflag) return freerflag
def Cad(self): '''Create a Cad wrapper from _Cad - set the working directory and log file stuff as a part of this...''' cad = _Cad() cad.set_working_directory(self.get_working_directory()) auto_logfiler(cad) return cad
def Mtz2various(self): '''Create a Mtz2various wrapper from _Mtz2various - set the working directory and log file stuff as a part of this...''' mtz2various = _Mtz2various() mtz2various.set_working_directory(self.get_working_directory()) auto_logfiler(mtz2various) return mtz2various
def spot_xds_to_reflection_pickle(spot_xds, working_directory): from xia2.Wrappers.Dials.ImportXDS import ImportXDS importer = ImportXDS() importer.set_working_directory(working_directory) auto_logfiler(importer) importer.set_spot_xds(spot_xds) importer.run() return importer.get_reflection_filename()
def _prepare_pointless_hklin(working_directory, hklin, phi_width): '''Prepare some data for pointless - this will take only 180 degrees of data if there is more than this (through a "rebatch" command) else will simply return hklin.''' # also remove blank images? if not Flags.get_microcrystal() and not Flags.get_small_molecule(): Debug.write('Excluding blank images') hklout = os.path.join( working_directory, '%s_noblank.mtz' % (os.path.split(hklin)[-1][:-4])) FileHandler.record_temporary_file(hklout) hklin = remove_blank(hklin, hklout) # find the number of batches md = Mtzdump() md.set_working_directory(working_directory) auto_logfiler(md) md.set_hklin(hklin) md.dump() batches = max(md.get_batches()) - min(md.get_batches()) phi_limit = 180 if batches * phi_width < phi_limit or Flags.get_small_molecule(): return hklin hklout = os.path.join( working_directory, '%s_prepointless.mtz' % (os.path.split(hklin)[-1][:-4])) rb = Rebatch() rb.set_working_directory(working_directory) auto_logfiler(rb) rb.set_hklin(hklin) rb.set_hklout(hklout) first = min(md.get_batches()) last = first + int(phi_limit / phi_width) Debug.write('Preparing data for pointless - %d batches (%d degrees)' % \ ((last - first), phi_limit)) rb.limit_batches(first, last) # we will want to delete this one exit FileHandler.record_temporary_file(hklout) return hklout
def CombineExperiments(self): combiner = _CombineExperiments() combiner.set_working_directory(self.get_working_directory()) auto_logfiler(combiner) for idxr in self._refinr_indexers.values(): combiner.add_experiments( idxr.get_indexer_payload("experiments_filename")) combiner.add_reflections(idxr.get_indexed_filename()) return combiner
def DialsSpotfinder(self): from xia2.Wrappers.Dials.Spotfinder import Spotfinder spotfinder = Spotfinder(params=PhilIndex.params.dials.find_spots) spotfinder.set_working_directory(self.get_working_directory()) spotfinder.setup_from_imageset(self.get_imageset()) auto_logfiler(spotfinder, 'SPOTFINDER') return spotfinder
def Import(self): importer = _Import() importer.set_working_directory(self.get_working_directory()) importer.setup_from_imageset(self.get_imageset()) auto_logfiler(importer) importer.set_mosflm_beam_centre(self.get_beam_centre()) importer.set_sweep_filename( os.path.join(self.get_working_directory(), '%s_datablock_import.json' %importer.get_xpid())) return importer
def transmogrify_matrix(lattice, matrix, target_lattice, wavelength, wd = None): '''Transmogrify a matrix for lattice X into a matrix for lattice Y. This should work find for Mosflm... Will also return the new unit cell.''' cell, a, u = parse_matrix(matrix) o = _Othercell() if wd: o.set_working_directory(wd) auto_logfiler(o) o.set_cell(cell) o.set_lattice(lattice) o.generate() new_cell = o.get_cell(target_lattice) op = symop_to_mat(o.get_reindex_op(target_lattice)) # HACK! again - bug # 3193 if 'lattice_symmetry' in o.get_executable(): op = transpose(op) # why is only one of the matrices inverted?! a = matmul(invert(op), a) u = matmul(op, u) # in here test that the given unit cell corresponds to the # one calculated from the A matrix. anew = matscl(a, 1.0 / wavelength) reala = transpose(invert(anew)) _a, _b, _c = mat2vec(reala) la = math.sqrt(dot(_a, _a)) lb = math.sqrt(dot(_b, _b)) lc = math.sqrt(dot(_c, _c)) if math.fabs(la - new_cell[0]) / new_cell[0] > 0.01: raise RuntimeError, 'cell check failed (wavelength != %f)' % \ wavelength if math.fabs(lb - new_cell[1]) / new_cell[1] > 0.01: raise RuntimeError, 'cell check failed (wavelength != %f)' % \ wavelength if math.fabs(lc - new_cell[2]) / new_cell[2] > 0.01: raise RuntimeError, 'cell check failed (wavelength != %f)' % \ wavelength return format_matrix(new_cell, a, u)
def Index(self): index = _Index() index.set_working_directory(self.get_working_directory()) params = PhilIndex.params.dials.index index.set_reflections_per_degree(params.reflections_per_degree) if params.fft3d.n_points is not None: index.set_fft3d_n_points(params.fft3d.n_points) auto_logfiler(index) index.set_outlier_algorithm(PhilIndex.params.dials.outlier.algorithm) index.set_histogram_binning(PhilIndex.params.dials.index.histogram_binning) return index
def ExportMtz(self): params = PhilIndex.params.dials.integrate export = _ExportMtz() export.set_working_directory(self.get_working_directory()) export.set_experiments_filename(self._intgr_experiments_filename) export.set_include_partials(params.include_partials) auto_logfiler(export, 'EXPORTMTZ') return export
def get_integrater_corrected_intensities(self): self.integrate() from xia2.Wrappers.Dials.ExportXDSASCII import ExportXDSASCII exporter = ExportXDSASCII() exporter.set_experiments_filename(self.get_integrated_experiments()) exporter.set_reflections_filename(self.get_integrated_reflections()) auto_logfiler(exporter) self._intgr_corrected_hklout = '%i_DIALS.HKL' %exporter.get_xpid() exporter.set_hkl_filename(self._intgr_corrected_hklout) exporter.run() assert os.path.exists(self._intgr_corrected_hklout) return self._intgr_corrected_hklout
def ExportMtz(self): params = PhilIndex.params.dials.integrate export = _ExportMtz() _, xname, _ = self.get_integrater_project_info() export.crystal_name = xname export.set_working_directory(self.get_working_directory()) export.set_experiments_filename(self._intgr_experiments_filename) export.set_combine_partials(params.combine_partials) export.set_partiality_threshold(params.partiality_threshold) if len(self.get_matching_images()) == 1: export.set_partiality_threshold(0.1) if (len(self.get_matching_images()) == 1 or PhilIndex.params.dials.fast_mode or not PhilIndex.params.xia2.settings.integration.profile_fitting ): # With no profiles available have to rely on summation alone export.set_intensity_choice("sum") auto_logfiler(export, "EXPORTMTZ") return export
def split_experiments(self, experiment, reflection, sweep_handler): """Split a multi-experiment dataset into individual datasets and set in the sweep handler.""" splitter = SplitExperiments() splitter.add_experiments(experiment) splitter.add_reflections(reflection) splitter.set_working_directory(self.get_working_directory()) auto_logfiler(splitter) splitter.run() nn = len(sweep_handler.get_epochs()) fmt = "%%0%dd" % (math.log10(nn) + 1) for i, epoch in enumerate(sweep_handler.get_epochs()): si = sweep_handler.get_sweep_information(epoch) nums = fmt % i si.set_reflections( os.path.join(self.get_working_directory(), "split_%s.refl" % nums) ) si.set_experiments( os.path.join(self.get_working_directory(), "split_%s.expt" % nums) ) return sweep_handler
def _scale_aimless(self, d_min=None): logger.debug('Scaling with aimless') PhilIndex.params.xia2.settings.multiprocessing.nproc = 1 scaler = Aimless() auto_logfiler(scaler) self._scaled_mtz = '%i_scaled.mtz' % scaler.get_xpid() scaler.set_surface_link(False) # multi-crystal scaler.set_hklin(self._sorted_mtz) scaler.set_hklout(self._scaled_mtz) scaler.set_surface_tie(self._params.scaling.surface_tie) if self._params.scaling.secondary.frame == 'camera': secondary = 'secondary' else: secondary = 'absorption' lmax = self._params.scaling.secondary.lmax scaler.set_secondary(mode=secondary, lmax=lmax) scaler.set_spacing(self._params.scaling.rotation.spacing) if d_min is not None: scaler.set_resolution(d_min) scaler.scale() self._scaled_unmerged_mtz \ = os.path.splitext(self._scaled_mtz)[0] + '_unmerged.mtz' return scaler
def Refine(self): refine = _Refine() params = PhilIndex.params.dials.refine refine.set_phil_file(params.phil_file) refine.set_working_directory(self.get_working_directory()) if PhilIndex.params.dials.fast_mode: # scan-static refinement in fast mode refine.set_scan_varying(False) else: refine.set_scan_varying(params.scan_varying) refine.set_reflections_per_degree(params.reflections_per_degree) refine.set_interval_width_degrees(params.interval_width_degrees) refine.set_outlier_algorithm(PhilIndex.params.dials.outlier.algorithm) if PhilIndex.params.dials.fix_geometry: refine.set_detector_fix("all") refine.set_beam_fix("all") elif PhilIndex.params.dials.fix_distance: refine.set_detector_fix("distance") refine.set_close_to_spindle_cutoff( PhilIndex.params.dials.close_to_spindle_cutoff) auto_logfiler(refine, "REFINE") return refine
def cosym(self): logger.debug("Running cosym analysis") experiments_filename = self._data_manager.export_experiments( "tmp.expt") reflections_filename = self._data_manager.export_reflections( "tmp.refl") cosym = DialsCosym() auto_logfiler(cosym) cosym.add_experiments_json(experiments_filename) cosym.add_reflections_file(reflections_filename) if self._params.symmetry.space_group is not None: cosym.set_space_group(self._params.symmetry.space_group.group()) if self._params.symmetry.laue_group is not None: cosym.set_space_group(self._params.symmetry.laue_group.group()) cosym.set_best_monoclinic_beta( self._params.symmetry.cosym.best_monoclinic_beta) cosym.run() self._cosym_analysis = cosym.get_cosym_analysis() self._experiments_filename = cosym.get_reindexed_experiments() self._reflections_filename = cosym.get_reindexed_reflections() self._data_manager.experiments = load.experiment_list( self._experiments_filename, check_format=False) self._data_manager.reflections = flex.reflection_table.from_file( self._reflections_filename) if not any([ self._params.symmetry.space_group, self._params.symmetry.laue_group ]): best_solution = cosym.get_best_solution() best_space_group = sgtbx.space_group( str(best_solution["patterson_group"]) ).build_derived_acentric_group() self._params.symmetry.laue_group = best_space_group.info() logger.info("Laue group determined by dials.cosym: %s" % best_space_group.info())
def multiplicity_plots(self): from xia2.Wrappers.XIA.PlotMultiplicity import PlotMultiplicity mult_json_files = {} mult_img_files = {} from xia2.lib.bits import auto_logfiler cwd = os.getcwd() try: os.chdir(self.report_dir) for axis in ('h', 'k', 'l'): pm = PlotMultiplicity() pm.set_mtz_filename(self.unmerged_mtz) pm.set_slice_axis(axis) pm.set_show_missing(True) auto_logfiler(pm) pm.run() mult_json_files[axis] = pm.get_json_filename() with open(pm.get_plot_filename(), 'rb') as fh: mult_img_files[axis] = fh.read().encode('base64').replace( '\n', '') return OrderedDict(('multiplicity_%s' % axis, mult_img_files[axis]) for axis in ('h', 'k', 'l')) finally: os.chdir(cwd)
def _decide_space_group_dials(self): logger.debug('Deciding space group with dials.symmetry') symmetry = DialsSymmetry() auto_logfiler(symmetry) self._sorted_mtz = '%i_sorted.mtz' % symmetry.get_xpid() self._experiments_filename = '%i_experiments_reindexed.json' % symmetry.get_xpid( ) self._reflections_filename = '%i_reflections_reindexed.pickle' % symmetry.get_xpid( ) experiments_filename = 'tmp_experiments.json' reflections_filename = 'tmp_reflections.pickle' self._data_manager.export_experiments(experiments_filename) self._data_manager.export_reflections(reflections_filename) symmetry.set_experiments_filename(experiments_filename) symmetry.set_reflections_filename(reflections_filename) symmetry.set_output_experiments_filename(self._experiments_filename) symmetry.set_output_reflections_filename(self._reflections_filename) symmetry.decide_pointgroup() space_group = sgtbx.space_group_info( symbol=str(symmetry.get_pointgroup())).group() cb_op = sgtbx.change_of_basis_op(symmetry.get_reindex_operator()) self._data_manager.experiments = load.experiment_list( self._experiments_filename, check_format=False) self._data_manager.reflections = flex.reflection_table.from_pickle( self._reflections_filename) # export reflections self._sorted_mtz = self._data_manager.export_mtz( filename=self._sorted_mtz) logger.info('Space group determined by dials.symmetry: %s' % space_group.info()) return space_group, cb_op
def Spotfinder(self): spotfinder = _Spotfinder() spotfinder.set_working_directory(self.get_working_directory()) auto_logfiler(spotfinder) spotfinder.set_hot_mask_prefix('%d_hot_mask' % spotfinder.get_xpid()) return spotfinder
def EstimateGain(self): estimater = _EstimateGain() estimater.set_working_directory(self.get_working_directory()) auto_logfiler(estimater) return estimater
def GenerateMask(self): genmask = _GenerateMask() genmask.set_working_directory(self.get_working_directory()) auto_logfiler(genmask) return genmask
def run(self): from xia2.Handlers.Streams import Chatter, Debug if self._reindexing_operators: Debug.write("Reindexing sweeps for dials.two_theta_refine") from xia2.lib.bits import auto_logfiler from xia2.Wrappers.Dials.Reindex import Reindex self._reindexed_experiments, self._reindexed_reflections = [], [] for e, p, op in zip( self._experiments, self._reflection_files, self._reindexing_operators, ): reindexer = Reindex() reindexer.set_cb_op(op) reindexer.set_experiments_filename(e) reindexer.set_indexed_filename(p) reindexer.set_working_directory( self.get_working_directory()) auto_logfiler(reindexer) reindexer.run() self._reindexed_experiments.append( reindexer.get_reindexed_experiments_filename()) self._reindexed_reflections.append( reindexer.get_reindexed_reflections_filename()) Debug.write("Running dials.two_theta_refine") self._output_cif = os.path.join( self.get_working_directory(), "%s_dials.two_theta_refine.cif" % self.get_xpid(), ) self._output_mmcif = os.path.join( self.get_working_directory(), "%s_dials.two_theta_refine.mmcif" % self.get_xpid(), ) if not self._output_p4p: self._output_p4p = os.path.join( self.get_working_directory(), "%s_dials.two_theta_refine.p4p" % self.get_xpid(), ) self._output_correlation_plot = os.path.join( self.get_working_directory(), "%s_dials.two_theta_refine.png" % self.get_xpid(), ) self._output_experiments = os.path.join( self.get_working_directory(), "%s_refined_cell.expt" % self.get_xpid()) self.clear_command_line() if self._reindexing_operators: for experiment in self._reindexed_experiments: self.add_command_line(experiment) for reflection_file in self._reindexed_reflections: self.add_command_line(reflection_file) else: for experiment in self._experiments: self.add_command_line(experiment) for reflection_file in self._reflection_files: self.add_command_line(reflection_file) self.add_command_line("combine_crystal_models=%s" % self._combine_crystal_models) self.add_command_line("output.cif=%s" % self._output_cif) self.add_command_line("output.mmcif=%s" % self._output_mmcif) self.add_command_line("output.p4p=%s" % self._output_p4p) if self._output_correlation_plot is not None: self.add_command_line("output.correlation_plot.filename=%s" % self._output_correlation_plot) if self._output_experiments is not None: self.add_command_line("output.experiments=%s" % self._output_experiments) if self._phil_file is not None: self.add_command_line("%s" % self._phil_file) self.start() self.close_wait() if not os.path.isfile(self._output_cif): Chatter.write( "TwoTheta refinement failed, see log file for more details:\n %s" % self.get_log_file()) raise RuntimeError("unit cell not refined") self.check_for_errors() from dxtbx.model.experiment_list import ExperimentListFactory experiments = ExperimentListFactory.from_json_file( self.get_output_experiments(), check_format=False) self._crystal = experiments.crystals()[0]
def _mosflm_parallel_integrate(self): '''Perform the integration as before, but this time as a number of parallel Mosflm jobs (hence, in separate directories) and including a step of pre-refinement of the mosaic spread and missets. This will all be kind of explicit and hence probably messy!''' refinr = self.get_integrater_refiner() lattice = refinr.get_refiner_lattice() spacegroup_number = lattice_to_spacegroup(lattice) mosaic = refinr.get_refiner_payload('mosaic') beam = refinr.get_refiner_payload('beam') distance = refinr.get_refiner_payload('distance') matrix = refinr.get_refiner_payload('mosflm_orientation_matrix') integration_params = refinr.get_refiner_payload( 'mosflm_integration_parameters') if integration_params: if 'separation' in integration_params: self.set_integrater_parameter( 'mosflm', 'separation', '%s %s' % tuple(integration_params['separation'])) if 'raster' in integration_params: self.set_integrater_parameter( 'mosflm', 'raster', '%d %d %d %d %d' % tuple(integration_params['raster'])) refinr.set_refiner_payload('mosflm_integration_parameters', None) pname, xname, dname = self.get_integrater_project_info() # what follows below should (i) be run in separate directories # and (ii) be repeated N=parallel times. nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc parallel = nproc # FIXME this is something of a kludge - if too few frames refinement # and integration does not work well... ideally want at least 15 # frames / chunk (say) nframes = self._intgr_wedge[1] - self._intgr_wedge[0] + 1 if parallel > nframes / 15: parallel = nframes // 15 if not parallel: raise RuntimeError('parallel not set') if parallel < 2: raise RuntimeError('parallel not parallel: %s' % parallel) jobs = [] hklouts = [] nref = 0 # calculate the chunks to use offset = self.get_frame_offset() start = self._intgr_wedge[0] - offset end = self._intgr_wedge[1] - offset left_images = 1 + end - start left_chunks = parallel chunks = [] while left_images > 0: size = left_images // left_chunks chunks.append((start, start + size - 1)) start += size left_images -= size left_chunks -= 1 summary_files = [] for j in range(parallel): # make some working directories, as necessary - chunk-(0:N-1) wd = os.path.join(self.get_working_directory(), 'chunk-%d' % j) if not os.path.exists(wd): os.makedirs(wd) job = MosflmIntegrate() job.set_working_directory(wd) auto_logfiler(job) l = refinr.get_refiner_lattice() # create the starting point f = open(os.path.join(wd, 'xiaintegrate-%s.mat' % l), 'w') for m in matrix: f.write(m) f.close() spacegroup_number = lattice_to_spacegroup(lattice) job.set_refine_profiles(self._mosflm_refine_profiles) # N.B. for harvesting need to append N to dname. if pname is not None and xname is not None and dname is not None: Debug.write('Harvesting: %s/%s/%s' % (pname, xname, dname)) harvest_dir = self.get_working_directory() temp_dname = '%s_%s' % \ (dname, self.get_integrater_sweep_name()) job.set_pname_xname_dname(pname, xname, temp_dname) job.set_template(os.path.basename(self.get_template())) job.set_directory(self.get_directory()) # check for ice - and if so, exclude (ranges taken from # XDS documentation) if self.get_integrater_ice() != 0: Debug.write('Excluding ice rings') job.set_exclude_ice(True) # exclude specified resolution ranges if len(self.get_integrater_excluded_regions()) != 0: regions = self.get_integrater_excluded_regions() Debug.write('Excluding regions: %s' % repr(regions)) job.set_exclude_regions(regions) mask = standard_mask(self.get_detector()) for m in mask: job.add_instruction(m) job.set_input_mat_file('xiaintegrate-%s.mat' % l) job.set_beam_centre(beam) job.set_distance(distance) job.set_space_group_number(spacegroup_number) job.set_mosaic(mosaic) if self.get_wavelength_prov() == 'user': job.set_wavelength(self.get_wavelength()) parameters = self.get_integrater_parameters('mosflm') job.update_parameters(parameters) if self._mosflm_gain: job.set_gain(self._mosflm_gain) # check for resolution limits if self._intgr_reso_high > 0.0: job.set_d_min(self._intgr_reso_high) if self._intgr_reso_low: job.set_d_max(self._intgr_reso_low) if PhilIndex.params.general.backstop_mask: from xia2.Toolkit.BackstopMask import BackstopMask mask = BackstopMask(PhilIndex.params.general.backstop_mask) mask = mask.calculate_mask_mosflm(self.get_header()) job.set_mask(mask) detector = self.get_detector() detector_width, detector_height = detector[0].get_image_size_mm() lim_x = 0.5 * detector_width lim_y = 0.5 * detector_height Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y)) job.set_limits(lim_x, lim_y) job.set_fix_mosaic(self._mosflm_postref_fix_mosaic) job.set_pre_refinement(True) job.set_image_range(chunks[j]) # these are now running so ... jobs.append(job) continue # ok, at this stage I need to ... # # (i) accumulate the statistics as a function of batch # (ii) mong them into a single block # # This is likely to be a pain in the arse! first_integrated_batch = 1.0e6 last_integrated_batch = -1.0e6 all_residuals = [] threads = [] for j in range(parallel): job = jobs[j] # now wait for them to finish - first wait will really be the # first one, then all should be finished... thread = Background(job, 'run') thread.start() threads.append(thread) mosaics = [] postref_result = {} integrated_images_first = 1.0e6 integrated_images_last = -1.0e6 self._intgr_per_image_statistics = {} for j in range(parallel): thread = threads[j] thread.stop() job = jobs[j] # get the log file output = job.get_all_output() # record a copy of it, perhaps - though not if parallel if self.get_integrater_sweep_name() and False: pname, xname, dname = self.get_integrater_project_info() FileHandler.record_log_file( '%s %s %s %s mosflm integrate' % \ (self.get_integrater_sweep_name(), pname, xname, '%s_%d' % (dname, j)), job.get_log_file()) # look for things that we want to know... # that is, the output reflection file name, the updated # value for the gain (if present,) any warnings, errors, # or just interesting facts. batches = job.get_batches_out() integrated_images_first = min(batches[0], integrated_images_first) integrated_images_last = max(batches[1], integrated_images_last) mosaics.extend(job.get_mosaic_spreads()) if min(mosaics) < 0: raise IntegrationError('negative mosaic spread: %s' % min(mosaic)) if (job.get_detector_gain_error() and not (self.get_imageset().get_detector()[0].get_type() == 'SENSOR_PAD')): gain = job.get_suggested_gain() if gain is not None: self.set_integrater_parameter('mosflm', 'gain', gain) self.set_integrater_export_parameter( 'mosflm', 'gain', gain) if self._mosflm_gain: Debug.write('GAIN updated to %f' % gain) else: Debug.write('GAIN found to be %f' % gain) self._mosflm_gain = gain self._mosflm_rerun_integration = True hklout = job.get_hklout() Debug.write('Integration output: %s' % hklout) hklouts.append(hklout) nref += job.get_nref() # if a BGSIG error happened try not refining the # profile and running again... if job.get_bgsig_too_large(): if not self._mosflm_refine_profiles: raise RuntimeError('BGSIG error with profiles fixed') Debug.write('BGSIG error detected - try fixing profile...') self._mosflm_refine_profiles = False self.set_integrater_done(False) return if job.get_getprof_error(): Debug.write('GETPROF error detected - try fixing profile...') self._mosflm_refine_profiles = False self.set_integrater_done(False) return # here # write the report for each image as .*-#$ to Chatter - # detailed report will be written automagically to science... self._intgr_per_image_statistics.update( job.get_per_image_statistics()) postref_result.update(job.get_postref_result()) # inspect the output for e.g. very high weighted residuals all_residuals.extend(job.get_residuals()) self._intgr_batches_out = (integrated_images_first, integrated_images_last) if mosaics and len(mosaics) > 0: self.set_integrater_mosaic_min_mean_max( min(mosaics), sum(mosaics) / len(mosaics), max(mosaics)) else: m = indxr.get_indexer_mosaic() self.set_integrater_mosaic_min_mean_max(m, m, m) Chatter.write(self.show_per_image_statistics()) Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \ self.get_integrater_mosaic_min_mean_max()) # gather the statistics from the postrefinement for all sweeps # now write this to a postrefinement log postref_log = os.path.join(self.get_working_directory(), 'postrefinement.log') fout = open(postref_log, 'w') fout.write('$TABLE: Postrefinement for %s:\n' % \ self._intgr_sweep_name) fout.write('$GRAPHS: Missetting angles:A:1, 2, 3, 4: $$\n') fout.write('Batch PhiX PhiY PhiZ $$ Batch PhiX PhiY PhiZ $$\n') for image in sorted(postref_result): phix = postref_result[image].get('phix', 0.0) phiy = postref_result[image].get('phiy', 0.0) phiz = postref_result[image].get('phiz', 0.0) fout.write('%d %5.2f %5.2f %5.2f\n' % \ (image, phix, phiy, phiz)) fout.write('$$\n') fout.close() if self.get_integrater_sweep_name(): pname, xname, dname = self.get_integrater_project_info() FileHandler.record_log_file('%s %s %s %s postrefinement' % \ (self.get_integrater_sweep_name(), pname, xname, dname), postref_log) hklouts.sort() hklout = os.path.join(self.get_working_directory(), os.path.split(hklouts[0])[-1]) Debug.write('Sorting data to %s' % hklout) for hklin in hklouts: Debug.write('<= %s' % hklin) sortmtz = Sortmtz() sortmtz.set_hklout(hklout) for hklin in hklouts: sortmtz.add_hklin(hklin) sortmtz.sort() self._mosflm_hklout = hklout return self._mosflm_hklout
def run(): try: check_environment() except Exception as e: with open("xia2-error.txt", "w") as fh: traceback.print_exc(file=fh) logger.error('Status: error "%s"', str(e)) if len(sys.argv) < 2 or "-help" in sys.argv: help() sys.exit() cwd = os.getcwd() try: from .xia2_main import xia2_main xia2_main(stop_after="integrate") logger.info("Status: normal termination") wd = os.path.join(cwd, "strategy") if not os.path.exists(wd): os.mkdir(wd) os.chdir(wd) CommandLine = get_command_line() xinfo = CommandLine.get_xinfo() crystals = xinfo.get_crystals() assert len(crystals) == 1 crystal = list(crystals.values())[0] assert len(crystal.get_wavelength_names()) == 1 wavelength = crystal.get_xwavelength(crystal.get_wavelength_names()[0]) sweeps = wavelength.get_sweeps() from xia2.Handlers.Phil import PhilIndex params = PhilIndex.get_python_object() strategy_params = params.strategy if not len(strategy_params): strategy_params = [ PhilIndex.get_scope_by_name("strategy")[0].extract() ] from dxtbx.model import MultiAxisGoniometer gonio = sweeps[0].get_imageset().get_goniometer() if (isinstance(gonio, MultiAxisGoniometer) and len(gonio.get_axes()) == 3 and gonio.get_scan_axis() == 2): from xia2.Wrappers.Dials.AlignCrystal import AlignCrystal align_crystal = AlignCrystal() align_crystal.set_experiments_filename( sweeps[0]._get_integrater().get_integrated_experiments()) align_crystal.set_working_directory(wd) auto_logfiler(align_crystal) align_crystal.set_json_filename("%i_align_crystal.json" % align_crystal.get_xpid()) align_crystal.run() logger.info("".join(align_crystal.get_all_output())) results_all = {} def process_one_strategy(args): assert len(args) == 4 experiments, reflections, strategy, t_ref = args from xia2.Wrappers.EMBL import Best best = Best.BestStrategy() for isweep, (expt, refl) in enumerate(zip(experiments, reflections)): from xia2.Wrappers.Dials.ExportBest import ExportBest export = ExportBest() export.set_experiments_filename(expt) export.set_reflections_filename(refl) export.set_working_directory(wd) auto_logfiler(export) prefix = "%i_best" % export.get_xpid() export.set_prefix(prefix) export.run() if isweep == 0: best.set_t_ref(t_ref) best.set_mos_dat("%s.dat" % prefix) best.set_mos_par("%s.par" % prefix) best.add_mos_hkl("%s.hkl" % prefix) best.set_i2s(strategy.i_over_sigi) best.set_T_max(strategy.max_total_exposure) best.set_t_min(strategy.min_exposure) # best.set_trans_ref(25.0) best.set_S_max(strategy.max_rotation_speed) best.set_w_min(strategy.min_oscillation_width) best.set_M_min(strategy.multiplicity) best.set_C_min(strategy.completeness) best.set_GpS(strategy.dose_rate) best.set_shape(strategy.shape) best.set_susceptibility(strategy.susceptibility) best.set_anomalous(strategy.anomalous) best.set_detector("pilatus6m") best.set_working_directory(wd) auto_logfiler(best) xmlout = os.path.join(best.get_working_directory(), "%i_best.xml" % best.get_xpid()) best.set_xmlout(xmlout) best.strategy() results = best.get_results_dict() results["description"] = strategy.description if "phi_end" not in results: results["phi_end"] = str( float(results["phi_start"]) + float(results["number_of_images"]) * float(results["phi_width"])) from dxtbx.serialize import load expt = load.experiment_list(experiments[0])[0] results["spacegroup"] = ( expt.crystal.get_space_group().type().lookup_symbol()) return results args = [] for istrategy, strategy in enumerate(strategy_params): imageset = sweeps[0].get_imageset() scan = imageset.get_scan() experiments = [ sweep._get_integrater().get_integrated_experiments() for sweep in sweeps ] reflections = [ sweep._get_integrater().get_integrated_reflections() for sweep in sweeps ] t_ref = scan.get_exposure_times()[0] args.append((experiments, reflections, strategy, t_ref)) nproc = params.xia2.settings.multiprocessing.nproc from libtbx import easy_mp results = easy_mp.parallel_map( process_one_strategy, args, processes=nproc, method="multiprocessing", preserve_order=True, preserve_exception_message=True, ) for istrategy, (result, strategy) in enumerate(zip(results, strategy_params)): name = strategy.name description = strategy.description if name is None: name = "Strategy%i" % (istrategy + 1) results_all[name] = result multiplicity = result["redundancy"] try: multiplicity = "%.2f" % multiplicity except TypeError: pass logger.info("Strategy %i", istrategy) if description is not None: logger.info(description) logger.info( "Start / end / width: %.2f/%.2f/%.2f", float(result["phi_start"]), float(result["phi_end"]), float(result["phi_width"]), ) logger.info( "Completeness / multiplicity / resolution: %.2f/%s/%.2f", float(result["completeness"]), multiplicity, float(result["resolution"]), ) logger.info( "Transmission / exposure %.3f/%.3f", float(result["transmission"]), float(result["exposure_time"]), ) with open("strategies.json", "wb") as f: json.dump(results_all, f, indent=2) except Exception as e: with open(os.path.join(cwd, "xia2-error.txt"), "w") as fh: traceback.print_exc(file=fh) logger.error('Status: error "%s"', str(e)) os.chdir(cwd)
def _index(self): '''Actually index the diffraction pattern. Note well that this is not going to compute the matrix...''' # acknowledge this program Citations.cite('labelit') Citations.cite('distl') #self.reset() _images = [] for i in self._indxr_images: for j in i: if not j in _images: _images.append(j) _images.sort() images_str = '%d' % _images[0] for i in _images[1:]: images_str += ', %d' % i cell_str = None if self._indxr_input_cell: cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \ self._indxr_input_cell if self._indxr_sweep_name: # then this is a proper autoindexing run - describe this # to the journal entry #if len(self._fp_directory) <= 50: #dirname = self._fp_directory #else: #dirname = '...%s' % self._fp_directory[-46:] dirname = os.path.dirname(self.get_imageset().get_template()) Journal.block( 'autoindexing', self._indxr_sweep_name, 'labelit', { 'images': images_str, 'target cell': cell_str, 'target lattice': self._indxr_input_lattice, 'template': self.get_imageset().get_template(), 'directory': dirname }) if len(_images) > 4: raise RuntimeError('cannot use more than 4 images') from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex index = LabelitIndex() index.set_working_directory(self.get_working_directory()) auto_logfiler(index) #task = 'Autoindex from images:' #for i in _images: #task += ' %s' % self.get_image_name(i) #self.set_task(task) Debug.write('Indexing from images:') for i in _images: index.add_image(self.get_image_name(i)) Debug.write('%s' % self.get_image_name(i)) xsweep = self.get_indexer_sweep() if xsweep is not None: if xsweep.get_distance() is not None: index.set_distance(xsweep.get_distance()) #if self.get_wavelength_prov() == 'user': #index.set_wavelength(self.get_wavelength()) if xsweep.get_beam_centre() is not None: index.set_beam_centre(xsweep.get_beam_centre()) if self._refine_beam is False: index.set_refine_beam(False) else: index.set_refine_beam(True) index.set_beam_search_scope(self._beam_search_scope) if ((math.fabs(self.get_wavelength() - 1.54) < 0.01) or (math.fabs(self.get_wavelength() - 2.29) < 0.01)): index.set_Cu_KA_or_Cr_KA(True) #sweep = self.get_indexer_sweep_name() #FileHandler.record_log_file( #'%s INDEX' % (sweep), self.get_log_file()) try: index.run() except RuntimeError as e: if self._refine_beam is False: raise e # can we improve the situation? if self._beam_search_scope < 4.0: self._beam_search_scope += 4.0 # try repeating the indexing! self.set_indexer_done(False) return 'failed' # otherwise this is beyond redemption raise e self._solutions = index.get_solutions() # FIXME this needs to check the smilie status e.g. # ":)" or ";(" or " ". # FIXME need to check the value of the RMSD and raise an # exception if the P1 solution has an RMSD > 1.0... # Change 27/FEB/08 to support user assigned spacegroups # (euugh!) have to "ignore" solutions with higher symmetry # otherwise the rest of xia will override us. Bummer. for i, solution in self._solutions.iteritems(): if self._indxr_user_input_lattice: if (lattice_to_spacegroup(solution['lattice']) > lattice_to_spacegroup(self._indxr_input_lattice)): Debug.write('Ignoring solution: %s' % solution['lattice']) del self._solutions[i] # check the RMSD from the triclinic unit cell if self._solutions[1]['rmsd'] > 1.0 and False: # don't know when this is useful - but I know when it is not! raise RuntimeError('high RMSD for triclinic solution') # configure the "right" solution self._solution = self.get_solution() # now store also all of the other solutions... keyed by the # lattice - however these should only be added if they # have a smiley in the appropriate record, perhaps? for solution in self._solutions.keys(): lattice = self._solutions[solution]['lattice'] if lattice in self._indxr_other_lattice_cell: if self._indxr_other_lattice_cell[lattice]['goodness'] < \ self._solutions[solution]['metric']: continue self._indxr_other_lattice_cell[lattice] = { 'goodness': self._solutions[solution]['metric'], 'cell': self._solutions[solution]['cell'] } self._indxr_lattice = self._solution['lattice'] self._indxr_cell = tuple(self._solution['cell']) self._indxr_mosaic = self._solution['mosaic'] lms = LabelitMosflmScript() lms.set_working_directory(self.get_working_directory()) lms.set_solution(self._solution['number']) self._indxr_payload['mosflm_orientation_matrix'] = lms.calculate() # get the beam centre from the mosflm script - mosflm # may have inverted the beam centre and labelit will know # this! mosflm_beam_centre = lms.get_mosflm_beam() if mosflm_beam_centre: self._indxr_payload['mosflm_beam_centre'] = tuple( mosflm_beam_centre) import copy detector = copy.deepcopy(self.get_detector()) beam = copy.deepcopy(self.get_beam()) from dxtbx.model.detector_helpers import set_mosflm_beam_centre set_mosflm_beam_centre(detector, beam, mosflm_beam_centre) from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number from scitbx import matrix from cctbx import sgtbx, uctbx from dxtbx.model import CrystalFactory mosflm_matrix = matrix.sqr([ float(i) for line in lms.calculate() for i in line.replace("-", " -").split() ][:9]) space_group = sgtbx.space_group_info( lattice_to_spacegroup_number(self._solution['lattice'])).group() crystal_model = CrystalFactory.from_mosflm_matrix( mosflm_matrix, unit_cell=uctbx.unit_cell(tuple(self._solution['cell'])), space_group=space_group) from dxtbx.model import Experiment, ExperimentList experiment = Experiment( beam=beam, detector=detector, goniometer=self.get_goniometer(), scan=self.get_scan(), crystal=crystal_model, ) experiment_list = ExperimentList([experiment]) self.set_indexer_experiment_list(experiment_list) # also get an estimate of the resolution limit from the # labelit.stats_distl output... FIXME the name is wrong! lsd = LabelitStats_distl() lsd.set_working_directory(self.get_working_directory()) lsd.stats_distl() resolution = 1.0e6 for i in _images: stats = lsd.get_statistics(self.get_image_name(i)) resol = 0.5 * (stats['resol_one'] + stats['resol_two']) if resol < resolution: resolution = resol self._indxr_resolution_estimate = resolution return 'ok'
def dials_symmetry(self): symmetry = _DialsSymmetry() symmetry.set_working_directory(self.get_working_directory()) auto_logfiler(symmetry) return symmetry
def _index_prepare(self): """Prepare to do autoindexing - in XDS terms this will mean calling xycorr, init and colspot on the input images.""" # decide on images to work with logger.debug("XDS INDEX PREPARE:") logger.debug("Wavelength: %.6f", self.get_wavelength()) logger.debug("Distance: %.2f", self.get_distance()) if self._indxr_images == []: _select_images_function = getattr( self, "_index_select_images_%s" % self._index_select_images ) wedges = _select_images_function() for wedge in wedges: self.add_indexer_image_wedge(wedge) self.set_indexer_prepare_done(True) all_images = self.get_matching_images() first = min(all_images) last = max(all_images) # next start to process these - first xycorr xycorr = self.Xycorr() xycorr.set_data_range(first, last) xycorr.set_background_range(self._indxr_images[0][0], self._indxr_images[0][1]) converter = to_xds(self.get_imageset()) xds_beam_centre = converter.detector_origin xycorr.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1]) for block in self._indxr_images: xycorr.add_spot_range(block[0], block[1]) # FIXME need to set the origin here xycorr.run() for file in ["X-CORRECTIONS.cbf", "Y-CORRECTIONS.cbf"]: self._indxr_payload[file] = xycorr.get_output_data_file(file) # next start to process these - then init if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing: imageset = self._indxr_imagesets[0] masker = ( imageset.get_format_class() .get_instance(imageset.paths()[0]) .get_masker() ) if masker is None: # disable dynamic_shadowing PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing: # find the region of the scan with the least predicted shadow # to use for background determination in XDS INIT step from dxtbx.model.experiment_list import ExperimentListFactory imageset = self._indxr_imagesets[0] xsweep = self._indxr_sweeps[0] sweep_filename = os.path.join( self.get_working_directory(), "%s_indexed.expt" % xsweep.get_name() ) ExperimentListFactory.from_imageset_and_crystal(imageset, None).as_file( sweep_filename ) from xia2.Wrappers.Dials.ShadowPlot import ShadowPlot shadow_plot = ShadowPlot() shadow_plot.set_working_directory(self.get_working_directory()) auto_logfiler(shadow_plot) shadow_plot.set_sweep_filename(sweep_filename) shadow_plot.set_json_filename( os.path.join( self.get_working_directory(), "%s_shadow_plot.json" % shadow_plot.get_xpid(), ) ) shadow_plot.run() results = shadow_plot.get_results() fraction_shadowed = flex.double(results["fraction_shadowed"]) if flex.max(fraction_shadowed) == 0: PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False else: scan_points = flex.double(results["scan_points"]) scan = imageset.get_scan() oscillation = scan.get_oscillation() if self._background_images is not None: bg_images = self._background_images bg_range_deg = ( scan.get_angle_from_image_index(bg_images[0]), scan.get_angle_from_image_index(bg_images[1]), ) bg_range_width = bg_range_deg[1] - bg_range_deg[0] min_shadow = 100 best_bg_range = bg_range_deg from libtbx.utils import frange for bg_range_start in frange( flex.min(scan_points), flex.max(scan_points) - bg_range_width, step=oscillation[1], ): bg_range_deg = (bg_range_start, bg_range_start + bg_range_width) sel = (scan_points >= bg_range_deg[0]) & ( scan_points <= bg_range_deg[1] ) mean_shadow = flex.mean(fraction_shadowed.select(sel)) if mean_shadow < min_shadow: min_shadow = mean_shadow best_bg_range = bg_range_deg self._background_images = ( scan.get_image_index_from_angle(best_bg_range[0]), scan.get_image_index_from_angle(best_bg_range[1]), ) logger.debug( "Setting background images: %s -> %s" % self._background_images ) init = self.Init() for file in ["X-CORRECTIONS.cbf", "Y-CORRECTIONS.cbf"]: init.set_input_data_file(file, self._indxr_payload[file]) init.set_data_range(first, last) if self._background_images: init.set_background_range( self._background_images[0], self._background_images[1] ) else: init.set_background_range( self._indxr_images[0][0], self._indxr_images[0][1] ) for block in self._indxr_images: init.add_spot_range(block[0], block[1]) init.run() # at this stage, need to (perhaps) modify the BKGINIT.cbf image # to mark out the back stop if PhilIndex.params.xds.backstop_mask: logger.debug("Applying mask to BKGINIT.pck") # copy the original file cbf_old = os.path.join(init.get_working_directory(), "BKGINIT.cbf") cbf_save = os.path.join(init.get_working_directory(), "BKGINIT.sav") shutil.copyfile(cbf_old, cbf_save) # modify the file to give the new mask from xia2.Toolkit.BackstopMask import BackstopMask mask = BackstopMask(PhilIndex.params.xds.backstop_mask) mask.apply_mask_xds(self.get_header(), cbf_save, cbf_old) init.reload() for file in ["BLANK.cbf", "BKGINIT.cbf", "GAIN.cbf"]: self._indxr_payload[file] = init.get_output_data_file(file) if PhilIndex.params.xia2.settings.developmental.use_dials_spotfinder: spotfinder = self.DialsSpotfinder() for block in self._indxr_images: spotfinder.add_spot_range(block[0], block[1]) spotfinder.run() export = self.DialsExportSpotXDS() export.set_input_data_file( "observations.refl", spotfinder.get_output_data_file("observations.refl"), ) export.run() for file in ["SPOT.XDS"]: self._indxr_payload[file] = export.get_output_data_file(file) else: # next start to process these - then colspot colspot = self.Colspot() for file in ( "X-CORRECTIONS.cbf", "Y-CORRECTIONS.cbf", "BLANK.cbf", "BKGINIT.cbf", "GAIN.cbf", ): colspot.set_input_data_file(file, self._indxr_payload[file]) colspot.set_data_range(first, last) colspot.set_background_range( self._indxr_images[0][0], self._indxr_images[0][1] ) for block in self._indxr_images: colspot.add_spot_range(block[0], block[1]) colspot.run() for file in ["SPOT.XDS"]: self._indxr_payload[file] = colspot.get_output_data_file(file)
def _index(self): """Actually index the diffraction pattern. Note well that this is not going to compute the matrix...""" # acknowledge this program if not self._indxr_images: raise RuntimeError("No good spots found on any images") Citations.cite("labelit") Citations.cite("distl") _images = [] for i in self._indxr_images: for j in i: if not j in _images: _images.append(j) _images.sort() images_str = "%d" % _images[0] for i in _images[1:]: images_str += ", %d" % i cell_str = None if self._indxr_input_cell: cell_str = "%.2f %.2f %.2f %.2f %.2f %.2f" % self._indxr_input_cell if self._indxr_sweep_name: # then this is a proper autoindexing run - describe this # to the journal entry if len(self._fp_directory) <= 50: dirname = self._fp_directory else: dirname = "...%s" % self._fp_directory[-46:] Journal.block( "autoindexing", self._indxr_sweep_name, "labelit", { "images": images_str, "target cell": cell_str, "target lattice": self._indxr_input_lattice, "template": self._fp_template, "directory": dirname, }, ) # auto_logfiler(self) from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex index = LabelitIndex() index.set_working_directory(self.get_working_directory()) auto_logfiler(index) # task = 'Autoindex from images:' # for i in _images: # task += ' %s' % self.get_image_name(i) # self.set_task(task) # self.add_command_line('--index_only') Debug.write("Indexing from images:") for i in _images: index.add_image(self.get_image_name(i)) Debug.write("%s" % self.get_image_name(i)) if self._primitive_unit_cell: index.set_primitive_unit_cell(self._primitive_unit_cell) if self._indxr_input_cell: index.set_max_cell(1.25 * max(self._indxr_input_cell[:3])) xsweep = self.get_indexer_sweep() if xsweep is not None: if xsweep.get_distance() is not None: index.set_distance(xsweep.get_distance()) # if self.get_wavelength_prov() == 'user': # index.set_wavelength(self.get_wavelength()) if xsweep.get_beam_centre() is not None: index.set_beam_centre(xsweep.get_beam_centre()) if self._refine_beam is False: index.set_refine_beam(False) else: index.set_refine_beam(True) index.set_beam_search_scope(self._beam_search_scope) if (math.fabs(self.get_wavelength() - 1.54) < 0.01) or (math.fabs(self.get_wavelength() - 2.29) < 0.01): index.set_Cu_KA_or_Cr_KA(True) try: index.run() except RuntimeError as e: if self._refine_beam is False: raise e # can we improve the situation? if self._beam_search_scope < 4.0: self._beam_search_scope += 4.0 # try repeating the indexing! self.set_indexer_done(False) return "failed" # otherwise this is beyond redemption raise e self._solutions = index.get_solutions() # FIXME this needs to check the smilie status e.g. # ":)" or ";(" or " ". # FIXME need to check the value of the RMSD and raise an # exception if the P1 solution has an RMSD > 1.0... # Change 27/FEB/08 to support user assigned spacegroups # (euugh!) have to "ignore" solutions with higher symmetry # otherwise the rest of xia will override us. Bummer. for i, solution in self._solutions.iteritems(): if self._indxr_user_input_lattice: if lattice_to_spacegroup( solution["lattice"]) > lattice_to_spacegroup( self._indxr_input_lattice): Debug.write("Ignoring solution: %s" % solution["lattice"]) del self._solutions[i] # configure the "right" solution self._solution = self.get_solution() # now store also all of the other solutions... keyed by the # lattice - however these should only be added if they # have a smiley in the appropriate record, perhaps? for solution in self._solutions.keys(): lattice = self._solutions[solution]["lattice"] if lattice in self._indxr_other_lattice_cell: if (self._indxr_other_lattice_cell[lattice]["goodness"] < self._solutions[solution]["metric"]): continue self._indxr_other_lattice_cell[lattice] = { "goodness": self._solutions[solution]["metric"], "cell": self._solutions[solution]["cell"], } self._indxr_lattice = self._solution["lattice"] self._indxr_cell = tuple(self._solution["cell"]) self._indxr_mosaic = self._solution["mosaic"] lms = LabelitMosflmMatrix() lms.set_working_directory(self.get_working_directory()) lms.set_solution(self._solution["number"]) self._indxr_payload["mosflm_orientation_matrix"] = lms.calculate() # get the beam centre from the mosflm script - mosflm # may have inverted the beam centre and labelit will know # this! mosflm_beam_centre = lms.get_mosflm_beam() if mosflm_beam_centre: self._indxr_payload["mosflm_beam_centre"] = tuple( mosflm_beam_centre) detector = copy.deepcopy(self.get_detector()) beam = copy.deepcopy(self.get_beam()) from dxtbx.model.detector_helpers import set_mosflm_beam_centre set_mosflm_beam_centre(detector, beam, mosflm_beam_centre) from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number from scitbx import matrix from cctbx import sgtbx, uctbx from dxtbx.model import CrystalFactory mosflm_matrix = matrix.sqr([ float(i) for line in lms.calculate() for i in line.replace("-", " -").split() ][:9]) space_group = sgtbx.space_group_info( lattice_to_spacegroup_number(self._solution["lattice"])).group() crystal_model = CrystalFactory.from_mosflm_matrix( mosflm_matrix, unit_cell=uctbx.unit_cell(tuple(self._solution["cell"])), space_group=space_group, ) from dxtbx.model import Experiment, ExperimentList experiment = Experiment( beam=beam, detector=detector, goniometer=self.get_goniometer(), scan=self.get_scan(), crystal=crystal_model, ) experiment_list = ExperimentList([experiment]) self.set_indexer_experiment_list(experiment_list) # also get an estimate of the resolution limit from the # labelit.stats_distl output... FIXME the name is wrong! lsd = LabelitStats_distl() lsd.set_working_directory(self.get_working_directory()) lsd.stats_distl() resolution = 1.0e6 for i in _images: stats = lsd.get_statistics(self.get_image_name(i)) resol = 0.5 * (stats["resol_one"] + stats["resol_two"]) if resol < resolution: resolution = resol self._indxr_resolution_estimate = resolution return "ok"
def process_one_sweep(args): assert len(args) == 1 args = args[0] # stop_after = args.stop_after command_line_args = args.command_line_args nproc = args.nproc crystal_id = args.crystal_id wavelength_id = args.wavelength_id sweep_id = args.sweep_id failover = args.failover driver_type = args.driver_type default_driver_type = DriverFactory.get_driver_type() DriverFactory.set_driver_type(driver_type) curdir = os.path.abspath(os.curdir) if "-xinfo" in command_line_args: idx = command_line_args.index("-xinfo") del command_line_args[idx + 1] del command_line_args[idx] xia2_integrate = XIA2Integrate() # import tempfile # tmpdir = tempfile.mkdtemp(dir=curdir) tmpdir = os.path.join(curdir, str(uuid.uuid4())) os.makedirs(tmpdir) xia2_integrate.set_working_directory(tmpdir) xia2_integrate.add_command_line_args(args.command_line_args) xia2_integrate.set_phil_file(os.path.join(curdir, "xia2-working.phil")) xia2_integrate.add_command_line_args(["sweep.id=%s" % sweep_id]) xia2_integrate.set_nproc(nproc) xia2_integrate.set_njob(1) xia2_integrate.set_mp_mode("serial") auto_logfiler(xia2_integrate) sweep_tmp_dir = os.path.join(tmpdir, crystal_id, wavelength_id, sweep_id) sweep_target_dir = os.path.join(curdir, crystal_id, wavelength_id, sweep_id) output = None success = False xsweep_dict = None try: xia2_integrate.run() output = get_sweep_output_only(xia2_integrate.get_all_output()) success = True except Exception as e: logger.warning("Processing sweep %s failed: %s", sweep_id, str(e)) if not failover: raise finally: from xia2.Schema.XProject import XProject xia2_json = os.path.join(tmpdir, "xia2.json") json_files = glob.glob(os.path.join(sweep_tmp_dir, "*", "*.json")) json_files.extend(glob.glob(os.path.join(sweep_tmp_dir, "*", "*.expt"))) if os.path.exists(xia2_json): json_files.append(xia2_json) import fileinput for line in fileinput.FileInput(files=json_files, inplace=1): line = line.replace(sweep_tmp_dir, sweep_target_dir) print(line) if os.path.exists(xia2_json): new_json = os.path.join(curdir, "xia2-%s.json" % sweep_id) shutil.copyfile(xia2_json, new_json) move_output_folder(sweep_tmp_dir, sweep_target_dir) if success: xinfo = XProject.from_json(new_json) xcryst = list(xinfo.get_crystals().values())[0] xsweep = xcryst.get_xwavelength(wavelength_id).get_sweeps()[0] xsweep_dict = xsweep.to_dict() shutil.rmtree(tmpdir, ignore_errors=True) if os.path.exists(tmpdir): shutil.rmtree(tmpdir, ignore_errors=True) DriverFactory.set_driver_type(default_driver_type) return success, output, xsweep_dict
def ExportXDS(self): from xia2.Wrappers.Dials.ExportXDS import ExportXDS as _ExportXDS export_xds = _ExportXDS() export_xds.set_working_directory(self.get_working_directory()) auto_logfiler(export_xds) return export_xds
def DetectBlanks(self): detectblanks = _DetectBlanks() detectblanks.set_working_directory(self.get_working_directory()) auto_logfiler(detectblanks) return detectblanks
def Report(self): report = _Report() report.set_working_directory(self.get_working_directory()) auto_logfiler(report, 'REPORT') return report
def _mosflm_test_refine_cell(self, idxr, test_lattice): '''Test performing cell refinement in with a different lattice to the one which was selected by the autoindex procedure. This should not change anything in the class.''' #idxr = self.get_integrater_indexer() lattice = idxr.get_indexer_lattice() mosaic = idxr.get_indexer_mosaic() beam_centre = idxr.get_indexer_beam_centre() distance = idxr.get_indexer_distance() matrix = idxr.get_indexer_payload('mosflm_orientation_matrix') phi_width = idxr.get_phi_width() if mosaic < 0.25 * phi_width: mosaic = 0.25 * phi_width input_matrix = '' for m in matrix: input_matrix += '%s\n' % m new_matrix = transmogrify_matrix(lattice, input_matrix, test_lattice, idxr.get_wavelength(), self.get_working_directory()) spacegroup_number = lattice_to_spacegroup(test_lattice) if not self._mosflm_cell_ref_images: raise RuntimeError('wedges must be assigned already') open( os.path.join(self.get_working_directory(), 'test-xiaindex-%s.mat' % lattice), 'w').write(new_matrix) refiner = MosflmRefineCell() refiner.set_working_directory(self.get_working_directory()) auto_logfiler(refiner) if self._mosflm_gain: refiner.set_gain(self._mosflm_gain) refiner.set_template(os.path.basename(idxr.get_template())) refiner.set_directory(idxr.get_directory()) refiner.set_input_mat_file('test-xiaindex-%s.mat' % lattice) refiner.set_output_mat_file('test-xiarefine.mat') refiner.set_beam_centre(beam_centre) refiner.set_distance(distance) refiner.set_space_group_number(spacegroup_number) # FIXME 18/JUN/08 - it may help to have an overestimate # of the mosaic spread in here as it *may* refine down # better than up... - this is not a good idea as it may # also not refine at all! - 12972 # integration failed # Bug # 3103 if self._mosflm_cell_ref_double_mosaic: mosaic *= 2.0 refiner.set_mosaic(mosaic) # if set, use the resolution for cell refinement - see # bug # 2078... if self._mosflm_cell_ref_resolution: refiner.set_resolution(self._mosflm_cell_ref_resolution) refiner.set_fix_mosaic(self._mosflm_postref_fix_mosaic) # note well that the beam centre is coming from indexing so # should be already properly handled #if idxr.get_wavelength_prov() == 'user': #refiner.set_wavelength(idxr.get_wavelength()) # belt + braces mode - only to be used when considering failover, # will run an additional step of autoindexing prior to cell # refinement, to be used only after proving that not going it # will result in cell refinement failure - will use the first # wedge... N.B. this is only useful if the indexer is Labelit # not Mosflm... refiner.set_add_autoindex(self._mosflm_cell_ref_add_autoindex) # XXX # get all of the stored parameter values parameters = self.get_refiner_parameters('mosflm') refiner.update_parameters(parameters) detector = idxr.get_detector() detector_width, detector_height = detector[0].get_image_size_mm() lim_x = 0.5 * detector_width lim_y = 0.5 * detector_height Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y)) refiner.set_limits(lim_x, lim_y) refiner.set_images(self._mosflm_cell_ref_images) refiner.run() rms_values = refiner.get_rms_values() background_residual = refiner.get_background_residual() return rms_values, background_residual
def SearchBeamPosition(self): discovery = _SearchBeamPosition() discovery.set_working_directory(self.get_working_directory()) # params = PhilIndex.params.dials.index auto_logfiler(discovery) return discovery
def DiscoverBetterExperimentalModel(self): discovery = _DiscoverBetterExperimentalModel() discovery.set_working_directory(self.get_working_directory()) #params = PhilIndex.params.dials.index auto_logfiler(discovery) return discovery
def _mosflm_integrate(self): '''Perform the actual integration, based on the results of the cell refinement or indexing (they have the equivalent form.)''' refinr = self.get_integrater_refiner() if not refinr.get_refiner_payload('mosflm_orientation_matrix'): raise RuntimeError('unexpected situation in indexing') lattice = refinr.get_refiner_lattice() spacegroup_number = lattice_to_spacegroup(lattice) mosaic = refinr.get_refiner_payload('mosaic') beam = refinr.get_refiner_payload('beam') distance = refinr.get_refiner_payload('distance') matrix = refinr.get_refiner_payload('mosflm_orientation_matrix') integration_params = refinr.get_refiner_payload( 'mosflm_integration_parameters') if integration_params: if 'separation' in integration_params: self.set_integrater_parameter( 'mosflm', 'separation', '%s %s' % tuple(integration_params['separation'])) if 'raster' in integration_params: self.set_integrater_parameter( 'mosflm', 'raster', '%d %d %d %d %d' % tuple(integration_params['raster'])) refinr.set_refiner_payload('mosflm_integration_parameters', None) f = open( os.path.join(self.get_working_directory(), 'xiaintegrate.mat'), 'w') for m in matrix: f.write(m) f.close() # then start the integration integrater = MosflmIntegrate() integrater.set_working_directory(self.get_working_directory()) auto_logfiler(integrater) integrater.set_refine_profiles(self._mosflm_refine_profiles) pname, xname, dname = self.get_integrater_project_info() if pname is not None and xname is not None and dname is not None: Debug.write('Harvesting: %s/%s/%s' % (pname, xname, dname)) harvest_dir = self.get_working_directory() # harvest file name will be %s.mosflm_run_start_end % dname temp_dname = '%s_%s' % \ (dname, self.get_integrater_sweep_name()) integrater.set_pname_xname_dname(pname, xname, temp_dname) integrater.set_template(os.path.basename(self.get_template())) integrater.set_directory(self.get_directory()) # check for ice - and if so, exclude (ranges taken from # XDS documentation) if self.get_integrater_ice() != 0: Debug.write('Excluding ice rings') integrater.set_exclude_ice(True) # exclude specified resolution ranges if len(self.get_integrater_excluded_regions()) != 0: regions = self.get_integrater_excluded_regions() Debug.write('Excluding regions: %s' % repr(regions)) integrater.set_exclude_regions(regions) mask = standard_mask(self.get_detector()) for m in mask: integrater.add_instruction(m) integrater.set_input_mat_file('xiaintegrate.mat') integrater.set_beam_centre(beam) integrater.set_distance(distance) integrater.set_space_group_number(spacegroup_number) integrater.set_mosaic(mosaic) if self.get_wavelength_prov() == 'user': integrater.set_wavelength(self.get_wavelength()) parameters = self.get_integrater_parameters('mosflm') integrater.update_parameters(parameters) if self._mosflm_gain: integrater.set_gain(self._mosflm_gain) # check for resolution limits if self._intgr_reso_high > 0.0: integrater.set_d_min(self._intgr_reso_high) if self._intgr_reso_low: integrater.set_d_max(self._intgr_reso_low) if PhilIndex.params.general.backstop_mask: from xia2.Toolkit.BackstopMask import BackstopMask mask = BackstopMask(PhilIndex.params.general.backstop_mask) mask = mask.calculate_mask_mosflm(self.get_header()) integrater.set_mask(mask) detector = self.get_detector() detector_width, detector_height = detector[0].get_image_size_mm() lim_x = 0.5 * detector_width lim_y = 0.5 * detector_height Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y)) integrater.set_limits(lim_x, lim_y) integrater.set_fix_mosaic(self._mosflm_postref_fix_mosaic) offset = self.get_frame_offset() integrater.set_image_range( (self._intgr_wedge[0] - offset, self._intgr_wedge[1] - offset)) try: integrater.run() except RuntimeError as e: if 'integration failed: reason unknown' in str(e): Chatter.write('Mosflm has failed in integration') message = 'The input was:\n\n' for input in integrater.get_all_input(): message += ' %s' % input Chatter.write(message) raise FileHandler.record_log_file( '%s %s %s %s mosflm integrate' % \ (self.get_integrater_sweep_name(), pname, xname, dname), integrater.get_log_file()) self._intgr_per_image_statistics = integrater.get_per_image_statistics( ) self._mosflm_hklout = integrater.get_hklout() Debug.write('Integration output: %s' % self._mosflm_hklout) self._intgr_n_ref = integrater.get_nref() # if a BGSIG error happened try not refining the # profile and running again... if integrater.get_bgsig_too_large(): if not self._mosflm_refine_profiles: raise RuntimeError('BGSIG error with profiles fixed') Debug.write('BGSIG error detected - try fixing profile...') self._mosflm_refine_profiles = False self.set_integrater_done(False) return if integrater.get_getprof_error(): Debug.write('GETPROF error detected - try fixing profile...') self._mosflm_refine_profiles = False self.set_integrater_done(False) return if (integrater.get_detector_gain_error() and not (self.get_imageset().get_detector()[0].get_type() == 'SENSOR_PAD')): gain = integrater.get_suggested_gain() if gain is not None: self.set_integrater_parameter('mosflm', 'gain', gain) self.set_integrater_export_parameter('mosflm', 'gain', gain) if self._mosflm_gain: Debug.write('GAIN updated to %f' % gain) else: Debug.write('GAIN found to be %f' % gain) self._mosflm_gain = gain self._mosflm_rerun_integration = True if not self._mosflm_hklout: raise RuntimeError('processing abandoned') self._intgr_batches_out = integrater.get_batches_out() mosaics = integrater.get_mosaic_spreads() if mosaics and len(mosaics) > 0: self.set_integrater_mosaic_min_mean_max( min(mosaics), sum(mosaics) / len(mosaics), max(mosaics)) else: m = indxr.get_indexer_mosaic() self.set_integrater_mosaic_min_mean_max(m, m, m) # write the report for each image as .*-#$ to Chatter - # detailed report will be written automagically to science... Chatter.write(self.show_per_image_statistics()) Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \ self.get_integrater_mosaic_min_mean_max()) # gather the statistics from the postrefinement postref_result = integrater.get_postref_result() # now write this to a postrefinement log postref_log = os.path.join(self.get_working_directory(), 'postrefinement.log') fout = open(postref_log, 'w') fout.write('$TABLE: Postrefinement for %s:\n' % \ self._intgr_sweep_name) fout.write('$GRAPHS: Missetting angles:A:1, 2, 3, 4: $$\n') fout.write('Batch PhiX PhiY PhiZ $$ Batch PhiX PhiY PhiZ $$\n') for image in sorted(postref_result): phix = postref_result[image].get('phix', 0.0) phiy = postref_result[image].get('phiy', 0.0) phiz = postref_result[image].get('phiz', 0.0) fout.write('%d %5.2f %5.2f %5.2f\n' % \ (image, phix, phiy, phiz)) fout.write('$$\n') fout.close() if self.get_integrater_sweep_name(): pname, xname, dname = self.get_integrater_project_info() FileHandler.record_log_file('%s %s %s %s postrefinement' % \ (self.get_integrater_sweep_name(), pname, xname, dname), postref_log) return self._mosflm_hklout
def CheckIndexingSymmetry(self): checksym = _CheckIndexingSymmetry() checksym.set_working_directory(self.get_working_directory()) auto_logfiler(checksym) return checksym
def Reindex(self): reindex = _Reindex() reindex.set_working_directory(self.get_working_directory()) auto_logfiler(reindex) return reindex
def _mosflm_refine_cell(self, idxr, set_spacegroup=None): '''Perform the refinement of the unit cell. This will populate all of the information needed to perform the integration.''' # FIXME this will die after #1285 #if not self.get_integrater_indexer(): #Debug.write('Replacing indexer of %s with self at %d' % \ #(str(self.get_integrater_indexer()), __line__)) #self.set_integrater_indexer(self) #idxr = self.get_integrater_indexer() if not idxr.get_indexer_payload('mosflm_orientation_matrix'): raise RuntimeError('unexpected situation in indexing') lattice = idxr.get_indexer_lattice() mosaic = idxr.get_indexer_mosaic() cell = idxr.get_indexer_cell() beam_centre = idxr.get_indexer_beam_centre() # bug # 3174 - if mosaic is very small (here defined to be # 0.25 x osc_width) then set to this minimum value. phi_width = idxr.get_phi_width() if mosaic < 0.25 * phi_width: mosaic = 0.25 * phi_width if idxr.get_indexer_payload('mosflm_beam_centre'): beam_centre = idxr.get_indexer_payload('mosflm_beam_centre') distance = idxr.get_indexer_distance() matrix = idxr.get_indexer_payload('mosflm_orientation_matrix') integration_params = idxr.get_indexer_payload( 'mosflm_integration_parameters') if integration_params is None: integration_params = {} if integration_params: if 'separation' in integration_params: self.set_refiner_parameter( 'mosflm', 'separation', '%f %f' % tuple(integration_params['separation'])) if 'raster' in integration_params: self.set_refiner_parameter( 'mosflm', 'raster', '%d %d %d %d %d' % tuple(integration_params['raster'])) idxr.set_indexer_payload('mosflm_integration_parameters', None) spacegroup_number = lattice_to_spacegroup(lattice) # copy these into myself for later reference, if indexer # is not myself - everything else is copied via the # cell refinement process... from cctbx import sgtbx from dxtbx.model import Crystal from dxtbx.model.detector_helpers import set_mosflm_beam_centre experiment = idxr.get_indexer_experiment_list()[0] set_mosflm_beam_centre(experiment.detector, experiment.beam, beam_centre) space_group = sgtbx.space_group_info(number=spacegroup_number).group() a, b, c = experiment.crystal.get_real_space_vectors() experiment.crystal = Crystal(a, b, c, space_group=space_group) # FIXME surely these have been assigned further up?! if not self._mosflm_cell_ref_images: self._mosflm_cell_ref_images = self._refine_select_images(mosaic) f = open( os.path.join(self.get_working_directory(), 'xiaindex-%s.mat' % lattice), 'w') for m in matrix: f.write(m) f.close() # then start the cell refinement refiner = MosflmRefineCell() refiner.set_working_directory(self.get_working_directory()) auto_logfiler(refiner) if self._mosflm_gain: refiner.set_gain(self._mosflm_gain) refiner.set_template(os.path.basename(idxr.get_template())) refiner.set_directory(idxr.get_directory()) refiner.set_input_mat_file('xiaindex-%s.mat' % lattice) refiner.set_output_mat_file('xiarefine.mat') refiner.set_beam_centre(beam_centre) refiner.set_unit_cell(cell) refiner.set_distance(distance) if set_spacegroup: refiner.set_space_group_number(set_spacegroup) else: refiner.set_space_group_number(spacegroup_number) # FIXME 18/JUN/08 - it may help to have an overestimate # of the mosaic spread in here as it *may* refine down # better than up... - this is not a good idea as it may # also not refine at all! - 12972 # integration failed # Bug # 3103 if self._mosflm_cell_ref_double_mosaic: mosaic *= 2.0 refiner.set_mosaic(mosaic) # if set, use the resolution for cell refinement - see # bug # 2078... if self._mosflm_cell_ref_resolution: refiner.set_resolution(self._mosflm_cell_ref_resolution) refiner.set_fix_mosaic(self._mosflm_postref_fix_mosaic) # note well that the beam centre is coming from indexing so # should be already properly handled #if idxr.get_wavelength_prov() == 'user': #refiner.set_wavelength(idxr.get_wavelength()) # belt + braces mode - only to be used when considering failover, # will run an additional step of autoindexing prior to cell # refinement, to be used only after proving that not going it # will result in cell refinement failure - will use the first # wedge... N.B. this is only useful if the indexer is Labelit # not Mosflm... refiner.set_add_autoindex(self._mosflm_cell_ref_add_autoindex) # get all of the stored parameter values parameters = self.get_refiner_parameters('mosflm') refiner.update_parameters(parameters) detector = idxr.get_detector() detector_width, detector_height = detector[0].get_image_size_mm() lim_x = 0.5 * detector_width lim_y = 0.5 * detector_height Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y)) refiner.set_limits(lim_x, lim_y) refiner.set_images(self._mosflm_cell_ref_images) failover = PhilIndex.params.xia2.settings.failover if failover and not self._mosflm_cell_ref_add_autoindex: refiner.set_ignore_cell_refinement_failure(True) refiner.run() # then look to see if the cell refinement worked ok - if it # didn't then this may indicate that the lattice was wrongly # selected. cell_refinement_ok = refiner.cell_refinement_ok() if not cell_refinement_ok: Debug.write('Repeating cell refinement...') self.set_integrater_prepare_done(False) self._mosflm_cell_ref_add_autoindex = True return [0.0], [0.0] rms_values = refiner.get_rms_values() background_residual = refiner.get_background_residual() self._refinr_cell = refiner.get_refined_unit_cell() distance = refiner.get_refined_distance2() experiment = idxr.get_indexer_experiment_list()[0] from xia2.Wrappers.Mosflm.AutoindexHelpers import set_distance set_distance(experiment.detector, distance) self.set_refiner_parameter('mosflm', 'distortion yscale', refiner.get_refined_distortion_yscale()) self.set_refiner_parameter('mosflm', 'raster', refiner.get_raster()) #integration_params['distortion yscale'] \ #= refiner.get_refined_distortion_yscale() #integration_params['raster'] = refiner.get_raster() separation = refiner.get_separation() if separation is not None: self.set_refiner_parameter('mosflm', 'separation', '%s %s' % refiner.get_separation()) #integration_params['separation'] = refiner.get_separation() self.set_refiner_parameter('mosflm', 'beam', '%s %s' % refiner.get_refined_beam_centre()) self.set_refiner_parameter('mosflm', 'distance', refiner.get_refined_distance()) self.set_refiner_parameter('mosflm', 'distortion tilt', refiner.get_refined_distortion_tilt()) self.set_refiner_parameter('mosflm', 'distortion twist', refiner.get_refined_distortion_twist()) integration_params['beam'] = tuple( float(b) for b in refiner.get_refined_beam_centre()) integration_params['distance'] = refiner.get_refined_distance() integration_params[ 'distortion tilt'] = refiner.get_refined_distortion_tilt() integration_params[ 'distortion twist'] = refiner.get_refined_distortion_twist() idxr._indxr_mosaic = refiner.get_refined_mosaic() idxr.set_indexer_payload( 'mosflm_orientation_matrix', open(os.path.join(self.get_working_directory(), 'xiarefine.mat'), 'r').readlines()) self.set_refiner_payload( 'mosflm_orientation_matrix', idxr.get_indexer_payload('mosflm_orientation_matrix')) self.set_refiner_payload('mosaic', refiner.get_refined_mosaic()) self.set_refiner_payload('beam', integration_params['beam']) self.set_refiner_payload('distance', integration_params['distance']) from xia2.Wrappers.Mosflm.AutoindexHelpers import crystal_model_from_mosflm_mat # make a dxtbx crystal_model object from the mosflm matrix experiment = idxr.get_indexer_experiment_list()[0] crystal_model = crystal_model_from_mosflm_mat( idxr._indxr_payload['mosflm_orientation_matrix'], unit_cell=refiner.get_refined_unit_cell(), space_group=experiment.crystal.get_space_group()) experiment.crystal = crystal_model #self.set_refiner_payload( #'mosflm_integration_parameters', integration_params) self._refinr_refined_experiment_list = ExperimentList([experiment]) return rms_values, background_residual
def _index(self): '''Implement the indexer interface.''' Citations.cite('mosflm') indexer = MosflmIndex() indexer.set_working_directory(self.get_working_directory()) auto_logfiler(indexer) from xia2.lib.bits import unique_elements _images = unique_elements(self._indxr_images) indexer.set_images(_images) images_str = ', '.join(map(str, _images)) cell_str = None if self._indxr_input_cell: cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \ self._indxr_input_cell if self._indxr_sweep_name: #if len(self._fp_directory) <= 50: #dirname = self._fp_directory #else: #dirname = '...%s' % self._fp_directory[-46:] dirname = os.path.dirname(self.get_imageset().get_template()) Journal.block( 'autoindexing', self._indxr_sweep_name, 'mosflm', { 'images': images_str, 'target cell': self._indxr_input_cell, 'target lattice': self._indxr_input_lattice, 'template': self.get_imageset().get_template(), 'directory': dirname }) #task = 'Autoindex from images:' #for i in _images: #task += ' %s' % self.get_image_name(i) #self.set_task(task) indexer.set_template(os.path.basename(self.get_template())) indexer.set_directory(self.get_directory()) xsweep = self.get_indexer_sweep() if xsweep is not None: if xsweep.get_distance() is not None: indexer.set_distance(xsweep.get_distance()) #if self.get_wavelength_prov() == 'user': #index.set_wavelength(self.get_wavelength()) if xsweep.get_beam_centre() is not None: indexer.set_beam_centre(xsweep.get_beam_centre()) if self._indxr_input_cell: indexer.set_unit_cell(self._indxr_input_cell) if self._indxr_input_lattice is not None: spacegroup_number = lattice_to_spacegroup( self._indxr_input_lattice) indexer.set_space_group_number(spacegroup_number) if not self._mosflm_autoindex_thresh: try: min_peaks = 200 Debug.write('Aiming for at least %d spots...' % min_peaks) thresholds = [] for i in _images: p = Printpeaks() p.set_working_directory(self.get_working_directory()) auto_logfiler(p) p.set_image(self.get_image_name(i)) thresh = p.threshold(min_peaks) Debug.write('Autoindex threshold for image %d: %d' % \ (i, thresh)) thresholds.append(thresh) thresh = min(thresholds) self._mosflm_autoindex_thresh = thresh except Exception as e: print str(e) #XXX this should disappear! Debug.write('Error computing threshold: %s' % str(e)) Debug.write('Using default of 20.0') thresh = 20.0 else: thresh = self._mosflm_autoindex_thresh Debug.write('Using autoindex threshold: %d' % thresh) if self._mosflm_autoindex_sol: indexer.set_solution_number(self._mosflm_autoindex_sol) indexer.set_threshold(thresh) # now forget this to prevent weird things happening later on if self._mosflm_autoindex_sol: self._mosflm_autoindex_sol = 0 indexer.run() indxr_cell = indexer.get_refined_unit_cell() self._indxr_lattice = indexer.get_lattice() space_group_number = indexer.get_indexed_space_group_number() detector_distance = indexer.get_refined_distance() beam_centre = indexer.get_refined_beam_centre() mosaic_spreads = indexer.get_mosaic_spreads() if min(list(indxr_cell)) < 10.0 and \ indxr_cell[2] / indxr_cell[0] > 6: Debug.write('Unrealistic autoindexing solution: ' + '%.2f %.2f %.2f %.2f %.2f %.2f' % indxr_cell) # tweak some parameters and try again... self._mosflm_autoindex_thresh *= 1.5 self.set_indexer_done(False) return intgr_params = {} # look up other possible indexing solutions (not well - in # standard settings only!) This is moved earlier as it could # result in returning if Mosflm has selected the wrong # solution! try: self._indxr_other_lattice_cell = indexer.get_solutions() # Change 27/FEB/08 to support user assigned spacegroups if self._indxr_user_input_lattice: lattice_to_spacegroup_dict = { 'aP': 1, 'mP': 3, 'mC': 5, 'oP': 16, 'oC': 20, 'oF': 22, 'oI': 23, 'tP': 75, 'tI': 79, 'hP': 143, 'hR': 146, 'cP': 195, 'cF': 196, 'cI': 197 } for k in self._indxr_other_lattice_cell.keys(): if lattice_to_spacegroup_dict[k] > \ lattice_to_spacegroup_dict[ self._indxr_input_lattice]: del (self._indxr_other_lattice_cell[k]) # check that the selected unit cell matches - and if # not raise a "horrible" exception if self._indxr_input_cell: assert indxr_cell is not None for j in range(6): if math.fabs(self._indxr_input_cell[j] - indxr_cell[j]) > 2.0: Chatter.write('Mosflm autoindexing did not select ' + 'correct (target) unit cell') raise RuntimeError( 'something horrible happened in indexing') except RuntimeError as e: # check if mosflm rejected a solution we have it if 'horribl' in str(e): # ok it did - time to break out the big guns... if not self._indxr_input_cell: raise RuntimeError( 'error in solution selection when not preset') # XXX FIXME self._mosflm_autoindex_sol = _get_indexing_solution_number( indexer.get_all_output(), self._indxr_input_cell, self._indxr_input_lattice) # set the fact that we are not done... self.set_indexer_done(False) # and return - hopefully this will restart everything return else: raise e if len(mosaic_spreads) == 0: # then consider setting it do a default value... # equal to the oscillation width (a good guess) phi_width = self.get_phi_width() Chatter.write( 'Mosaic estimation failed, so guessing at %4.2f' % \ phi_width) # only consider this if we have thus far no idea on the # mosaic spread... mosaic_spreads.append(phi_width) intgr_params['raster'] = indexer.get_raster() intgr_params['separation'] = indexer.get_separation() self._indxr_resolution_estimate = indexer.get_resolution_estimate() # compute mosaic as mean(mosaic_spreads) self._indxr_mosaic = sum(mosaic_spreads) / len(mosaic_spreads) self._indxr_payload['mosflm_integration_parameters'] = intgr_params self._indxr_payload['mosflm_orientation_matrix'] = open( os.path.join(self.get_working_directory(), 'xiaindex.mat'), 'r').readlines() import copy from dxtbx.model.detector_helpers import set_mosflm_beam_centre from xia2.Wrappers.Mosflm.AutoindexHelpers import set_distance from xia2.Wrappers.Mosflm.AutoindexHelpers import crystal_model_from_mosflm_mat from cctbx import sgtbx, uctbx # update the beam centre (i.e. shift the origin of the detector) detector = copy.deepcopy(self.get_detector()) beam = copy.deepcopy(self.get_beam()) set_mosflm_beam_centre(detector, beam, beam_centre) if detector_distance is not None: set_distance(detector, detector_distance) # make a dxtbx crystal_model object from the mosflm matrix space_group = sgtbx.space_group_info(number=space_group_number).group() crystal_model = crystal_model_from_mosflm_mat( self._indxr_payload['mosflm_orientation_matrix'], unit_cell=uctbx.unit_cell(tuple(indxr_cell)), space_group=space_group) # construct an experiment_list from dxtbx.model import Experiment, ExperimentList experiment = Experiment(beam=beam, detector=detector, goniometer=self.get_goniometer(), scan=self.get_scan(), crystal=crystal_model) experiment_list = ExperimentList([experiment]) self.set_indexer_experiment_list(experiment_list)
def estimate_resolution_limit(self): # see also xia2/Modules/Scaler/CommonScaler.py: CommonScaler._estimate_resolution_limit() params = self._params.resolution m = EstimateResolution() auto_logfiler(m) # use the scaled .refl and .expt file if self._experiments_filename and self._reflections_filename: m.set_reflections(self._reflections_filename) m.set_experiments(self._experiments_filename) else: m.set_hklin(self._scaled_unmerged_mtz) m.set_limit_rmerge(params.rmerge) m.set_limit_completeness(params.completeness) m.set_limit_cc_half(params.cc_half) m.set_cc_half_fit(params.cc_half_fit) m.set_cc_half_significance_level(params.cc_half_significance_level) m.set_limit_isigma(params.isigma) m.set_limit_misigma(params.misigma) m.set_labels(params.labels) # if batch_range is not None: # start, end = batch_range # m.set_batch_range(start, end) m.run() resolution_limits = [] reasoning = [] if params.completeness is not None: r_comp = m.get_resolution_completeness() resolution_limits.append(r_comp) reasoning.append("completeness > %s" % params.completeness) if params.cc_half is not None: r_cc_half = m.get_resolution_cc_half() resolution_limits.append(r_cc_half) reasoning.append("cc_half > %s" % params.cc_half) if params.rmerge is not None: r_rm = m.get_resolution_rmerge() resolution_limits.append(r_rm) reasoning.append("rmerge > %s" % params.rmerge) if params.isigma is not None: r_uis = m.get_resolution_isigma() resolution_limits.append(r_uis) reasoning.append("unmerged <I/sigI> > %s" % params.isigma) if params.misigma is not None: r_mis = m.get_resolution_misigma() resolution_limits.append(r_mis) reasoning.append("merged <I/sigI> > %s" % params.misigma) if len(resolution_limits): resolution = max(resolution_limits) reasoning = [ reason for limit, reason in zip(resolution_limits, reasoning) if limit >= resolution ] reasoning = ", ".join(reasoning) else: resolution = 0.0 reasoning = None return resolution, reasoning