def exercise_dials_wrappers(nproc=None): if not have_dials_regression: print "Skipping exercise_dials_wrappers(): dials_regression not configured" return if nproc is not None: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc from xia2.Wrappers.Dials.Import import Import from xia2.Wrappers.Dials.Spotfinder import Spotfinder from xia2.Wrappers.Dials.Index import Index from xia2.Wrappers.Dials.Integrate import Integrate from xia2.Wrappers.Dials.RefineBravaisSettings import RefineBravaisSettings from xia2.Wrappers.Dials.Refine import Refine from xia2.Wrappers.Dials.Reindex import Reindex from xia2.Wrappers.Dials.ExportMtz import ExportMtz from xia2.Wrappers.Dials.ExportXDSASCII import ExportXDSASCII xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data") template = os.path.join(xia2_demo_data, "insulin_1_%03i.img") scan_ranges = [(1, 45)] image_range = (1, 45) cwd = os.path.abspath(os.curdir) tmp_dir = os.path.abspath(open_tmp_directory()) os.chdir(tmp_dir) print "Begin importing" importer = Import() importer.setup_from_image(template %1) importer.set_image_range(image_range) importer.run() print ''.join(importer.get_all_output()) print "Done importing" print "Begin spotfinding" spotfinder = Spotfinder() spotfinder.set_input_sweep_filename(importer.get_sweep_filename()) spotfinder.set_scan_ranges(scan_ranges) spotfinder.run() print ''.join(spotfinder.get_all_output()) print "Done spotfinding" print "Begin indexing" indexer = Index() indexer.add_spot_filename(spotfinder.get_spot_filename()) indexer.add_sweep_filename(importer.get_sweep_filename()) indexer.run('fft3d') print ''.join(indexer.get_all_output()) print "Done indexing" print "Begin refining" rbs = RefineBravaisSettings() rbs.set_experiments_filename(indexer.get_experiments_filename()) rbs.set_indexed_filename(indexer.get_indexed_filename()) rbs.run() print ''.join(rbs.get_all_output()) print "Done refining" bravais_setting_22 = rbs.get_bravais_summary()[22] assert bravais_setting_22['bravais'] == 'cI' assert bravais_setting_22['cb_op'] == 'b+c,a+c,a+b' assert approx_equal(bravais_setting_22['unit_cell'], (78.14,78.14,78.14,90,90,90), eps=1e-1) bravais_setting_22_json = bravais_setting_22['experiments_file'] assert os.path.exists(bravais_setting_22_json) print "Begin reindexing" reindexer = Reindex() reindexer.set_experiments_filename(indexer.get_experiments_filename()) reindexer.set_indexed_filename(indexer.get_indexed_filename()) reindexer.set_cb_op(bravais_setting_22['cb_op']) reindexer.run() assert os.path.exists(reindexer.get_reindexed_experiments_filename()) assert os.path.exists(reindexer.get_reindexed_reflections_filename()) print ''.join(reindexer.get_all_output()) print "Done reindexing" print "Begin refining" refiner = Refine() refiner.set_experiments_filename(bravais_setting_22_json) refiner.set_indexed_filename(reindexer.get_reindexed_reflections_filename()) refiner.set_scan_varying(True) refiner.run() assert os.path.exists(refiner.get_refined_experiments_filename()) print ''.join(refiner.get_all_output()) print "Done refining" print "Begin integrating" integrater = Integrate() integrater.set_experiments_filename(refiner.get_refined_experiments_filename()) integrater.set_reflections_filename(reindexer.get_reindexed_reflections_filename()) integrater.run() print ''.join(integrater.get_all_output()) print "Done integrating" print "Begin exporting" exporter = ExportMtz() exporter.set_experiments_filename(integrater.get_integrated_experiments()) exporter.set_reflections_filename(integrater.get_integrated_reflections()) exporter.run() print ''.join(exporter.get_all_output()) print "Done exporting" assert os.path.exists(exporter.get_mtz_filename()) print "Begin exporting xds_ascii" exporter = ExportXDSASCII() exporter.set_experiments_filename(integrater.get_integrated_experiments()) exporter.set_reflections_filename(integrater.get_integrated_reflections()) exporter.run() print ''.join(exporter.get_all_output()) print "Done exporting" assert os.path.exists(exporter.get_hkl_filename()) import shutil shutil.copy(indexer.get_experiments_filename(), "copy.json") shutil.copy(indexer.get_indexed_filename(), "copy.pickle") print "Begin combining" from xia2.Wrappers.Dials.CombineExperiments import CombineExperiments exporter = CombineExperiments() exporter.add_experiments(indexer.get_experiments_filename()) exporter.add_experiments("copy.json") exporter.add_reflections(indexer.get_indexed_filename()) exporter.add_reflections("copy.pickle") exporter.run() print ''.join(exporter.get_all_output()) print "Done combining" assert os.path.exists(exporter.get_combined_experiments_filename()) assert os.path.exists(exporter.get_combined_reflections_filename())
def exercise_dials_wrappers(template, nproc=None): if nproc is not None: PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc scan_ranges = [(1, 45)] image_range = (1, 45) print("Begin importing") importer = Import() importer.setup_from_image(template % 1) importer.set_image_range(image_range) importer.run() print("".join(importer.get_all_output())) print("Done importing") print("Begin spotfinding") spotfinder = Spotfinder() spotfinder.set_input_sweep_filename(importer.get_sweep_filename()) spotfinder.set_scan_ranges(scan_ranges) spotfinder.run() print("".join(spotfinder.get_all_output())) print("Done spotfinding") print("Begin indexing") indexer = Index() indexer.add_spot_filename(spotfinder.get_spot_filename()) indexer.add_sweep_filename(importer.get_sweep_filename()) indexer.run("fft3d") print("".join(indexer.get_all_output())) print("Done indexing") print("Begin refining") rbs = RefineBravaisSettings() rbs.set_experiments_filename(indexer.get_experiments_filename()) rbs.set_indexed_filename(indexer.get_indexed_filename()) rbs.run() print("".join(rbs.get_all_output())) print("Done refining") bravais_setting_22 = rbs.get_bravais_summary()[22] assert bravais_setting_22["bravais"] == "cI" assert bravais_setting_22["cb_op"] == "b+c,a+c,a+b" assert bravais_setting_22["unit_cell"] == pytest.approx( (78.14, 78.14, 78.14, 90, 90, 90), abs=1e-1 ) bravais_setting_22_json = bravais_setting_22["experiments_file"] assert os.path.exists(bravais_setting_22_json) print("Begin reindexing") reindexer = Reindex() reindexer.set_experiments_filename(indexer.get_experiments_filename()) reindexer.set_indexed_filename(indexer.get_indexed_filename()) reindexer.set_cb_op(bravais_setting_22["cb_op"]) reindexer.run() assert os.path.exists(reindexer.get_reindexed_experiments_filename()) assert os.path.exists(reindexer.get_reindexed_reflections_filename()) print("".join(reindexer.get_all_output())) print("Done reindexing") print("Begin refining") refiner = Refine() refiner.set_experiments_filename(bravais_setting_22_json) refiner.set_indexed_filename(reindexer.get_reindexed_reflections_filename()) refiner.set_scan_varying(True) refiner.run() assert os.path.exists(refiner.get_refined_experiments_filename()) print("".join(refiner.get_all_output())) print("Done refining") print("Begin integrating") integrater = Integrate() integrater.set_experiments_filename(refiner.get_refined_experiments_filename()) integrater.set_reflections_filename(reindexer.get_reindexed_reflections_filename()) integrater.run() print("".join(integrater.get_all_output())) print("Done integrating") print("Begin exporting") exporter = ExportMtz() exporter.set_experiments_filename(integrater.get_integrated_experiments()) exporter.set_reflections_filename(integrater.get_integrated_reflections()) exporter.run() print("".join(exporter.get_all_output())) print("Done exporting") assert os.path.exists(exporter.get_mtz_filename()) print("Begin exporting xds_ascii") exporter = ExportXDSASCII() exporter.set_experiments_filename(integrater.get_integrated_experiments()) exporter.set_reflections_filename(integrater.get_integrated_reflections()) exporter.run() print("".join(exporter.get_all_output())) print("Done exporting") assert os.path.exists(exporter.get_hkl_filename()) # Test combine experiments wrapper. Duplicate the file and adjust the # identifier to allow combination. shutil.copy(indexer.get_experiments_filename(), "copy.expt") shutil.copy(indexer.get_indexed_filename(), "copy.refl") expts = load.experiment_list("copy.expt") expts[0].identifier = "0" expts.as_file("copy.expt") refls = flex.reflection_table.from_file("copy.refl") refls.experiment_identifiers()[0] = "0" refls.as_file("copy.refl") exporter = CombineExperiments() exporter.add_experiments(indexer.get_experiments_filename()) exporter.add_experiments("copy.expt") exporter.add_reflections(indexer.get_indexed_filename()) exporter.add_reflections("copy.refl") exporter.run() print("".join(exporter.get_all_output())) print("Done combining") assert os.path.exists(exporter.get_combined_experiments_filename()) assert os.path.exists(exporter.get_combined_reflections_filename())
def exercise_dials_wrappers(dials_regression, nproc=None): if nproc is not None: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc from xia2.Wrappers.Dials.Import import Import from xia2.Wrappers.Dials.Spotfinder import Spotfinder from xia2.Wrappers.Dials.Index import Index from xia2.Wrappers.Dials.Integrate import Integrate from xia2.Wrappers.Dials.RefineBravaisSettings import RefineBravaisSettings from xia2.Wrappers.Dials.Refine import Refine from xia2.Wrappers.Dials.Reindex import Reindex from xia2.Wrappers.Dials.ExportMtz import ExportMtz from xia2.Wrappers.Dials.ExportXDSASCII import ExportXDSASCII xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data") template = os.path.join(xia2_demo_data, "insulin_1_%03i.img") scan_ranges = [(1, 45)] image_range = (1, 45) print("Begin importing") importer = Import() importer.setup_from_image(template % 1) importer.set_image_range(image_range) importer.run() print(''.join(importer.get_all_output())) print("Done importing") print("Begin spotfinding") spotfinder = Spotfinder() spotfinder.set_input_sweep_filename(importer.get_sweep_filename()) spotfinder.set_scan_ranges(scan_ranges) spotfinder.run() print(''.join(spotfinder.get_all_output())) print("Done spotfinding") print("Begin indexing") indexer = Index() indexer.add_spot_filename(spotfinder.get_spot_filename()) indexer.add_sweep_filename(importer.get_sweep_filename()) indexer.run('fft3d') print(''.join(indexer.get_all_output())) print("Done indexing") print("Begin refining") rbs = RefineBravaisSettings() rbs.set_experiments_filename(indexer.get_experiments_filename()) rbs.set_indexed_filename(indexer.get_indexed_filename()) rbs.run() print(''.join(rbs.get_all_output())) print("Done refining") bravais_setting_22 = rbs.get_bravais_summary()[22] assert bravais_setting_22['bravais'] == 'cI' assert bravais_setting_22['cb_op'] == 'b+c,a+c,a+b' assert approx_equal(bravais_setting_22['unit_cell'], (78.14, 78.14, 78.14, 90, 90, 90), eps=1e-1) bravais_setting_22_json = bravais_setting_22['experiments_file'] assert os.path.exists(bravais_setting_22_json) print("Begin reindexing") reindexer = Reindex() reindexer.set_experiments_filename(indexer.get_experiments_filename()) reindexer.set_indexed_filename(indexer.get_indexed_filename()) reindexer.set_cb_op(bravais_setting_22['cb_op']) reindexer.run() assert os.path.exists(reindexer.get_reindexed_experiments_filename()) assert os.path.exists(reindexer.get_reindexed_reflections_filename()) print(''.join(reindexer.get_all_output())) print("Done reindexing") print("Begin refining") refiner = Refine() refiner.set_experiments_filename(bravais_setting_22_json) refiner.set_indexed_filename( reindexer.get_reindexed_reflections_filename()) refiner.set_scan_varying(True) refiner.run() assert os.path.exists(refiner.get_refined_experiments_filename()) print(''.join(refiner.get_all_output())) print("Done refining") print("Begin integrating") integrater = Integrate() integrater.set_experiments_filename( refiner.get_refined_experiments_filename()) integrater.set_reflections_filename( reindexer.get_reindexed_reflections_filename()) integrater.run() print(''.join(integrater.get_all_output())) print("Done integrating") print("Begin exporting") exporter = ExportMtz() exporter.set_experiments_filename(integrater.get_integrated_experiments()) exporter.set_reflections_filename(integrater.get_integrated_reflections()) exporter.run() print(''.join(exporter.get_all_output())) print("Done exporting") assert os.path.exists(exporter.get_mtz_filename()) print("Begin exporting xds_ascii") exporter = ExportXDSASCII() exporter.set_experiments_filename(integrater.get_integrated_experiments()) exporter.set_reflections_filename(integrater.get_integrated_reflections()) exporter.run() print(''.join(exporter.get_all_output())) print("Done exporting") assert os.path.exists(exporter.get_hkl_filename()) import shutil shutil.copy(indexer.get_experiments_filename(), "copy.json") shutil.copy(indexer.get_indexed_filename(), "copy.pickle") print("Begin combining") from xia2.Wrappers.Dials.CombineExperiments import CombineExperiments exporter = CombineExperiments() exporter.add_experiments(indexer.get_experiments_filename()) exporter.add_experiments("copy.json") exporter.add_reflections(indexer.get_indexed_filename()) exporter.add_reflections("copy.pickle") exporter.run() print(''.join(exporter.get_all_output())) print("Done combining") assert os.path.exists(exporter.get_combined_experiments_filename()) assert os.path.exists(exporter.get_combined_reflections_filename())
def run(self): from xia2.Handlers.Streams import Chatter, Debug if self._reindexing_operators: Debug.write("Reindexing sweeps for dials.two_theta_refine") from xia2.lib.bits import auto_logfiler from xia2.Wrappers.Dials.Reindex import Reindex self._reindexed_experiments, self._reindexed_reflections = [], [] for e, p, op in zip( self._experiments, self._reflection_files, self._reindexing_operators, ): reindexer = Reindex() reindexer.set_cb_op(op) reindexer.set_experiments_filename(e) reindexer.set_indexed_filename(p) reindexer.set_working_directory( self.get_working_directory()) auto_logfiler(reindexer) reindexer.run() self._reindexed_experiments.append( reindexer.get_reindexed_experiments_filename()) self._reindexed_reflections.append( reindexer.get_reindexed_reflections_filename()) Debug.write("Running dials.two_theta_refine") self._output_cif = os.path.join( self.get_working_directory(), "%s_dials.two_theta_refine.cif" % self.get_xpid(), ) self._output_mmcif = os.path.join( self.get_working_directory(), "%s_dials.two_theta_refine.mmcif" % self.get_xpid(), ) if not self._output_p4p: self._output_p4p = os.path.join( self.get_working_directory(), "%s_dials.two_theta_refine.p4p" % self.get_xpid(), ) self._output_correlation_plot = os.path.join( self.get_working_directory(), "%s_dials.two_theta_refine.png" % self.get_xpid(), ) self._output_experiments = os.path.join( self.get_working_directory(), "%s_refined_cell.expt" % self.get_xpid()) self.clear_command_line() if self._reindexing_operators: for experiment in self._reindexed_experiments: self.add_command_line(experiment) for reflection_file in self._reindexed_reflections: self.add_command_line(reflection_file) else: for experiment in self._experiments: self.add_command_line(experiment) for reflection_file in self._reflection_files: self.add_command_line(reflection_file) self.add_command_line("combine_crystal_models=%s" % self._combine_crystal_models) self.add_command_line("output.cif=%s" % self._output_cif) self.add_command_line("output.mmcif=%s" % self._output_mmcif) self.add_command_line("output.p4p=%s" % self._output_p4p) if self._output_correlation_plot is not None: self.add_command_line("output.correlation_plot.filename=%s" % self._output_correlation_plot) if self._output_experiments is not None: self.add_command_line("output.experiments=%s" % self._output_experiments) if self._phil_file is not None: self.add_command_line("%s" % self._phil_file) self.start() self.close_wait() if not os.path.isfile(self._output_cif): Chatter.write( "TwoTheta refinement failed, see log file for more details:\n %s" % self.get_log_file()) raise RuntimeError("unit cell not refined") self.check_for_errors() from dxtbx.model.experiment_list import ExperimentListFactory experiments = ExperimentListFactory.from_json_file( self.get_output_experiments(), check_format=False) self._crystal = experiments.crystals()[0]
def _determine_scaled_pointgroup(self): """Rerun symmetry after scaling to check for consistent space group. If not, then new space group should be used and data rescaled.""" from cctbx import crystal exp_crystal = load.experiment_list(self._scaler.get_scaled_experiments())[ 0 ].crystal cs = crystal.symmetry( space_group=exp_crystal.get_space_group(), unit_cell=exp_crystal.get_unit_cell(), ) cs_ref = cs.as_reference_setting() current_pointgroup = cs_ref.space_group() current_patt_group = ( current_pointgroup.build_derived_patterson_group().type().lookup_symbol() ) Debug.write( "Space group used in scaling: %s" % current_pointgroup.type().lookup_symbol() ) first = self._sweep_handler.get_epochs()[0] si = self._sweep_handler.get_sweep_information(first) refiner = si.get_integrater().get_integrater_refiner() point_group, reindex_op, _, _, reind_refl, reind_exp, reindex_initial = self._dials_symmetry_indexer_jiffy( [self._scaler.get_scaled_experiments()], [self._scaler.get_scaled_reflections()], [refiner], ) Debug.write( "Point group determined by dials.symmetry on scaled dataset: %s" % point_group ) sginfo = space_group_info(symbol=point_group) patt_group = ( sginfo.group().build_derived_patterson_group().type().lookup_symbol() ) self._scaler_symmetry_check_count += 1 if patt_group != current_patt_group: if reindex_initial: reindexer = DialsReindex() reindexer.set_working_directory(self.get_working_directory()) auto_logfiler(reindexer) reindexer.set_experiments_filename( self._scaler.get_scaled_experiments() ) reindexer.set_indexed_filename(self._scaler.get_scaled_reflections()) reindexer.set_cb_op(reindex_op) reindexer.run() self._scaler.set_scaled_experiments( reindexer.get_reindexed_experiments_filename() ) self._scaler.set_scaled_reflections( reindexer.get_reindexed_reflections_filename() ) else: self._scaler.set_scaled_experiments(reind_exp) self._scaler.set_scaled_reflections(reind_refl) self.set_scaler_done(False) Chatter.write( """Inconsistent space groups determined before and after scaling: %s, %s \n Data will be rescaled in new point group""" % (current_patt_group, patt_group) ) return else: Chatter.write("Consistent space group determined before and after scaling")
def apply_reindex_operator_to_sweep_info(self, si, reindex_operator, reason): """Use a reindex operator to reindex the data. Take the data from the sweep info, reindex using dials.reindex, and set the new data into the si. """ reindexer = DialsReindex() reindexer.set_working_directory(self.get_working_directory()) auto_logfiler(reindexer) reindexer.set_indexed_filename(si.get_reflections()) reindexer.set_experiments_filename(si.get_experiments()) reindexer.set_cb_op(reindex_operator) reindexer.run() si.set_reflections(reindexer.get_reindexed_reflections_filename()) si.set_experiments(reindexer.get_reindexed_experiments_filename()) Debug.write( "Reindexed with operator %s, reason is %s" % (reindex_operator, reason) )
def _scale_prepare(self): """Perform all of the preparation required to deliver the scaled data. This should sort together the reflection files, ensure that they are correctly indexed (via dials.symmetry) and generally tidy things up.""" # AIM discover symmetry and reindex with dials.symmetry, and set the correct # reflections in si.reflections, si.experiments self._helper.set_working_directory(self.get_working_directory()) self._factory.set_working_directory(self.get_working_directory()) need_to_return = False self._sweep_handler = SweepInformationHandler(self._scalr_integraters) p, x = self._sweep_handler.get_project_info() self._scalr_pname = p self._scalr_xname = x self._helper.set_pname_xname(p, x) Journal.block( "gathering", self.get_scaler_xcrystal().get_name(), "Dials", {"working directory": self.get_working_directory()}, ) # First do stuff to work out if excluding any data # Note - does this actually work? I couldn't seem to get it to work # in either this pipeline or the standard dials pipeline for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) intgr = si.get_integrater() _, xname, dname = si.get_project_info() sname = si.get_sweep_name() exclude_sweep = False for sweep in PhilIndex.params.xia2.settings.sweep: if sweep.id == sname and sweep.exclude: exclude_sweep = True break if exclude_sweep: self._sweep_handler.remove_epoch(epoch) Debug.write("Excluding sweep %s" % sname) else: Journal.entry({"adding data from": "%s/%s/%s" % (xname, dname, sname)}) # If multiple files, want to run symmetry to check for consistent indexing # also # try to reproduce what CCP4ScalerA is doing # first assign identifiers to avoid dataset-id collisions # Idea is that this should be called anytime you get data anew from the # integrater, to intercept and assign unique ids, then set in the # sweep_information (si) and always use si.set_reflections/ # si.get_reflections as we process. # self._sweep_handler = self._helper.assign_and_return_datasets( # self._sweep_handler # ) symmetry now sorts out identifiers. need_to_return = False if self._scalr_input_pointgroup: self._input_pointgroup_scale_prepare() elif ( len(self._sweep_handler.get_epochs()) > 1 and PhilIndex.params.xia2.settings.multi_sweep_indexing ): need_to_return = self._multi_sweep_scale_prepare() else: need_to_return = self._standard_scale_prepare() if need_to_return: self.set_scaler_done(False) self.set_scaler_prepare_done(False) return ### After this point, point group is good and only need to ### reindex to consistent setting. Don't need to call back to the ### integator, just use the data in the sweep info. # First work out if we're going to reindex against external reference param = PhilIndex.params.xia2.settings.scale using_external_references = False reference_refl = None reference_expt = None if param.reference_reflection_file: if not param.reference_experiment_file: Chatter.write( """ No DIALS reference experiments file provided, reference reflection file will not be used. Reference mtz files for reindexing not currently supported for pipeline=dials (supported for pipeline=dials-aimless). """ ) else: reference_refl = param.reference_reflection_file reference_expt = param.reference_experiment_file using_external_references = True Debug.write("Using reference reflections %s" % reference_refl) Debug.write("Using reference experiments %s" % reference_expt) if len(self._sweep_handler.get_epochs()) > 1: if PhilIndex.params.xia2.settings.unify_setting: self.unify_setting() if PhilIndex.params.xia2.settings.use_brehm_diederichs: self.brehm_diederichs_reindexing() # If not using Brehm-deidrichs reindexing, set reference as first # sweep, unless using external reference. elif not using_external_references: Debug.write("First sweep will be used as reference for reindexing") first = self._sweep_handler.get_epochs()[0] si = self._sweep_handler.get_sweep_information(first) reference_expt = si.get_experiments() reference_refl = si.get_reflections() # Now reindex to be consistent with first dataset - run reindex on each # dataset with reference (unless did brehm diederichs and didn't supply # a reference file) if reference_refl and reference_expt: exp = load.experiment_list(reference_expt) reference_cell = exp[0].crystal.get_unit_cell().parameters() # ---------- REINDEX TO CORRECT (REFERENCE) SETTING ---------- Chatter.write("Reindexing all datasets to common reference") if using_external_references: epochs = self._sweep_handler.get_epochs() else: epochs = self._sweep_handler.get_epochs()[1:] for epoch in epochs: # if we are working with unified UB matrix then this should not # be a problem here (note, *if*; *should*) # what about e.g. alternative P1 settings? # see JIRA MXSW-904 if PhilIndex.params.xia2.settings.unify_setting: continue reindexer = DialsReindex() reindexer.set_working_directory(self.get_working_directory()) auto_logfiler(reindexer) si = self._sweep_handler.get_sweep_information(epoch) reindexer.set_reference_filename(reference_expt) reindexer.set_reference_reflections(reference_refl) reindexer.set_indexed_filename(si.get_reflections()) reindexer.set_experiments_filename(si.get_experiments()) reindexer.run() # At this point, CCP4ScalerA would reset in integrator so that # the integrater calls reindex, no need to do that here as # have access to the files and will never need to reintegrate. si.set_reflections(reindexer.get_reindexed_reflections_filename()) si.set_experiments(reindexer.get_reindexed_experiments_filename()) # FIXME how to get some indication of the reindexing used? exp = load.experiment_list( reindexer.get_reindexed_experiments_filename() ) cell = exp[0].crystal.get_unit_cell().parameters() # Note - no lattice check as this will already be caught by reindex Debug.write("Cell: %.2f %.2f %.2f %.2f %.2f %.2f" % cell) Debug.write("Ref: %.2f %.2f %.2f %.2f %.2f %.2f" % reference_cell) for j in range(6): if ( math.fabs((cell[j] - reference_cell[j]) / reference_cell[j]) > 0.1 ): raise RuntimeError( "unit cell parameters differ in %s and %s" % (reference_expt, si.get_reflections()) ) # Now make sure all batches ok before finish preparing # This should be made safer, currently after dials.scale there is no # concept of 'batch', dials.export uses the calculate_batch_offsets # to assign batches, giving the same result as below. experiments_to_rebatch = [] for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) experiment = si.get_experiments() experiments_to_rebatch.append(load.experiment_list(experiment)[0]) offsets = calculate_batch_offsets(experiments_to_rebatch) for i, epoch in enumerate(self._sweep_handler.get_epochs()): si = self._sweep_handler.get_sweep_information(epoch) r = si.get_batch_range() si.set_batch_offset(offsets[i]) si.set_batches([r[0] + offsets[i], r[1] + offsets[i]])
def load_sweeps_with_common_indexing(): assert os.path.exists('xia2.json') from xia2.Schema.XProject import XProject xinfo = XProject.from_json(filename='xia2.json') import dials # required for gaussian_rs warning from xia2.Wrappers.Dials.Reindex import Reindex Citations.cite('dials') from dxtbx.model.experiment.experiment_list import ExperimentListFactory import cPickle as pickle crystals = xinfo.get_crystals() assert len(crystals) == 1 crystal = next(crystals.itervalues()) working_directory = Environment.generate_directory([crystal.get_name(), 'analysis']) os.chdir(working_directory) scaler = crystal._get_scaler() epoch_to_batches = {} epoch_to_integrated_intensities = {} epoch_to_sweep_name = {} # Aimless only epochs = scaler._sweep_handler.get_epochs() reference_cell = None reference_lattice = None reference_vectors = None reference_wavelength = None # Reindex each sweep to same setting all_miller_indices = flex.miller_index() all_two_thetas = flex.double() for epoch in epochs: si = scaler._sweep_handler.get_sweep_information(epoch) Chatter.smallbanner(si.get_sweep_name(), True) Debug.smallbanner(si.get_sweep_name(), True) intgr = si.get_integrater() experiments_filename = intgr.get_integrated_experiments() reflections_filename = intgr.get_integrated_reflections() refiner = intgr.get_integrater_refiner() Debug.write('experiment: %s' % experiments_filename) Debug.write('reflection: %s' % reflections_filename) # Use setting of first sweep as reference if reference_vectors is None: reference_vectors = experiments_filename # Assume that all sweeps have the same lattice system if reference_lattice is None: reference_lattice = refiner.get_refiner_lattice() else: assert reference_lattice == refiner.get_refiner_lattice() Debug.write("lattice: %s" % refiner.get_refiner_lattice()) # Read .json file for sweep db = ExperimentListFactory.from_json_file(experiments_filename) # Assume that each file only contains a single experiment assert (len(db) == 1) db = db[0] # Get beam vector s0 = db.beam.get_unit_s0() # Use the unit cell of the first sweep as reference if reference_cell is None: reference_cell = db.crystal.get_unit_cell() Debug.write("Reference cell: %s" % str(reference_cell)) dials_reindex = Reindex() dials_reindex.set_working_directory(working_directory) dials_reindex.set_cb_op("auto") dials_reindex.set_reference_filename(reference_vectors) dials_reindex.set_experiments_filename(experiments_filename) dials_reindex.set_indexed_filename(reflections_filename) auto_logfiler(dials_reindex) dials_reindex.run() # Assume that all data are collected at same wavelength if reference_wavelength is None: reference_wavelength = intgr.get_wavelength() else: assert abs(reference_wavelength - intgr.get_wavelength()) < 0.01 Debug.write("wavelength: %f A" % intgr.get_wavelength()) Debug.write("distance: %f mm" % intgr.get_distance()) # Get integrated reflection data import dials with open(dials_reindex.get_reindexed_reflections_filename(), 'rb') as fh: reflections = pickle.load(fh) selection = reflections.get_flags(reflections.flags.used_in_refinement) Chatter.write("Found %d reflections used in refinement (out of %d entries)" % (selection.count(True), len(reflections['miller_index']))) reflections = reflections.select(selection) # Filter bad reflections selection = reflections['intensity.sum.variance'] <= 0 if selection.count(True) > 0: reflections.del_selected(selection) print 'Removing %d reflections with negative variance' % \ selection.count(True) if 'intensity.prf.variance' in reflections: selection = reflections['intensity.prf.variance'] <= 0 if selection.count(True) > 0: reflections.del_selected(selection) print 'Removing %d profile reflections with negative variance' % \ selection.count(True) # Find the observed 2theta angles miller_indices = flex.miller_index() two_thetas_obs = flex.double() for pixel, panel, hkl in zip(reflections['xyzobs.px.value'], reflections['panel'], reflections['miller_index']): assert hkl != (0, 0, 0) two_thetas_obs.append(db.detector[panel].get_two_theta_at_pixel(s0, pixel[0:2])) miller_indices.append(hkl) # Convert observed 2theta angles to degrees two_thetas_obs = two_thetas_obs * 180 / 3.14159265359 Chatter.write("Remaining %d reflections are in 2theta range %.3f - %.3f deg" % (len(miller_indices), min(two_thetas_obs), max(two_thetas_obs))) all_miller_indices.extend(miller_indices) all_two_thetas.extend(two_thetas_obs) return all_miller_indices, all_two_thetas, reference_cell, reference_lattice, reference_wavelength
def run(self): from xia2.Handlers.Streams import Chatter, Debug if self._reindexing_operator: Debug.write('Reindexing sweeps for dials.two_theta_refine') from xia2.lib.bits import auto_logfiler from xia2.Wrappers.Dials.Reindex import Reindex self._reindexed_experiments, self._reindexed_reflections = [], [] for e, p in zip(self._experiments, self._pickles): reindexer = Reindex() reindexer.set_cb_op(self._reindexing_operator) reindexer.set_experiments_filename(e) reindexer.set_indexed_filename(p) reindexer.set_working_directory(self.get_working_directory()) auto_logfiler(reindexer) reindexer.run() self._reindexed_experiments.append(reindexer.get_reindexed_experiments_filename()) self._reindexed_reflections.append(reindexer.get_reindexed_reflections_filename()) Debug.write('Running dials.two_theta_refine') self._output_cif = os.path.join( self.get_working_directory(), '%s_dials.two_theta_refine.cif' % self.get_xpid()) self._output_mmcif = os.path.join( self.get_working_directory(), '%s_dials.two_theta_refine.mmcif' % self.get_xpid()) self._output_correlation_plot = os.path.join( self.get_working_directory(), '%s_dials.two_theta_refine.png' % self.get_xpid()) self._output_experiments = os.path.join( self.get_working_directory(), '%s_refined_cell.json' % self.get_xpid()) self.clear_command_line() if self._reindexing_operator: for experiment in self._reindexed_experiments: self.add_command_line(experiment) for pickle in self._reindexed_reflections: self.add_command_line(pickle) else: for experiment in self._experiments: self.add_command_line(experiment) for pickle in self._pickles: self.add_command_line(pickle) self.add_command_line('output.cif=%s' % self._output_cif) self.add_command_line('output.mmcif=%s' % self._output_mmcif) if self._output_correlation_plot is not None: self.add_command_line( 'output.correlation_plot.filename=%s' % self._output_correlation_plot) if self._output_experiments is not None: self.add_command_line( 'output.experiments=%s' % self._output_experiments) if self._phil_file is not None: self.add_command_line('%s' %self._phil_file) self.start() self.close_wait() if not os.path.isfile(self._output_cif): Chatter.write( "TwoTheta refinement failed, see log file for more details:\n %s" % self.get_log_file()) raise RuntimeError, 'unit cell not refined' self.check_for_errors() from dxtbx.model.experiment.experiment_list import ExperimentListFactory experiments = ExperimentListFactory.from_json_file(self.get_output_experiments()) self._crystal = experiments.crystals()[0]