def _extract_or_read_imagesets(params): """ Return a list of ImageSets, importing them via alternative means if necessary. The "Alternative Means" means via params.input.template or .directory, if the images to import haven't been specified directly. Args: params: The phil.scope_extract from dials.import Returns: A list of ImageSet objects """ # Get the experiments experiments = flatten_experiments(params.input.experiments) # Check we have some filenames if len(experiments) == 0: # FIXME Should probably make this smarter since it requires editing here # and in dials.import phil scope try: format_kwargs = { "dynamic_shadowing": params.format.dynamic_shadowing, "multi_panel": params.format.multi_panel, } except AttributeError: format_kwargs = None # Check if a template has been set and print help if not, otherwise try to # import the images based on the template input if len(params.input.template) > 0: importer = ExperimentListTemplateImporter( params.input.template, image_range=params.geometry.scan.image_range, format_kwargs=format_kwargs, ) experiments = importer.experiments if len(experiments) == 0: raise Sorry("No experiments found matching template %s" % params.input.experiments) elif len(params.input.directory) > 0: experiments = ExperimentListFactory.from_filenames( params.input.directory, format_kwargs=format_kwargs) if len(experiments) == 0: raise Sorry("No experiments found in directories %s" % params.input.directory) else: raise Sorry("No experiments found") # TODO (Nick): This looks redundant as the experiments are immediately discarded. # verify this, and remove if it is. if params.identifier_type: generate_experiment_identifiers(experiments, params.identifier_type) # Get a list of all imagesets imageset_list = experiments.imagesets() # Return the experiments return imageset_list
def test_labelit_indexer_II(regression_test, ccp4, dials_data, run_in_tmpdir): template = dials_data("insulin").join("insulin_1_###.img").strpath from xia2.Modules.Indexer.LabelitIndexerII import LabelitIndexerII from xia2.DriverExceptions.NotAvailableError import NotAvailableError try: ls = LabelitIndexerII(indxr_print=True) except NotAvailableError: pytest.skip("labelit not found") ls.set_working_directory(run_in_tmpdir.strpath) from dxtbx.model.experiment_list import ExperimentListTemplateImporter importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] ls.add_indexer_imageset(imageset) ls.set_indexer_input_cell((78, 78, 78, 90, 90, 90)) ls.set_indexer_user_input_lattice(True) ls.set_indexer_input_lattice("cI") ls.index() assert ls.get_indexer_cell() == pytest.approx( (78.52, 78.52, 78.52, 90, 90, 90), abs=1e-1) solution = ls.get_solution() assert solution["rmsd"] == pytest.approx(0.079, abs=1e-1) assert solution["metric"] <= 0.1762 assert solution["number"] == 22 assert solution["lattice"] == "cI" assert solution["mosaic"] <= 0.2 assert solution["nspots"] == pytest.approx(5509, abs=50) beam_centre = ls.get_indexer_beam_centre() assert beam_centre == pytest.approx((94.3286, 94.4662), abs=5e-1) assert ls.get_indexer_images() == [ (1, 1), (3, 3), (5, 5), (7, 7), (9, 9), (11, 11), (13, 13), (15, 15), (17, 17), (19, 19), (21, 21), (23, 23), (25, 25), (27, 27), (29, 29), (31, 31), (33, 33), (35, 35), (37, 37), (39, 39), ] print(ls.get_indexer_experiment_list()[0].crystal) print(ls.get_indexer_experiment_list()[0].detector)
def exercise_xds_indexer(dials_data, tmp_dir, nproc=None): if nproc is not None: PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc template = dials_data("insulin").join("insulin_1_###.img").strpath indexer = XDSIndexer() indexer.set_working_directory(tmp_dir) importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] indexer.add_indexer_imageset(imageset) cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, indexer.get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) indexer.index() assert indexer.get_indexer_cell() == pytest.approx( (78.076, 78.076, 78.076, 90, 90, 90), abs=1), indexer.get_indexer_cell() experiment = indexer.get_indexer_experiment_list()[0] sgi = experiment.crystal.get_space_group().info() assert sgi.type().number() == 197 beam_centre = indexer.get_indexer_beam_centre() assert beam_centre == pytest.approx((94.4221, 94.5096), abs=1e-1) assert indexer.get_indexer_images() == [(1, 5), (20, 24), (41, 45)] print(indexer.get_indexer_experiment_list()[0].crystal) print(indexer.get_indexer_experiment_list()[0].detector) # test serialization of indexer json_str = indexer.as_json() print(json_str) indexer2 = XDSIndexer.from_json(string=json_str) indexer2.index() assert indexer.get_indexer_cell() == pytest.approx( indexer2.get_indexer_cell()) assert indexer.get_indexer_beam_centre() == pytest.approx( indexer2.get_indexer_beam_centre()) assert indexer.get_indexer_images() == [ tuple(i) for i in indexer2.get_indexer_images() ] indexer.eliminate() indexer2.eliminate() assert indexer.get_indexer_cell() == pytest.approx( indexer2.get_indexer_cell()) assert indexer.get_indexer_lattice() == "hR" assert indexer2.get_indexer_lattice() == "hR"
def __call__(self): """ Import the experiments """ # Get the experiments experiments = flatten_experiments(self.params.input.experiments) # Check we have some filenames if len(experiments) == 0: # FIXME Should probably make this smarter since it requires editing here # and in dials.import phil scope try: format_kwargs = { "dynamic_shadowing": self.params.format.dynamic_shadowing, "multi_panel": self.params.format.multi_panel, } except AttributeError: format_kwargs = None # Check if a template has been set and print help if not, otherwise try to # import the images based on the template input if len(self.params.input.template) > 0: importer = ExperimentListTemplateImporter( self.params.input.template, image_range=self.params.geometry.scan.image_range, format_kwargs=format_kwargs, ) experiments = importer.experiments if len(experiments) == 0: raise Sorry( "No experiments found matching template %s" % self.params.input.experiments ) elif len(self.params.input.directory) > 0: experiments = ExperimentListFactory.from_filenames( self.params.input.directory, format_kwargs=format_kwargs ) if len(experiments) == 0: raise Sorry( "No experiments found in directories %s" % self.params.input.directory ) else: raise Sorry("No experiments found") if self.params.identifier_type: generate_experiment_identifiers(experiments, self.params.identifier_type) # Get a list of all imagesets imageset_list = experiments.imagesets() # Return the experiments return imageset_list
def exercise_dials_indexer(dials_data, tmp_dir, nproc=None): if nproc is not None: PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc template = dials_data("insulin").join("insulin_1_###.img").strpath indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] indexer.add_indexer_imageset(imageset) cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) indexer.index() assert indexer.get_indexer_cell() == pytest.approx( (78.14, 78.14, 78.14, 90, 90, 90), rel=1e-3) solution = indexer.get_solution() assert solution["rmsd"] == pytest.approx(0.03545, abs=1e-3) assert solution["metric"] == pytest.approx(0.02517, abs=5e-3) assert solution["number"] == 22 assert solution["lattice"] == "cI" beam_centre = indexer.get_indexer_beam_centre() assert beam_centre == pytest.approx((94.41567208118963, 94.51337522659865), abs=1e-3) print(indexer.get_indexer_experiment_list()[0].crystal) print(indexer.get_indexer_experiment_list()[0].detector) # test serialization of indexer json_str = indexer.as_json() indexer2 = DialsIndexer.from_json(string=json_str) indexer2.index() assert indexer.get_indexer_cell() == pytest.approx( indexer2.get_indexer_cell()) assert indexer.get_indexer_beam_centre() == pytest.approx( indexer2.get_indexer_beam_centre()) indexer.eliminate() indexer2.eliminate() assert indexer.get_indexer_cell() == pytest.approx( indexer2.get_indexer_cell()) assert indexer.get_indexer_lattice() == "hR" assert indexer2.get_indexer_lattice() == "hR"
def test_mosflm_indexer(regression_test, ccp4, dials_data, run_in_tmpdir): template = dials_data("insulin").join("insulin_1_###.img").strpath from xia2.Modules.Indexer.MosflmIndexer import MosflmIndexer indexer = MosflmIndexer() indexer.set_working_directory(run_in_tmpdir.strpath) from dxtbx.model.experiment_list import ExperimentListTemplateImporter importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] indexer.add_indexer_imageset(imageset) with mock.patch.object( sys, "argv", []): # otherwise indexing fails when running pytest with '--runslow' indexer.index() assert indexer.get_indexer_cell() == pytest.approx( (78.6657, 78.6657, 78.6657, 90.0, 90.0, 90.0), abs=1e-3) experiment = indexer.get_indexer_experiment_list()[0] sgi = experiment.crystal.get_space_group().info() assert sgi.type().number() == 197 beam_centre = indexer.get_indexer_beam_centre() assert beam_centre == pytest.approx((94.34, 94.57), abs=1e-2) assert indexer.get_indexer_images() == [(1, 1), (22, 22), (45, 45)] print(indexer.get_indexer_experiment_list()[0].crystal) print(indexer.get_indexer_experiment_list()[0].detector) # test serialization of indexer json_str = indexer.as_json() print(json_str) indexer2 = MosflmIndexer.from_json(string=json_str) indexer2.index() assert indexer.get_indexer_cell() == pytest.approx( indexer2.get_indexer_cell(), abs=1e-6) assert indexer.get_indexer_beam_centre() == pytest.approx( indexer2.get_indexer_beam_centre(), abs=1e-6) assert indexer2.get_indexer_images() == [ [1, 1], [22, 22], [45, 45], ] # coming from json these are now lists indexer.eliminate() indexer2.eliminate() assert indexer.get_indexer_cell() == pytest.approx( indexer2.get_indexer_cell(), abs=1e-6) assert indexer.get_indexer_lattice() == "hR" assert indexer2.get_indexer_lattice() == "hR"
def exercise_serialization(dials_data, tmp_dir): base_path = pathlib.Path(tmp_dir) template = dials_data("insulin").join("insulin_1_###.img").strpath from xia2.Modules.Indexer.DialsIndexer import DialsIndexer from xia2.Modules.Refiner.DialsRefiner import DialsRefiner from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater from xia2.Modules.Scaler.CCP4ScalerA import CCP4ScalerA from dxtbx.model.experiment_list import ExperimentListTemplateImporter importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] from xia2.Schema.XProject import XProject from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample proj = XProject(base_path=base_path) proj._name = "PROJ1" cryst = XCrystal("CRYST1", proj) wav = XWavelength("WAVE1", cryst, wavelength=0.98) samp = XSample("X1", cryst) cryst.add_wavelength(wav) cryst.set_ha_info({"atom": "S"}) cryst.add_sample(samp) directory, image = os.path.split(imageset.get_path(1)) sweep = wav.add_sweep(name="SWEEP1", sample=samp, directory=directory, image=image) samp.add_sweep(sweep) from dxtbx.serialize.load import _decode_dict indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) indexer.add_indexer_imageset(imageset) indexer.set_indexer_sweep(sweep) sweep._indexer = indexer refiner = DialsRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) refiner.add_refiner_sweep(sweep) sweep._refiner = refiner integrater = DialsIntegrater() integrater.set_output_format("hkl") integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) # integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.set_integrater_epoch(sweep.get_epoch(1)) integrater.set_integrater_sweep_name(sweep.get_name()) integrater.set_integrater_project_info( cryst.get_name(), wav.get_name(), sweep.get_name() ) sweep._integrater = integrater scaler = CCP4ScalerA(base_path=base_path) scaler.add_scaler_integrater(integrater) scaler.set_scaler_xcrystal(cryst) scaler.set_scaler_project_info(cryst.get_name(), wav.get_name()) scaler._scalr_xcrystal = cryst cryst._scaler = scaler proj.add_crystal(cryst) s_dict = sweep.to_dict() s_str = json.dumps(s_dict, ensure_ascii=True) s_dict = json.loads(s_str, object_hook=_decode_dict) xsweep = XSweep.from_dict(s_dict) assert xsweep w_dict = wav.to_dict() w_str = json.dumps(w_dict, ensure_ascii=True) w_dict = json.loads(w_str, object_hook=_decode_dict) xwav = XWavelength.from_dict(w_dict) assert xwav.get_sweeps()[0].get_wavelength() is xwav c_dict = cryst.to_dict() c_str = json.dumps(c_dict, ensure_ascii=True) c_dict = json.loads(c_str, object_hook=_decode_dict) xcryst = XCrystal.from_dict(c_dict) assert ( xcryst.get_xwavelength(xcryst.get_wavelength_names()[0]).get_crystal() is xcryst ) p_dict = proj.to_dict() p_str = json.dumps(p_dict, ensure_ascii=True) p_dict = json.loads(p_str, object_hook=_decode_dict) xproj = XProject.from_dict(p_dict) assert xproj.path == base_path assert list(xproj.get_crystals().values())[0].get_project() is xproj assert list(xproj.get_crystals().values())[0]._scaler._base_path == base_path json_str = proj.as_json() xproj = XProject.from_json(string=json_str) assert xproj.path == base_path assert list(xproj.get_crystals().values())[0].get_project() is xproj print(xproj.get_output()) print("\n".join(xproj.summarise())) json_str = xproj.as_json() xproj = XProject.from_json(string=json_str) assert xproj.path == base_path # Test that we can serialize to json and back again xproj = XProject.from_json(string=xproj.as_json()) assert xproj.path == base_path xcryst = list(xproj.get_crystals().values())[0] assert xcryst.get_project() is xproj intgr = xcryst._get_integraters()[0] assert intgr.get_integrater_finish_done() assert ( xcryst._get_scaler() ._sweep_handler.get_sweep_information(intgr.get_integrater_epoch()) .get_integrater() is intgr ) print(xproj.get_output()) print("\n".join(xproj.summarise()))
def exercise_xds_integrater(dials_data, tmp_dir, nproc=None): if nproc: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc template = dials_data("insulin").join("insulin_1_###.img").strpath from xia2.Modules.Indexer.XDSIndexer import XDSIndexer from xia2.Modules.Integrater.XDSIntegrater import XDSIntegrater indexer = XDSIndexer() indexer.set_working_directory(tmp_dir) from dxtbx.model.experiment_list import ExperimentListTemplateImporter importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, indexer.get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) from xia2.Modules.Refiner.XDSRefiner import XDSRefiner refiner = XDSRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) # refiner.refine() integrater = XDSIntegrater() integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) integrater.set_integrater_sweep(sweep) integrater.integrate() from iotbx.reflection_file_reader import any_reflection_file integrater_intensities = integrater.get_integrater_intensities() assert os.path.exists(integrater_intensities) reader = any_reflection_file(integrater_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert approx_equal(mtz_object.n_reflections(), 50000, eps=400) assert mtz_object.column_labels() == [ "H", "K", "L", "M_ISYM", "BATCH", "I", "SIGI", "FRACTIONCALC", "XDET", "YDET", "ROT", "LP", "FLAG", ] corrected_intensities = integrater.get_integrater_corrected_intensities() assert os.path.exists(corrected_intensities) reader = any_reflection_file(corrected_intensities) assert reader.file_type() == "xds_ascii" ma = reader.as_miller_arrays(merge_equivalents=False)[0] assert approx_equal(ma.size(), 50000, eps=400) assert integrater.get_integrater_wedge() == (1, 45) assert approx_equal( integrater.get_integrater_cell(), [78.066, 78.066, 78.066, 90, 90, 90], eps=1 ) assert approx_equal( integrater.get_integrater_mosaic_min_mean_max(), (0.180, 0.180, 0.180), eps=1e-1 ) # test serialization of integrater json_str = integrater.as_json() # print(json_str) integrater2 = XDSIntegrater.from_json(string=json_str) integrater2.set_integrater_sweep(sweep, reset=False) integrater2_intensities = integrater.get_integrater_intensities() assert integrater2_intensities == integrater_intensities integrater2.set_integrater_finish_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert approx_equal(mtz_object.n_reflections(), 50000, eps=400) integrater2.set_integrater_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert approx_equal(mtz_object.n_reflections(), 50000, eps=450) integrater2.set_integrater_prepare_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert approx_equal(mtz_object.n_reflections(), 50100, eps=400)
def exercise_xds_indexer(dials_data, tmp_dir, nproc=None): if nproc is not None: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc template = dials_data("insulin").join("insulin_1_###.img").strpath from xia2.Modules.Indexer.XDSIndexerII import XDSIndexerII indexer = XDSIndexerII() indexer.set_working_directory(tmp_dir) from dxtbx.model.experiment_list import ExperimentListTemplateImporter importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) indexer.index() assert approx_equal(indexer.get_indexer_cell(), (78.054, 78.054, 78.054, 90, 90, 90), eps=1), indexer.get_indexer_cell() experiment = indexer.get_indexer_experiment_list()[0] sgi = experiment.crystal.get_space_group().info() assert sgi.type().number() == 197 beam_centre = indexer.get_indexer_beam_centre() assert approx_equal(beam_centre, (94.4239, 94.5110), eps=1e-1) assert indexer.get_indexer_images() == [(1, 45)] print(indexer.get_indexer_experiment_list()[0].crystal) print(indexer.get_indexer_experiment_list()[0].detector) # test serialization of indexer json_str = indexer.as_json() print(json_str) indexer2 = XDSIndexerII.from_json(string=json_str) indexer2.index() assert approx_equal(indexer.get_indexer_cell(), indexer2.get_indexer_cell()) assert approx_equal(indexer.get_indexer_beam_centre(), indexer2.get_indexer_beam_centre()) assert approx_equal(indexer.get_indexer_images(), indexer2.get_indexer_images()) indexer.eliminate() indexer2.eliminate() assert approx_equal(indexer.get_indexer_cell(), indexer2.get_indexer_cell()) assert indexer.get_indexer_lattice() == "hR" assert indexer2.get_indexer_lattice() == "hR"
def test_ccp4_scalerA(regression_test, ccp4, dials_data, run_in_tmpdir, nproc): if nproc is not None: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc template = dials_data("insulin").join("insulin_1_###.img").strpath tmpdir = run_in_tmpdir.strpath from xia2.Modules.Indexer.DialsIndexer import DialsIndexer from xia2.Modules.Refiner.DialsRefiner import DialsRefiner from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater from xia2.Modules.Scaler.CCP4ScalerA import CCP4ScalerA indexer = DialsIndexer() indexer.set_working_directory(tmpdir) from dxtbx.model.experiment_list import ExperimentListTemplateImporter importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) with mock.patch.object(sys, "argv", []): sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) refiner = DialsRefiner() refiner.set_working_directory(tmpdir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) integrater = DialsIntegrater() integrater.set_working_directory(tmpdir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) # integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.set_integrater_sweep_name("SWEEP1") integrater.set_integrater_project_info("CRYST1", "WAVE1", "SWEEP1") scaler = CCP4ScalerA() scaler.add_scaler_integrater(integrater) scaler.set_scaler_xcrystal(cryst) scaler.set_scaler_project_info("CRYST1", "WAVE1") check_scaler_files_exist(scaler) # test serialization of scaler json_str = scaler.as_json() # print json_str scaler2 = CCP4ScalerA.from_json(string=json_str) scaler2.set_scaler_xcrystal(cryst) check_scaler_files_exist(scaler2) scaler2.set_scaler_finish_done(False) check_scaler_files_exist(scaler2) scaler2.set_scaler_done(False) check_scaler_files_exist(scaler2) scaler2._scalr_integraters = {} # XXX scaler2.add_scaler_integrater(integrater) scaler2.set_scaler_prepare_done(False) check_scaler_files_exist(scaler2)
def exercise_dials_indexer(dials_data, tmp_dir, nproc=None): if nproc is not None: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc template = dials_data("insulin").join("insulin_1_###.img").strpath from xia2.Modules.Indexer.DialsIndexer import DialsIndexer indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) from dxtbx.model.experiment_list import ExperimentListTemplateImporter importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) indexer.index() assert approx_equal(indexer.get_indexer_cell(), (78.14, 78.14, 78.14, 90, 90, 90), eps=1e-1) solution = indexer.get_solution() assert approx_equal(solution["rmsd"], 0.041, eps=1e-2) assert approx_equal(solution["metric"], 0.027, eps=1e-2) assert solution["number"] == 22 assert solution["lattice"] == "cI" beam_centre = indexer.get_indexer_beam_centre() assert approx_equal(beam_centre, (94.4223, 94.5097), eps=1e-2) print(indexer.get_indexer_experiment_list()[0].crystal) print(indexer.get_indexer_experiment_list()[0].detector) # test serialization of indexer json_str = indexer.as_json() # print(json_str) indexer2 = DialsIndexer.from_json(string=json_str) indexer2.index() assert approx_equal(indexer.get_indexer_cell(), indexer2.get_indexer_cell()) assert approx_equal(indexer.get_indexer_beam_centre(), indexer2.get_indexer_beam_centre()) indexer.eliminate() indexer2.eliminate() assert approx_equal(indexer.get_indexer_cell(), indexer2.get_indexer_cell()) assert indexer.get_indexer_lattice() == "hR" assert indexer2.get_indexer_lattice() == "hR"
def load_imagesets( template, directory, id_image=None, image_range=None, use_cache=True, reversephi=False, ): global imageset_cache from dxtbx.model.experiment_list import ExperimentListFactory from xia2.Applications.xia2setup import known_hdf5_extensions from dxtbx.imageset import ImageSweep full_template_path = os.path.join(directory, template) if full_template_path not in imageset_cache or not use_cache: from dxtbx.model.experiment_list import BeamComparison from dxtbx.model.experiment_list import DetectorComparison from dxtbx.model.experiment_list import GoniometerComparison params = PhilIndex.params.xia2.settings compare_beam = BeamComparison( wavelength_tolerance=params.input.tolerance.beam.wavelength, direction_tolerance=params.input.tolerance.beam.direction, polarization_normal_tolerance=params.input.tolerance.beam.polarization_normal, polarization_fraction_tolerance=params.input.tolerance.beam.polarization_fraction, ) compare_detector = DetectorComparison( fast_axis_tolerance=params.input.tolerance.detector.fast_axis, slow_axis_tolerance=params.input.tolerance.detector.slow_axis, origin_tolerance=params.input.tolerance.detector.origin, ) compare_goniometer = GoniometerComparison( rotation_axis_tolerance=params.input.tolerance.goniometer.rotation_axis, fixed_rotation_tolerance=params.input.tolerance.goniometer.fixed_rotation, setting_rotation_tolerance=params.input.tolerance.goniometer.setting_rotation, ) scan_tolerance = params.input.tolerance.scan.oscillation format_kwargs = { "dynamic_shadowing": params.input.format.dynamic_shadowing, "multi_panel": params.input.format.multi_panel, } if os.path.splitext(full_template_path)[-1] in known_hdf5_extensions: # if we are passed the correct file, use this, else look for a master # file (i.e. something_master.h5) if os.path.exists(full_template_path) and os.path.isfile( full_template_path ): master_file = full_template_path else: import glob g = glob.glob(os.path.join(directory, "*_master.h5")) master_file = None for p in g: substr = longest_common_substring(template, p) if substr: if master_file is None or ( len(substr) > len(longest_common_substring(template, master_file)) ): master_file = p if master_file is None: raise RuntimeError("Can't find master file for %s" % full_template_path) unhandled = [] experiments = ExperimentListFactory.from_filenames( [master_file], verbose=False, unhandled=unhandled, compare_beam=compare_beam, compare_detector=compare_detector, compare_goniometer=compare_goniometer, scan_tolerance=scan_tolerance, format_kwargs=format_kwargs, ) assert len(unhandled) == 0, ( "unhandled image files identified: %s" % unhandled ) else: from dxtbx.sweep_filenames import locate_files_matching_template_string params = PhilIndex.get_python_object() read_all_image_headers = params.xia2.settings.read_all_image_headers if read_all_image_headers: paths = sorted( locate_files_matching_template_string(full_template_path) ) unhandled = [] experiments = ExperimentListFactory.from_filenames( paths, verbose=False, unhandled=unhandled, compare_beam=compare_beam, compare_detector=compare_detector, compare_goniometer=compare_goniometer, scan_tolerance=scan_tolerance, format_kwargs=format_kwargs, ) assert len(unhandled) == 0, ( "unhandled image files identified: %s" % unhandled ) else: from dxtbx.model.experiment_list import ExperimentListTemplateImporter importer = ExperimentListTemplateImporter( [full_template_path], format_kwargs=format_kwargs ) experiments = importer.experiments imagesets = [ iset for iset in experiments.imagesets() if isinstance(iset, ImageSweep) ] assert len(imagesets) > 0, "no imageset found" imageset_cache[full_template_path] = collections.OrderedDict() if reversephi: for imageset in imagesets: goniometer = imageset.get_goniometer() goniometer.set_rotation_axis( tuple(-g for g in goniometer.get_rotation_axis()) ) reference_geometry = PhilIndex.params.xia2.settings.input.reference_geometry if reference_geometry is not None and len(reference_geometry) > 0: update_with_reference_geometry(imagesets, reference_geometry) # Update the geometry params = PhilIndex.params.xia2.settings update_geometry = [] from dials.command_line.dials_import import ManualGeometryUpdater from dials.util.options import geometry_phil_scope # Then add manual geometry work_phil = geometry_phil_scope.format(params.input) diff_phil = geometry_phil_scope.fetch_diff(source=work_phil) if diff_phil.as_str() != "": update_geometry.append(ManualGeometryUpdater(params.input)) imageset_list = [] for imageset in imagesets: for updater in update_geometry: imageset = updater(imageset) imageset_list.append(imageset) imagesets = imageset_list from scitbx.array_family import flex for imageset in imagesets: scan = imageset.get_scan() exposure_times = scan.get_exposure_times() epochs = scan.get_epochs() if exposure_times.all_eq(0) or exposure_times[0] == 0: exposure_times = flex.double(exposure_times.size(), 1) scan.set_exposure_times(exposure_times) elif not exposure_times.all_gt(0): exposure_times = flex.double(exposure_times.size(), exposure_times[0]) scan.set_exposure_times(exposure_times) if epochs.size() > 1 and not epochs.all_gt(0): if epochs[0] == 0: epochs[0] = 1 for i in range(1, epochs.size()): epochs[i] = epochs[i - 1] + exposure_times[i - 1] scan.set_epochs(epochs) _id_image = scan.get_image_range()[0] imageset_cache[full_template_path][_id_image] = imageset if id_image is not None: return [imageset_cache[full_template_path][id_image]] elif image_range is not None: for imageset in imageset_cache[full_template_path].values(): scan = imageset.get_scan() scan_image_range = scan.get_image_range() if ( image_range[0] >= scan_image_range[0] and image_range[1] <= scan_image_range[1] ): imagesets = [ imageset[ image_range[0] - scan_image_range[0] : image_range[1] + 1 - scan_image_range[0] ] ] assert len(imagesets[0]) == image_range[1] - image_range[0] + 1, len( imagesets[0] ) return imagesets return imageset_cache[full_template_path].values()
def _run_dials_import(self): """ Perform a minimal version of dials.import to get an experiment list. Use some filleted bits of dials.import and dials.util.options.Importer. """ # Get some key data format arguments. try: format_kwargs = { "dynamic_shadowing": self.params.dials_import.format.dynamic_shadowing, "multi_panel": self.params.dials_import.format.multi_panel, } except AttributeError: format_kwargs = {} # If filenames contain wildcards, expand args = [] for arg in self.params.dials_import.input.experiments: if "*" in arg: args.extend(glob(arg)) else: args.append(arg) if args: # Are compare{beam,detector,goniometer} and scan_tolerance necessary? # They are cargo-culted from the DIALS option parser. tol_params = self.params.dials_import.input.tolerance compare_beam = BeamComparison( wavelength_tolerance=tol_params.beam.wavelength, direction_tolerance=tol_params.beam.direction, polarization_normal_tolerance=tol_params.beam. polarization_normal, polarization_fraction_tolerance=tol_params.beam. polarization_fraction, ) compare_detector = DetectorComparison( fast_axis_tolerance=tol_params.detector.fast_axis, slow_axis_tolerance=tol_params.detector.slow_axis, origin_tolerance=tol_params.detector.origin, ) compare_goniometer = GoniometerComparison( rotation_axis_tolerance=tol_params.goniometer.rotation_axis, fixed_rotation_tolerance=tol_params.goniometer.fixed_rotation, setting_rotation_tolerance=tol_params.goniometer. setting_rotation, ) scan_tolerance = tol_params.scan.oscillation # Import an experiment list from image data. try: experiments = ExperimentListFactory.from_filenames( args, compare_beam=compare_beam, compare_detector=compare_detector, compare_goniometer=compare_goniometer, scan_tolerance=scan_tolerance, format_kwargs=format_kwargs, ) except IOError as e: warning("%s '%s'", e.strerror, e.filename) sys.exit(1) # Record the imported experiments for use elsewhere. # Quit if there aren't any. self.expts.extend(experiments) if not self.expts: warning("No images found.") sys.exit(1) else: # Use the template importer. if len(self.params.dials_import.input.template) > 0: importer = ExperimentListTemplateImporter( self.params.dials_import.input.template, format_kwargs=format_kwargs) # Record the imported experiments for use elsewhere. # Quit if there aren't any. self.expts.extend(importer.experiments) if not self.expts: warning("No images found matching template %s" % self.params.dials_import.input.template[0]) sys.exit(1) # Setup the metadata updater metadata_updater = MetaDataUpdater(self.params.dials_import) # Extract the experiments and loop through self.expts = metadata_updater(self.expts.imagesets())
def test_labelit_indexer(regression_test, ccp4, dials_data, run_in_tmpdir): template = dials_data("insulin").join("insulin_1_###.img").strpath from xia2.Modules.Indexer.LabelitIndexer import LabelitIndexer from xia2.DriverExceptions.NotAvailableError import NotAvailableError try: ls = LabelitIndexer(indxr_print=True) except NotAvailableError: pytest.skip("labelit not found") ls.set_working_directory(run_in_tmpdir.strpath) from dxtbx.model.experiment_list import ExperimentListTemplateImporter importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] ls.add_indexer_imageset(imageset) ls.index() assert ls.get_indexer_cell() == pytest.approx( (78.58, 78.58, 78.58, 90, 90, 90), abs=0.5) solution = ls.get_solution() assert solution["rmsd"] <= 0.2 assert solution["metric"] <= 0.16 assert solution["number"] == 22 assert solution["lattice"] == "cI" assert solution["mosaic"] <= 0.25 assert solution["nspots"] == pytest.approx(860, abs=30) beam_centre = ls.get_indexer_beam_centre() assert beam_centre == pytest.approx((94.3416, 94.4994), abs=2e-1) assert ls.get_indexer_images() == [(1, 1), (22, 22), (45, 45)] print(ls.get_indexer_experiment_list()[0].crystal) print(ls.get_indexer_experiment_list()[0].detector) json_str = ls.as_json() # print(json_str) ls1 = LabelitIndexer.from_json(string=json_str) ls1.index() print(ls.get_indexer_experiment_list()[0].crystal) assert ls.get_indexer_beam_centre() == ls1.get_indexer_beam_centre() assert ls1.get_indexer_images() == [ [1, 1], [22, 22], [45, 45], ] # in JSON tuples become lists assert ls.get_distance() == ls1.get_distance() ls.eliminate() ls1.eliminate() print(ls1.get_indexer_experiment_list()[0].crystal) assert ls.get_indexer_beam_centre() == ls1.get_indexer_beam_centre() assert ls.get_indexer_images() == [(1, 1), (22, 22), (45, 45)] assert ls1.get_indexer_images() == [ [1, 1], [22, 22], [45, 45], ] # in JSON tuples become lists assert ls.get_distance() == ls1.get_distance() print(ls1.get_indexer_cell()) print(ls1.get_solution()) assert ls.get_indexer_cell() == pytest.approx( (111.11, 111.11, 68.08, 90.0, 90.0, 120.0), abs=5e-1) solution = ls1.get_solution() assert solution["rmsd"] >= 0.07, solution["rmsd"] assert solution["metric"] == pytest.approx(0.1291, abs=1e-1) assert solution["lattice"] == "hR", solution["lattice"] assert solution["mosaic"] <= 0.3, solution["mosaic"] assert solution["nspots"] == pytest.approx(856, abs=30)
def exercise_dials_integrater(dials_data, tmp_dir, nproc=None): if nproc: PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc template = dials_data("insulin").join("insulin_1_###.img").strpath indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] indexer.add_indexer_imageset(imageset) cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) refiner = DialsRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) # refiner.refine() integrater = DialsIntegrater() integrater.set_output_format("hkl") integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) # integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.integrate() integrater_intensities = integrater.get_integrater_intensities() assert os.path.exists(integrater_intensities) reader = any_reflection_file(integrater_intensities) assert reader.file_type() == "ccp4_mtz", repr(integrater_intensities) mtz_object = reader.file_content() expected_reflections = 47623 assert (abs(mtz_object.n_reflections() - expected_reflections) < 300), mtz_object.n_reflections() assert mtz_object.column_labels() == [ "H", "K", "L", "M_ISYM", "BATCH", "IPR", "SIGIPR", "I", "SIGI", "BG", "SIGBG", "FRACTIONCALC", "XDET", "YDET", "ROT", "LP", "QE", ] assert integrater.get_integrater_wedge() == (1, 45) assert integrater.get_integrater_cell() == pytest.approx( (78.14, 78.14, 78.14, 90, 90, 90), abs=1e-1) # test serialization of integrater json_str = integrater.as_json() # print(json_str) integrater2 = DialsIntegrater.from_json(string=json_str) integrater2.set_integrater_sweep(sweep, reset=False) integrater2_intensities = integrater.get_integrater_intensities() assert integrater2_intensities == integrater_intensities integrater2.set_integrater_finish_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - expected_reflections) < 300), mtz_object.n_reflections() integrater2.set_integrater_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - expected_reflections) < 300), mtz_object.n_reflections() integrater2.set_integrater_prepare_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - expected_reflections) < 300), mtz_object.n_reflections() # Test that diamond anvil cell attenuation correction does something. # That it does the right thing is left as a matter for the DIALS tests. integrater3 = DialsIntegrater.from_json(string=json_str) integrater3.set_integrater_sweep(sweep, reset=False) integrater3.set_integrater_done(False) integrater3.high_pressure = True # Don't get .hkl output because we're applying the attenuation correction to data # that weren't actually collected with a diamond anvil cell and some integrated # intensities will be rather nonsensical, which causes an error # 'cctbx Error: Inconsistent observation/sigma pair in columns: IPR, SIGIPR', # when some internal .hkl consistency checks are run, which is not meaningful here. integrater3.set_output_format("pickle") # Compare the first ten profile-fitted integrated intensities without correction. control_reflections = flex.reflection_table.from_file( integrater2.get_integrated_reflections()) valid = control_reflections.get_flags( control_reflections.flags.integrated_prf) valid = valid.iselection()[:10] control_reflections = control_reflections.select(valid) # Get the first ten profile-fitted integrated intensities with DAC correction. corrected_reflections = flex.reflection_table.from_file( integrater3.get_integrated_reflections()) valid = corrected_reflections.get_flags( corrected_reflections.flags.integrated_prf) valid = valid.iselection()[:10] corrected_reflections = corrected_reflections.select(valid) # Check that we're comparing equivalent reflections. assert control_reflections["miller_index"] == corrected_reflections[ "miller_index"] control_intensities = control_reflections["intensity.prf.value"] corrected_intensities = corrected_reflections["intensity.prf.value"] # Check that the reflection intensities are not the same. assert pytest.approx(control_intensities) != corrected_intensities
def exercise_mosflm_integrater(dials_data, tmp_dir, nproc): from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc template = dials_data("insulin").join("insulin_1_###.img").strpath # otherwise if this test is running multiple times simultaneously two mosflm # processes try to write to the same genfile os.environ["CCP4_SCR"] = tmp_dir from xia2.Modules.Indexer.MosflmIndexer import MosflmIndexer from xia2.Modules.Integrater.MosflmIntegrater import MosflmIntegrater from dxtbx.model.experiment_list import ExperimentListTemplateImporter indexer = MosflmIndexer() indexer.set_working_directory(tmp_dir) importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, indexer.get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) from xia2.Modules.Refiner.MosflmRefiner import MosflmRefiner refiner = MosflmRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) nref_error = 500 integrater = MosflmIntegrater() integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) # integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.integrate() integrater_intensities = integrater.get_integrater_intensities() assert os.path.exists(integrater_intensities) from iotbx.reflection_file_reader import any_reflection_file reader = any_reflection_file(integrater_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - 81116) < nref_error), mtz_object.n_reflections() assert mtz_object.column_labels() == [ "H", "K", "L", "M_ISYM", "BATCH", "I", "SIGI", "IPR", "SIGIPR", "FRACTIONCALC", "XDET", "YDET", "ROT", "WIDTH", "LP", "MPART", "FLAG", "BGPKRATIOS", ] assert integrater.get_integrater_wedge() == (1, 45) assert approx_equal( integrater.get_integrater_cell(), (78.014, 78.014, 78.014, 90.0, 90.0, 90.0), eps=1e-2, ) # test serialization of integrater json_str = integrater.as_json() # print json_str integrater2 = MosflmIntegrater.from_json(string=json_str) integrater2.set_integrater_sweep(sweep, reset=False) integrater2_intensities = integrater.get_integrater_intensities() assert integrater2_intensities == integrater_intensities integrater2.set_integrater_finish_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - 81116) < nref_error), mtz_object.n_reflections() integrater2.set_integrater_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - 81116) < nref_error), mtz_object.n_reflections() integrater2.set_integrater_prepare_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - 81116) < nref_error), mtz_object.n_reflections()