def exercise_dials_integrater(nproc=None): if not have_dials_regression: print "Skipping exercise_dials_integrater(): dials_regression not configured" return if nproc is not None: from xia2.Handlers.Flags import Flags Flags.set_parallel(nproc) xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data") template = os.path.join(xia2_demo_data, "insulin_1_###.img") cwd = os.path.abspath(os.curdir) tmp_dir = os.path.abspath(open_tmp_directory()) os.chdir(tmp_dir) from xia2.Modules.Indexer.DialsIndexer import DialsIndexer from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater from dxtbx.datablock import DataBlockTemplateImporter indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) importer = DataBlockTemplateImporter([template]) datablocks = importer.datablocks imageset = datablocks[0].extract_imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) from xia2.Modules.Refiner.DialsRefiner import DialsRefiner refiner = DialsRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) #refiner.refine() integrater = DialsIntegrater() integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) #integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.integrate() integrater_intensities = integrater.get_integrater_intensities() assert os.path.exists(integrater_intensities) from iotbx.reflection_file_reader import any_reflection_file reader = any_reflection_file(integrater_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - 48117) < 100, mtz_object.n_reflections() assert mtz_object.column_labels() == [ 'H', 'K', 'L', 'M_ISYM', 'BATCH', 'IPR', 'SIGIPR', 'I', 'SIGI', 'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'LP', 'DQE'] assert integrater.get_integrater_wedge() == (1, 45) assert approx_equal(integrater.get_integrater_cell(), (78.14, 78.14, 78.14, 90, 90, 90), eps=1e-1) # test serialization of integrater json_str = integrater.as_json() #print json_str integrater2 = DialsIntegrater.from_json(string=json_str) integrater2.set_integrater_sweep(sweep, reset=False) integrater2_intensities = integrater.get_integrater_intensities() assert integrater2_intensities == integrater_intensities integrater2.set_integrater_finish_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - 48117) < 100, mtz_object.n_reflections() integrater2.set_integrater_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - 48117) < 100, mtz_object.n_reflections() integrater2.set_integrater_prepare_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - 48117) < 100, mtz_object.n_reflections()
def exercise_dials_integrater(xia2_regression_build, tmp_dir, nproc=None): if nproc: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc xia2_demo_data = os.path.join(xia2_regression_build, "test_data", "insulin") template = os.path.join(xia2_demo_data, "insulin_1_###.img") from xia2.Modules.Indexer.DialsIndexer import DialsIndexer from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater from dxtbx.datablock import DataBlockTemplateImporter indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) importer = DataBlockTemplateImporter([template]) datablocks = importer.datablocks imageset = datablocks[0].extract_imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) from xia2.Modules.Refiner.DialsRefiner import DialsRefiner refiner = DialsRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) #refiner.refine() integrater = DialsIntegrater() integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) #integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.integrate() integrater_intensities = integrater.get_integrater_intensities() assert os.path.exists(integrater_intensities) from iotbx.reflection_file_reader import any_reflection_file reader = any_reflection_file(integrater_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() expected_reflections = 48456 assert abs(mtz_object.n_reflections() - expected_reflections) < 300, mtz_object.n_reflections() assert mtz_object.column_labels() == [ 'H', 'K', 'L', 'M_ISYM', 'BATCH', 'IPR', 'SIGIPR', 'I', 'SIGI', 'BG', 'SIGBG', 'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'LP', 'QE' ] assert integrater.get_integrater_wedge() == (1, 45) assert approx_equal(integrater.get_integrater_cell(), (78.14, 78.14, 78.14, 90, 90, 90), eps=1e-1) # test serialization of integrater json_str = integrater.as_json() #print(json_str) integrater2 = DialsIntegrater.from_json(string=json_str) integrater2.set_integrater_sweep(sweep, reset=False) integrater2_intensities = integrater.get_integrater_intensities() assert integrater2_intensities == integrater_intensities integrater2.set_integrater_finish_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - expected_reflections) < 300, mtz_object.n_reflections() integrater2.set_integrater_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - expected_reflections) < 300, mtz_object.n_reflections() integrater2.set_integrater_prepare_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - expected_reflections) < 300, mtz_object.n_reflections()
def exercise_dials_integrater(dials_data, tmp_dir, nproc=None): if nproc: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc template = dials_data("insulin").join("insulin_1_###.img").strpath from xia2.Modules.Indexer.DialsIndexer import DialsIndexer from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater from dxtbx.model.experiment_list import ExperimentListTemplateImporter indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) from xia2.Modules.Refiner.DialsRefiner import DialsRefiner refiner = DialsRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) # refiner.refine() integrater = DialsIntegrater() integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) # integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.integrate() integrater_intensities = integrater.get_integrater_intensities() assert os.path.exists(integrater_intensities) from iotbx.reflection_file_reader import any_reflection_file reader = any_reflection_file(integrater_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() expected_reflections = 47623 assert (abs(mtz_object.n_reflections() - expected_reflections) < 300), mtz_object.n_reflections() assert mtz_object.column_labels() == [ "H", "K", "L", "M_ISYM", "BATCH", "IPR", "SIGIPR", "I", "SIGI", "BG", "SIGBG", "FRACTIONCALC", "XDET", "YDET", "ROT", "LP", "QE", ] assert integrater.get_integrater_wedge() == (1, 45) assert approx_equal(integrater.get_integrater_cell(), (78.14, 78.14, 78.14, 90, 90, 90), eps=1e-1) # test serialization of integrater json_str = integrater.as_json() # print(json_str) integrater2 = DialsIntegrater.from_json(string=json_str) integrater2.set_integrater_sweep(sweep, reset=False) integrater2_intensities = integrater.get_integrater_intensities() assert integrater2_intensities == integrater_intensities integrater2.set_integrater_finish_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - expected_reflections) < 300), mtz_object.n_reflections() integrater2.set_integrater_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - expected_reflections) < 300), mtz_object.n_reflections() integrater2.set_integrater_prepare_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - expected_reflections) < 300), mtz_object.n_reflections()
def exercise_dials_integrater(dials_data, tmp_dir, nproc=None): if nproc: PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc template = dials_data("insulin").join("insulin_1_###.img").strpath indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] indexer.add_indexer_imageset(imageset) cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) refiner = DialsRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) # refiner.refine() integrater = DialsIntegrater() integrater.set_output_format("hkl") integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) # integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.integrate() integrater_intensities = integrater.get_integrater_intensities() assert os.path.exists(integrater_intensities) reader = any_reflection_file(integrater_intensities) assert reader.file_type() == "ccp4_mtz", repr(integrater_intensities) mtz_object = reader.file_content() expected_reflections = 47623 assert (abs(mtz_object.n_reflections() - expected_reflections) < 300), mtz_object.n_reflections() assert mtz_object.column_labels() == [ "H", "K", "L", "M_ISYM", "BATCH", "IPR", "SIGIPR", "I", "SIGI", "BG", "SIGBG", "FRACTIONCALC", "XDET", "YDET", "ROT", "LP", "QE", ] assert integrater.get_integrater_wedge() == (1, 45) assert integrater.get_integrater_cell() == pytest.approx( (78.14, 78.14, 78.14, 90, 90, 90), abs=1e-1) # test serialization of integrater json_str = integrater.as_json() # print(json_str) integrater2 = DialsIntegrater.from_json(string=json_str) integrater2.set_integrater_sweep(sweep, reset=False) integrater2_intensities = integrater.get_integrater_intensities() assert integrater2_intensities == integrater_intensities integrater2.set_integrater_finish_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - expected_reflections) < 300), mtz_object.n_reflections() integrater2.set_integrater_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - expected_reflections) < 300), mtz_object.n_reflections() integrater2.set_integrater_prepare_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - expected_reflections) < 300), mtz_object.n_reflections() # Test that diamond anvil cell attenuation correction does something. # That it does the right thing is left as a matter for the DIALS tests. integrater3 = DialsIntegrater.from_json(string=json_str) integrater3.set_integrater_sweep(sweep, reset=False) integrater3.set_integrater_done(False) integrater3.high_pressure = True # Don't get .hkl output because we're applying the attenuation correction to data # that weren't actually collected with a diamond anvil cell and some integrated # intensities will be rather nonsensical, which causes an error # 'cctbx Error: Inconsistent observation/sigma pair in columns: IPR, SIGIPR', # when some internal .hkl consistency checks are run, which is not meaningful here. integrater3.set_output_format("pickle") # Compare the first ten profile-fitted integrated intensities without correction. control_reflections = flex.reflection_table.from_file( integrater2.get_integrated_reflections()) valid = control_reflections.get_flags( control_reflections.flags.integrated_prf) valid = valid.iselection()[:10] control_reflections = control_reflections.select(valid) # Get the first ten profile-fitted integrated intensities with DAC correction. corrected_reflections = flex.reflection_table.from_file( integrater3.get_integrated_reflections()) valid = corrected_reflections.get_flags( corrected_reflections.flags.integrated_prf) valid = valid.iselection()[:10] corrected_reflections = corrected_reflections.select(valid) # Check that we're comparing equivalent reflections. assert control_reflections["miller_index"] == corrected_reflections[ "miller_index"] control_intensities = control_reflections["intensity.prf.value"] corrected_intensities = corrected_reflections["intensity.prf.value"] # Check that the reflection intensities are not the same. assert pytest.approx(control_intensities) != corrected_intensities