def add_sweep(self, name, sample, directory=None, image=None, beam=None, reversephi=False, distance=None, gain=0.0, dmin=0.0, dmax=0.0, polarization=0.0, frames_to_process=None, user_lattice=None, user_cell=None, epoch=0, ice=False, excluded_regions=None): '''Add a sweep to this wavelength.''' if excluded_regions is None: excluded_regions = [] xsweep = XSweep(name, self, sample=sample, directory=directory, image=image, beam=beam, reversephi=reversephi, distance=distance, gain=gain, dmin=dmin, dmax=dmax, polarization=polarization, frames_to_process=frames_to_process, user_lattice=user_lattice, user_cell=user_cell, epoch=epoch, ice=ice, excluded_regions=excluded_regions) self._sweeps.append(xsweep) return xsweep
def exercise_xds_indexer(dials_data, tmp_dir, nproc=None): if nproc is not None: PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc template = dials_data("insulin").join("insulin_1_###.img").strpath indexer = XDSIndexer() indexer.set_working_directory(tmp_dir) importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] indexer.add_indexer_imageset(imageset) cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, indexer.get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) indexer.index() assert indexer.get_indexer_cell() == pytest.approx( (78.076, 78.076, 78.076, 90, 90, 90), abs=1), indexer.get_indexer_cell() experiment = indexer.get_indexer_experiment_list()[0] sgi = experiment.crystal.get_space_group().info() assert sgi.type().number() == 197 beam_centre = indexer.get_indexer_beam_centre() assert beam_centre == pytest.approx((94.4221, 94.5096), abs=1e-1) assert indexer.get_indexer_images() == [(1, 5), (20, 24), (41, 45)] print(indexer.get_indexer_experiment_list()[0].crystal) print(indexer.get_indexer_experiment_list()[0].detector) # test serialization of indexer json_str = indexer.as_json() print(json_str) indexer2 = XDSIndexer.from_json(string=json_str) indexer2.index() assert indexer.get_indexer_cell() == pytest.approx( indexer2.get_indexer_cell()) assert indexer.get_indexer_beam_centre() == pytest.approx( indexer2.get_indexer_beam_centre()) assert indexer.get_indexer_images() == [ tuple(i) for i in indexer2.get_indexer_images() ] indexer.eliminate() indexer2.eliminate() assert indexer.get_indexer_cell() == pytest.approx( indexer2.get_indexer_cell()) assert indexer.get_indexer_lattice() == "hR" assert indexer2.get_indexer_lattice() == "hR"
def from_dict(cls, obj): assert obj['__id__'] == 'XWavelength' return_obj = cls(name=None, crystal=None, wavelength=None) for k, v in obj.iteritems(): if k == '_sweeps': v = [XSweep.from_dict(s_dict) for s_dict in v] for sweep in v: sweep._wavelength = return_obj setattr(return_obj, k, v) return return_obj
def exercise_dials_indexer(dials_data, tmp_dir, nproc=None): if nproc is not None: PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc template = dials_data("insulin").join("insulin_1_###.img").strpath indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] indexer.add_indexer_imageset(imageset) cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) indexer.index() assert indexer.get_indexer_cell() == pytest.approx( (78.14, 78.14, 78.14, 90, 90, 90), rel=1e-3) solution = indexer.get_solution() assert solution["rmsd"] == pytest.approx(0.03545, abs=1e-3) assert solution["metric"] == pytest.approx(0.02517, abs=5e-3) assert solution["number"] == 22 assert solution["lattice"] == "cI" beam_centre = indexer.get_indexer_beam_centre() assert beam_centre == pytest.approx((94.41567208118963, 94.51337522659865), abs=1e-3) print(indexer.get_indexer_experiment_list()[0].crystal) print(indexer.get_indexer_experiment_list()[0].detector) # test serialization of indexer json_str = indexer.as_json() indexer2 = DialsIndexer.from_json(string=json_str) indexer2.index() assert indexer.get_indexer_cell() == pytest.approx( indexer2.get_indexer_cell()) assert indexer.get_indexer_beam_centre() == pytest.approx( indexer2.get_indexer_beam_centre()) indexer.eliminate() indexer2.eliminate() assert indexer.get_indexer_cell() == pytest.approx( indexer2.get_indexer_cell()) assert indexer.get_indexer_lattice() == "hR" assert indexer2.get_indexer_lattice() == "hR"
def exercise_dials_multi_indexer(nproc=None): template = "/Volumes/touro/data/i19/4sweep/56_Reza-H2-normalhem_100K_3.75mmAl/56-RezaH2-hem_%02i_#####.cbf" cwd = os.path.abspath(os.curdir) tmp_dir = os.path.abspath(open_tmp_directory()) os.chdir(tmp_dir) cryst = None wav = None samp = None from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.trust_beam_centre = True from xia2.Modules.Indexer.DialsIndexer import DialsIndexer from xia2.Modules.Indexer.DialsIndexer import DialsIndexer indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) for i in range(4): from dxtbx.datablock import DataBlockTemplateImporter importer = DataBlockTemplateImporter([template % (i + 1)]) datablocks = importer.datablocks imageset = datablocks[0].extract_imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample if cryst is None or wav is None: cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image) indexer.add_indexer_sweep(sweep) indexer.index() assert approx_equal(indexer.get_indexer_cell(), (9.088, 12.415, 17.420, 90.000, 90.000, 90.000), eps=1e-2)
def exercise_xds_integrater(xia2_regression_build, tmp_dir, nproc=None): if nproc: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc xia2_demo_data = os.path.join(xia2_regression_build, "test_data", "insulin") template = os.path.join(xia2_demo_data, "insulin_1_###.img") from xia2.Modules.Indexer.XDSIndexer import XDSIndexer from xia2.Modules.Integrater.XDSIntegrater import XDSIntegrater indexer = XDSIndexer() indexer.set_working_directory(tmp_dir) from dxtbx.datablock import DataBlockTemplateImporter importer = DataBlockTemplateImporter([template]) datablocks = importer.datablocks imageset = datablocks[0].extract_imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, indexer.get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) from xia2.Modules.Refiner.XDSRefiner import XDSRefiner refiner = XDSRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) #refiner.refine() integrater = XDSIntegrater() integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) integrater.set_integrater_sweep(sweep) integrater.integrate() from iotbx.reflection_file_reader import any_reflection_file integrater_intensities = integrater.get_integrater_intensities() assert os.path.exists(integrater_intensities) reader = any_reflection_file(integrater_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert approx_equal(mtz_object.n_reflections(), 50000, eps=400) assert mtz_object.column_labels() == [ 'H', 'K', 'L', 'M_ISYM', 'BATCH', 'I', 'SIGI', 'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'LP', 'FLAG' ] corrected_intensities = integrater.get_integrater_corrected_intensities() assert os.path.exists(corrected_intensities) reader = any_reflection_file(corrected_intensities) assert reader.file_type() == "xds_ascii" ma = reader.as_miller_arrays(merge_equivalents=False)[0] assert approx_equal(ma.size(), 50000, eps=400) assert integrater.get_integrater_wedge() == (1, 45) assert approx_equal(integrater.get_integrater_cell(), [78.066, 78.066, 78.066, 90, 90, 90], eps=1) assert approx_equal(integrater.get_integrater_mosaic_min_mean_max(), (0.180, 0.180, 0.180), eps=1e-1) # test serialization of integrater json_str = integrater.as_json() #print(json_str) integrater2 = XDSIntegrater.from_json(string=json_str) integrater2.set_integrater_sweep(sweep, reset=False) integrater2_intensities = integrater.get_integrater_intensities() assert integrater2_intensities == integrater_intensities integrater2.set_integrater_finish_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert approx_equal(mtz_object.n_reflections(), 50000, eps=400) integrater2.set_integrater_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert approx_equal(mtz_object.n_reflections(), 50000, eps=450) integrater2.set_integrater_prepare_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert approx_equal(mtz_object.n_reflections(), 50100, eps=400)
def exercise_xds_integrater(nproc=None): if not have_dials_regression: print "Skipping exercise_xds_integrater(): dials_regression not configured" return if nproc is not None: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data") template = os.path.join(xia2_demo_data, "insulin_1_###.img") cwd = os.path.abspath(os.curdir) tmp_dir = os.path.abspath(open_tmp_directory()) os.chdir(tmp_dir) from xia2.Modules.Indexer.XDSIndexer import XDSIndexer from xia2.Modules.Integrater.XDSIntegrater import XDSIntegrater indexer = XDSIndexer() indexer.set_working_directory(tmp_dir) from dxtbx.datablock import DataBlockTemplateImporter importer = DataBlockTemplateImporter([template]) datablocks = importer.datablocks imageset = datablocks[0].extract_imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, indexer.get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) from xia2.Modules.Refiner.XDSRefiner import XDSRefiner refiner = XDSRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) #refiner.refine() integrater = XDSIntegrater() integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) #integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.integrate() integrater_intensities = integrater.get_integrater_intensities() assert os.path.exists(integrater_intensities) from iotbx.reflection_file_reader import any_reflection_file reader = any_reflection_file(integrater_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert approx_equal(mtz_object.n_reflections(), 50000, eps=400) assert mtz_object.column_labels() == [ 'H', 'K', 'L', 'M_ISYM', 'BATCH', 'I', 'SIGI', 'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'LP', 'FLAG'] corrected_intensities = integrater.get_integrater_corrected_intensities() assert os.path.exists(corrected_intensities) from iotbx.reflection_file_reader import any_reflection_file reader = any_reflection_file(corrected_intensities) assert reader.file_type() == "xds_ascii" ma = reader.as_miller_arrays(merge_equivalents=False)[0] assert approx_equal(ma.size(), 50000, eps=400) #assert integrater.get_integrater_reindex_operator() == 'x,z,-y' assert integrater.get_integrater_wedge() == (1, 45) assert approx_equal(integrater.get_integrater_cell(), [78.066, 78.066, 78.066, 90, 90, 90], eps=3e-2) assert approx_equal(integrater.get_integrater_mosaic_min_mean_max(), (0.180, 0.180, 0.180), eps=1e-3) # test serialization of integrater json_str = integrater.as_json() #print json_str integrater2 = XDSIntegrater.from_json(string=json_str) integrater2.set_integrater_sweep(sweep, reset=False) integrater2_intensities = integrater.get_integrater_intensities() assert integrater2_intensities == integrater_intensities integrater2.set_integrater_finish_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert approx_equal(mtz_object.n_reflections(), 50000, eps=400) integrater2.set_integrater_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert approx_equal(mtz_object.n_reflections(), 50000, eps=350) integrater2.set_integrater_prepare_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert approx_equal(mtz_object.n_reflections(), 50000, eps=300)
def exercise_mosflm_integrater(dials_data, tmp_dir, nproc): from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc template = dials_data("insulin").join("insulin_1_###.img").strpath # otherwise if this test is running multiple times simultaneously two mosflm # processes try to write to the same genfile os.environ["CCP4_SCR"] = tmp_dir from xia2.Modules.Indexer.MosflmIndexer import MosflmIndexer from xia2.Modules.Integrater.MosflmIntegrater import MosflmIntegrater from dxtbx.model.experiment_list import ExperimentListTemplateImporter indexer = MosflmIndexer() indexer.set_working_directory(tmp_dir) importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, indexer.get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) from xia2.Modules.Refiner.MosflmRefiner import MosflmRefiner refiner = MosflmRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) nref_error = 500 integrater = MosflmIntegrater() integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) # integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.integrate() integrater_intensities = integrater.get_integrater_intensities() assert os.path.exists(integrater_intensities) from iotbx.reflection_file_reader import any_reflection_file reader = any_reflection_file(integrater_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - 81116) < nref_error), mtz_object.n_reflections() assert mtz_object.column_labels() == [ "H", "K", "L", "M_ISYM", "BATCH", "I", "SIGI", "IPR", "SIGIPR", "FRACTIONCALC", "XDET", "YDET", "ROT", "WIDTH", "LP", "MPART", "FLAG", "BGPKRATIOS", ] assert integrater.get_integrater_wedge() == (1, 45) assert approx_equal( integrater.get_integrater_cell(), (78.014, 78.014, 78.014, 90.0, 90.0, 90.0), eps=1e-2, ) # test serialization of integrater json_str = integrater.as_json() # print json_str integrater2 = MosflmIntegrater.from_json(string=json_str) integrater2.set_integrater_sweep(sweep, reset=False) integrater2_intensities = integrater.get_integrater_intensities() assert integrater2_intensities == integrater_intensities integrater2.set_integrater_finish_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - 81116) < nref_error), mtz_object.n_reflections() integrater2.set_integrater_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - 81116) < nref_error), mtz_object.n_reflections() integrater2.set_integrater_prepare_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - 81116) < nref_error), mtz_object.n_reflections()
def test_xds_scalerA(ccp4, xia2_regression_build, tmpdir, nproc): if nproc is not None: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc template = os.path.join(xia2_regression_build, "test_data", "insulin", "insulin_1_###.img") tmpdir.chdir() tmpdir = tmpdir.strpath from xia2.Modules.Indexer.XDSIndexer import XDSIndexer from xia2.Modules.Integrater.XDSIntegrater import XDSIntegrater from xia2.Modules.Scaler.XDSScalerA import XDSScalerA indexer = XDSIndexer() indexer.set_working_directory(tmpdir) from dxtbx.datablock import DataBlockTemplateImporter importer = DataBlockTemplateImporter([template]) datablocks = importer.datablocks imageset = datablocks[0].extract_imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) with mock.patch.object(sys, 'argv', []): sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) from xia2.Modules.Refiner.XDSRefiner import XDSRefiner refiner = XDSRefiner() refiner.set_working_directory(tmpdir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) integrater = XDSIntegrater() integrater.set_working_directory(tmpdir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) #integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.set_integrater_sweep_name('SWEEP1') integrater.set_integrater_project_info('CRYST1', 'WAVE1', 'SWEEP1') scaler = XDSScalerA() scaler.add_scaler_integrater(integrater) scaler.set_scaler_xcrystal(cryst) scaler.set_scaler_project_info('CRYST1', 'WAVE1') check_scaler_files_exist(scaler) # test serialization of scaler json_str = scaler.as_json() #print json_str scaler2 = XDSScalerA.from_json(string=json_str) scaler2.set_scaler_xcrystal(cryst) check_scaler_files_exist(scaler2) scaler2.set_scaler_finish_done(False) check_scaler_files_exist(scaler2) scaler2.set_scaler_done(False) check_scaler_files_exist(scaler2) scaler2._scalr_integraters = {} # XXX scaler2.add_scaler_integrater(integrater) scaler2.set_scaler_prepare_done(False) check_scaler_files_exist(scaler2)
def exercise_dials_integrater(nproc=None): if not have_dials_regression: print "Skipping exercise_dials_integrater(): dials_regression not configured" return if nproc is not None: from xia2.Handlers.Flags import Flags Flags.set_parallel(nproc) xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data") template = os.path.join(xia2_demo_data, "insulin_1_###.img") cwd = os.path.abspath(os.curdir) tmp_dir = os.path.abspath(open_tmp_directory()) os.chdir(tmp_dir) from xia2.Modules.Indexer.DialsIndexer import DialsIndexer from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater from dxtbx.datablock import DataBlockTemplateImporter indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) importer = DataBlockTemplateImporter([template]) datablocks = importer.datablocks imageset = datablocks[0].extract_imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) from xia2.Modules.Refiner.DialsRefiner import DialsRefiner refiner = DialsRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) #refiner.refine() integrater = DialsIntegrater() integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) #integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.integrate() integrater_intensities = integrater.get_integrater_intensities() assert os.path.exists(integrater_intensities) from iotbx.reflection_file_reader import any_reflection_file reader = any_reflection_file(integrater_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - 48117) < 100, mtz_object.n_reflections() assert mtz_object.column_labels() == [ 'H', 'K', 'L', 'M_ISYM', 'BATCH', 'IPR', 'SIGIPR', 'I', 'SIGI', 'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'LP', 'DQE'] assert integrater.get_integrater_wedge() == (1, 45) assert approx_equal(integrater.get_integrater_cell(), (78.14, 78.14, 78.14, 90, 90, 90), eps=1e-1) # test serialization of integrater json_str = integrater.as_json() #print json_str integrater2 = DialsIntegrater.from_json(string=json_str) integrater2.set_integrater_sweep(sweep, reset=False) integrater2_intensities = integrater.get_integrater_intensities() assert integrater2_intensities == integrater_intensities integrater2.set_integrater_finish_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - 48117) < 100, mtz_object.n_reflections() integrater2.set_integrater_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - 48117) < 100, mtz_object.n_reflections() integrater2.set_integrater_prepare_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - 48117) < 100, mtz_object.n_reflections()
def exercise_dials_indexer(xia2_regression_build, tmp_dir, nproc=None): if nproc is not None: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc xia2_demo_data = os.path.join(xia2_regression_build, "test_data", "insulin") template = os.path.join(xia2_demo_data, "insulin_1_###.img") from xia2.Modules.Indexer.DialsIndexer import DialsIndexer indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) from dxtbx.datablock import DataBlockTemplateImporter importer = DataBlockTemplateImporter([template]) datablocks = importer.datablocks imageset = datablocks[0].extract_imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) indexer.index() assert approx_equal(indexer.get_indexer_cell(), (78.14, 78.14, 78.14, 90, 90, 90), eps=1e-1) solution = indexer.get_solution() assert approx_equal(solution['rmsd'], 0.041, eps=1e-2) assert approx_equal(solution['metric'], 0.027, eps=1e-2) assert solution['number'] == 22 assert solution['lattice'] == 'cI' beam_centre = indexer.get_indexer_beam_centre() assert approx_equal(beam_centre, (94.4223, 94.5097), eps=1e-2) print(indexer.get_indexer_experiment_list()[0].crystal) print(indexer.get_indexer_experiment_list()[0].detector) # test serialization of indexer json_str = indexer.as_json() #print(json_str) indexer2 = DialsIndexer.from_json(string=json_str) indexer2.index() assert approx_equal(indexer.get_indexer_cell(), indexer2.get_indexer_cell()) assert approx_equal(indexer.get_indexer_beam_centre(), indexer2.get_indexer_beam_centre()) indexer.eliminate() indexer2.eliminate() assert approx_equal(indexer.get_indexer_cell(), indexer2.get_indexer_cell()) assert indexer.get_indexer_lattice() == 'hR' assert indexer2.get_indexer_lattice() == 'hR'
def exercise_dials_integrater(dials_data, tmp_dir, nproc=None): if nproc: PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc template = dials_data("insulin").join("insulin_1_###.img").strpath indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] indexer.add_indexer_imageset(imageset) cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) refiner = DialsRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) # refiner.refine() integrater = DialsIntegrater() integrater.set_output_format("hkl") integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) # integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.integrate() integrater_intensities = integrater.get_integrater_intensities() assert os.path.exists(integrater_intensities) reader = any_reflection_file(integrater_intensities) assert reader.file_type() == "ccp4_mtz", repr(integrater_intensities) mtz_object = reader.file_content() expected_reflections = 47623 assert (abs(mtz_object.n_reflections() - expected_reflections) < 300), mtz_object.n_reflections() assert mtz_object.column_labels() == [ "H", "K", "L", "M_ISYM", "BATCH", "IPR", "SIGIPR", "I", "SIGI", "BG", "SIGBG", "FRACTIONCALC", "XDET", "YDET", "ROT", "LP", "QE", ] assert integrater.get_integrater_wedge() == (1, 45) assert integrater.get_integrater_cell() == pytest.approx( (78.14, 78.14, 78.14, 90, 90, 90), abs=1e-1) # test serialization of integrater json_str = integrater.as_json() # print(json_str) integrater2 = DialsIntegrater.from_json(string=json_str) integrater2.set_integrater_sweep(sweep, reset=False) integrater2_intensities = integrater.get_integrater_intensities() assert integrater2_intensities == integrater_intensities integrater2.set_integrater_finish_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - expected_reflections) < 300), mtz_object.n_reflections() integrater2.set_integrater_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - expected_reflections) < 300), mtz_object.n_reflections() integrater2.set_integrater_prepare_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert (abs(mtz_object.n_reflections() - expected_reflections) < 300), mtz_object.n_reflections() # Test that diamond anvil cell attenuation correction does something. # That it does the right thing is left as a matter for the DIALS tests. integrater3 = DialsIntegrater.from_json(string=json_str) integrater3.set_integrater_sweep(sweep, reset=False) integrater3.set_integrater_done(False) integrater3.high_pressure = True # Don't get .hkl output because we're applying the attenuation correction to data # that weren't actually collected with a diamond anvil cell and some integrated # intensities will be rather nonsensical, which causes an error # 'cctbx Error: Inconsistent observation/sigma pair in columns: IPR, SIGIPR', # when some internal .hkl consistency checks are run, which is not meaningful here. integrater3.set_output_format("pickle") # Compare the first ten profile-fitted integrated intensities without correction. control_reflections = flex.reflection_table.from_file( integrater2.get_integrated_reflections()) valid = control_reflections.get_flags( control_reflections.flags.integrated_prf) valid = valid.iselection()[:10] control_reflections = control_reflections.select(valid) # Get the first ten profile-fitted integrated intensities with DAC correction. corrected_reflections = flex.reflection_table.from_file( integrater3.get_integrated_reflections()) valid = corrected_reflections.get_flags( corrected_reflections.flags.integrated_prf) valid = valid.iselection()[:10] corrected_reflections = corrected_reflections.select(valid) # Check that we're comparing equivalent reflections. assert control_reflections["miller_index"] == corrected_reflections[ "miller_index"] control_intensities = control_reflections["intensity.prf.value"] corrected_intensities = corrected_reflections["intensity.prf.value"] # Check that the reflection intensities are not the same. assert pytest.approx(control_intensities) != corrected_intensities
def exercise_mosflm_integrater(dials_regression, tmp_dir, nproc): from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data") template = os.path.join(xia2_demo_data, "insulin_1_###.img") # otherwise if this test is running multiple times simultaneously two mosflm # processes try to write to the same genfile os.environ['CCP4_SCR'] = tmp_dir from xia2.Modules.Indexer.MosflmIndexer import MosflmIndexer from xia2.Modules.Integrater.MosflmIntegrater import MosflmIntegrater from dxtbx.datablock import DataBlockTemplateImporter indexer = MosflmIndexer() indexer.set_working_directory(tmp_dir) importer = DataBlockTemplateImporter([template]) datablocks = importer.datablocks imageset = datablocks[0].extract_imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, indexer.get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) from xia2.Modules.Refiner.MosflmRefiner import MosflmRefiner refiner = MosflmRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) nref_error = 500 integrater = MosflmIntegrater() integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) #integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.integrate() integrater_intensities = integrater.get_integrater_intensities() assert os.path.exists(integrater_intensities) from iotbx.reflection_file_reader import any_reflection_file reader = any_reflection_file(integrater_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - 81116) < nref_error, \ mtz_object.n_reflections() assert mtz_object.column_labels() == [ 'H', 'K', 'L', 'M_ISYM', 'BATCH', 'I', 'SIGI', 'IPR', 'SIGIPR', 'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'WIDTH', 'LP', 'MPART', 'FLAG', 'BGPKRATIOS'] assert integrater.get_integrater_wedge() == (1, 45) assert approx_equal(integrater.get_integrater_cell(), (78.014, 78.014, 78.014, 90.0, 90.0, 90.0), eps=1e-2) # test serialization of integrater json_str = integrater.as_json() #print json_str integrater2 = MosflmIntegrater.from_json(string=json_str) integrater2.set_integrater_sweep(sweep, reset=False) integrater2_intensities = integrater.get_integrater_intensities() assert integrater2_intensities == integrater_intensities integrater2.set_integrater_finish_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - 81116) < nref_error, \ mtz_object.n_reflections() integrater2.set_integrater_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - 81116) < nref_error, \ mtz_object.n_reflections() integrater2.set_integrater_prepare_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - 81116) < nref_error, \ mtz_object.n_reflections()
def exercise_dials_refiner(nproc=None): if not have_xia2_regression: print "Skipping exercise_dials_refiner(): xia2_regression not configured" return if nproc is not None: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc xia2_demo_data = os.path.join(xia2_regression, "test_data", "mad_example") template = os.path.join(xia2_demo_data, "12287_1_E%i_%03i.img") cwd = os.path.abspath(os.curdir) tmp_dir = os.path.abspath(open_tmp_directory()) os.chdir(tmp_dir) from xia2.Modules.Indexer.DialsIndexer import DialsIndexer from xia2.Modules.Refiner.DialsRefiner import DialsRefiner from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater indexer1 = DialsIndexer() indexer1.set_working_directory(tmp_dir) indexer1.setup_from_image(template % (1, 1)) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav1 = XWavelength("WAVE1", cryst, indexer1.get_wavelength()) samp1 = XSample("X1", cryst) directory, image = os.path.split(template % (1, 1)) sweep1 = XSweep('SWEEP1', wav1, samp1, directory=directory, image=image) indexer1.set_indexer_sweep(sweep1) indexer2 = DialsIndexer() indexer2.set_working_directory(tmp_dir) indexer2.setup_from_image(template % (2, 1)) wav2 = XWavelength("WAVE2", cryst, indexer2.get_wavelength()) samp2 = XSample("X2", cryst) directory, image = os.path.split(template % (2, 1)) sweep2 = XSweep('SWEEP2', wav2, samp2, directory=directory, image=image) indexer2.set_indexer_sweep(sweep2) refiner = DialsRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(indexer1) refiner.add_refiner_indexer(indexer2) refined_experiment_list = refiner.get_refined_experiment_list() assert refined_experiment_list is not None assert len(refined_experiment_list.detectors()) == 1 refined_detector = refined_experiment_list[0].detector # test serialization of refiner json_str = refiner.as_json() #print json_str refiner2 = DialsRefiner.from_json(string=json_str) refined_expts_2 = refiner2.get_refined_experiment_list() assert refined_expts_2[0].detector == refined_detector refiner2.set_refiner_finish_done(False) refined_expts_2 = refiner2.get_refined_experiment_list() assert refined_expts_2[0].detector == refined_detector refiner2.set_refiner_done(False) refined_expts_2 = refiner2.get_refined_experiment_list() assert refined_expts_2[0].detector == refined_detector refiner2.set_refiner_prepare_done(False) refined_expts_2 = refiner2.get_refined_experiment_list() assert refined_expts_2[0].detector == refined_detector assert (indexer1.get_indexer_experiment_list()[0].detector != indexer2.get_indexer_experiment_list()[0].detector) assert (indexer1.get_indexer_experiment_list()[0].beam != indexer2.get_indexer_experiment_list()[0].beam) indexer1.set_indexer_experiment_list(refined_experiment_list[0:1]) indexer2.set_indexer_experiment_list(refined_experiment_list[1:2]) assert (indexer1.get_indexer_experiment_list()[0].detector == indexer2.get_indexer_experiment_list()[0].detector) assert (indexer1.get_indexer_experiment_list()[0].goniometer == indexer2.get_indexer_experiment_list()[0].goniometer)
def exercise_ccp4_scaler(nproc=None): if not have_dials_regression: print "Skipping exercise_ccp4_scaler(): dials_regression not configured" return if nproc is not None: from xia2.Handlers.Flags import Flags Flags.set_parallel(nproc) xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data") template = os.path.join(xia2_demo_data, "insulin_1_###.img") cwd = os.path.abspath(os.curdir) tmp_dir = os.path.abspath(open_tmp_directory()) os.chdir(tmp_dir) from xia2.Modules.Indexer.DialsIndexer import DialsIndexer from xia2.Modules.Refiner.DialsRefiner import DialsRefiner from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater from xia2.Modules.Scaler.CCP4ScalerA import CCP4ScalerA indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) from dxtbx.datablock import DataBlockTemplateImporter importer = DataBlockTemplateImporter([template]) datablocks = importer.datablocks imageset = datablocks[0].extract_imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) refiner = DialsRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) integrater = DialsIntegrater() integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) #integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.set_integrater_sweep_name('SWEEP1') integrater.set_integrater_project_info('CRYST1', 'WAVE1', 'SWEEP1') scaler = CCP4ScalerA() scaler.add_scaler_integrater(integrater) scaler.set_scaler_xcrystal(cryst) scaler.set_scaler_project_info('CRYST1', 'WAVE1') check_scaler_files_exist(scaler) # test serialization of scaler json_str = scaler.as_json() #print json_str scaler2 = CCP4ScalerA.from_json(string=json_str) scaler2.set_scaler_xcrystal(cryst) check_scaler_files_exist(scaler2) scaler2.set_scaler_finish_done(False) check_scaler_files_exist(scaler2) scaler2.set_scaler_done(False) check_scaler_files_exist(scaler2) scaler2._scalr_integraters = {} # XXX scaler2.add_scaler_integrater(integrater) scaler2.set_scaler_prepare_done(False) check_scaler_files_exist(scaler2)
self.integrate() from xia2.Wrappers.Dials.ExportXDSASCII import ExportXDSASCII exporter = ExportXDSASCII() exporter.set_experiments_filename(self.get_integrated_experiments()) exporter.set_reflections_filename(self.get_integrated_reflections()) exporter.set_working_directory(self.get_working_directory()) auto_logfiler(exporter) self._intgr_corrected_hklout = os.path.join( self.get_working_directory(), '%i_DIALS.HKL' % exporter.get_xpid()) exporter.set_hkl_filename(self._intgr_corrected_hklout) exporter.run() assert os.path.exists(self._intgr_corrected_hklout) return self._intgr_corrected_hklout if __name__ == '__main__': # run a demo test di = DialsIntegrater() di.setup_from_image(sys.argv[1]) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, di.get_wavelength()) directory, image = os.path.split(sys.argv[1]) sweep = XSweep('SWEEP1', wav, directory=directory, image=image) di.set_integrater_sweep(sweep) di.integrate()
def xia2_main(stop_after=None): '''Actually process something...''' Citations.cite('xia2') # print versions of related software from dials.util.version import dials_version Chatter.write(dials_version()) start_time = time.time() CommandLine = get_command_line() start_dir = Flags.get_starting_directory() # check that something useful has been assigned for processing... xtals = CommandLine.get_xinfo().get_crystals() no_images = True for name in xtals.keys(): xtal = xtals[name] if not xtal.get_all_image_names(): Chatter.write('-----------------------------------' + \ '-' * len(name)) Chatter.write('| No images assigned for crystal %s |' % name) Chatter.write('-----------------------------------' + '-' \ * len(name)) else: no_images = False args = [] from xia2.Handlers.Phil import PhilIndex params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing njob = mp_params.njob from libtbx import group_args xinfo = CommandLine.get_xinfo() if os.path.exists('xia2.json'): from xia2.Schema.XProject import XProject xinfo_new = xinfo xinfo = XProject.from_json(filename='xia2.json') crystals = xinfo.get_crystals() crystals_new = xinfo_new.get_crystals() for crystal_id in crystals_new.keys(): if crystal_id not in crystals: crystals[crystal_id] = crystals_new[crystal_id] continue crystals[crystal_id]._scaler = None # reset scaler for wavelength_id in crystals_new[crystal_id].get_wavelength_names(): wavelength_new = crystals_new[crystal_id].get_xwavelength(wavelength_id) if wavelength_id not in crystals[crystal_id].get_wavelength_names(): crystals[crystal_id].add_wavelength( crystals_new[crystal_id].get_xwavelength(wavelength_new)) continue wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) sweeps_new = wavelength_new.get_sweeps() sweeps = wavelength.get_sweeps() sweep_names = [s.get_name() for s in sweeps] sweep_keys = [ (s.get_directory(), s.get_template(), s.get_image_range()) for s in sweeps] for sweep in sweeps_new: if ((sweep.get_directory(), sweep.get_template(), sweep.get_image_range()) not in sweep_keys): if sweep.get_name() in sweep_names: i = 1 while 'SWEEEP%i' %i in sweep_names: i += 1 sweep._name = 'SWEEP%i' %i break wavelength.add_sweep( name=sweep.get_name(), directory=sweep.get_directory(), image=sweep.get_image(), beam=sweep.get_beam_centre(), reversephi=sweep.get_reversephi(), distance=sweep.get_distance(), gain=sweep.get_gain(), dmin=sweep.get_resolution_high(), dmax=sweep.get_resolution_low(), polarization=sweep.get_polarization(), frames_to_process=sweep.get_frames_to_process(), user_lattice=sweep.get_user_lattice(), user_cell=sweep.get_user_cell(), epoch=sweep._epoch, ice=sweep._ice, excluded_regions=sweep._excluded_regions, ) sweep_names.append(sweep.get_name()) crystals = xinfo.get_crystals() failover = params.xia2.settings.failover if njob > 1: driver_type = mp_params.type command_line_args = CommandLine.get_argv()[1:] for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) sweeps = wavelength.get_sweeps() for sweep in sweeps: sweep._get_indexer() sweep._get_refiner() sweep._get_integrater() args.append(( group_args( driver_type=driver_type, stop_after=stop_after, failover=failover, command_line_args=command_line_args, nproc=mp_params.nproc, crystal_id=crystal_id, wavelength_id=wavelength_id, sweep_id=sweep.get_name(), ),)) from xia2.Driver.DriverFactory import DriverFactory default_driver_type = DriverFactory.get_driver_type() # run every nth job on the current computer (no need to submit to qsub) for i_job, arg in enumerate(args): if (i_job % njob) == 0: arg[0].driver_type = default_driver_type if mp_params.type == "qsub": method = "sge" else: method = "multiprocessing" nproc = mp_params.nproc qsub_command = mp_params.qsub_command if not qsub_command: qsub_command = 'qsub' qsub_command = '%s -V -cwd -pe smp %d' %(qsub_command, nproc) from libtbx import easy_mp results = easy_mp.parallel_map( process_one_sweep, args, processes=njob, #method=method, method="multiprocessing", qsub_command=qsub_command, preserve_order=True, preserve_exception_message=True) # Hack to update sweep with the serialized indexers/refiners/integraters i_sweep = 0 for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: success, output, xsweep_dict = results[i_sweep] assert xsweep_dict is not None if output is not None: Chatter.write(output) if not success: Chatter.write('Sweep failed: removing %s' %sweep.get_name()) remove_sweeps.append(sweep) else: Chatter.write('Loading sweep: %s' % sweep.get_name()) from xia2.Schema.XSweep import XSweep new_sweep = XSweep.from_dict(xsweep_dict) sweep._indexer = new_sweep._indexer sweep._refiner = new_sweep._refiner sweep._integrater = new_sweep._integrater i_sweep += 1 for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep) else: for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: try: if stop_after == 'index': sweep.get_indexer_cell() else: sweep.get_integrater_intensities() sweep.serialize() except Exception, e: if failover: Chatter.write('Processing sweep %s failed: %s' % \ (sweep.get_name(), str(e))) remove_sweeps.append(sweep) else: raise for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep)
exporter = ExportXDSASCII() exporter.set_experiments_filename(self.get_integrated_experiments()) exporter.set_reflections_filename(self.get_integrated_reflections()) exporter.set_working_directory(self.get_working_directory()) auto_logfiler(exporter) self._intgr_corrected_hklout = os.path.join( self.get_working_directory(), "%i_DIALS.HKL" % exporter.get_xpid()) exporter.set_hkl_filename(self._intgr_corrected_hklout) exporter.run() assert os.path.exists(self._intgr_corrected_hklout) return self._intgr_corrected_hklout if __name__ == "__main__": # run a demo test di = DialsIntegrater() di.setup_from_image(sys.argv[1]) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, di.get_wavelength()) directory, image = os.path.split(sys.argv[1]) sweep = XSweep("SWEEP1", wav, directory=directory, image=image) di.set_integrater_sweep(sweep) di.integrate()
def xia2_main(stop_after=None): """Actually process something...""" Citations.cite("xia2") # print versions of related software logger.info(dials_version()) ccp4_version = get_ccp4_version() if ccp4_version: logger.info("CCP4 %s", ccp4_version) start_time = time.time() CommandLine = get_command_line() # check that something useful has been assigned for processing... xtals = CommandLine.get_xinfo().get_crystals() for name, xtal in xtals.items(): if not xtal.get_all_image_names(): logger.info("-----------------------------------" + "-" * len(name)) logger.info("| No images assigned for crystal %s |", name) logger.info("-----------------------------------" + "-" * len(name)) from xia2.Handlers.Phil import PhilIndex params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing njob = mp_params.njob xinfo = CommandLine.get_xinfo() logger.info("Project directory: %s", xinfo.path) if (params.xia2.settings.developmental.continue_from_previous_job and os.path.exists("xia2.json")): logger.debug("==== Starting from existing xia2.json ====") xinfo_new = xinfo xinfo = XProject.from_json(filename="xia2.json") crystals = xinfo.get_crystals() crystals_new = xinfo_new.get_crystals() for crystal_id in crystals_new: if crystal_id not in crystals: crystals[crystal_id] = crystals_new[crystal_id] continue crystals[crystal_id]._scaler = None # reset scaler for wavelength_id in crystals_new[crystal_id].get_wavelength_names( ): wavelength_new = crystals_new[crystal_id].get_xwavelength( wavelength_id) if wavelength_id not in crystals[ crystal_id].get_wavelength_names(): crystals[crystal_id].add_wavelength( crystals_new[crystal_id].get_xwavelength( wavelength_new)) continue wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) sweeps_new = wavelength_new.get_sweeps() sweeps = wavelength.get_sweeps() sweep_names = {s.get_name() for s in sweeps} sweep_keys = {(s.get_directory(), s.get_template(), s.get_image_range()) for s in sweeps} for sweep in sweeps_new: if ( sweep.get_directory(), sweep.get_template(), sweep.get_image_range(), ) not in sweep_keys: if sweep.get_name() in sweep_names: i = 1 while "SWEEEP%i" % i in sweep_names: i += 1 sweep._name = "SWEEP%i" % i break wavelength.add_sweep( name=sweep.get_name(), sample=sweep.sample, directory=sweep.get_directory(), image=sweep.get_image(), beam=sweep.get_beam_centre(), reversephi=sweep.get_reversephi(), distance=sweep.get_distance(), gain=sweep.get_gain(), dmin=sweep.get_resolution_high(), dmax=sweep.get_resolution_low(), polarization=sweep.get_polarization(), frames_to_process=sweep.get_frames_to_process(), user_lattice=sweep.get_user_lattice(), user_cell=sweep.get_user_cell(), epoch=sweep._epoch, ice=sweep._ice, excluded_regions=sweep._excluded_regions, ) sweep_names.add(sweep.get_name()) crystals = xinfo.get_crystals() failover = params.xia2.settings.failover with cleanup(xinfo.path): if mp_params.mode == "parallel" and njob > 1: driver_type = mp_params.type command_line_args = CommandLine.get_argv()[1:] jobs = [] for crystal_id in crystals: for wavelength_id in crystals[crystal_id].get_wavelength_names( ): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) sweeps = wavelength.get_sweeps() for sweep in sweeps: sweep._get_indexer() sweep._get_refiner() sweep._get_integrater() jobs.append((group_args( driver_type=driver_type, stop_after=stop_after, failover=failover, command_line_args=command_line_args, nproc=mp_params.nproc, crystal_id=crystal_id, wavelength_id=wavelength_id, sweep_id=sweep.get_name(), ), )) from xia2.Driver.DriverFactory import DriverFactory default_driver_type = DriverFactory.get_driver_type() # run every nth job on the current computer (no need to submit to qsub) for i_job, arg in enumerate(jobs): if (i_job % njob) == 0: arg[0].driver_type = default_driver_type nproc = mp_params.nproc qsub_command = mp_params.qsub_command or "qsub" qsub_command = "%s -V -cwd -pe smp %d" % (qsub_command, nproc) from libtbx import easy_mp results = easy_mp.parallel_map( process_one_sweep, jobs, processes=njob, method="multiprocessing", qsub_command=qsub_command, preserve_order=True, preserve_exception_message=True, ) # Hack to update sweep with the serialized indexers/refiners/integraters i_sweep = 0 for crystal_id in crystals: for wavelength_id in crystals[crystal_id].get_wavelength_names( ): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: success, output, xsweep_dict = results[i_sweep] if output is not None: logger.info(output) if not success: logger.info("Sweep failed: removing %s", sweep.get_name()) remove_sweeps.append(sweep) else: assert xsweep_dict is not None logger.info("Loading sweep: %s", sweep.get_name()) new_sweep = XSweep.from_dict(xsweep_dict) sweep._indexer = new_sweep._indexer sweep._refiner = new_sweep._refiner sweep._integrater = new_sweep._integrater i_sweep += 1 for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.sample sample.remove_sweep(sweep) else: for crystal_id in list(crystals.keys()): for wavelength_id in crystals[crystal_id].get_wavelength_names( ): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: from dials.command_line.show import show_experiments from dxtbx.model.experiment_list import ExperimentListFactory logger.debug(sweep.get_name()) logger.debug( show_experiments( ExperimentListFactory. from_imageset_and_crystal( sweep.get_imageset(), None))) Citations.cite("dials") try: if stop_after == "index": sweep.get_indexer_cell() else: sweep.get_integrater_intensities() sweep.serialize() except Exception as e: if failover: logger.info( "Processing sweep %s failed: %s", sweep.get_name(), str(e), ) remove_sweeps.append(sweep) else: raise for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.sample sample.remove_sweep(sweep) # save intermediate xia2.json file in case scaling step fails xinfo.as_json(filename="xia2.json") if stop_after not in ("index", "integrate"): logger.info(xinfo.get_output()) for crystal in list(crystals.values()): crystal.serialize() # save final xia2.json file in case report generation fails xinfo.as_json(filename="xia2.json") if stop_after not in ("index", "integrate"): # and the summary file with open("xia2-summary.dat", "w") as fh: for record in xinfo.summarise(): fh.write("%s\n" % record) # looks like this import overwrites the initial command line # Phil overrides so... for https://github.com/xia2/xia2/issues/150 from xia2.command_line.html import generate_xia2_html if params.xia2.settings.small_molecule: params.xia2.settings.report.xtriage_analysis = False params.xia2.settings.report.include_radiation_damage = False with xia2.Driver.timing.record_step("xia2.report"): generate_xia2_html(xinfo, filename="xia2.html", params=params.xia2.settings.report) duration = time.time() - start_time # write out the time taken in a human readable way logger.info("Processing took %s", time.strftime("%Hh %Mm %Ss", time.gmtime(duration))) write_citations()
def exercise_dials_indexer(dials_data, tmp_dir, nproc=None): if nproc is not None: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc template = dials_data("insulin").join("insulin_1_###.img").strpath from xia2.Modules.Indexer.DialsIndexer import DialsIndexer indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) from dxtbx.model.experiment_list import ExperimentListTemplateImporter importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) indexer.index() assert approx_equal(indexer.get_indexer_cell(), (78.14, 78.14, 78.14, 90, 90, 90), eps=1e-1) solution = indexer.get_solution() assert approx_equal(solution["rmsd"], 0.041, eps=1e-2) assert approx_equal(solution["metric"], 0.027, eps=1e-2) assert solution["number"] == 22 assert solution["lattice"] == "cI" beam_centre = indexer.get_indexer_beam_centre() assert approx_equal(beam_centre, (94.4223, 94.5097), eps=1e-2) print(indexer.get_indexer_experiment_list()[0].crystal) print(indexer.get_indexer_experiment_list()[0].detector) # test serialization of indexer json_str = indexer.as_json() # print(json_str) indexer2 = DialsIndexer.from_json(string=json_str) indexer2.index() assert approx_equal(indexer.get_indexer_cell(), indexer2.get_indexer_cell()) assert approx_equal(indexer.get_indexer_beam_centre(), indexer2.get_indexer_beam_centre()) indexer.eliminate() indexer2.eliminate() assert approx_equal(indexer.get_indexer_cell(), indexer2.get_indexer_cell()) assert indexer.get_indexer_lattice() == "hR" assert indexer2.get_indexer_lattice() == "hR"
def exercise_xds_indexer(xia2_regression_build, tmp_dir, nproc=None): if nproc is not None: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc xia2_demo_data = os.path.join(xia2_regression_build, "test_data", "insulin") template = os.path.join(xia2_demo_data, "insulin_1_###.img") from xia2.Modules.Indexer.XDSIndexer import XDSIndexer indexer = XDSIndexer() indexer.set_working_directory(tmp_dir) from dxtbx.datablock import DataBlockTemplateImporter importer = DataBlockTemplateImporter([template]) datablocks = importer.datablocks imageset = datablocks[0].extract_imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, indexer.get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) indexer.index() assert approx_equal(indexer.get_indexer_cell(), (78.076, 78.076, 78.076, 90, 90, 90), eps=1), indexer.get_indexer_cell() experiment = indexer.get_indexer_experiment_list()[0] sgi = experiment.crystal.get_space_group().info() assert sgi.type().number() == 197 beam_centre = indexer.get_indexer_beam_centre() assert approx_equal(beam_centre, (94.4221, 94.5096), eps=1e-1) assert indexer.get_indexer_images() == [(1, 5), (20, 24), (41, 45)] print(indexer.get_indexer_experiment_list()[0].crystal) print(indexer.get_indexer_experiment_list()[0].detector) # test serialization of indexer json_str = indexer.as_json() print(json_str) indexer2 = XDSIndexer.from_json(string=json_str) indexer2.index() assert approx_equal(indexer.get_indexer_cell(), indexer2.get_indexer_cell()) assert approx_equal(indexer.get_indexer_beam_centre(), indexer2.get_indexer_beam_centre()) assert approx_equal(indexer.get_indexer_images(), indexer2.get_indexer_images()) indexer.eliminate() indexer2.eliminate() assert approx_equal(indexer.get_indexer_cell(), indexer2.get_indexer_cell()) assert indexer.get_indexer_lattice() == 'hR' assert indexer2.get_indexer_lattice() == 'hR'
def test_ccp4_scalerA(regression_test, ccp4, dials_data, run_in_tmpdir, nproc): if nproc is not None: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc template = dials_data("insulin").join("insulin_1_###.img").strpath tmpdir = run_in_tmpdir.strpath from xia2.Modules.Indexer.DialsIndexer import DialsIndexer from xia2.Modules.Refiner.DialsRefiner import DialsRefiner from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater from xia2.Modules.Scaler.CCP4ScalerA import CCP4ScalerA indexer = DialsIndexer() indexer.set_working_directory(tmpdir) from dxtbx.model.experiment_list import ExperimentListTemplateImporter importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) with mock.patch.object(sys, "argv", []): sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) refiner = DialsRefiner() refiner.set_working_directory(tmpdir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) integrater = DialsIntegrater() integrater.set_working_directory(tmpdir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) # integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.set_integrater_sweep_name("SWEEP1") integrater.set_integrater_project_info("CRYST1", "WAVE1", "SWEEP1") scaler = CCP4ScalerA() scaler.add_scaler_integrater(integrater) scaler.set_scaler_xcrystal(cryst) scaler.set_scaler_project_info("CRYST1", "WAVE1") check_scaler_files_exist(scaler) # test serialization of scaler json_str = scaler.as_json() # print json_str scaler2 = CCP4ScalerA.from_json(string=json_str) scaler2.set_scaler_xcrystal(cryst) check_scaler_files_exist(scaler2) scaler2.set_scaler_finish_done(False) check_scaler_files_exist(scaler2) scaler2.set_scaler_done(False) check_scaler_files_exist(scaler2) scaler2._scalr_integraters = {} # XXX scaler2.add_scaler_integrater(integrater) scaler2.set_scaler_prepare_done(False) check_scaler_files_exist(scaler2)
def exercise_serialization(dials_data, tmp_dir): base_path = pathlib.Path(tmp_dir) template = dials_data("insulin").join("insulin_1_###.img").strpath from xia2.Modules.Indexer.DialsIndexer import DialsIndexer from xia2.Modules.Refiner.DialsRefiner import DialsRefiner from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater from xia2.Modules.Scaler.CCP4ScalerA import CCP4ScalerA from dxtbx.model.experiment_list import ExperimentListTemplateImporter importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] from xia2.Schema.XProject import XProject from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample proj = XProject(base_path=base_path) proj._name = "PROJ1" cryst = XCrystal("CRYST1", proj) wav = XWavelength("WAVE1", cryst, wavelength=0.98) samp = XSample("X1", cryst) cryst.add_wavelength(wav) cryst.set_ha_info({"atom": "S"}) cryst.add_sample(samp) directory, image = os.path.split(imageset.get_path(1)) sweep = wav.add_sweep(name="SWEEP1", sample=samp, directory=directory, image=image) samp.add_sweep(sweep) from dxtbx.serialize.load import _decode_dict indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) indexer.add_indexer_imageset(imageset) indexer.set_indexer_sweep(sweep) sweep._indexer = indexer refiner = DialsRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) refiner.add_refiner_sweep(sweep) sweep._refiner = refiner integrater = DialsIntegrater() integrater.set_output_format("hkl") integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) # integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.set_integrater_epoch(sweep.get_epoch(1)) integrater.set_integrater_sweep_name(sweep.get_name()) integrater.set_integrater_project_info( cryst.get_name(), wav.get_name(), sweep.get_name() ) sweep._integrater = integrater scaler = CCP4ScalerA(base_path=base_path) scaler.add_scaler_integrater(integrater) scaler.set_scaler_xcrystal(cryst) scaler.set_scaler_project_info(cryst.get_name(), wav.get_name()) scaler._scalr_xcrystal = cryst cryst._scaler = scaler proj.add_crystal(cryst) s_dict = sweep.to_dict() s_str = json.dumps(s_dict, ensure_ascii=True) s_dict = json.loads(s_str, object_hook=_decode_dict) xsweep = XSweep.from_dict(s_dict) assert xsweep w_dict = wav.to_dict() w_str = json.dumps(w_dict, ensure_ascii=True) w_dict = json.loads(w_str, object_hook=_decode_dict) xwav = XWavelength.from_dict(w_dict) assert xwav.get_sweeps()[0].get_wavelength() is xwav c_dict = cryst.to_dict() c_str = json.dumps(c_dict, ensure_ascii=True) c_dict = json.loads(c_str, object_hook=_decode_dict) xcryst = XCrystal.from_dict(c_dict) assert ( xcryst.get_xwavelength(xcryst.get_wavelength_names()[0]).get_crystal() is xcryst ) p_dict = proj.to_dict() p_str = json.dumps(p_dict, ensure_ascii=True) p_dict = json.loads(p_str, object_hook=_decode_dict) xproj = XProject.from_dict(p_dict) assert xproj.path == base_path assert list(xproj.get_crystals().values())[0].get_project() is xproj assert list(xproj.get_crystals().values())[0]._scaler._base_path == base_path json_str = proj.as_json() xproj = XProject.from_json(string=json_str) assert xproj.path == base_path assert list(xproj.get_crystals().values())[0].get_project() is xproj print(xproj.get_output()) print("\n".join(xproj.summarise())) json_str = xproj.as_json() xproj = XProject.from_json(string=json_str) assert xproj.path == base_path # Test that we can serialize to json and back again xproj = XProject.from_json(string=xproj.as_json()) assert xproj.path == base_path xcryst = list(xproj.get_crystals().values())[0] assert xcryst.get_project() is xproj intgr = xcryst._get_integraters()[0] assert intgr.get_integrater_finish_done() assert ( xcryst._get_scaler() ._sweep_handler.get_sweep_information(intgr.get_integrater_epoch()) .get_integrater() is intgr ) print(xproj.get_output()) print("\n".join(xproj.summarise()))
def exercise_serialization(): if not have_dials_regression: print "Skipping exercise_serialization(): dials_regression not configured" return from xia2.Handlers.CommandLine import CommandLine xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data") template = os.path.join(xia2_demo_data, "insulin_1_###.img") cwd = os.path.abspath(os.curdir) tmp_dir = os.path.abspath(open_tmp_directory()) os.chdir(tmp_dir) from xia2.Modules.Indexer.DialsIndexer import DialsIndexer from xia2.Modules.Refiner.DialsRefiner import DialsRefiner from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater from xia2.Modules.Scaler.CCP4ScalerA import CCP4ScalerA from dxtbx.datablock import DataBlockTemplateImporter importer = DataBlockTemplateImporter([template]) datablocks = importer.datablocks imageset = datablocks[0].extract_imagesets()[0] from xia2.Schema.XProject import XProject from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample proj = XProject() proj._name = "PROJ1" cryst = XCrystal("CRYST1", proj) proj.add_crystal(cryst) wav = XWavelength("WAVE1", cryst, wavelength=0.98) samp = XSample("X1", cryst) cryst.add_wavelength(wav) cryst.set_ha_info({'atom': 'S'}) directory, image = os.path.split(imageset.get_path(1)) wav.add_sweep(name='SWEEP1', sample=samp, directory=directory, image=image) import json from dxtbx.serialize.load import _decode_dict sweep = wav.get_sweeps()[0] indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) indexer.add_indexer_imageset(imageset) indexer.set_indexer_sweep(sweep) sweep._indexer = indexer refiner = DialsRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) sweep._refiner = refiner integrater = DialsIntegrater() integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) #integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.set_integrater_epoch(sweep.get_epoch(1)) integrater.set_integrater_sweep_name(sweep.get_name()) integrater.set_integrater_project_info( cryst.get_name(), wav.get_name(), sweep.get_name()) sweep._integrater = integrater scaler = CCP4ScalerA() scaler.add_scaler_integrater(integrater) scaler.set_scaler_xcrystal(cryst) scaler.set_scaler_project_info(cryst.get_name(), wav.get_name()) scaler._scalr_xcrystal = cryst cryst._scaler = scaler s_dict = sweep.to_dict() s_str = json.dumps(s_dict, ensure_ascii=True) s_dict = json.loads(s_str, object_hook=_decode_dict) xsweep = XSweep.from_dict(s_dict) w_dict = wav.to_dict() w_str = json.dumps(w_dict, ensure_ascii=True) w_dict = json.loads(w_str, object_hook=_decode_dict) xwav = XWavelength.from_dict(w_dict) assert xwav.get_sweeps()[0].get_wavelength() is xwav c_dict = cryst.to_dict() c_str = json.dumps(c_dict, ensure_ascii=True) c_dict = json.loads(c_str, object_hook=_decode_dict) xcryst = XCrystal.from_dict(c_dict) assert xcryst.get_xwavelength(xcryst.get_wavelength_names()[0]).get_crystal() is xcryst p_dict = proj.to_dict() p_str = json.dumps(p_dict, ensure_ascii=True) p_dict = json.loads(p_str, object_hook=_decode_dict) xproj = XProject.from_dict(p_dict) assert xproj.get_crystals().values()[0].get_project() is xproj json_str = proj.as_json() xproj = XProject.from_json(string=json_str) assert xproj.get_crystals().values()[0].get_project() is xproj print xproj.get_output() print "\n".join(xproj.summarise()) json_str = xproj.as_json() xproj = XProject.from_json(string=json_str) assert xproj.get_crystals().values()[0].get_project() is xproj xcryst = xproj.get_crystals().values()[0] intgr = xcryst._get_integraters()[0] assert intgr.get_integrater_finish_done() assert xcryst._get_scaler()._sweep_handler.get_sweep_information( intgr.get_integrater_epoch()).get_integrater() is intgr print xproj.get_output() print "\n".join(xproj.summarise()) print
def exercise_mosflm_integrater(nproc=None): if not have_dials_regression: print "Skipping exercise_mosflm_integrater(): dials_regression not configured" return if nproc is not None: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data") template = os.path.join(xia2_demo_data, "insulin_1_###.img") cwd = os.path.abspath(os.curdir) tmp_dir = os.path.abspath(open_tmp_directory()) os.chdir(tmp_dir) # otherwise if this test is running multiple times simultaneously two mosflm # processes try to write to the same genfile os.environ['CCP4_SCR'] = tmp_dir from xia2.Modules.Indexer.MosflmIndexer import MosflmIndexer from xia2.Modules.Integrater.MosflmIntegrater import MosflmIntegrater from dxtbx.datablock import DataBlockTemplateImporter indexer = MosflmIndexer() indexer.set_working_directory(tmp_dir) importer = DataBlockTemplateImporter([template]) datablocks = importer.datablocks imageset = datablocks[0].extract_imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, indexer.get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) from xia2.Modules.Refiner.MosflmRefiner import MosflmRefiner refiner = MosflmRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) nref_error = 500 integrater = MosflmIntegrater() integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) #integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.integrate() integrater_intensities = integrater.get_integrater_intensities() assert os.path.exists(integrater_intensities) from iotbx.reflection_file_reader import any_reflection_file reader = any_reflection_file(integrater_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - 81116) < nref_error, \ mtz_object.n_reflections() assert mtz_object.column_labels() == [ 'H', 'K', 'L', 'M_ISYM', 'BATCH', 'I', 'SIGI', 'IPR', 'SIGIPR', 'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'WIDTH', 'LP', 'MPART', 'FLAG', 'BGPKRATIOS'] assert integrater.get_integrater_wedge() == (1, 45) assert approx_equal(integrater.get_integrater_cell(), (78.014, 78.014, 78.014, 90.0, 90.0, 90.0), eps=1e-2) # test serialization of integrater json_str = integrater.as_json() #print json_str integrater2 = MosflmIntegrater.from_json(string=json_str) integrater2.set_integrater_sweep(sweep, reset=False) integrater2_intensities = integrater.get_integrater_intensities() assert integrater2_intensities == integrater_intensities integrater2.set_integrater_finish_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - 81116) < nref_error, \ mtz_object.n_reflections() integrater2.set_integrater_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - 81116) < nref_error, \ mtz_object.n_reflections() integrater2.set_integrater_prepare_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - 81116) < nref_error, \ mtz_object.n_reflections()
def exercise_xds_indexer(dials_data, tmp_dir, nproc=None): if nproc is not None: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc template = dials_data("insulin").join("insulin_1_###.img").strpath from xia2.Modules.Indexer.XDSIndexerII import XDSIndexerII indexer = XDSIndexerII() indexer.set_working_directory(tmp_dir) from dxtbx.model.experiment_list import ExperimentListTemplateImporter importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) indexer.index() assert approx_equal(indexer.get_indexer_cell(), (78.054, 78.054, 78.054, 90, 90, 90), eps=1), indexer.get_indexer_cell() experiment = indexer.get_indexer_experiment_list()[0] sgi = experiment.crystal.get_space_group().info() assert sgi.type().number() == 197 beam_centre = indexer.get_indexer_beam_centre() assert approx_equal(beam_centre, (94.4239, 94.5110), eps=1e-1) assert indexer.get_indexer_images() == [(1, 45)] print(indexer.get_indexer_experiment_list()[0].crystal) print(indexer.get_indexer_experiment_list()[0].detector) # test serialization of indexer json_str = indexer.as_json() print(json_str) indexer2 = XDSIndexerII.from_json(string=json_str) indexer2.index() assert approx_equal(indexer.get_indexer_cell(), indexer2.get_indexer_cell()) assert approx_equal(indexer.get_indexer_beam_centre(), indexer2.get_indexer_beam_centre()) assert approx_equal(indexer.get_indexer_images(), indexer2.get_indexer_images()) indexer.eliminate() indexer2.eliminate() assert approx_equal(indexer.get_indexer_cell(), indexer2.get_indexer_cell()) assert indexer.get_indexer_lattice() == "hR" assert indexer2.get_indexer_lattice() == "hR"
def xia2_main(stop_after=None): '''Actually process something...''' Citations.cite('xia2') # print versions of related software Chatter.write(dials_version()) ccp4_version = get_ccp4_version() if ccp4_version is not None: Chatter.write('CCP4 %s' % ccp4_version) start_time = time.time() CommandLine = get_command_line() start_dir = Flags.get_starting_directory() # check that something useful has been assigned for processing... xtals = CommandLine.get_xinfo().get_crystals() no_images = True for name in xtals.keys(): xtal = xtals[name] if not xtal.get_all_image_names(): Chatter.write('-----------------------------------' + \ '-' * len(name)) Chatter.write('| No images assigned for crystal %s |' % name) Chatter.write('-----------------------------------' + '-' \ * len(name)) else: no_images = False args = [] from xia2.Handlers.Phil import PhilIndex params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing njob = mp_params.njob from libtbx import group_args xinfo = CommandLine.get_xinfo() if os.path.exists('xia2.json'): from xia2.Schema.XProject import XProject xinfo_new = xinfo xinfo = XProject.from_json(filename='xia2.json') crystals = xinfo.get_crystals() crystals_new = xinfo_new.get_crystals() for crystal_id in crystals_new.keys(): if crystal_id not in crystals: crystals[crystal_id] = crystals_new[crystal_id] continue crystals[crystal_id]._scaler = None # reset scaler for wavelength_id in crystals_new[crystal_id].get_wavelength_names( ): wavelength_new = crystals_new[crystal_id].get_xwavelength( wavelength_id) if wavelength_id not in crystals[ crystal_id].get_wavelength_names(): crystals[crystal_id].add_wavelength( crystals_new[crystal_id].get_xwavelength( wavelength_new)) continue wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) sweeps_new = wavelength_new.get_sweeps() sweeps = wavelength.get_sweeps() sweep_names = [s.get_name() for s in sweeps] sweep_keys = [(s.get_directory(), s.get_template(), s.get_image_range()) for s in sweeps] for sweep in sweeps_new: if ((sweep.get_directory(), sweep.get_template(), sweep.get_image_range()) not in sweep_keys): if sweep.get_name() in sweep_names: i = 1 while 'SWEEEP%i' % i in sweep_names: i += 1 sweep._name = 'SWEEP%i' % i break wavelength.add_sweep( name=sweep.get_name(), sample=sweep.get_xsample(), directory=sweep.get_directory(), image=sweep.get_image(), beam=sweep.get_beam_centre(), reversephi=sweep.get_reversephi(), distance=sweep.get_distance(), gain=sweep.get_gain(), dmin=sweep.get_resolution_high(), dmax=sweep.get_resolution_low(), polarization=sweep.get_polarization(), frames_to_process=sweep.get_frames_to_process(), user_lattice=sweep.get_user_lattice(), user_cell=sweep.get_user_cell(), epoch=sweep._epoch, ice=sweep._ice, excluded_regions=sweep._excluded_regions, ) sweep_names.append(sweep.get_name()) crystals = xinfo.get_crystals() failover = params.xia2.settings.failover if mp_params.mode == 'parallel' and njob > 1: driver_type = mp_params.type command_line_args = CommandLine.get_argv()[1:] for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) sweeps = wavelength.get_sweeps() for sweep in sweeps: sweep._get_indexer() sweep._get_refiner() sweep._get_integrater() args.append((group_args( driver_type=driver_type, stop_after=stop_after, failover=failover, command_line_args=command_line_args, nproc=mp_params.nproc, crystal_id=crystal_id, wavelength_id=wavelength_id, sweep_id=sweep.get_name(), ), )) from xia2.Driver.DriverFactory import DriverFactory default_driver_type = DriverFactory.get_driver_type() # run every nth job on the current computer (no need to submit to qsub) for i_job, arg in enumerate(args): if (i_job % njob) == 0: arg[0].driver_type = default_driver_type if mp_params.type == "qsub": method = "sge" else: method = "multiprocessing" nproc = mp_params.nproc qsub_command = mp_params.qsub_command if not qsub_command: qsub_command = 'qsub' qsub_command = '%s -V -cwd -pe smp %d' % (qsub_command, nproc) from libtbx import easy_mp results = easy_mp.parallel_map( process_one_sweep, args, processes=njob, #method=method, method="multiprocessing", qsub_command=qsub_command, preserve_order=True, preserve_exception_message=True) # Hack to update sweep with the serialized indexers/refiners/integraters i_sweep = 0 for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: success, output, xsweep_dict = results[i_sweep] if output is not None: Chatter.write(output) if not success: Chatter.write('Sweep failed: removing %s' % sweep.get_name()) remove_sweeps.append(sweep) else: assert xsweep_dict is not None Chatter.write('Loading sweep: %s' % sweep.get_name()) from xia2.Schema.XSweep import XSweep new_sweep = XSweep.from_dict(xsweep_dict) sweep._indexer = new_sweep._indexer sweep._refiner = new_sweep._refiner sweep._integrater = new_sweep._integrater i_sweep += 1 for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep) else: for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: from dials.command_line.show import show_datablocks from dxtbx.datablock import DataBlock Debug.write(sweep.get_name()) Debug.write( show_datablocks([DataBlock([sweep.get_imageset()])])) try: if stop_after == 'index': sweep.get_indexer_cell() else: sweep.get_integrater_intensities() sweep.serialize() except Exception as e: if failover: Chatter.write('Processing sweep %s failed: %s' % \ (sweep.get_name(), str(e))) remove_sweeps.append(sweep) else: raise for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep) # save intermediate xia2.json file in case scaling step fails xinfo.as_json(filename='xia2.json') if stop_after not in ('index', 'integrate'): Chatter.write(xinfo.get_output(), strip=False) for crystal in crystals.values(): crystal.serialize() # save final xia2.json file in case report generation fails xinfo.as_json(filename='xia2.json') duration = time.time() - start_time # write out the time taken in a human readable way Chatter.write('Processing took %s' % \ time.strftime("%Hh %Mm %Ss", time.gmtime(duration))) if stop_after not in ('index', 'integrate'): # and the summary file with open('xia2-summary.dat', 'w') as fh: for record in xinfo.summarise(): fh.write('%s\n' % record) # looks like this import overwrites the initial command line # Phil overrides so... for https://github.com/xia2/xia2/issues/150 from xia2.command_line.html import generate_xia2_html if params.xia2.settings.small_molecule == True: params.xia2.settings.report.xtriage_analysis = False params.xia2.settings.report.include_radiation_damage = False generate_xia2_html(xinfo, filename='xia2.html', params=params.xia2.settings.report) write_citations() # delete all of the temporary mtz files... cleanup() Environment.cleanup()
def exercise_xds_integrater(dials_data, tmp_dir, nproc=None): if nproc: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc template = dials_data("insulin").join("insulin_1_###.img").strpath from xia2.Modules.Indexer.XDSIndexer import XDSIndexer from xia2.Modules.Integrater.XDSIntegrater import XDSIntegrater indexer = XDSIndexer() indexer.set_working_directory(tmp_dir) from dxtbx.model.experiment_list import ExperimentListTemplateImporter importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, indexer.get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) from xia2.Modules.Refiner.XDSRefiner import XDSRefiner refiner = XDSRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) # refiner.refine() integrater = XDSIntegrater() integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) integrater.set_integrater_sweep(sweep) integrater.integrate() from iotbx.reflection_file_reader import any_reflection_file integrater_intensities = integrater.get_integrater_intensities() assert os.path.exists(integrater_intensities) reader = any_reflection_file(integrater_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert approx_equal(mtz_object.n_reflections(), 50000, eps=400) assert mtz_object.column_labels() == [ "H", "K", "L", "M_ISYM", "BATCH", "I", "SIGI", "FRACTIONCALC", "XDET", "YDET", "ROT", "LP", "FLAG", ] corrected_intensities = integrater.get_integrater_corrected_intensities() assert os.path.exists(corrected_intensities) reader = any_reflection_file(corrected_intensities) assert reader.file_type() == "xds_ascii" ma = reader.as_miller_arrays(merge_equivalents=False)[0] assert approx_equal(ma.size(), 50000, eps=400) assert integrater.get_integrater_wedge() == (1, 45) assert approx_equal( integrater.get_integrater_cell(), [78.066, 78.066, 78.066, 90, 90, 90], eps=1 ) assert approx_equal( integrater.get_integrater_mosaic_min_mean_max(), (0.180, 0.180, 0.180), eps=1e-1 ) # test serialization of integrater json_str = integrater.as_json() # print(json_str) integrater2 = XDSIntegrater.from_json(string=json_str) integrater2.set_integrater_sweep(sweep, reset=False) integrater2_intensities = integrater.get_integrater_intensities() assert integrater2_intensities == integrater_intensities integrater2.set_integrater_finish_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert approx_equal(mtz_object.n_reflections(), 50000, eps=400) integrater2.set_integrater_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert approx_equal(mtz_object.n_reflections(), 50000, eps=450) integrater2.set_integrater_prepare_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert approx_equal(mtz_object.n_reflections(), 50100, eps=400)
def exercise_serialization(dials_regression, tmp_dir): xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data") template = os.path.join(xia2_demo_data, "insulin_1_###.img") os.chdir(tmp_dir) from xia2.Modules.Indexer.DialsIndexer import DialsIndexer from xia2.Modules.Refiner.DialsRefiner import DialsRefiner from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater from xia2.Modules.Scaler.CCP4ScalerA import CCP4ScalerA from dxtbx.datablock import DataBlockTemplateImporter importer = DataBlockTemplateImporter([template]) datablocks = importer.datablocks imageset = datablocks[0].extract_imagesets()[0] from xia2.Schema.XProject import XProject from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample proj = XProject() proj._name = "PROJ1" cryst = XCrystal("CRYST1", proj) proj.add_crystal(cryst) wav = XWavelength("WAVE1", cryst, wavelength=0.98) samp = XSample("X1", cryst) cryst.add_wavelength(wav) cryst.set_ha_info({'atom': 'S'}) directory, image = os.path.split(imageset.get_path(1)) wav.add_sweep(name='SWEEP1', sample=samp, directory=directory, image=image) import json from dxtbx.serialize.load import _decode_dict sweep = wav.get_sweeps()[0] indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) indexer.add_indexer_imageset(imageset) indexer.set_indexer_sweep(sweep) sweep._indexer = indexer refiner = DialsRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) sweep._refiner = refiner integrater = DialsIntegrater() integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) #integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.set_integrater_epoch(sweep.get_epoch(1)) integrater.set_integrater_sweep_name(sweep.get_name()) integrater.set_integrater_project_info(cryst.get_name(), wav.get_name(), sweep.get_name()) sweep._integrater = integrater scaler = CCP4ScalerA() scaler.add_scaler_integrater(integrater) scaler.set_scaler_xcrystal(cryst) scaler.set_scaler_project_info(cryst.get_name(), wav.get_name()) scaler._scalr_xcrystal = cryst cryst._scaler = scaler s_dict = sweep.to_dict() s_str = json.dumps(s_dict, ensure_ascii=True) s_dict = json.loads(s_str, object_hook=_decode_dict) xsweep = XSweep.from_dict(s_dict) w_dict = wav.to_dict() w_str = json.dumps(w_dict, ensure_ascii=True) w_dict = json.loads(w_str, object_hook=_decode_dict) xwav = XWavelength.from_dict(w_dict) assert xwav.get_sweeps()[0].get_wavelength() is xwav c_dict = cryst.to_dict() c_str = json.dumps(c_dict, ensure_ascii=True) c_dict = json.loads(c_str, object_hook=_decode_dict) xcryst = XCrystal.from_dict(c_dict) assert xcryst.get_xwavelength( xcryst.get_wavelength_names()[0]).get_crystal() is xcryst p_dict = proj.to_dict() p_str = json.dumps(p_dict, ensure_ascii=True) p_dict = json.loads(p_str, object_hook=_decode_dict) xproj = XProject.from_dict(p_dict) assert xproj.get_crystals().values()[0].get_project() is xproj json_str = proj.as_json() xproj = XProject.from_json(string=json_str) assert xproj.get_crystals().values()[0].get_project() is xproj print(xproj.get_output()) print("\n".join(xproj.summarise())) json_str = xproj.as_json() xproj = XProject.from_json(string=json_str) assert xproj.get_crystals().values()[0].get_project() is xproj xcryst = xproj.get_crystals().values()[0] intgr = xcryst._get_integraters()[0] assert intgr.get_integrater_finish_done() assert xcryst._get_scaler()._sweep_handler.get_sweep_information( intgr.get_integrater_epoch()).get_integrater() is intgr print(xproj.get_output()) print("\n".join(xproj.summarise()))
def exercise_dials_integrater(dials_regression, tmp_dir, nproc=None): if nproc: from xia2.Handlers.Phil import PhilIndex PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data") template = os.path.join(xia2_demo_data, "insulin_1_###.img") from xia2.Modules.Indexer.DialsIndexer import DialsIndexer from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater from dxtbx.datablock import DataBlockTemplateImporter indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) importer = DataBlockTemplateImporter([template]) datablocks = importer.datablocks imageset = datablocks[0].extract_imagesets()[0] indexer.add_indexer_imageset(imageset) from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample cryst = XCrystal("CRYST1", None) wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength()) samp = XSample("X1", cryst) directory, image = os.path.split(imageset.get_path(1)) sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image) indexer.set_indexer_sweep(sweep) from xia2.Modules.Refiner.DialsRefiner import DialsRefiner refiner = DialsRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) #refiner.refine() integrater = DialsIntegrater() integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) #integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.integrate() integrater_intensities = integrater.get_integrater_intensities() assert os.path.exists(integrater_intensities) from iotbx.reflection_file_reader import any_reflection_file reader = any_reflection_file(integrater_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() expected_reflections = 48456 assert abs(mtz_object.n_reflections() - expected_reflections) < 300, mtz_object.n_reflections() assert mtz_object.column_labels() == [ 'H', 'K', 'L', 'M_ISYM', 'BATCH', 'IPR', 'SIGIPR', 'I', 'SIGI', 'BG', 'SIGBG', 'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'LP', 'DQE'] assert integrater.get_integrater_wedge() == (1, 45) assert approx_equal(integrater.get_integrater_cell(), (78.14, 78.14, 78.14, 90, 90, 90), eps=1e-1) # test serialization of integrater json_str = integrater.as_json() #print(json_str) integrater2 = DialsIntegrater.from_json(string=json_str) integrater2.set_integrater_sweep(sweep, reset=False) integrater2_intensities = integrater.get_integrater_intensities() assert integrater2_intensities == integrater_intensities integrater2.set_integrater_finish_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - expected_reflections) < 300, mtz_object.n_reflections() integrater2.set_integrater_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - expected_reflections) < 300, mtz_object.n_reflections() integrater2.set_integrater_prepare_done(False) integrater2_intensities = integrater2.get_integrater_intensities() assert os.path.exists(integrater2_intensities) reader = any_reflection_file(integrater2_intensities) assert reader.file_type() == "ccp4_mtz" mtz_object = reader.file_content() assert abs(mtz_object.n_reflections() - expected_reflections) < 300, mtz_object.n_reflections()