Ejemplo n.º 1
0
def _extract_or_read_imagesets(params):
    """
    Return a list of ImageSets, importing them via alternative means if necessary.

    The "Alternative Means" means via params.input.template or .directory,
    if the images to import haven't been specified directly.

    Args:
        params: The phil.scope_extract from dials.import

    Returns: A list of ImageSet objects
    """

    # Get the experiments
    experiments = flatten_experiments(params.input.experiments)

    # Check we have some filenames
    if len(experiments) == 0:

        # FIXME Should probably make this smarter since it requires editing here
        # and in dials.import phil scope
        try:
            format_kwargs = {
                "dynamic_shadowing": params.format.dynamic_shadowing,
                "multi_panel": params.format.multi_panel,
            }
        except AttributeError:
            format_kwargs = None

        # Check if a template has been set and print help if not, otherwise try to
        # import the images based on the template input
        if len(params.input.template) > 0:
            importer = ExperimentListTemplateImporter(
                params.input.template,
                image_range=params.geometry.scan.image_range,
                format_kwargs=format_kwargs,
            )
            experiments = importer.experiments
            if len(experiments) == 0:
                raise Sorry("No experiments found matching template %s" %
                            params.input.experiments)
        elif len(params.input.directory) > 0:
            experiments = ExperimentListFactory.from_filenames(
                params.input.directory, format_kwargs=format_kwargs)
            if len(experiments) == 0:
                raise Sorry("No experiments found in directories %s" %
                            params.input.directory)
        else:
            raise Sorry("No experiments found")

    # TODO (Nick):  This looks redundant as the experiments are immediately discarded.
    #               verify this, and remove if it is.
    if params.identifier_type:
        generate_experiment_identifiers(experiments, params.identifier_type)

    # Get a list of all imagesets
    imageset_list = experiments.imagesets()

    # Return the experiments
    return imageset_list
Ejemplo n.º 2
0
def test_labelit_indexer_II(regression_test, ccp4, dials_data, run_in_tmpdir):
    template = dials_data("insulin").join("insulin_1_###.img").strpath

    from xia2.Modules.Indexer.LabelitIndexerII import LabelitIndexerII
    from xia2.DriverExceptions.NotAvailableError import NotAvailableError

    try:
        ls = LabelitIndexerII(indxr_print=True)
    except NotAvailableError:
        pytest.skip("labelit not found")
    ls.set_working_directory(run_in_tmpdir.strpath)
    from dxtbx.model.experiment_list import ExperimentListTemplateImporter

    importer = ExperimentListTemplateImporter([template])
    experiments = importer.experiments
    imageset = experiments.imagesets()[0]
    ls.add_indexer_imageset(imageset)
    ls.set_indexer_input_cell((78, 78, 78, 90, 90, 90))
    ls.set_indexer_user_input_lattice(True)
    ls.set_indexer_input_lattice("cI")
    ls.index()

    assert ls.get_indexer_cell() == pytest.approx(
        (78.52, 78.52, 78.52, 90, 90, 90), abs=1e-1)
    solution = ls.get_solution()
    assert solution["rmsd"] == pytest.approx(0.079, abs=1e-1)
    assert solution["metric"] <= 0.1762
    assert solution["number"] == 22
    assert solution["lattice"] == "cI"
    assert solution["mosaic"] <= 0.2
    assert solution["nspots"] == pytest.approx(5509, abs=50)

    beam_centre = ls.get_indexer_beam_centre()
    assert beam_centre == pytest.approx((94.3286, 94.4662), abs=5e-1)
    assert ls.get_indexer_images() == [
        (1, 1),
        (3, 3),
        (5, 5),
        (7, 7),
        (9, 9),
        (11, 11),
        (13, 13),
        (15, 15),
        (17, 17),
        (19, 19),
        (21, 21),
        (23, 23),
        (25, 25),
        (27, 27),
        (29, 29),
        (31, 31),
        (33, 33),
        (35, 35),
        (37, 37),
        (39, 39),
    ]
    print(ls.get_indexer_experiment_list()[0].crystal)
    print(ls.get_indexer_experiment_list()[0].detector)
Ejemplo n.º 3
0
def exercise_xds_indexer(dials_data, tmp_dir, nproc=None):
    if nproc is not None:
        PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc

    template = dials_data("insulin").join("insulin_1_###.img").strpath

    indexer = XDSIndexer()
    indexer.set_working_directory(tmp_dir)

    importer = ExperimentListTemplateImporter([template])
    experiments = importer.experiments
    imageset = experiments.imagesets()[0]
    indexer.add_indexer_imageset(imageset)

    cryst = XCrystal("CRYST1", None)
    wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
    samp = XSample("X1", cryst)
    directory, image = os.path.split(imageset.get_path(1))
    sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
    indexer.set_indexer_sweep(sweep)

    indexer.index()

    assert indexer.get_indexer_cell() == pytest.approx(
        (78.076, 78.076, 78.076, 90, 90, 90),
        abs=1), indexer.get_indexer_cell()
    experiment = indexer.get_indexer_experiment_list()[0]
    sgi = experiment.crystal.get_space_group().info()
    assert sgi.type().number() == 197

    beam_centre = indexer.get_indexer_beam_centre()
    assert beam_centre == pytest.approx((94.4221, 94.5096), abs=1e-1)
    assert indexer.get_indexer_images() == [(1, 5), (20, 24), (41, 45)]
    print(indexer.get_indexer_experiment_list()[0].crystal)
    print(indexer.get_indexer_experiment_list()[0].detector)

    # test serialization of indexer
    json_str = indexer.as_json()
    print(json_str)
    indexer2 = XDSIndexer.from_json(string=json_str)
    indexer2.index()

    assert indexer.get_indexer_cell() == pytest.approx(
        indexer2.get_indexer_cell())
    assert indexer.get_indexer_beam_centre() == pytest.approx(
        indexer2.get_indexer_beam_centre())
    assert indexer.get_indexer_images() == [
        tuple(i) for i in indexer2.get_indexer_images()
    ]

    indexer.eliminate()
    indexer2.eliminate()

    assert indexer.get_indexer_cell() == pytest.approx(
        indexer2.get_indexer_cell())
    assert indexer.get_indexer_lattice() == "hR"
    assert indexer2.get_indexer_lattice() == "hR"
Ejemplo n.º 4
0
    def __call__(self):
        """
        Import the experiments
        """

        # Get the experiments
        experiments = flatten_experiments(self.params.input.experiments)

        # Check we have some filenames
        if len(experiments) == 0:

            # FIXME Should probably make this smarter since it requires editing here
            # and in dials.import phil scope
            try:
                format_kwargs = {
                    "dynamic_shadowing": self.params.format.dynamic_shadowing,
                    "multi_panel": self.params.format.multi_panel,
                }
            except AttributeError:
                format_kwargs = None

            # Check if a template has been set and print help if not, otherwise try to
            # import the images based on the template input
            if len(self.params.input.template) > 0:
                importer = ExperimentListTemplateImporter(
                    self.params.input.template,
                    image_range=self.params.geometry.scan.image_range,
                    format_kwargs=format_kwargs,
                )
                experiments = importer.experiments
                if len(experiments) == 0:
                    raise Sorry(
                        "No experiments found matching template %s"
                        % self.params.input.experiments
                    )
            elif len(self.params.input.directory) > 0:
                experiments = ExperimentListFactory.from_filenames(
                    self.params.input.directory, format_kwargs=format_kwargs
                )
                if len(experiments) == 0:
                    raise Sorry(
                        "No experiments found in directories %s"
                        % self.params.input.directory
                    )
            else:
                raise Sorry("No experiments found")

        if self.params.identifier_type:
            generate_experiment_identifiers(experiments, self.params.identifier_type)

        # Get a list of all imagesets
        imageset_list = experiments.imagesets()

        # Return the experiments
        return imageset_list
Ejemplo n.º 5
0
def exercise_dials_indexer(dials_data, tmp_dir, nproc=None):
    if nproc is not None:
        PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc

    template = dials_data("insulin").join("insulin_1_###.img").strpath

    indexer = DialsIndexer()
    indexer.set_working_directory(tmp_dir)

    importer = ExperimentListTemplateImporter([template])
    experiments = importer.experiments
    imageset = experiments.imagesets()[0]
    indexer.add_indexer_imageset(imageset)

    cryst = XCrystal("CRYST1", None)
    wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
    samp = XSample("X1", cryst)
    directory, image = os.path.split(imageset.get_path(1))
    sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
    indexer.set_indexer_sweep(sweep)

    indexer.index()

    assert indexer.get_indexer_cell() == pytest.approx(
        (78.14, 78.14, 78.14, 90, 90, 90), rel=1e-3)
    solution = indexer.get_solution()
    assert solution["rmsd"] == pytest.approx(0.03545, abs=1e-3)
    assert solution["metric"] == pytest.approx(0.02517, abs=5e-3)
    assert solution["number"] == 22
    assert solution["lattice"] == "cI"

    beam_centre = indexer.get_indexer_beam_centre()
    assert beam_centre == pytest.approx((94.41567208118963, 94.51337522659865),
                                        abs=1e-3)
    print(indexer.get_indexer_experiment_list()[0].crystal)
    print(indexer.get_indexer_experiment_list()[0].detector)

    # test serialization of indexer
    json_str = indexer.as_json()
    indexer2 = DialsIndexer.from_json(string=json_str)
    indexer2.index()

    assert indexer.get_indexer_cell() == pytest.approx(
        indexer2.get_indexer_cell())
    assert indexer.get_indexer_beam_centre() == pytest.approx(
        indexer2.get_indexer_beam_centre())

    indexer.eliminate()
    indexer2.eliminate()

    assert indexer.get_indexer_cell() == pytest.approx(
        indexer2.get_indexer_cell())
    assert indexer.get_indexer_lattice() == "hR"
    assert indexer2.get_indexer_lattice() == "hR"
Ejemplo n.º 6
0
def test_mosflm_indexer(regression_test, ccp4, dials_data, run_in_tmpdir):
    template = dials_data("insulin").join("insulin_1_###.img").strpath

    from xia2.Modules.Indexer.MosflmIndexer import MosflmIndexer

    indexer = MosflmIndexer()
    indexer.set_working_directory(run_in_tmpdir.strpath)
    from dxtbx.model.experiment_list import ExperimentListTemplateImporter

    importer = ExperimentListTemplateImporter([template])
    experiments = importer.experiments
    imageset = experiments.imagesets()[0]
    indexer.add_indexer_imageset(imageset)

    with mock.patch.object(
            sys, "argv",
        []):  # otherwise indexing fails when running pytest with '--runslow'
        indexer.index()

    assert indexer.get_indexer_cell() == pytest.approx(
        (78.6657, 78.6657, 78.6657, 90.0, 90.0, 90.0), abs=1e-3)
    experiment = indexer.get_indexer_experiment_list()[0]
    sgi = experiment.crystal.get_space_group().info()
    assert sgi.type().number() == 197

    beam_centre = indexer.get_indexer_beam_centre()
    assert beam_centre == pytest.approx((94.34, 94.57), abs=1e-2)
    assert indexer.get_indexer_images() == [(1, 1), (22, 22), (45, 45)]
    print(indexer.get_indexer_experiment_list()[0].crystal)
    print(indexer.get_indexer_experiment_list()[0].detector)

    # test serialization of indexer
    json_str = indexer.as_json()
    print(json_str)
    indexer2 = MosflmIndexer.from_json(string=json_str)
    indexer2.index()

    assert indexer.get_indexer_cell() == pytest.approx(
        indexer2.get_indexer_cell(), abs=1e-6)
    assert indexer.get_indexer_beam_centre() == pytest.approx(
        indexer2.get_indexer_beam_centre(), abs=1e-6)
    assert indexer2.get_indexer_images() == [
        [1, 1],
        [22, 22],
        [45, 45],
    ]  # coming from json these are now lists

    indexer.eliminate()
    indexer2.eliminate()

    assert indexer.get_indexer_cell() == pytest.approx(
        indexer2.get_indexer_cell(), abs=1e-6)
    assert indexer.get_indexer_lattice() == "hR"
    assert indexer2.get_indexer_lattice() == "hR"
Ejemplo n.º 7
0
def exercise_serialization(dials_data, tmp_dir):
    base_path = pathlib.Path(tmp_dir)
    template = dials_data("insulin").join("insulin_1_###.img").strpath

    from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
    from xia2.Modules.Refiner.DialsRefiner import DialsRefiner
    from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater
    from xia2.Modules.Scaler.CCP4ScalerA import CCP4ScalerA

    from dxtbx.model.experiment_list import ExperimentListTemplateImporter

    importer = ExperimentListTemplateImporter([template])
    experiments = importer.experiments
    imageset = experiments.imagesets()[0]

    from xia2.Schema.XProject import XProject
    from xia2.Schema.XCrystal import XCrystal
    from xia2.Schema.XWavelength import XWavelength
    from xia2.Schema.XSweep import XSweep
    from xia2.Schema.XSample import XSample

    proj = XProject(base_path=base_path)
    proj._name = "PROJ1"
    cryst = XCrystal("CRYST1", proj)
    wav = XWavelength("WAVE1", cryst, wavelength=0.98)
    samp = XSample("X1", cryst)
    cryst.add_wavelength(wav)
    cryst.set_ha_info({"atom": "S"})
    cryst.add_sample(samp)
    directory, image = os.path.split(imageset.get_path(1))
    sweep = wav.add_sweep(name="SWEEP1", sample=samp, directory=directory, image=image)
    samp.add_sweep(sweep)

    from dxtbx.serialize.load import _decode_dict

    indexer = DialsIndexer()
    indexer.set_working_directory(tmp_dir)
    indexer.add_indexer_imageset(imageset)
    indexer.set_indexer_sweep(sweep)
    sweep._indexer = indexer

    refiner = DialsRefiner()
    refiner.set_working_directory(tmp_dir)
    refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
    refiner.add_refiner_sweep(sweep)
    sweep._refiner = refiner

    integrater = DialsIntegrater()
    integrater.set_output_format("hkl")
    integrater.set_working_directory(tmp_dir)
    integrater.setup_from_image(imageset.get_path(1))
    integrater.set_integrater_refiner(refiner)
    # integrater.set_integrater_indexer(indexer)
    integrater.set_integrater_sweep(sweep)
    integrater.set_integrater_epoch(sweep.get_epoch(1))
    integrater.set_integrater_sweep_name(sweep.get_name())
    integrater.set_integrater_project_info(
        cryst.get_name(), wav.get_name(), sweep.get_name()
    )
    sweep._integrater = integrater

    scaler = CCP4ScalerA(base_path=base_path)
    scaler.add_scaler_integrater(integrater)
    scaler.set_scaler_xcrystal(cryst)
    scaler.set_scaler_project_info(cryst.get_name(), wav.get_name())
    scaler._scalr_xcrystal = cryst
    cryst._scaler = scaler

    proj.add_crystal(cryst)

    s_dict = sweep.to_dict()
    s_str = json.dumps(s_dict, ensure_ascii=True)
    s_dict = json.loads(s_str, object_hook=_decode_dict)
    xsweep = XSweep.from_dict(s_dict)
    assert xsweep

    w_dict = wav.to_dict()
    w_str = json.dumps(w_dict, ensure_ascii=True)
    w_dict = json.loads(w_str, object_hook=_decode_dict)
    xwav = XWavelength.from_dict(w_dict)
    assert xwav.get_sweeps()[0].get_wavelength() is xwav

    c_dict = cryst.to_dict()
    c_str = json.dumps(c_dict, ensure_ascii=True)
    c_dict = json.loads(c_str, object_hook=_decode_dict)
    xcryst = XCrystal.from_dict(c_dict)
    assert (
        xcryst.get_xwavelength(xcryst.get_wavelength_names()[0]).get_crystal() is xcryst
    )

    p_dict = proj.to_dict()
    p_str = json.dumps(p_dict, ensure_ascii=True)
    p_dict = json.loads(p_str, object_hook=_decode_dict)
    xproj = XProject.from_dict(p_dict)
    assert xproj.path == base_path
    assert list(xproj.get_crystals().values())[0].get_project() is xproj
    assert list(xproj.get_crystals().values())[0]._scaler._base_path == base_path

    json_str = proj.as_json()
    xproj = XProject.from_json(string=json_str)
    assert xproj.path == base_path
    assert list(xproj.get_crystals().values())[0].get_project() is xproj
    print(xproj.get_output())
    print("\n".join(xproj.summarise()))
    json_str = xproj.as_json()
    xproj = XProject.from_json(string=json_str)
    assert xproj.path == base_path
    # Test that we can serialize to json and back again
    xproj = XProject.from_json(string=xproj.as_json())
    assert xproj.path == base_path
    xcryst = list(xproj.get_crystals().values())[0]
    assert xcryst.get_project() is xproj
    intgr = xcryst._get_integraters()[0]
    assert intgr.get_integrater_finish_done()
    assert (
        xcryst._get_scaler()
        ._sweep_handler.get_sweep_information(intgr.get_integrater_epoch())
        .get_integrater()
        is intgr
    )

    print(xproj.get_output())
    print("\n".join(xproj.summarise()))
def exercise_xds_integrater(dials_data, tmp_dir, nproc=None):
    if nproc:
        from xia2.Handlers.Phil import PhilIndex

        PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc

    template = dials_data("insulin").join("insulin_1_###.img").strpath

    from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
    from xia2.Modules.Integrater.XDSIntegrater import XDSIntegrater

    indexer = XDSIndexer()
    indexer.set_working_directory(tmp_dir)
    from dxtbx.model.experiment_list import ExperimentListTemplateImporter

    importer = ExperimentListTemplateImporter([template])
    experiments = importer.experiments
    imageset = experiments.imagesets()[0]
    indexer.add_indexer_imageset(imageset)

    from xia2.Schema.XCrystal import XCrystal
    from xia2.Schema.XWavelength import XWavelength
    from xia2.Schema.XSweep import XSweep
    from xia2.Schema.XSample import XSample

    cryst = XCrystal("CRYST1", None)
    wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
    samp = XSample("X1", cryst)
    directory, image = os.path.split(imageset.get_path(1))
    sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
    indexer.set_indexer_sweep(sweep)

    from xia2.Modules.Refiner.XDSRefiner import XDSRefiner

    refiner = XDSRefiner()
    refiner.set_working_directory(tmp_dir)
    refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
    # refiner.refine()

    integrater = XDSIntegrater()
    integrater.set_working_directory(tmp_dir)
    integrater.setup_from_image(imageset.get_path(1))
    integrater.set_integrater_refiner(refiner)
    integrater.set_integrater_sweep(sweep)
    integrater.integrate()

    from iotbx.reflection_file_reader import any_reflection_file

    integrater_intensities = integrater.get_integrater_intensities()
    assert os.path.exists(integrater_intensities)
    reader = any_reflection_file(integrater_intensities)
    assert reader.file_type() == "ccp4_mtz"
    mtz_object = reader.file_content()
    assert approx_equal(mtz_object.n_reflections(), 50000, eps=400)
    assert mtz_object.column_labels() == [
        "H",
        "K",
        "L",
        "M_ISYM",
        "BATCH",
        "I",
        "SIGI",
        "FRACTIONCALC",
        "XDET",
        "YDET",
        "ROT",
        "LP",
        "FLAG",
    ]

    corrected_intensities = integrater.get_integrater_corrected_intensities()
    assert os.path.exists(corrected_intensities)
    reader = any_reflection_file(corrected_intensities)
    assert reader.file_type() == "xds_ascii"
    ma = reader.as_miller_arrays(merge_equivalents=False)[0]
    assert approx_equal(ma.size(), 50000, eps=400)

    assert integrater.get_integrater_wedge() == (1, 45)
    assert approx_equal(
        integrater.get_integrater_cell(), [78.066, 78.066, 78.066, 90, 90, 90], eps=1
    )
    assert approx_equal(
        integrater.get_integrater_mosaic_min_mean_max(), (0.180, 0.180, 0.180), eps=1e-1
    )

    # test serialization of integrater
    json_str = integrater.as_json()
    # print(json_str)
    integrater2 = XDSIntegrater.from_json(string=json_str)
    integrater2.set_integrater_sweep(sweep, reset=False)
    integrater2_intensities = integrater.get_integrater_intensities()
    assert integrater2_intensities == integrater_intensities

    integrater2.set_integrater_finish_done(False)
    integrater2_intensities = integrater2.get_integrater_intensities()
    assert os.path.exists(integrater2_intensities)
    reader = any_reflection_file(integrater2_intensities)
    assert reader.file_type() == "ccp4_mtz"
    mtz_object = reader.file_content()
    assert approx_equal(mtz_object.n_reflections(), 50000, eps=400)

    integrater2.set_integrater_done(False)
    integrater2_intensities = integrater2.get_integrater_intensities()
    assert os.path.exists(integrater2_intensities)
    reader = any_reflection_file(integrater2_intensities)
    assert reader.file_type() == "ccp4_mtz"
    mtz_object = reader.file_content()
    assert approx_equal(mtz_object.n_reflections(), 50000, eps=450)

    integrater2.set_integrater_prepare_done(False)
    integrater2_intensities = integrater2.get_integrater_intensities()
    assert os.path.exists(integrater2_intensities)
    reader = any_reflection_file(integrater2_intensities)
    assert reader.file_type() == "ccp4_mtz"
    mtz_object = reader.file_content()
    assert approx_equal(mtz_object.n_reflections(), 50100, eps=400)
def exercise_xds_indexer(dials_data, tmp_dir, nproc=None):
    if nproc is not None:
        from xia2.Handlers.Phil import PhilIndex

        PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc

    template = dials_data("insulin").join("insulin_1_###.img").strpath

    from xia2.Modules.Indexer.XDSIndexerII import XDSIndexerII

    indexer = XDSIndexerII()
    indexer.set_working_directory(tmp_dir)
    from dxtbx.model.experiment_list import ExperimentListTemplateImporter

    importer = ExperimentListTemplateImporter([template])
    experiments = importer.experiments
    imageset = experiments.imagesets()[0]
    indexer.add_indexer_imageset(imageset)

    from xia2.Schema.XCrystal import XCrystal
    from xia2.Schema.XWavelength import XWavelength
    from xia2.Schema.XSweep import XSweep
    from xia2.Schema.XSample import XSample

    cryst = XCrystal("CRYST1", None)
    wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
    samp = XSample("X1", cryst)
    directory, image = os.path.split(imageset.get_path(1))
    sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
    indexer.set_indexer_sweep(sweep)

    indexer.index()

    assert approx_equal(indexer.get_indexer_cell(),
                        (78.054, 78.054, 78.054, 90, 90, 90),
                        eps=1), indexer.get_indexer_cell()
    experiment = indexer.get_indexer_experiment_list()[0]
    sgi = experiment.crystal.get_space_group().info()
    assert sgi.type().number() == 197

    beam_centre = indexer.get_indexer_beam_centre()
    assert approx_equal(beam_centre, (94.4239, 94.5110), eps=1e-1)
    assert indexer.get_indexer_images() == [(1, 45)]
    print(indexer.get_indexer_experiment_list()[0].crystal)
    print(indexer.get_indexer_experiment_list()[0].detector)

    # test serialization of indexer
    json_str = indexer.as_json()
    print(json_str)
    indexer2 = XDSIndexerII.from_json(string=json_str)
    indexer2.index()

    assert approx_equal(indexer.get_indexer_cell(),
                        indexer2.get_indexer_cell())
    assert approx_equal(indexer.get_indexer_beam_centre(),
                        indexer2.get_indexer_beam_centre())
    assert approx_equal(indexer.get_indexer_images(),
                        indexer2.get_indexer_images())

    indexer.eliminate()
    indexer2.eliminate()

    assert approx_equal(indexer.get_indexer_cell(),
                        indexer2.get_indexer_cell())
    assert indexer.get_indexer_lattice() == "hR"
    assert indexer2.get_indexer_lattice() == "hR"
Ejemplo n.º 10
0
def test_ccp4_scalerA(regression_test, ccp4, dials_data, run_in_tmpdir, nproc):
    if nproc is not None:
        from xia2.Handlers.Phil import PhilIndex

        PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc

    template = dials_data("insulin").join("insulin_1_###.img").strpath

    tmpdir = run_in_tmpdir.strpath

    from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
    from xia2.Modules.Refiner.DialsRefiner import DialsRefiner
    from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater
    from xia2.Modules.Scaler.CCP4ScalerA import CCP4ScalerA

    indexer = DialsIndexer()
    indexer.set_working_directory(tmpdir)
    from dxtbx.model.experiment_list import ExperimentListTemplateImporter

    importer = ExperimentListTemplateImporter([template])
    experiments = importer.experiments
    imageset = experiments.imagesets()[0]
    indexer.add_indexer_imageset(imageset)

    from xia2.Schema.XCrystal import XCrystal
    from xia2.Schema.XWavelength import XWavelength
    from xia2.Schema.XSweep import XSweep
    from xia2.Schema.XSample import XSample

    cryst = XCrystal("CRYST1", None)
    wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
    samp = XSample("X1", cryst)
    directory, image = os.path.split(imageset.get_path(1))
    with mock.patch.object(sys, "argv", []):
        sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
    indexer.set_indexer_sweep(sweep)

    refiner = DialsRefiner()
    refiner.set_working_directory(tmpdir)
    refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)

    integrater = DialsIntegrater()
    integrater.set_working_directory(tmpdir)
    integrater.setup_from_image(imageset.get_path(1))
    integrater.set_integrater_refiner(refiner)
    # integrater.set_integrater_indexer(indexer)
    integrater.set_integrater_sweep(sweep)
    integrater.set_integrater_sweep_name("SWEEP1")
    integrater.set_integrater_project_info("CRYST1", "WAVE1", "SWEEP1")

    scaler = CCP4ScalerA()
    scaler.add_scaler_integrater(integrater)
    scaler.set_scaler_xcrystal(cryst)
    scaler.set_scaler_project_info("CRYST1", "WAVE1")

    check_scaler_files_exist(scaler)

    # test serialization of scaler
    json_str = scaler.as_json()
    # print json_str
    scaler2 = CCP4ScalerA.from_json(string=json_str)
    scaler2.set_scaler_xcrystal(cryst)

    check_scaler_files_exist(scaler2)

    scaler2.set_scaler_finish_done(False)
    check_scaler_files_exist(scaler2)

    scaler2.set_scaler_done(False)
    check_scaler_files_exist(scaler2)

    scaler2._scalr_integraters = {}  # XXX
    scaler2.add_scaler_integrater(integrater)
    scaler2.set_scaler_prepare_done(False)
    check_scaler_files_exist(scaler2)
def exercise_dials_indexer(dials_data, tmp_dir, nproc=None):
    if nproc is not None:
        from xia2.Handlers.Phil import PhilIndex

        PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc

    template = dials_data("insulin").join("insulin_1_###.img").strpath

    from xia2.Modules.Indexer.DialsIndexer import DialsIndexer

    indexer = DialsIndexer()
    indexer.set_working_directory(tmp_dir)
    from dxtbx.model.experiment_list import ExperimentListTemplateImporter

    importer = ExperimentListTemplateImporter([template])
    experiments = importer.experiments
    imageset = experiments.imagesets()[0]
    indexer.add_indexer_imageset(imageset)

    from xia2.Schema.XCrystal import XCrystal
    from xia2.Schema.XWavelength import XWavelength
    from xia2.Schema.XSweep import XSweep
    from xia2.Schema.XSample import XSample

    cryst = XCrystal("CRYST1", None)
    wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
    samp = XSample("X1", cryst)
    directory, image = os.path.split(imageset.get_path(1))
    sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
    indexer.set_indexer_sweep(sweep)

    indexer.index()

    assert approx_equal(indexer.get_indexer_cell(),
                        (78.14, 78.14, 78.14, 90, 90, 90),
                        eps=1e-1)
    solution = indexer.get_solution()
    assert approx_equal(solution["rmsd"], 0.041, eps=1e-2)
    assert approx_equal(solution["metric"], 0.027, eps=1e-2)
    assert solution["number"] == 22
    assert solution["lattice"] == "cI"

    beam_centre = indexer.get_indexer_beam_centre()
    assert approx_equal(beam_centre, (94.4223, 94.5097), eps=1e-2)
    print(indexer.get_indexer_experiment_list()[0].crystal)
    print(indexer.get_indexer_experiment_list()[0].detector)

    # test serialization of indexer
    json_str = indexer.as_json()
    # print(json_str)
    indexer2 = DialsIndexer.from_json(string=json_str)
    indexer2.index()

    assert approx_equal(indexer.get_indexer_cell(),
                        indexer2.get_indexer_cell())
    assert approx_equal(indexer.get_indexer_beam_centre(),
                        indexer2.get_indexer_beam_centre())

    indexer.eliminate()
    indexer2.eliminate()

    assert approx_equal(indexer.get_indexer_cell(),
                        indexer2.get_indexer_cell())
    assert indexer.get_indexer_lattice() == "hR"
    assert indexer2.get_indexer_lattice() == "hR"
Ejemplo n.º 12
0
def load_imagesets(
    template,
    directory,
    id_image=None,
    image_range=None,
    use_cache=True,
    reversephi=False,
):
    global imageset_cache
    from dxtbx.model.experiment_list import ExperimentListFactory
    from xia2.Applications.xia2setup import known_hdf5_extensions
    from dxtbx.imageset import ImageSweep

    full_template_path = os.path.join(directory, template)

    if full_template_path not in imageset_cache or not use_cache:

        from dxtbx.model.experiment_list import BeamComparison
        from dxtbx.model.experiment_list import DetectorComparison
        from dxtbx.model.experiment_list import GoniometerComparison

        params = PhilIndex.params.xia2.settings
        compare_beam = BeamComparison(
            wavelength_tolerance=params.input.tolerance.beam.wavelength,
            direction_tolerance=params.input.tolerance.beam.direction,
            polarization_normal_tolerance=params.input.tolerance.beam.polarization_normal,
            polarization_fraction_tolerance=params.input.tolerance.beam.polarization_fraction,
        )
        compare_detector = DetectorComparison(
            fast_axis_tolerance=params.input.tolerance.detector.fast_axis,
            slow_axis_tolerance=params.input.tolerance.detector.slow_axis,
            origin_tolerance=params.input.tolerance.detector.origin,
        )
        compare_goniometer = GoniometerComparison(
            rotation_axis_tolerance=params.input.tolerance.goniometer.rotation_axis,
            fixed_rotation_tolerance=params.input.tolerance.goniometer.fixed_rotation,
            setting_rotation_tolerance=params.input.tolerance.goniometer.setting_rotation,
        )
        scan_tolerance = params.input.tolerance.scan.oscillation

        format_kwargs = {
            "dynamic_shadowing": params.input.format.dynamic_shadowing,
            "multi_panel": params.input.format.multi_panel,
        }

        if os.path.splitext(full_template_path)[-1] in known_hdf5_extensions:
            # if we are passed the correct file, use this, else look for a master
            # file (i.e. something_master.h5)

            if os.path.exists(full_template_path) and os.path.isfile(
                full_template_path
            ):
                master_file = full_template_path
            else:
                import glob

                g = glob.glob(os.path.join(directory, "*_master.h5"))
                master_file = None
                for p in g:
                    substr = longest_common_substring(template, p)
                    if substr:
                        if master_file is None or (
                            len(substr)
                            > len(longest_common_substring(template, master_file))
                        ):
                            master_file = p

            if master_file is None:
                raise RuntimeError("Can't find master file for %s" % full_template_path)

            unhandled = []
            experiments = ExperimentListFactory.from_filenames(
                [master_file],
                verbose=False,
                unhandled=unhandled,
                compare_beam=compare_beam,
                compare_detector=compare_detector,
                compare_goniometer=compare_goniometer,
                scan_tolerance=scan_tolerance,
                format_kwargs=format_kwargs,
            )

            assert len(unhandled) == 0, (
                "unhandled image files identified: %s" % unhandled
            )

        else:

            from dxtbx.sweep_filenames import locate_files_matching_template_string

            params = PhilIndex.get_python_object()
            read_all_image_headers = params.xia2.settings.read_all_image_headers

            if read_all_image_headers:
                paths = sorted(
                    locate_files_matching_template_string(full_template_path)
                )
                unhandled = []
                experiments = ExperimentListFactory.from_filenames(
                    paths,
                    verbose=False,
                    unhandled=unhandled,
                    compare_beam=compare_beam,
                    compare_detector=compare_detector,
                    compare_goniometer=compare_goniometer,
                    scan_tolerance=scan_tolerance,
                    format_kwargs=format_kwargs,
                )
                assert len(unhandled) == 0, (
                    "unhandled image files identified: %s" % unhandled
                )

            else:
                from dxtbx.model.experiment_list import ExperimentListTemplateImporter

                importer = ExperimentListTemplateImporter(
                    [full_template_path], format_kwargs=format_kwargs
                )
                experiments = importer.experiments

        imagesets = [
            iset for iset in experiments.imagesets() if isinstance(iset, ImageSweep)
        ]
        assert len(imagesets) > 0, "no imageset found"

        imageset_cache[full_template_path] = collections.OrderedDict()
        if reversephi:
            for imageset in imagesets:
                goniometer = imageset.get_goniometer()
                goniometer.set_rotation_axis(
                    tuple(-g for g in goniometer.get_rotation_axis())
                )

        reference_geometry = PhilIndex.params.xia2.settings.input.reference_geometry
        if reference_geometry is not None and len(reference_geometry) > 0:
            update_with_reference_geometry(imagesets, reference_geometry)

        # Update the geometry
        params = PhilIndex.params.xia2.settings
        update_geometry = []

        from dials.command_line.dials_import import ManualGeometryUpdater
        from dials.util.options import geometry_phil_scope

        # Then add manual geometry
        work_phil = geometry_phil_scope.format(params.input)
        diff_phil = geometry_phil_scope.fetch_diff(source=work_phil)
        if diff_phil.as_str() != "":
            update_geometry.append(ManualGeometryUpdater(params.input))

        imageset_list = []
        for imageset in imagesets:
            for updater in update_geometry:
                imageset = updater(imageset)
            imageset_list.append(imageset)
        imagesets = imageset_list

        from scitbx.array_family import flex

        for imageset in imagesets:
            scan = imageset.get_scan()
            exposure_times = scan.get_exposure_times()
            epochs = scan.get_epochs()
            if exposure_times.all_eq(0) or exposure_times[0] == 0:
                exposure_times = flex.double(exposure_times.size(), 1)
                scan.set_exposure_times(exposure_times)
            elif not exposure_times.all_gt(0):
                exposure_times = flex.double(exposure_times.size(), exposure_times[0])
                scan.set_exposure_times(exposure_times)
            if epochs.size() > 1 and not epochs.all_gt(0):
                if epochs[0] == 0:
                    epochs[0] = 1
                for i in range(1, epochs.size()):
                    epochs[i] = epochs[i - 1] + exposure_times[i - 1]
                scan.set_epochs(epochs)
            _id_image = scan.get_image_range()[0]
            imageset_cache[full_template_path][_id_image] = imageset

    if id_image is not None:
        return [imageset_cache[full_template_path][id_image]]
    elif image_range is not None:
        for imageset in imageset_cache[full_template_path].values():
            scan = imageset.get_scan()
            scan_image_range = scan.get_image_range()
            if (
                image_range[0] >= scan_image_range[0]
                and image_range[1] <= scan_image_range[1]
            ):
                imagesets = [
                    imageset[
                        image_range[0]
                        - scan_image_range[0] : image_range[1]
                        + 1
                        - scan_image_range[0]
                    ]
                ]
                assert len(imagesets[0]) == image_range[1] - image_range[0] + 1, len(
                    imagesets[0]
                )
                return imagesets
    return imageset_cache[full_template_path].values()
Ejemplo n.º 13
0
    def _run_dials_import(self):
        """
        Perform a minimal version of dials.import to get an experiment list.

        Use some filleted bits of dials.import and dials.util.options.Importer.
        """
        # Get some key data format arguments.
        try:
            format_kwargs = {
                "dynamic_shadowing":
                self.params.dials_import.format.dynamic_shadowing,
                "multi_panel": self.params.dials_import.format.multi_panel,
            }
        except AttributeError:
            format_kwargs = {}

        # If filenames contain wildcards, expand
        args = []
        for arg in self.params.dials_import.input.experiments:
            if "*" in arg:
                args.extend(glob(arg))
            else:
                args.append(arg)

        if args:
            # Are compare{beam,detector,goniometer} and scan_tolerance necessary?
            # They are cargo-culted from the DIALS option parser.
            tol_params = self.params.dials_import.input.tolerance
            compare_beam = BeamComparison(
                wavelength_tolerance=tol_params.beam.wavelength,
                direction_tolerance=tol_params.beam.direction,
                polarization_normal_tolerance=tol_params.beam.
                polarization_normal,
                polarization_fraction_tolerance=tol_params.beam.
                polarization_fraction,
            )
            compare_detector = DetectorComparison(
                fast_axis_tolerance=tol_params.detector.fast_axis,
                slow_axis_tolerance=tol_params.detector.slow_axis,
                origin_tolerance=tol_params.detector.origin,
            )
            compare_goniometer = GoniometerComparison(
                rotation_axis_tolerance=tol_params.goniometer.rotation_axis,
                fixed_rotation_tolerance=tol_params.goniometer.fixed_rotation,
                setting_rotation_tolerance=tol_params.goniometer.
                setting_rotation,
            )
            scan_tolerance = tol_params.scan.oscillation

            # Import an experiment list from image data.
            try:
                experiments = ExperimentListFactory.from_filenames(
                    args,
                    compare_beam=compare_beam,
                    compare_detector=compare_detector,
                    compare_goniometer=compare_goniometer,
                    scan_tolerance=scan_tolerance,
                    format_kwargs=format_kwargs,
                )
            except IOError as e:
                warning("%s '%s'", e.strerror, e.filename)
                sys.exit(1)

            # Record the imported experiments for use elsewhere.
            # Quit if there aren't any.
            self.expts.extend(experiments)
            if not self.expts:
                warning("No images found.")
                sys.exit(1)

        else:
            # Use the template importer.
            if len(self.params.dials_import.input.template) > 0:
                importer = ExperimentListTemplateImporter(
                    self.params.dials_import.input.template,
                    format_kwargs=format_kwargs)
                # Record the imported experiments for use elsewhere.
                # Quit if there aren't any.
                self.expts.extend(importer.experiments)
                if not self.expts:
                    warning("No images found matching template %s" %
                            self.params.dials_import.input.template[0])
                    sys.exit(1)

        # Setup the metadata updater
        metadata_updater = MetaDataUpdater(self.params.dials_import)

        # Extract the experiments and loop through
        self.expts = metadata_updater(self.expts.imagesets())
def test_labelit_indexer(regression_test, ccp4, dials_data, run_in_tmpdir):
    template = dials_data("insulin").join("insulin_1_###.img").strpath

    from xia2.Modules.Indexer.LabelitIndexer import LabelitIndexer
    from xia2.DriverExceptions.NotAvailableError import NotAvailableError

    try:
        ls = LabelitIndexer(indxr_print=True)
    except NotAvailableError:
        pytest.skip("labelit not found")
    ls.set_working_directory(run_in_tmpdir.strpath)
    from dxtbx.model.experiment_list import ExperimentListTemplateImporter

    importer = ExperimentListTemplateImporter([template])
    experiments = importer.experiments
    imageset = experiments.imagesets()[0]
    ls.add_indexer_imageset(imageset)
    ls.index()

    assert ls.get_indexer_cell() == pytest.approx(
        (78.58, 78.58, 78.58, 90, 90, 90), abs=0.5)
    solution = ls.get_solution()
    assert solution["rmsd"] <= 0.2
    assert solution["metric"] <= 0.16
    assert solution["number"] == 22
    assert solution["lattice"] == "cI"
    assert solution["mosaic"] <= 0.25
    assert solution["nspots"] == pytest.approx(860, abs=30)

    beam_centre = ls.get_indexer_beam_centre()
    assert beam_centre == pytest.approx((94.3416, 94.4994), abs=2e-1)
    assert ls.get_indexer_images() == [(1, 1), (22, 22), (45, 45)]
    print(ls.get_indexer_experiment_list()[0].crystal)
    print(ls.get_indexer_experiment_list()[0].detector)

    json_str = ls.as_json()
    # print(json_str)
    ls1 = LabelitIndexer.from_json(string=json_str)
    ls1.index()

    print(ls.get_indexer_experiment_list()[0].crystal)
    assert ls.get_indexer_beam_centre() == ls1.get_indexer_beam_centre()
    assert ls1.get_indexer_images() == [
        [1, 1],
        [22, 22],
        [45, 45],
    ]  # in JSON tuples become lists
    assert ls.get_distance() == ls1.get_distance()

    ls.eliminate()
    ls1.eliminate()

    print(ls1.get_indexer_experiment_list()[0].crystal)
    assert ls.get_indexer_beam_centre() == ls1.get_indexer_beam_centre()
    assert ls.get_indexer_images() == [(1, 1), (22, 22), (45, 45)]
    assert ls1.get_indexer_images() == [
        [1, 1],
        [22, 22],
        [45, 45],
    ]  # in JSON tuples become lists
    assert ls.get_distance() == ls1.get_distance()

    print(ls1.get_indexer_cell())
    print(ls1.get_solution())
    assert ls.get_indexer_cell() == pytest.approx(
        (111.11, 111.11, 68.08, 90.0, 90.0, 120.0), abs=5e-1)
    solution = ls1.get_solution()
    assert solution["rmsd"] >= 0.07, solution["rmsd"]
    assert solution["metric"] == pytest.approx(0.1291, abs=1e-1)
    assert solution["lattice"] == "hR", solution["lattice"]
    assert solution["mosaic"] <= 0.3, solution["mosaic"]
    assert solution["nspots"] == pytest.approx(856, abs=30)
Ejemplo n.º 15
0
def exercise_dials_integrater(dials_data, tmp_dir, nproc=None):
    if nproc:
        PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc

    template = dials_data("insulin").join("insulin_1_###.img").strpath

    indexer = DialsIndexer()
    indexer.set_working_directory(tmp_dir)
    importer = ExperimentListTemplateImporter([template])
    experiments = importer.experiments
    imageset = experiments.imagesets()[0]
    indexer.add_indexer_imageset(imageset)

    cryst = XCrystal("CRYST1", None)
    wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
    samp = XSample("X1", cryst)
    directory, image = os.path.split(imageset.get_path(1))
    sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
    indexer.set_indexer_sweep(sweep)

    refiner = DialsRefiner()
    refiner.set_working_directory(tmp_dir)
    refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
    # refiner.refine()

    integrater = DialsIntegrater()
    integrater.set_output_format("hkl")
    integrater.set_working_directory(tmp_dir)
    integrater.setup_from_image(imageset.get_path(1))
    integrater.set_integrater_refiner(refiner)
    # integrater.set_integrater_indexer(indexer)
    integrater.set_integrater_sweep(sweep)
    integrater.integrate()

    integrater_intensities = integrater.get_integrater_intensities()
    assert os.path.exists(integrater_intensities)

    reader = any_reflection_file(integrater_intensities)
    assert reader.file_type() == "ccp4_mtz", repr(integrater_intensities)
    mtz_object = reader.file_content()
    expected_reflections = 47623
    assert (abs(mtz_object.n_reflections() - expected_reflections) <
            300), mtz_object.n_reflections()

    assert mtz_object.column_labels() == [
        "H",
        "K",
        "L",
        "M_ISYM",
        "BATCH",
        "IPR",
        "SIGIPR",
        "I",
        "SIGI",
        "BG",
        "SIGBG",
        "FRACTIONCALC",
        "XDET",
        "YDET",
        "ROT",
        "LP",
        "QE",
    ]

    assert integrater.get_integrater_wedge() == (1, 45)
    assert integrater.get_integrater_cell() == pytest.approx(
        (78.14, 78.14, 78.14, 90, 90, 90), abs=1e-1)

    # test serialization of integrater
    json_str = integrater.as_json()
    # print(json_str)
    integrater2 = DialsIntegrater.from_json(string=json_str)
    integrater2.set_integrater_sweep(sweep, reset=False)
    integrater2_intensities = integrater.get_integrater_intensities()
    assert integrater2_intensities == integrater_intensities

    integrater2.set_integrater_finish_done(False)
    integrater2_intensities = integrater2.get_integrater_intensities()
    assert os.path.exists(integrater2_intensities)
    reader = any_reflection_file(integrater2_intensities)
    assert reader.file_type() == "ccp4_mtz"
    mtz_object = reader.file_content()
    assert (abs(mtz_object.n_reflections() - expected_reflections) <
            300), mtz_object.n_reflections()

    integrater2.set_integrater_done(False)
    integrater2_intensities = integrater2.get_integrater_intensities()
    assert os.path.exists(integrater2_intensities)
    reader = any_reflection_file(integrater2_intensities)
    assert reader.file_type() == "ccp4_mtz"
    mtz_object = reader.file_content()
    assert (abs(mtz_object.n_reflections() - expected_reflections) <
            300), mtz_object.n_reflections()

    integrater2.set_integrater_prepare_done(False)
    integrater2_intensities = integrater2.get_integrater_intensities()
    assert os.path.exists(integrater2_intensities)
    reader = any_reflection_file(integrater2_intensities)
    assert reader.file_type() == "ccp4_mtz"
    mtz_object = reader.file_content()
    assert (abs(mtz_object.n_reflections() - expected_reflections) <
            300), mtz_object.n_reflections()

    # Test that diamond anvil cell attenuation correction does something.
    # That it does the right thing is left as a matter for the DIALS tests.
    integrater3 = DialsIntegrater.from_json(string=json_str)
    integrater3.set_integrater_sweep(sweep, reset=False)
    integrater3.set_integrater_done(False)
    integrater3.high_pressure = True
    # Don't get .hkl output because we're applying the attenuation correction to data
    # that weren't actually collected with a diamond anvil cell and some integrated
    # intensities will be rather nonsensical, which causes an error
    # 'cctbx Error: Inconsistent observation/sigma pair in columns: IPR, SIGIPR',
    # when some internal .hkl consistency checks are run, which is not meaningful here.
    integrater3.set_output_format("pickle")
    # Compare the first ten profile-fitted integrated intensities without correction.
    control_reflections = flex.reflection_table.from_file(
        integrater2.get_integrated_reflections())
    valid = control_reflections.get_flags(
        control_reflections.flags.integrated_prf)
    valid = valid.iselection()[:10]
    control_reflections = control_reflections.select(valid)
    # Get the first ten profile-fitted integrated intensities with DAC correction.
    corrected_reflections = flex.reflection_table.from_file(
        integrater3.get_integrated_reflections())
    valid = corrected_reflections.get_flags(
        corrected_reflections.flags.integrated_prf)
    valid = valid.iselection()[:10]
    corrected_reflections = corrected_reflections.select(valid)
    # Check that we're comparing equivalent reflections.
    assert control_reflections["miller_index"] == corrected_reflections[
        "miller_index"]
    control_intensities = control_reflections["intensity.prf.value"]
    corrected_intensities = corrected_reflections["intensity.prf.value"]
    # Check that the reflection intensities are not the same.
    assert pytest.approx(control_intensities) != corrected_intensities
def exercise_mosflm_integrater(dials_data, tmp_dir, nproc):
    from xia2.Handlers.Phil import PhilIndex

    PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc

    template = dials_data("insulin").join("insulin_1_###.img").strpath

    # otherwise if this test is running multiple times simultaneously two mosflm
    # processes try to write to the same genfile
    os.environ["CCP4_SCR"] = tmp_dir

    from xia2.Modules.Indexer.MosflmIndexer import MosflmIndexer
    from xia2.Modules.Integrater.MosflmIntegrater import MosflmIntegrater
    from dxtbx.model.experiment_list import ExperimentListTemplateImporter

    indexer = MosflmIndexer()
    indexer.set_working_directory(tmp_dir)
    importer = ExperimentListTemplateImporter([template])
    experiments = importer.experiments
    imageset = experiments.imagesets()[0]
    indexer.add_indexer_imageset(imageset)

    from xia2.Schema.XCrystal import XCrystal
    from xia2.Schema.XWavelength import XWavelength
    from xia2.Schema.XSweep import XSweep
    from xia2.Schema.XSample import XSample

    cryst = XCrystal("CRYST1", None)
    wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
    samp = XSample("X1", cryst)
    directory, image = os.path.split(imageset.get_path(1))
    sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
    indexer.set_indexer_sweep(sweep)

    from xia2.Modules.Refiner.MosflmRefiner import MosflmRefiner

    refiner = MosflmRefiner()
    refiner.set_working_directory(tmp_dir)
    refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)

    nref_error = 500

    integrater = MosflmIntegrater()
    integrater.set_working_directory(tmp_dir)
    integrater.setup_from_image(imageset.get_path(1))
    integrater.set_integrater_refiner(refiner)
    # integrater.set_integrater_indexer(indexer)
    integrater.set_integrater_sweep(sweep)
    integrater.integrate()

    integrater_intensities = integrater.get_integrater_intensities()
    assert os.path.exists(integrater_intensities)
    from iotbx.reflection_file_reader import any_reflection_file

    reader = any_reflection_file(integrater_intensities)
    assert reader.file_type() == "ccp4_mtz"
    mtz_object = reader.file_content()
    assert (abs(mtz_object.n_reflections() - 81116) <
            nref_error), mtz_object.n_reflections()
    assert mtz_object.column_labels() == [
        "H",
        "K",
        "L",
        "M_ISYM",
        "BATCH",
        "I",
        "SIGI",
        "IPR",
        "SIGIPR",
        "FRACTIONCALC",
        "XDET",
        "YDET",
        "ROT",
        "WIDTH",
        "LP",
        "MPART",
        "FLAG",
        "BGPKRATIOS",
    ]

    assert integrater.get_integrater_wedge() == (1, 45)
    assert approx_equal(
        integrater.get_integrater_cell(),
        (78.014, 78.014, 78.014, 90.0, 90.0, 90.0),
        eps=1e-2,
    )

    # test serialization of integrater
    json_str = integrater.as_json()
    # print json_str
    integrater2 = MosflmIntegrater.from_json(string=json_str)
    integrater2.set_integrater_sweep(sweep, reset=False)
    integrater2_intensities = integrater.get_integrater_intensities()
    assert integrater2_intensities == integrater_intensities

    integrater2.set_integrater_finish_done(False)
    integrater2_intensities = integrater2.get_integrater_intensities()
    assert os.path.exists(integrater2_intensities)
    reader = any_reflection_file(integrater2_intensities)
    assert reader.file_type() == "ccp4_mtz"
    mtz_object = reader.file_content()
    assert (abs(mtz_object.n_reflections() - 81116) <
            nref_error), mtz_object.n_reflections()

    integrater2.set_integrater_done(False)
    integrater2_intensities = integrater2.get_integrater_intensities()
    assert os.path.exists(integrater2_intensities)
    reader = any_reflection_file(integrater2_intensities)
    assert reader.file_type() == "ccp4_mtz"
    mtz_object = reader.file_content()
    assert (abs(mtz_object.n_reflections() - 81116) <
            nref_error), mtz_object.n_reflections()

    integrater2.set_integrater_prepare_done(False)
    integrater2_intensities = integrater2.get_integrater_intensities()
    assert os.path.exists(integrater2_intensities)
    reader = any_reflection_file(integrater2_intensities)
    assert reader.file_type() == "ccp4_mtz"
    mtz_object = reader.file_content()
    assert (abs(mtz_object.n_reflections() - 81116) <
            nref_error), mtz_object.n_reflections()