Beispiel #1
0
def exercise_dials_wrappers(nproc=None):
  if not have_dials_regression:
    print "Skipping exercise_dials_wrappers(): dials_regression not configured"
    return

  if nproc is not None:
    from xia2.Handlers.Phil import PhilIndex
    PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc

  from xia2.Wrappers.Dials.Import import Import
  from xia2.Wrappers.Dials.Spotfinder import Spotfinder
  from xia2.Wrappers.Dials.Index import Index
  from xia2.Wrappers.Dials.Integrate import Integrate
  from xia2.Wrappers.Dials.RefineBravaisSettings import RefineBravaisSettings
  from xia2.Wrappers.Dials.Refine import Refine
  from xia2.Wrappers.Dials.Reindex import Reindex
  from xia2.Wrappers.Dials.ExportMtz import ExportMtz
  from xia2.Wrappers.Dials.ExportXDSASCII import ExportXDSASCII

  xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data")
  template = os.path.join(xia2_demo_data, "insulin_1_%03i.img")
  scan_ranges = [(1, 45)]
  image_range = (1, 45)

  cwd = os.path.abspath(os.curdir)
  tmp_dir = os.path.abspath(open_tmp_directory())
  os.chdir(tmp_dir)

  print "Begin importing"
  importer = Import()
  importer.setup_from_image(template %1)
  importer.set_image_range(image_range)
  importer.run()
  print ''.join(importer.get_all_output())
  print "Done importing"

  print "Begin spotfinding"
  spotfinder = Spotfinder()
  spotfinder.set_input_sweep_filename(importer.get_sweep_filename())
  spotfinder.set_scan_ranges(scan_ranges)
  spotfinder.run()
  print ''.join(spotfinder.get_all_output())
  print "Done spotfinding"

  print "Begin indexing"
  indexer = Index()
  indexer.add_spot_filename(spotfinder.get_spot_filename())
  indexer.add_sweep_filename(importer.get_sweep_filename())
  indexer.run('fft3d')
  print ''.join(indexer.get_all_output())
  print "Done indexing"

  print "Begin refining"
  rbs = RefineBravaisSettings()
  rbs.set_experiments_filename(indexer.get_experiments_filename())
  rbs.set_indexed_filename(indexer.get_indexed_filename())
  rbs.run()
  print ''.join(rbs.get_all_output())
  print "Done refining"
  bravais_setting_22 = rbs.get_bravais_summary()[22]
  assert bravais_setting_22['bravais'] == 'cI'
  assert bravais_setting_22['cb_op'] == 'b+c,a+c,a+b'
  assert approx_equal(bravais_setting_22['unit_cell'],
                      (78.14,78.14,78.14,90,90,90), eps=1e-1)
  bravais_setting_22_json = bravais_setting_22['experiments_file']
  assert os.path.exists(bravais_setting_22_json)

  print "Begin reindexing"
  reindexer = Reindex()
  reindexer.set_experiments_filename(indexer.get_experiments_filename())
  reindexer.set_indexed_filename(indexer.get_indexed_filename())
  reindexer.set_cb_op(bravais_setting_22['cb_op'])
  reindexer.run()
  assert os.path.exists(reindexer.get_reindexed_experiments_filename())
  assert os.path.exists(reindexer.get_reindexed_reflections_filename())
  print ''.join(reindexer.get_all_output())
  print "Done reindexing"

  print "Begin refining"
  refiner = Refine()
  refiner.set_experiments_filename(bravais_setting_22_json)
  refiner.set_indexed_filename(reindexer.get_reindexed_reflections_filename())
  refiner.set_scan_varying(True)
  refiner.run()
  assert os.path.exists(refiner.get_refined_experiments_filename())
  print ''.join(refiner.get_all_output())
  print "Done refining"

  print "Begin integrating"
  integrater = Integrate()
  integrater.set_experiments_filename(refiner.get_refined_experiments_filename())
  integrater.set_reflections_filename(reindexer.get_reindexed_reflections_filename())
  integrater.run()
  print ''.join(integrater.get_all_output())
  print "Done integrating"

  print "Begin exporting"
  exporter = ExportMtz()
  exporter.set_experiments_filename(integrater.get_integrated_experiments())
  exporter.set_reflections_filename(integrater.get_integrated_reflections())
  exporter.run()
  print ''.join(exporter.get_all_output())
  print "Done exporting"
  assert os.path.exists(exporter.get_mtz_filename())

  print "Begin exporting xds_ascii"
  exporter = ExportXDSASCII()
  exporter.set_experiments_filename(integrater.get_integrated_experiments())
  exporter.set_reflections_filename(integrater.get_integrated_reflections())
  exporter.run()
  print ''.join(exporter.get_all_output())
  print "Done exporting"
  assert os.path.exists(exporter.get_hkl_filename())

  import shutil
  shutil.copy(indexer.get_experiments_filename(), "copy.json")
  shutil.copy(indexer.get_indexed_filename(), "copy.pickle")
  print "Begin combining"
  from xia2.Wrappers.Dials.CombineExperiments import CombineExperiments
  exporter = CombineExperiments()
  exporter.add_experiments(indexer.get_experiments_filename())
  exporter.add_experiments("copy.json")
  exporter.add_reflections(indexer.get_indexed_filename())
  exporter.add_reflections("copy.pickle")
  exporter.run()
  print ''.join(exporter.get_all_output())
  print "Done combining"
  assert os.path.exists(exporter.get_combined_experiments_filename())
  assert os.path.exists(exporter.get_combined_reflections_filename())
def exercise_dials_wrappers(template, nproc=None):
    if nproc is not None:
        PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc

    scan_ranges = [(1, 45)]
    image_range = (1, 45)

    print("Begin importing")
    importer = Import()
    importer.setup_from_image(template % 1)
    importer.set_image_range(image_range)
    importer.run()
    print("".join(importer.get_all_output()))
    print("Done importing")

    print("Begin spotfinding")
    spotfinder = Spotfinder()
    spotfinder.set_input_sweep_filename(importer.get_sweep_filename())
    spotfinder.set_scan_ranges(scan_ranges)
    spotfinder.run()
    print("".join(spotfinder.get_all_output()))
    print("Done spotfinding")

    print("Begin indexing")
    indexer = Index()
    indexer.add_spot_filename(spotfinder.get_spot_filename())
    indexer.add_sweep_filename(importer.get_sweep_filename())
    indexer.run("fft3d")
    print("".join(indexer.get_all_output()))
    print("Done indexing")

    print("Begin refining")
    rbs = RefineBravaisSettings()
    rbs.set_experiments_filename(indexer.get_experiments_filename())
    rbs.set_indexed_filename(indexer.get_indexed_filename())
    rbs.run()
    print("".join(rbs.get_all_output()))
    print("Done refining")
    bravais_setting_22 = rbs.get_bravais_summary()[22]
    assert bravais_setting_22["bravais"] == "cI"
    assert bravais_setting_22["cb_op"] == "b+c,a+c,a+b"
    assert bravais_setting_22["unit_cell"] == pytest.approx(
        (78.14, 78.14, 78.14, 90, 90, 90), abs=1e-1
    )
    bravais_setting_22_json = bravais_setting_22["experiments_file"]
    assert os.path.exists(bravais_setting_22_json)

    print("Begin reindexing")
    reindexer = Reindex()
    reindexer.set_experiments_filename(indexer.get_experiments_filename())
    reindexer.set_indexed_filename(indexer.get_indexed_filename())
    reindexer.set_cb_op(bravais_setting_22["cb_op"])
    reindexer.run()
    assert os.path.exists(reindexer.get_reindexed_experiments_filename())
    assert os.path.exists(reindexer.get_reindexed_reflections_filename())
    print("".join(reindexer.get_all_output()))
    print("Done reindexing")

    print("Begin refining")
    refiner = Refine()
    refiner.set_experiments_filename(bravais_setting_22_json)
    refiner.set_indexed_filename(reindexer.get_reindexed_reflections_filename())
    refiner.set_scan_varying(True)
    refiner.run()
    assert os.path.exists(refiner.get_refined_experiments_filename())
    print("".join(refiner.get_all_output()))
    print("Done refining")

    print("Begin integrating")
    integrater = Integrate()
    integrater.set_experiments_filename(refiner.get_refined_experiments_filename())
    integrater.set_reflections_filename(reindexer.get_reindexed_reflections_filename())
    integrater.run()
    print("".join(integrater.get_all_output()))
    print("Done integrating")

    print("Begin exporting")
    exporter = ExportMtz()
    exporter.set_experiments_filename(integrater.get_integrated_experiments())
    exporter.set_reflections_filename(integrater.get_integrated_reflections())
    exporter.run()
    print("".join(exporter.get_all_output()))
    print("Done exporting")
    assert os.path.exists(exporter.get_mtz_filename())

    print("Begin exporting xds_ascii")
    exporter = ExportXDSASCII()
    exporter.set_experiments_filename(integrater.get_integrated_experiments())
    exporter.set_reflections_filename(integrater.get_integrated_reflections())
    exporter.run()
    print("".join(exporter.get_all_output()))
    print("Done exporting")
    assert os.path.exists(exporter.get_hkl_filename())

    # Test combine experiments wrapper. Duplicate the file and adjust the
    # identifier to allow combination.
    shutil.copy(indexer.get_experiments_filename(), "copy.expt")
    shutil.copy(indexer.get_indexed_filename(), "copy.refl")

    expts = load.experiment_list("copy.expt")
    expts[0].identifier = "0"
    expts.as_file("copy.expt")
    refls = flex.reflection_table.from_file("copy.refl")
    refls.experiment_identifiers()[0] = "0"
    refls.as_file("copy.refl")

    exporter = CombineExperiments()
    exporter.add_experiments(indexer.get_experiments_filename())
    exporter.add_experiments("copy.expt")
    exporter.add_reflections(indexer.get_indexed_filename())
    exporter.add_reflections("copy.refl")
    exporter.run()
    print("".join(exporter.get_all_output()))
    print("Done combining")
    assert os.path.exists(exporter.get_combined_experiments_filename())
    assert os.path.exists(exporter.get_combined_reflections_filename())
Beispiel #3
0
def exercise_dials_wrappers(dials_regression, nproc=None):
    if nproc is not None:
        from xia2.Handlers.Phil import PhilIndex
        PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc

    from xia2.Wrappers.Dials.Import import Import
    from xia2.Wrappers.Dials.Spotfinder import Spotfinder
    from xia2.Wrappers.Dials.Index import Index
    from xia2.Wrappers.Dials.Integrate import Integrate
    from xia2.Wrappers.Dials.RefineBravaisSettings import RefineBravaisSettings
    from xia2.Wrappers.Dials.Refine import Refine
    from xia2.Wrappers.Dials.Reindex import Reindex
    from xia2.Wrappers.Dials.ExportMtz import ExportMtz
    from xia2.Wrappers.Dials.ExportXDSASCII import ExportXDSASCII

    xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data")
    template = os.path.join(xia2_demo_data, "insulin_1_%03i.img")
    scan_ranges = [(1, 45)]
    image_range = (1, 45)

    print("Begin importing")
    importer = Import()
    importer.setup_from_image(template % 1)
    importer.set_image_range(image_range)
    importer.run()
    print(''.join(importer.get_all_output()))
    print("Done importing")

    print("Begin spotfinding")
    spotfinder = Spotfinder()
    spotfinder.set_input_sweep_filename(importer.get_sweep_filename())
    spotfinder.set_scan_ranges(scan_ranges)
    spotfinder.run()
    print(''.join(spotfinder.get_all_output()))
    print("Done spotfinding")

    print("Begin indexing")
    indexer = Index()
    indexer.add_spot_filename(spotfinder.get_spot_filename())
    indexer.add_sweep_filename(importer.get_sweep_filename())
    indexer.run('fft3d')
    print(''.join(indexer.get_all_output()))
    print("Done indexing")

    print("Begin refining")
    rbs = RefineBravaisSettings()
    rbs.set_experiments_filename(indexer.get_experiments_filename())
    rbs.set_indexed_filename(indexer.get_indexed_filename())
    rbs.run()
    print(''.join(rbs.get_all_output()))
    print("Done refining")
    bravais_setting_22 = rbs.get_bravais_summary()[22]
    assert bravais_setting_22['bravais'] == 'cI'
    assert bravais_setting_22['cb_op'] == 'b+c,a+c,a+b'
    assert approx_equal(bravais_setting_22['unit_cell'],
                        (78.14, 78.14, 78.14, 90, 90, 90),
                        eps=1e-1)
    bravais_setting_22_json = bravais_setting_22['experiments_file']
    assert os.path.exists(bravais_setting_22_json)

    print("Begin reindexing")
    reindexer = Reindex()
    reindexer.set_experiments_filename(indexer.get_experiments_filename())
    reindexer.set_indexed_filename(indexer.get_indexed_filename())
    reindexer.set_cb_op(bravais_setting_22['cb_op'])
    reindexer.run()
    assert os.path.exists(reindexer.get_reindexed_experiments_filename())
    assert os.path.exists(reindexer.get_reindexed_reflections_filename())
    print(''.join(reindexer.get_all_output()))
    print("Done reindexing")

    print("Begin refining")
    refiner = Refine()
    refiner.set_experiments_filename(bravais_setting_22_json)
    refiner.set_indexed_filename(
        reindexer.get_reindexed_reflections_filename())
    refiner.set_scan_varying(True)
    refiner.run()
    assert os.path.exists(refiner.get_refined_experiments_filename())
    print(''.join(refiner.get_all_output()))
    print("Done refining")

    print("Begin integrating")
    integrater = Integrate()
    integrater.set_experiments_filename(
        refiner.get_refined_experiments_filename())
    integrater.set_reflections_filename(
        reindexer.get_reindexed_reflections_filename())
    integrater.run()
    print(''.join(integrater.get_all_output()))
    print("Done integrating")

    print("Begin exporting")
    exporter = ExportMtz()
    exporter.set_experiments_filename(integrater.get_integrated_experiments())
    exporter.set_reflections_filename(integrater.get_integrated_reflections())
    exporter.run()
    print(''.join(exporter.get_all_output()))
    print("Done exporting")
    assert os.path.exists(exporter.get_mtz_filename())

    print("Begin exporting xds_ascii")
    exporter = ExportXDSASCII()
    exporter.set_experiments_filename(integrater.get_integrated_experiments())
    exporter.set_reflections_filename(integrater.get_integrated_reflections())
    exporter.run()
    print(''.join(exporter.get_all_output()))
    print("Done exporting")
    assert os.path.exists(exporter.get_hkl_filename())

    import shutil
    shutil.copy(indexer.get_experiments_filename(), "copy.json")
    shutil.copy(indexer.get_indexed_filename(), "copy.pickle")
    print("Begin combining")
    from xia2.Wrappers.Dials.CombineExperiments import CombineExperiments
    exporter = CombineExperiments()
    exporter.add_experiments(indexer.get_experiments_filename())
    exporter.add_experiments("copy.json")
    exporter.add_reflections(indexer.get_indexed_filename())
    exporter.add_reflections("copy.pickle")
    exporter.run()
    print(''.join(exporter.get_all_output()))
    print("Done combining")
    assert os.path.exists(exporter.get_combined_experiments_filename())
    assert os.path.exists(exporter.get_combined_reflections_filename())
Beispiel #4
0
        def run(self):
            from xia2.Handlers.Streams import Chatter, Debug

            if self._reindexing_operators:
                Debug.write("Reindexing sweeps for dials.two_theta_refine")
                from xia2.lib.bits import auto_logfiler
                from xia2.Wrappers.Dials.Reindex import Reindex

                self._reindexed_experiments, self._reindexed_reflections = [], []
                for e, p, op in zip(
                        self._experiments,
                        self._reflection_files,
                        self._reindexing_operators,
                ):
                    reindexer = Reindex()
                    reindexer.set_cb_op(op)
                    reindexer.set_experiments_filename(e)
                    reindexer.set_indexed_filename(p)
                    reindexer.set_working_directory(
                        self.get_working_directory())
                    auto_logfiler(reindexer)
                    reindexer.run()
                    self._reindexed_experiments.append(
                        reindexer.get_reindexed_experiments_filename())
                    self._reindexed_reflections.append(
                        reindexer.get_reindexed_reflections_filename())

            Debug.write("Running dials.two_theta_refine")

            self._output_cif = os.path.join(
                self.get_working_directory(),
                "%s_dials.two_theta_refine.cif" % self.get_xpid(),
            )
            self._output_mmcif = os.path.join(
                self.get_working_directory(),
                "%s_dials.two_theta_refine.mmcif" % self.get_xpid(),
            )
            if not self._output_p4p:
                self._output_p4p = os.path.join(
                    self.get_working_directory(),
                    "%s_dials.two_theta_refine.p4p" % self.get_xpid(),
                )
            self._output_correlation_plot = os.path.join(
                self.get_working_directory(),
                "%s_dials.two_theta_refine.png" % self.get_xpid(),
            )
            self._output_experiments = os.path.join(
                self.get_working_directory(),
                "%s_refined_cell.expt" % self.get_xpid())

            self.clear_command_line()

            if self._reindexing_operators:
                for experiment in self._reindexed_experiments:
                    self.add_command_line(experiment)
                for reflection_file in self._reindexed_reflections:
                    self.add_command_line(reflection_file)
            else:
                for experiment in self._experiments:
                    self.add_command_line(experiment)
                for reflection_file in self._reflection_files:
                    self.add_command_line(reflection_file)
            self.add_command_line("combine_crystal_models=%s" %
                                  self._combine_crystal_models)
            self.add_command_line("output.cif=%s" % self._output_cif)
            self.add_command_line("output.mmcif=%s" % self._output_mmcif)
            self.add_command_line("output.p4p=%s" % self._output_p4p)
            if self._output_correlation_plot is not None:
                self.add_command_line("output.correlation_plot.filename=%s" %
                                      self._output_correlation_plot)
            if self._output_experiments is not None:
                self.add_command_line("output.experiments=%s" %
                                      self._output_experiments)
            if self._phil_file is not None:
                self.add_command_line("%s" % self._phil_file)

            self.start()
            self.close_wait()

            if not os.path.isfile(self._output_cif):
                Chatter.write(
                    "TwoTheta refinement failed, see log file for more details:\n  %s"
                    % self.get_log_file())
                raise RuntimeError("unit cell not refined")

            self.check_for_errors()

            from dxtbx.model.experiment_list import ExperimentListFactory

            experiments = ExperimentListFactory.from_json_file(
                self.get_output_experiments(), check_format=False)
            self._crystal = experiments.crystals()[0]
Beispiel #5
0
    def _determine_scaled_pointgroup(self):
        """Rerun symmetry after scaling to check for consistent space group. If not,
        then new space group should be used and data rescaled."""
        from cctbx import crystal

        exp_crystal = load.experiment_list(self._scaler.get_scaled_experiments())[
            0
        ].crystal
        cs = crystal.symmetry(
            space_group=exp_crystal.get_space_group(),
            unit_cell=exp_crystal.get_unit_cell(),
        )
        cs_ref = cs.as_reference_setting()
        current_pointgroup = cs_ref.space_group()
        current_patt_group = (
            current_pointgroup.build_derived_patterson_group().type().lookup_symbol()
        )
        Debug.write(
            "Space group used in scaling: %s"
            % current_pointgroup.type().lookup_symbol()
        )
        first = self._sweep_handler.get_epochs()[0]
        si = self._sweep_handler.get_sweep_information(first)
        refiner = si.get_integrater().get_integrater_refiner()
        point_group, reindex_op, _, _, reind_refl, reind_exp, reindex_initial = self._dials_symmetry_indexer_jiffy(
            [self._scaler.get_scaled_experiments()],
            [self._scaler.get_scaled_reflections()],
            [refiner],
        )
        Debug.write(
            "Point group determined by dials.symmetry on scaled dataset: %s"
            % point_group
        )
        sginfo = space_group_info(symbol=point_group)
        patt_group = (
            sginfo.group().build_derived_patterson_group().type().lookup_symbol()
        )
        self._scaler_symmetry_check_count += 1
        if patt_group != current_patt_group:
            if reindex_initial:
                reindexer = DialsReindex()
                reindexer.set_working_directory(self.get_working_directory())
                auto_logfiler(reindexer)
                reindexer.set_experiments_filename(
                    self._scaler.get_scaled_experiments()
                )
                reindexer.set_indexed_filename(self._scaler.get_scaled_reflections())
                reindexer.set_cb_op(reindex_op)
                reindexer.run()
                self._scaler.set_scaled_experiments(
                    reindexer.get_reindexed_experiments_filename()
                )
                self._scaler.set_scaled_reflections(
                    reindexer.get_reindexed_reflections_filename()
                )
            else:
                self._scaler.set_scaled_experiments(reind_exp)
                self._scaler.set_scaled_reflections(reind_refl)
            self.set_scaler_done(False)
            Chatter.write(
                """Inconsistent space groups determined before and after scaling: %s, %s \n
Data will be rescaled in new point group"""
                % (current_patt_group, patt_group)
            )
            return
        else:
            Chatter.write("Consistent space group determined before and after scaling")
Beispiel #6
0
    def apply_reindex_operator_to_sweep_info(self, si, reindex_operator, reason):
        """Use a reindex operator to reindex the data.

        Take the data from the sweep info, reindex using
        dials.reindex, and set the new data into the si.
        """
        reindexer = DialsReindex()
        reindexer.set_working_directory(self.get_working_directory())
        auto_logfiler(reindexer)

        reindexer.set_indexed_filename(si.get_reflections())
        reindexer.set_experiments_filename(si.get_experiments())
        reindexer.set_cb_op(reindex_operator)

        reindexer.run()

        si.set_reflections(reindexer.get_reindexed_reflections_filename())
        si.set_experiments(reindexer.get_reindexed_experiments_filename())

        Debug.write(
            "Reindexed with operator %s, reason is %s" % (reindex_operator, reason)
        )
Beispiel #7
0
    def _scale_prepare(self):
        """Perform all of the preparation required to deliver the scaled
        data. This should sort together the reflection files, ensure that
        they are correctly indexed (via dials.symmetry) and generally tidy
        things up."""

        # AIM discover symmetry and reindex with dials.symmetry, and set the correct
        # reflections in si.reflections, si.experiments

        self._helper.set_working_directory(self.get_working_directory())
        self._factory.set_working_directory(self.get_working_directory())

        need_to_return = False

        self._sweep_handler = SweepInformationHandler(self._scalr_integraters)

        p, x = self._sweep_handler.get_project_info()
        self._scalr_pname = p
        self._scalr_xname = x

        self._helper.set_pname_xname(p, x)

        Journal.block(
            "gathering",
            self.get_scaler_xcrystal().get_name(),
            "Dials",
            {"working directory": self.get_working_directory()},
        )

        # First do stuff to work out if excluding any data
        # Note - does this actually work? I couldn't seem to get it to work
        # in either this pipeline or the standard dials pipeline
        for epoch in self._sweep_handler.get_epochs():
            si = self._sweep_handler.get_sweep_information(epoch)
            intgr = si.get_integrater()
            _, xname, dname = si.get_project_info()
            sname = si.get_sweep_name()

            exclude_sweep = False

            for sweep in PhilIndex.params.xia2.settings.sweep:
                if sweep.id == sname and sweep.exclude:
                    exclude_sweep = True
                    break

            if exclude_sweep:
                self._sweep_handler.remove_epoch(epoch)
                Debug.write("Excluding sweep %s" % sname)
            else:
                Journal.entry({"adding data from": "%s/%s/%s" % (xname, dname, sname)})

        # If multiple files, want to run symmetry to check for consistent indexing
        # also

        # try to reproduce what CCP4ScalerA is doing

        # first assign identifiers to avoid dataset-id collisions
        # Idea is that this should be called anytime you get data anew from the
        # integrater, to intercept and assign unique ids, then set in the
        # sweep_information (si) and always use si.set_reflections/
        # si.get_reflections as we process.

        # self._sweep_handler = self._helper.assign_and_return_datasets(
        #    self._sweep_handler
        # ) symmetry now sorts out identifiers.

        need_to_return = False

        if self._scalr_input_pointgroup:
            self._input_pointgroup_scale_prepare()
        elif (
            len(self._sweep_handler.get_epochs()) > 1
            and PhilIndex.params.xia2.settings.multi_sweep_indexing
        ):
            need_to_return = self._multi_sweep_scale_prepare()
        else:
            need_to_return = self._standard_scale_prepare()

        if need_to_return:
            self.set_scaler_done(False)
            self.set_scaler_prepare_done(False)
            return

        ### After this point, point group is good and only need to
        ### reindex to consistent setting. Don't need to call back to the
        ### integator, just use the data in the sweep info.

        # First work out if we're going to reindex against external reference
        param = PhilIndex.params.xia2.settings.scale
        using_external_references = False
        reference_refl = None
        reference_expt = None
        if param.reference_reflection_file:
            if not param.reference_experiment_file:
                Chatter.write(
                    """
No DIALS reference experiments file provided, reference reflection file will
not be used. Reference mtz files for reindexing not currently supported for
pipeline=dials (supported for pipeline=dials-aimless).
"""
                )
            else:
                reference_refl = param.reference_reflection_file
                reference_expt = param.reference_experiment_file
                using_external_references = True
                Debug.write("Using reference reflections %s" % reference_refl)
                Debug.write("Using reference experiments %s" % reference_expt)

        if len(self._sweep_handler.get_epochs()) > 1:
            if PhilIndex.params.xia2.settings.unify_setting:
                self.unify_setting()

            if PhilIndex.params.xia2.settings.use_brehm_diederichs:
                self.brehm_diederichs_reindexing()
            # If not using Brehm-deidrichs reindexing, set reference as first
            # sweep, unless using external reference.
            elif not using_external_references:
                Debug.write("First sweep will be used as reference for reindexing")
                first = self._sweep_handler.get_epochs()[0]
                si = self._sweep_handler.get_sweep_information(first)
                reference_expt = si.get_experiments()
                reference_refl = si.get_reflections()

        # Now reindex to be consistent with first dataset - run reindex on each
        # dataset with reference (unless did brehm diederichs and didn't supply
        # a reference file)

        if reference_refl and reference_expt:
            exp = load.experiment_list(reference_expt)
            reference_cell = exp[0].crystal.get_unit_cell().parameters()

            # ---------- REINDEX TO CORRECT (REFERENCE) SETTING ----------
            Chatter.write("Reindexing all datasets to common reference")

            if using_external_references:
                epochs = self._sweep_handler.get_epochs()
            else:
                epochs = self._sweep_handler.get_epochs()[1:]
            for epoch in epochs:
                # if we are working with unified UB matrix then this should not
                # be a problem here (note, *if*; *should*)

                # what about e.g. alternative P1 settings?
                # see JIRA MXSW-904
                if PhilIndex.params.xia2.settings.unify_setting:
                    continue

                reindexer = DialsReindex()
                reindexer.set_working_directory(self.get_working_directory())
                auto_logfiler(reindexer)

                si = self._sweep_handler.get_sweep_information(epoch)
                reindexer.set_reference_filename(reference_expt)
                reindexer.set_reference_reflections(reference_refl)
                reindexer.set_indexed_filename(si.get_reflections())
                reindexer.set_experiments_filename(si.get_experiments())
                reindexer.run()

                # At this point, CCP4ScalerA would reset in integrator so that
                # the integrater calls reindex, no need to do that here as
                # have access to the files and will never need to reintegrate.

                si.set_reflections(reindexer.get_reindexed_reflections_filename())
                si.set_experiments(reindexer.get_reindexed_experiments_filename())

                # FIXME how to get some indication of the reindexing used?

                exp = load.experiment_list(
                    reindexer.get_reindexed_experiments_filename()
                )
                cell = exp[0].crystal.get_unit_cell().parameters()

                # Note - no lattice check as this will already be caught by reindex
                Debug.write("Cell: %.2f %.2f %.2f %.2f %.2f %.2f" % cell)
                Debug.write("Ref:  %.2f %.2f %.2f %.2f %.2f %.2f" % reference_cell)

                for j in range(6):
                    if (
                        math.fabs((cell[j] - reference_cell[j]) / reference_cell[j])
                        > 0.1
                    ):
                        raise RuntimeError(
                            "unit cell parameters differ in %s and %s"
                            % (reference_expt, si.get_reflections())
                        )

        # Now make sure all batches ok before finish preparing
        # This should be made safer, currently after dials.scale there is no
        # concept of 'batch', dials.export uses the calculate_batch_offsets
        # to assign batches, giving the same result as below.

        experiments_to_rebatch = []
        for epoch in self._sweep_handler.get_epochs():
            si = self._sweep_handler.get_sweep_information(epoch)
            experiment = si.get_experiments()
            experiments_to_rebatch.append(load.experiment_list(experiment)[0])
        offsets = calculate_batch_offsets(experiments_to_rebatch)

        for i, epoch in enumerate(self._sweep_handler.get_epochs()):
            si = self._sweep_handler.get_sweep_information(epoch)
            r = si.get_batch_range()
            si.set_batch_offset(offsets[i])
            si.set_batches([r[0] + offsets[i], r[1] + offsets[i]])
Beispiel #8
0
def load_sweeps_with_common_indexing():
  assert os.path.exists('xia2.json')
  from xia2.Schema.XProject import XProject
  xinfo = XProject.from_json(filename='xia2.json')

  import dials # required for gaussian_rs warning
  from xia2.Wrappers.Dials.Reindex import Reindex
  Citations.cite('dials')

  from dxtbx.model.experiment.experiment_list import ExperimentListFactory
  import cPickle as pickle
  crystals = xinfo.get_crystals()
  assert len(crystals) == 1
  crystal = next(crystals.itervalues())
  working_directory = Environment.generate_directory([crystal.get_name(), 'analysis'])
  os.chdir(working_directory)

  scaler = crystal._get_scaler()

  epoch_to_batches = {}
  epoch_to_integrated_intensities = {}
  epoch_to_sweep_name = {}

  # Aimless only
  epochs = scaler._sweep_handler.get_epochs()

  reference_cell = None
  reference_lattice = None
  reference_vectors = None
  reference_wavelength = None

  # Reindex each sweep to same setting
  all_miller_indices = flex.miller_index()
  all_two_thetas = flex.double()

  for epoch in epochs:
    si = scaler._sweep_handler.get_sweep_information(epoch)
    Chatter.smallbanner(si.get_sweep_name(), True)
    Debug.smallbanner(si.get_sweep_name(), True)

    intgr = si.get_integrater()
    experiments_filename = intgr.get_integrated_experiments()
    reflections_filename = intgr.get_integrated_reflections()
    refiner = intgr.get_integrater_refiner()
    Debug.write('experiment: %s' % experiments_filename)
    Debug.write('reflection: %s' % reflections_filename)

    # Use setting of first sweep as reference
    if reference_vectors is None:
      reference_vectors = experiments_filename

    # Assume that all sweeps have the same lattice system
    if reference_lattice is None:
      reference_lattice = refiner.get_refiner_lattice()
    else:
      assert reference_lattice == refiner.get_refiner_lattice()
    Debug.write("lattice: %s" % refiner.get_refiner_lattice())

    # Read .json file for sweep
    db = ExperimentListFactory.from_json_file(experiments_filename)

    # Assume that each file only contains a single experiment
    assert (len(db) == 1)
    db = db[0]

    # Get beam vector
    s0 = db.beam.get_unit_s0()

    # Use the unit cell of the first sweep as reference
    if reference_cell is None:
      reference_cell = db.crystal.get_unit_cell()
      Debug.write("Reference cell: %s" % str(reference_cell))

    dials_reindex = Reindex()
    dials_reindex.set_working_directory(working_directory)
    dials_reindex.set_cb_op("auto")
    dials_reindex.set_reference_filename(reference_vectors)
    dials_reindex.set_experiments_filename(experiments_filename)
    dials_reindex.set_indexed_filename(reflections_filename)
    auto_logfiler(dials_reindex)
    dials_reindex.run()

    # Assume that all data are collected at same wavelength
    if reference_wavelength is None:
      reference_wavelength = intgr.get_wavelength()
    else:
      assert abs(reference_wavelength - intgr.get_wavelength()) < 0.01
    Debug.write("wavelength: %f A" % intgr.get_wavelength())
    Debug.write("distance: %f mm" % intgr.get_distance())

    # Get integrated reflection data
    import dials
    with open(dials_reindex.get_reindexed_reflections_filename(), 'rb') as fh:
      reflections = pickle.load(fh)

    selection = reflections.get_flags(reflections.flags.used_in_refinement)
    Chatter.write("Found %d reflections used in refinement (out of %d entries)" % (selection.count(True), len(reflections['miller_index'])))
    reflections = reflections.select(selection)

    # Filter bad reflections
    selection = reflections['intensity.sum.variance'] <= 0
    if selection.count(True) > 0:
      reflections.del_selected(selection)
      print 'Removing %d reflections with negative variance' % \
        selection.count(True)

    if 'intensity.prf.variance' in reflections:
      selection = reflections['intensity.prf.variance'] <= 0
      if selection.count(True) > 0:
        reflections.del_selected(selection)
        print 'Removing %d profile reflections with negative variance' % \
          selection.count(True)

    # Find the observed 2theta angles
    miller_indices = flex.miller_index()
    two_thetas_obs = flex.double()
    for pixel, panel, hkl in zip(reflections['xyzobs.px.value'], reflections['panel'], reflections['miller_index']):
      assert hkl != (0, 0, 0)
      two_thetas_obs.append(db.detector[panel].get_two_theta_at_pixel(s0, pixel[0:2]))
      miller_indices.append(hkl)

    # Convert observed 2theta angles to degrees
    two_thetas_obs = two_thetas_obs * 180 / 3.14159265359
    Chatter.write("Remaining %d reflections are in 2theta range %.3f - %.3f deg" % (len(miller_indices), min(two_thetas_obs), max(two_thetas_obs)))

    all_miller_indices.extend(miller_indices)
    all_two_thetas.extend(two_thetas_obs)

  return all_miller_indices, all_two_thetas, reference_cell, reference_lattice, reference_wavelength
Beispiel #9
0
    def run(self):
      from xia2.Handlers.Streams import Chatter, Debug

      if self._reindexing_operator:
        Debug.write('Reindexing sweeps for dials.two_theta_refine')
        from xia2.lib.bits import auto_logfiler
        from xia2.Wrappers.Dials.Reindex import Reindex
        self._reindexed_experiments, self._reindexed_reflections = [], []
        for e, p in zip(self._experiments, self._pickles):
          reindexer = Reindex()
          reindexer.set_cb_op(self._reindexing_operator)
          reindexer.set_experiments_filename(e)
          reindexer.set_indexed_filename(p)
          reindexer.set_working_directory(self.get_working_directory())
          auto_logfiler(reindexer)
          reindexer.run()
          self._reindexed_experiments.append(reindexer.get_reindexed_experiments_filename())
          self._reindexed_reflections.append(reindexer.get_reindexed_reflections_filename())

      Debug.write('Running dials.two_theta_refine')

      self._output_cif = os.path.join(
        self.get_working_directory(),
        '%s_dials.two_theta_refine.cif' % self.get_xpid())
      self._output_mmcif = os.path.join(
        self.get_working_directory(),
        '%s_dials.two_theta_refine.mmcif' % self.get_xpid())
      self._output_correlation_plot = os.path.join(
        self.get_working_directory(),
        '%s_dials.two_theta_refine.png' % self.get_xpid())
      self._output_experiments = os.path.join(
        self.get_working_directory(),
        '%s_refined_cell.json' % self.get_xpid())

      self.clear_command_line()

      if self._reindexing_operator:
        for experiment in self._reindexed_experiments:
          self.add_command_line(experiment)
        for pickle in self._reindexed_reflections:
          self.add_command_line(pickle)
      else:
        for experiment in self._experiments:
          self.add_command_line(experiment)
        for pickle in self._pickles:
          self.add_command_line(pickle)
      self.add_command_line('output.cif=%s' % self._output_cif)
      self.add_command_line('output.mmcif=%s' % self._output_mmcif)
      if self._output_correlation_plot is not None:
        self.add_command_line(
          'output.correlation_plot.filename=%s' % self._output_correlation_plot)
      if self._output_experiments is not None:
        self.add_command_line(
          'output.experiments=%s' % self._output_experiments)
      if self._phil_file is not None:
        self.add_command_line('%s' %self._phil_file)

      self.start()
      self.close_wait()

      if not os.path.isfile(self._output_cif):
        Chatter.write(
          "TwoTheta refinement failed, see log file for more details:\n  %s" % self.get_log_file())
        raise RuntimeError, 'unit cell not refined'

      self.check_for_errors()

      from dxtbx.model.experiment.experiment_list import ExperimentListFactory
      experiments = ExperimentListFactory.from_json_file(self.get_output_experiments())
      self._crystal = experiments.crystals()[0]