Example #1
0
        def run(self, method):
            from xia2.Handlers.Streams import Debug

            Debug.write("Running dials.index")

            self.clear_command_line()
            for f in self._sweep_filenames:
                self.add_command_line(f)
            for f in self._spot_filenames:
                self.add_command_line(f)
            if len(self._sweep_filenames) > 1:
                self.add_command_line("auto_reduction.action=fix")
            self.add_command_line("indexing.method=%s" % method)
            nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
            self.set_cpu_threads(nproc)
            self.add_command_line("indexing.nproc=%i" % nproc)
            if PhilIndex.params.xia2.settings.small_molecule:
                self.add_command_line("filter_ice=false")
            if self._reflections_per_degree is not None:
                self.add_command_line("reflections_per_degree=%i" %
                                      self._reflections_per_degree)
            if self._fft3d_n_points is not None:
                self.add_command_line(
                    "fft3d.reciprocal_space_grid.n_points=%i" %
                    self._fft3d_n_points)
            if self._close_to_spindle_cutoff is not None:
                self.add_command_line("close_to_spindle_cutoff=%f" %
                                      self._close_to_spindle_cutoff)
            if self._outlier_algorithm:
                self.add_command_line("outlier.algorithm=%s" %
                                      self._outlier_algorithm)
            if self._max_cell:
                self.add_command_line("max_cell=%g" % self._max_cell)
            if self._max_cell_max_height_fraction is not None:
                self.add_command_line("max_height_fraction=%g" %
                                      self._max_cell_max_height_fraction)
            if self._min_cell:
                self.add_command_line("min_cell=%d" % self._min_cell)
            if self._histogram_binning is not None:
                self.add_command_line(
                    "max_cell_estimation.histogram_binning=%s" %
                    self._histogram_binning)
            if self._nearest_neighbor_percentile is not None:
                self.add_command_line(
                    "max_cell_estimation.nearest_neighbor_percentile=%s" %
                    self._nearest_neighbor_percentile)
            if self._d_min_start:
                self.add_command_line("d_min_start=%f" % self._d_min_start)
            if self._indxr_input_lattice is not None:
                from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number

                self._symm = lattice_to_spacegroup_number(
                    self._indxr_input_lattice)
                self.add_command_line("known_symmetry.space_group=%s" %
                                      self._symm)
            if self._indxr_input_cell is not None:
                self.add_command_line(
                    'known_symmetry.unit_cell="%s,%s,%s,%s,%s,%s"' %
                    self._indxr_input_cell)
            if self._maximum_spot_error:
                self.add_command_line("maximum_spot_error=%.f" %
                                      self._maximum_spot_error)
            if self._detector_fix:
                self.add_command_line("detector.fix=%s" % self._detector_fix)
            if self._beam_fix:
                self.add_command_line("beam.fix=%s" % self._beam_fix)
            if self._phil_file is not None:
                self.add_command_line("%s" % self._phil_file)

            self._experiment_filename = os.path.join(
                self.get_working_directory(),
                "%d_indexed.expt" % self.get_xpid())
            self._indexed_filename = os.path.join(
                self.get_working_directory(),
                "%d_indexed.refl" % self.get_xpid())
            self.add_command_line("output.experiments=%s" %
                                  self._experiment_filename)
            self.add_command_line("output.reflections=%s" %
                                  self._indexed_filename)

            self.start()
            self.close_wait()

            if not os.path.isfile(
                    self._experiment_filename) or not os.path.isfile(
                        self._indexed_filename):
                # Indexing failed
                with open(self.get_log_file(), "r") as fh:
                    if "No suitable lattice could be found" in fh.read():
                        raise libtbx.utils.Sorry(
                            "No suitable indexing solution could be found.\n\n"
                            "You can view the reciprocal space with:\n"
                            "dials.reciprocal_lattice_viewer %s" % " ".join(
                                os.path.normpath(
                                    os.path.join(self.get_working_directory(),
                                                 p))
                                for p in self._sweep_filenames +
                                self._spot_filenames))
                    else:
                        raise RuntimeError(
                            "dials.index failed, see log file for more details: %s"
                            % self.get_log_file())

            self.check_for_errors()

            for record in self.get_all_output():
                if "Too few reflections to parameterise" in record:
                    Debug.write(record.strip())

            from dials.array_family import flex
            from dxtbx.serialize import load

            self._experiment_list = load.experiment_list(
                self._experiment_filename)
            self._reflections = flex.reflection_table.from_file(
                self._indexed_filename)

            crystal = self._experiment_list.crystals()[0]
            self._p1_cell = crystal.get_unit_cell().parameters()

            refined_sel = self._reflections.get_flags(
                self._reflections.flags.used_in_refinement)
            refl = self._reflections.select(refined_sel)
            xc, yc, zc = refl["xyzcal.px"].parts()
            xo, yo, zo = refl["xyzobs.px.value"].parts()
            import math

            self._nref = refl.size()
            self._rmsd_x = math.sqrt(flex.mean(flex.pow2(xc - xo)))
            self._rmsd_y = math.sqrt(flex.mean(flex.pow2(yc - yo)))
            self._rmsd_z = math.sqrt(flex.mean(flex.pow2(zc - zo)))
        def run(self, ignore_errors=False):
            """Run idxref."""

            # image_header = self.get_header()

            ## crank through the header dictionary and replace incorrect
            ## information with updated values through the indexer
            ## interface if available...

            ## need to add distance, wavelength - that should be enough...

            # if self.get_distance():
            # image_header['distance'] = self.get_distance()

            # if self.get_wavelength():
            # image_header['wavelength'] = self.get_wavelength()

            # if self.get_two_theta():
            # image_header['two_theta'] = self.get_two_theta()

            header = imageset_to_xds(
                self.get_imageset(),
                refined_beam_vector=self._refined_beam_vector,
                refined_rotation_axis=self._refined_rotation_axis,
                refined_distance=self._refined_distance,
            )

            xds_inp = open(os.path.join(self.get_working_directory(), "XDS.INP"), "w")

            # what are we doing?
            xds_inp.write("JOB=IDXREF\n")
            xds_inp.write("MAXIMUM_NUMBER_OF_PROCESSORS=%d\n" % self._parallel)

            # FIXME this needs to be calculated from the beam centre...

            if self._refined_origin:
                xds_inp.write("ORGX=%f ORGY=%f\n" % tuple(self._refined_origin))
            else:
                xds_inp.write("ORGX=%f ORGY=%f\n" % tuple(self._org))

            # FIXME in here make sure sweep is wider than 5 degrees
            # before specifying AXIS: if <= 5 degrees replace AXIS with
            # nothing - base this on the maximum possible angular separation

            min_frame = self._spot_range[0][0]
            max_frame = self._spot_range[-1][1]

            refine_params = [p for p in self._params.refine]

            phi_width = self.get_phi_width()
            if "AXIS" in refine_params and (max_frame - min_frame) * phi_width < 5.0:
                refine_params.remove("AXIS")

            xds_inp.write("REFINE(IDXREF)=%s\n" % " ".join(refine_params))

            if self._starting_frame and self._starting_angle:
                xds_inp.write("STARTING_FRAME=%d\n" % self._starting_frame)
                xds_inp.write("STARTING_ANGLE=%f\n" % self._starting_angle)

            # FIXME this looks like a potential bug - what will
            # happen if the input lattice has not been set??
            if self._indxr_input_cell:
                self._cell = self._indxr_input_cell
            if self._indxr_input_lattice:
                self._symm = lattice_to_spacegroup_number(self._indxr_input_lattice)

            if self._cell:
                xds_inp.write("SPACE_GROUP_NUMBER=%d\n" % self._symm)
                cell_format = "%6.2f %6.2f %6.2f %6.2f %6.2f %6.2f"
                xds_inp.write("UNIT_CELL_CONSTANTS=%s\n" % cell_format % self._cell)

            if self._a_axis:
                xds_inp.write("UNIT_CELL_A-AXIS=%.2f %.2f %.2f\n" % tuple(self._a_axis))

            if self._b_axis:
                xds_inp.write("UNIT_CELL_B-AXIS=%.2f %.2f %.2f\n" % tuple(self._b_axis))

            if self._c_axis:
                xds_inp.write("UNIT_CELL_C-AXIS=%.2f %.2f %.2f\n" % tuple(self._c_axis))

            for record in header:
                xds_inp.write("%s\n" % record)

            name_template = template_to_xds(
                os.path.join(self.get_directory(), self.get_template())
            )

            record = "NAME_TEMPLATE_OF_DATA_FRAMES=%s\n" % name_template

            xds_inp.write(record)

            xds_inp.write("DATA_RANGE=%d %d\n" % self._data_range)
            for spot_range in self._spot_range:
                xds_inp.write("SPOT_RANGE=%d %d\n" % spot_range)
            xds_inp.write("BACKGROUND_RANGE=%d %d\n" % self._background_range)

            xds_inp.close()

            # copy the input file...
            shutil.copyfile(
                os.path.join(self.get_working_directory(), "XDS.INP"),
                os.path.join(
                    self.get_working_directory(), "%d_IDXREF.INP" % self.get_xpid()
                ),
            )

            # write the input data files...
            for file_name in self._input_data_files_list:
                src = self._input_data_files[file_name]
                dst = os.path.join(self.get_working_directory(), file_name)
                if src != dst:
                    shutil.copyfile(src, dst)

            self.start()
            self.close_wait()

            xds_check_version_supported(self.get_all_output())
            if not ignore_errors:
                xds_check_error(self.get_all_output())

            # If xds_check_error detects any errors it will raise an exception
            # The caller can then continue using the run_continue_from_error()
            # function. If XDS does not throw any errors we just plow on.

            return self.continue_from_error()
        def continue_from_error(self):
            # copy the LP file
            shutil.copyfile(
                os.path.join(self.get_working_directory(), "IDXREF.LP"),
                os.path.join(
                    self.get_working_directory(), "%d_IDXREF.LP" % self.get_xpid()
                ),
            )

            # parse the output
            lp = open(
                os.path.join(self.get_working_directory(), "IDXREF.LP"), "r"
            ).readlines()

            self._fraction_rmsd_rmsphi = _parse_idxref_lp_quality(lp)

            self._idxref_data = _parse_idxref_lp(lp)

            if not self._idxref_data:
                raise RuntimeError("indexing failed")

            st = _parse_idxref_lp_subtree(lp)

            if 2 in st:

                if st[2] > st[1] / 10.0:
                    Debug.write("Look closely at autoindexing solution!")
                    self._index_tree_problem = True
                    for j in sorted(st):
                        Debug.write("%2d: %5d" % (j, st[j]))

            # print out some (perhaps dire) warnings about the beam centre
            # if there is really any ambiguity...

            origins = _parse_idxref_index_origin(lp)

            assert (0, 0, 0) in origins

            quality_0 = origins[(0, 0, 0)][0]

            alternatives = []

            for hkl in origins:
                if hkl == (0, 0, 0):
                    continue
                if origins[hkl][0] < 4 * quality_0:
                    quality, delta, beam_x, beam_y = origins[hkl]
                    alternatives.append(
                        (hkl[0], hkl[1], hkl[2], quality, beam_x, beam_y)
                    )

            if alternatives:
                Debug.write("Alternative indexing possible:")
                for alternative in alternatives:
                    Debug.write("... %3d %3d %3d %4.1f %6.1f %6.1f" % alternative)

            # New algorithm in here - now use iotbx.lattice_symmetry with the
            # P1 indexing solution (solution #1) to determine the list of
            # allowable solutions - only consider those lattices in this
            # allowed list (unless we have user input)

            from xia2.Wrappers.Phenix.LatticeSymmetry import LatticeSymmetry

            ls = LatticeSymmetry()
            ls.set_lattice("aP")
            ls.set_cell(tuple(self._idxref_data[44]["cell"]))
            ls.generate()

            allowed_lattices = ls.get_lattices()

            for j in range(1, 45):
                if j not in self._idxref_data:
                    continue
                data = self._idxref_data[j]
                lattice = data["lattice"]
                fit = data["fit"]
                cell = data["cell"]
                mosaic = data["mosaic"]

                if self._symm and self._cell and self._indxr_user_input_lattice:

                    if (
                        self._compare_cell(self._cell, cell)
                        and lattice_to_spacegroup_number(lattice) == self._symm
                    ):
                        if lattice in self._indexing_solutions:
                            if self._indexing_solutions[lattice]["goodness"] < fit:
                                continue

                        self._indexing_solutions[lattice] = {
                            "goodness": fit,
                            "cell": cell,
                        }

                else:
                    if lattice in allowed_lattices or (self._symm and fit < 200.0):
                        # bug 2417 - if we have an input lattice then we
                        # don't want to include anything higher symmetry
                        # in the results table...

                        if self._symm:
                            if lattice_to_spacegroup_number(lattice) > self._symm:
                                Debug.write(
                                    "Ignoring solution with lattice %s" % lattice
                                )
                                continue

                        if lattice in self._indexing_solutions:
                            if self._indexing_solutions[lattice]["goodness"] < fit:
                                continue

                        self._indexing_solutions[lattice] = {
                            "goodness": fit,
                            "cell": cell,
                        }

            # postprocess this list, to remove lattice solutions which are
            # lower symmetry but higher penalty than the putative correct
            # one, if self._symm is set...

            if self._symm:
                assert self._indexing_solutions, (
                    "No remaining indexing solutions (%s, %s)"
                    % (s2l(self._symm), self._symm)
                )
            else:
                assert self._indexing_solutions, "No remaining indexing solutions"

            if self._symm:
                max_p = 2.0 * self._indexing_solutions[s2l(self._symm)]["goodness"]
                to_remove = []
                for lattice in self._indexing_solutions:
                    if self._indexing_solutions[lattice]["goodness"] > max_p:
                        to_remove.append(lattice)
                for lattice in to_remove:
                    Debug.write("Ignoring solution with lattice %s" % lattice)
                    del self._indexing_solutions[lattice]

            # get the highest symmetry "acceptable" solution

            items = [
                (k, self._indexing_solutions[k]["cell"])
                for k in self._indexing_solutions
            ]

            # if there was a preassigned cell and symmetry return now
            # with everything done, else select the "top" solution and
            # reindex, resetting the input cell and symmetry.

            if self._cell:

                # select the solution which matches the input unit cell
                # actually after the changes above this should now be the
                # only solution in the table..

                Debug.write(
                    "Target unit cell: %.2f %.2f %.2f %.2f %.2f %.2f" % self._cell
                )

                for l in items:
                    if lattice_to_spacegroup_number(l[0]) == self._symm:
                        # this should be the correct solution...
                        # check the unit cell...
                        cell = l[1]

                        cell_str = "%.2f %.2f %.2f %.2f %.2f %.2f" % cell
                        Debug.write("Chosen unit cell: %s" % cell_str)

                        self._indxr_lattice = l[0]
                        self._indxr_cell = l[1]
                        self._indxr_mosaic = mosaic

            else:

                # select the top solution as the input cell and reset the
                # "indexing done" flag

                sorted_list = SortLattices(items)
                #       print sorted_list

                self._symm = lattice_to_spacegroup_number(sorted_list[0][0])
                self._cell = sorted_list[0][1]

                return False

            # get the refined distance &c.

            beam, distance = _parse_idxref_lp_distance_etc(lp)

            self._refined_beam = beam
            self._refined_distance = distance

            # gather the output files

            for file in self._output_data_files_list:
                self._output_data_files[file] = os.path.join(
                    self.get_working_directory(), file
                )

            return True
Example #4
0
  def _integrate_finish(self):
    '''Finish off the integration by running correct.'''

    # first run the postrefinement etc with spacegroup P1
    # and the current unit cell - this will be used to
    # obtain a benchmark rmsd in pixels / phi and also
    # cell deviations (this is working towards spotting bad
    # indexing solutions) - only do this if we have no
    # reindex matrix... and no postrefined cell...

    p1_deviations = None

    # fix for bug # 3264 -
    # if we have not run integration with refined parameters, make it so...
    # erm? shouldn't this therefore return if this is the principle, or
    # set the flag after we have tested the lattice?

    if not self._xds_data_files.has_key('GXPARM.XDS') and \
      PhilIndex.params.xds.integrate.reintegrate:
      Debug.write(
          'Resetting integrater, to ensure refined orientation is used')
      self.set_integrater_done(False)

    if not self.get_integrater_reindex_matrix() and not self._intgr_cell \
           and not Flags.get_no_lattice_test() and \
           not self.get_integrater_sweep().get_user_lattice():
      correct = self.Correct()

      correct.set_data_range(self._intgr_wedge[0],
                             self._intgr_wedge[1])

      if self.get_polarization() > 0.0:
        correct.set_polarization(self.get_polarization())

      # FIXME should this be using the correctly transformed
      # cell or are the results ok without it?!

      correct.set_spacegroup_number(1)
      correct.set_cell(self._intgr_refiner_cell)

      correct.run()

      # record the log file -

      pname, xname, dname = self.get_integrater_project_info()
      sweep = self.get_integrater_sweep_name()
      FileHandler.record_log_file('%s %s %s %s CORRECT' % \
                                  (pname, xname, dname, sweep),
                                  os.path.join(
          self.get_working_directory(),
          'CORRECT.LP'))

      FileHandler.record_more_data_file(
          '%s %s %s %s CORRECT' % (pname, xname, dname, sweep),
          os.path.join(self.get_working_directory(), 'XDS_ASCII.HKL'))

      cell = correct.get_result('cell')
      cell_esd = correct.get_result('cell_esd')

      Debug.write('Postrefinement in P1 results:')
      Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
                  tuple(cell))
      Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
                  tuple(cell_esd))
      Debug.write('Deviations: %.2f pixels %.2f degrees' % \
                  (correct.get_result('rmsd_pixel'),
                   correct.get_result('rmsd_phi')))

      p1_deviations = (correct.get_result('rmsd_pixel'),
                       correct.get_result('rmsd_phi'))

    # next run the postrefinement etc with the given
    # cell / lattice - this will be the assumed result...

    correct = self.Correct()

    correct.set_data_range(self._intgr_wedge[0],
                           self._intgr_wedge[1])

    if self.get_polarization() > 0.0:
      correct.set_polarization(self.get_polarization())

    # BUG # 2695 probably comes from here - need to check...
    # if the pointless interface comes back with a different
    # crystal setting then the unit cell stored in self._intgr_cell
    # needs to be set to None...

    if self.get_integrater_spacegroup_number():
      correct.set_spacegroup_number(
          self.get_integrater_spacegroup_number())
      if not self._intgr_cell:
        raise RuntimeError, 'no unit cell to recycle'
      correct.set_cell(self._intgr_cell)

    # BUG # 3113 - new version of XDS will try and figure the
    # best spacegroup out from the intensities (and get it wrong!)
    # unless we set the spacegroup and cell explicitly

    if not self.get_integrater_spacegroup_number():
      cell = self._intgr_refiner_cell
      lattice = self._intgr_refiner.get_refiner_lattice()
      spacegroup_number = lattice_to_spacegroup_number(lattice)

      # this should not prevent the postrefinement from
      # working correctly, else what is above would not
      # work correctly (the postrefinement test)

      correct.set_spacegroup_number(spacegroup_number)
      correct.set_cell(cell)

      Debug.write('Setting spacegroup to: %d' % spacegroup_number)
      Debug.write(
        'Setting cell to: %.2f %.2f %.2f %.2f %.2f %.2f' % tuple(cell))

    if self.get_integrater_reindex_matrix():

      # bug! if the lattice is not primitive the values in this
      # reindex matrix need to be multiplied by a constant which
      # depends on the Bravais lattice centering.

      lattice = self._intgr_refiner.get_refiner_lattice()

      import scitbx.matrix
      matrix = self.get_integrater_reindex_matrix()
      matrix = scitbx.matrix.sqr(matrix).transpose().elems
      matrix = r_to_rt(matrix)

      if lattice[1] == 'P':
        mult = 1
      elif lattice[1] == 'C' or lattice[1] == 'I':
        mult = 2
      elif lattice[1] == 'R':
        mult = 3
      elif lattice[1] == 'F':
        mult = 4
      else:
        raise RuntimeError, 'unknown multiplier for lattice %s' % \
              lattice

      Debug.write('REIDX multiplier for lattice %s: %d' % \
                  (lattice, mult))

      mult_matrix = [mult * m for m in matrix]

      Debug.write('REIDX set to %d %d %d %d %d %d %d %d %d %d %d %d' % \
                  tuple(mult_matrix))
      correct.set_reindex_matrix(mult_matrix)

    correct.run()

    # erm. just to be sure
    if self.get_integrater_reindex_matrix() and \
           correct.get_reindex_used():
      raise RuntimeError, 'Reindex panic!'

    # get the reindex operation used, which may be useful if none was
    # set but XDS decided to apply one, e.g. #419.

    if not self.get_integrater_reindex_matrix() and \
           correct.get_reindex_used():
      # convert this reindex operation to h, k, l form: n.b. this
      # will involve dividing through by the lattice centring multiplier

      matrix = rt_to_r(correct.get_reindex_used())
      import scitbx.matrix
      matrix = scitbx.matrix.sqr(matrix).transpose().elems

      lattice = self._intgr_refiner.get_refiner_lattice()

      if lattice[1] == 'P':
        mult = 1.0
      elif lattice[1] == 'C' or lattice[1] == 'I':
        mult = 2.0
      elif lattice[1] == 'R':
        mult = 3.0
      elif lattice[1] == 'F':
        mult = 4.0

      matrix = [m / mult for m in matrix]

      reindex_op = mat_to_symop(matrix)

      # assign this to self: will this reset?! make for a leaky
      # abstraction and just assign this...

      # self.set_integrater_reindex_operator(reindex)

      self._intgr_reindex_operator = reindex_op


    # record the log file -

    pname, xname, dname = self.get_integrater_project_info()
    sweep = self.get_integrater_sweep_name()
    FileHandler.record_log_file('%s %s %s %s CORRECT' % \
                                (pname, xname, dname, sweep),
                                os.path.join(self.get_working_directory(),
                                             'CORRECT.LP'))

    # should get some interesting stuff from the XDS correct file
    # here, for instance the resolution range to use in integration
    # (which should be fed back if not fast) and so on...

    self._intgr_corrected_hklout = os.path.join(self.get_working_directory(),
                                'XDS_ASCII.HKL')

    # also record the batch range - needed for the analysis of the
    # radiation damage in chef...

    self._intgr_batches_out = (self._intgr_wedge[0],
                               self._intgr_wedge[1])

    # FIXME perhaps I should also feedback the GXPARM file here??
    for file in ['GXPARM.XDS']:
      self._xds_data_files[file] = correct.get_output_data_file(file)

    # record the postrefined cell parameters
    self._intgr_cell = correct.get_result('cell')
    self._intgr_n_ref = correct.get_result('n_ref')

    Debug.write('Postrefinement in "correct" spacegroup results:')
    Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
                tuple(correct.get_result('cell')))
    Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
                tuple(correct.get_result('cell_esd')))
    Debug.write('Deviations: %.2f pixels %.2f degrees' % \
                (correct.get_result('rmsd_pixel'),
                 correct.get_result('rmsd_phi')))

    Debug.write('Error correction parameters: A=%.3f B=%.3f' % \
                correct.get_result('sdcorrection'))

    # compute misorientation of axes

    xparm_file = os.path.join(self.get_working_directory(), 'GXPARM.XDS')

    from iotbx.xds import xparm
    handle = xparm.reader()
    handle.read_file(xparm_file)

    rotn = handle.rotation_axis
    beam = handle.beam_vector

    dot = sum([rotn[j] * beam[j] for j in range(3)])
    r = math.sqrt(sum([rotn[j] * rotn[j] for j in range(3)]))
    b = math.sqrt(sum([beam[j] * beam[j] for j in range(3)]))

    rtod = 180.0 / math.pi

    angle = rtod * math.fabs(0.5 * math.pi - math.acos(dot / (r * b)))

    Debug.write('Axis misalignment %.2f degrees' % angle)

    correct_deviations = (correct.get_result('rmsd_pixel'),
                          correct.get_result('rmsd_phi'))

    if p1_deviations:
      # compare and reject if both > 50% higher - though adding a little
      # flexibility - 0.5 pixel / osc width slack.

      pixel = p1_deviations[0]
      phi = math.sqrt(0.05 * 0.05 + \
                      p1_deviations[1] * p1_deviations[1])

      threshold = Flags.get_rejection_threshold()

      Debug.write('RMSD ratio: %.2f' % (correct_deviations[0] / pixel))
      Debug.write('RMSPhi ratio: %.2f' % (correct_deviations[1] / phi))

      if correct_deviations[0] / pixel > threshold and \
             correct_deviations[1] / phi > threshold:

        Chatter.write(
        'Eliminating this indexing solution as postrefinement')
        Chatter.write(
        'deviations rather high relative to triclinic')
        raise BadLatticeError, \
              'high relative deviations in postrefinement'

    if not Flags.get_quick() and Flags.get_remove():
      # check for alien reflections and perhaps recycle - removing them
      if len(correct.get_remove()) > 0:

        correct_remove = correct.get_remove()
        current_remove = []
        final_remove = []

        # first ensure that there are no duplicate entries...
        if os.path.exists(os.path.join(
            self.get_working_directory(),
            'REMOVE.HKL')):
          for line in open(os.path.join(
              self.get_working_directory(),
              'REMOVE.HKL'), 'r').readlines():
            h, k, l = map(int, line.split()[:3])
            z = float(line.split()[3])

            if not (h, k, l, z) in current_remove:
              current_remove.append((h, k, l, z))

          for c in correct_remove:
            if c in current_remove:
              continue
            final_remove.append(c)

          Debug.write(
              '%d alien reflections are already removed' % \
              (len(correct_remove) - len(final_remove)))
        else:
          # we want to remove all of the new dodgy reflections
          final_remove = correct_remove

        remove_hkl = open(os.path.join(
            self.get_working_directory(),
            'REMOVE.HKL'), 'w')

        z_min = Flags.get_z_min()
        rejected = 0

        # write in the old reflections
        for remove in current_remove:
          z = remove[3]
          if z >= z_min:
            remove_hkl.write('%d %d %d %f\n' % remove)
          else:
            rejected += 1
        Debug.write('Wrote %d old reflections to REMOVE.HKL' % \
                    (len(current_remove) - rejected))
        Debug.write('Rejected %d as z < %f' % \
                    (rejected, z_min))

        # and the new reflections
        rejected = 0
        used = 0
        for remove in final_remove:
          z = remove[3]
          if z >= z_min:
            used += 1
            remove_hkl.write('%d %d %d %f\n' % remove)
          else:
            rejected += 1
        Debug.write('Wrote %d new reflections to REMOVE.HKL' % \
                    (len(final_remove) - rejected))
        Debug.write('Rejected %d as z < %f' % \
                    (rejected, z_min))

        remove_hkl.close()

        # we want to rerun the finishing step so...
        # unless we have added no new reflections... or unless we
        # have not confirmed the point group (see SCI-398)

        if used and self.get_integrater_reindex_matrix():
          self.set_integrater_finish_done(False)

    else:
      Debug.write(
          'Going quickly so not removing %d outlier reflections...' % \
          len(correct.get_remove()))

    # Convert INTEGRATE.HKL to MTZ format and reapply any reindexing operations
    # spacegroup changes to allow use with CCP4 / Aimless for scaling

    integrate_hkl = os.path.join(
      self.get_working_directory(), 'INTEGRATE.HKL')

    hklout = os.path.splitext(integrate_hkl)[0] + ".mtz"
    self._factory.set_working_directory(self.get_working_directory())
    pointless = self._factory.Pointless()
    pointless.set_xdsin(integrate_hkl)
    pointless.set_hklout(hklout)
    pointless.xds_to_mtz()

    integrate_mtz = hklout

    if self.get_integrater_reindex_operator() or \
       self.get_integrater_spacegroup_number():

      Debug.write('Reindexing things to MTZ')

      reindex = Reindex()
      reindex.set_working_directory(self.get_working_directory())
      auto_logfiler(reindex)

      if self.get_integrater_reindex_operator():
        reindex.set_operator(self.get_integrater_reindex_operator())

      if self.get_integrater_spacegroup_number():
        reindex.set_spacegroup(self.get_integrater_spacegroup_number())

      hklout = '%s_reindex.mtz' % os.path.splitext(integrate_mtz)[0]

      reindex.set_hklin(integrate_mtz)
      reindex.set_hklout(hklout)
      reindex.reindex()
      integrate_mtz = hklout

    return integrate_mtz
Example #5
0
    def _index(self):
        if PhilIndex.params.dials.index.method in (libtbx.Auto, None):
            if self._indxr_input_cell is not None:
                indexer = self._do_indexing("real_space_grid_search")
            else:
                try:
                    indexer_fft3d = self._do_indexing(method="fft3d")
                    nref_3d, rmsd_3d = indexer_fft3d.get_nref_rmsds()
                except Exception as e:
                    nref_3d = None
                    rmsd_3d = None
                try:
                    indexer_fft1d = self._do_indexing(method="fft1d")
                    nref_1d, rmsd_1d = indexer_fft1d.get_nref_rmsds()
                except Exception as e:
                    nref_1d = None
                    rmsd_1d = None

                if (nref_1d is not None and nref_3d is None or
                    (nref_1d > nref_3d and rmsd_1d[0] < rmsd_3d[0]
                     and rmsd_1d[1] < rmsd_3d[1] and rmsd_1d[2] < rmsd_3d[2])):
                    indexer = indexer_fft1d
                elif nref_3d is not None:
                    indexer = indexer_fft3d
                else:
                    raise RuntimeError(e)

        else:
            indexer = self._do_indexing(
                method=PhilIndex.params.dials.index.method)

        # not strictly the P1 cell, rather the cell that was used in indexing
        self._p1_cell = indexer._p1_cell
        self.set_indexer_payload("indexed_filename",
                                 indexer.get_indexed_filename())

        from cctbx.sgtbx import bravais_types
        from dxtbx.serialize import load

        indexed_file = indexer.get_indexed_filename()
        indexed_experiments = indexer.get_experiments_filename()

        fast_mode = PhilIndex.params.dials.fast_mode
        trust_beam_centre = PhilIndex.params.xia2.settings.trust_beam_centre
        multi_sweep_indexing = PhilIndex.params.xia2.settings.multi_sweep_indexing == True

        if not (trust_beam_centre or fast_mode or multi_sweep_indexing):
            checksym = self.CheckIndexingSymmetry()
            checksym.set_experiments_filename(indexed_experiments)
            checksym.set_indexed_filename(indexed_file)
            checksym.set_grid_search_scope(1)
            checksym.run()
            hkl_offset = checksym.get_hkl_offset()
            Debug.write("hkl_offset: %s" % str(hkl_offset))
            if hkl_offset is not None and hkl_offset != (0, 0, 0):
                reindex = self.Reindex()
                reindex.set_hkl_offset(hkl_offset)
                reindex.set_indexed_filename(indexed_file)
                reindex.run()
                indexed_file = reindex.get_reindexed_reflections_filename()

                # do some scan-static refinement - run twice, first without outlier
                # rejection as the model is too far from reality to do a sensible job of
                # outlier rejection
                refiner = self.Refine()
                refiner.set_experiments_filename(indexed_experiments)
                refiner.set_indexed_filename(
                    reindex.get_reindexed_reflections_filename())
                refiner.set_outlier_algorithm(None)
                refiner.run()
                indexed_experiments = refiner.get_refined_experiments_filename(
                )

                # now again with outlier rejection (possibly)
                refiner = self.Refine()
                refiner.set_experiments_filename(indexed_experiments)
                refiner.set_indexed_filename(indexed_file)
                refiner.run()
                indexed_experiments = refiner.get_refined_experiments_filename(
                )

        if self._indxr_input_lattice is None:

            # FIXME in here should respect the input unit cell and lattice if provided

            # FIXME from this (i) populate the helper table,
            # (ii) try to avoid re-running the indexing
            # step if we eliminate a solution as we have all of the refined results
            # already available.

            rbs = self.RefineBravaisSettings()
            rbs.set_experiments_filename(indexed_experiments)
            rbs.set_indexed_filename(indexed_file)
            if PhilIndex.params.dials.fix_geometry:
                rbs.set_detector_fix('all')
                rbs.set_beam_fix('all')

            FileHandler.record_log_file(
                '%s LATTICE' % self.get_indexer_full_name(),
                rbs.get_log_file())
            rbs.run()

            from cctbx import crystal, sgtbx

            for k in sorted(rbs.get_bravais_summary()):
                summary = rbs.get_bravais_summary()[k]

                # FIXME need to do this better - for the moment only accept lattices
                # where R.M.S. deviation is less than twice P1 R.M.S. deviation.

                if self._indxr_input_lattice is None:
                    if not summary['recommended']:
                        continue

                experiments = load.experiment_list(summary['experiments_file'],
                                                   check_format=False)
                cryst = experiments.crystals()[0]
                cs = crystal.symmetry(unit_cell=cryst.get_unit_cell(),
                                      space_group=cryst.get_space_group())
                cb_op_best_to_ref = cs.change_of_basis_op_to_reference_setting(
                )
                cs_reference = cs.change_basis(cb_op_best_to_ref)
                lattice = str(
                    bravais_types.bravais_lattice(
                        group=cs_reference.space_group()))
                cb_op = cb_op_best_to_ref * sgtbx.change_of_basis_op(
                    str(summary['cb_op']))

                self._solutions[k] = {
                    'number': k,
                    'mosaic': 0.0,
                    'metric': summary['max_angular_difference'],
                    'rmsd': summary['rmsd'],
                    'nspots': summary['nspots'],
                    'lattice': lattice,
                    'cell': cs_reference.unit_cell().parameters(),
                    'experiments_file': summary['experiments_file'],
                    'cb_op': str(cb_op)
                }

            self._solution = self.get_solution()
            self._indxr_lattice = self._solution['lattice']

            for solution in self._solutions.keys():
                lattice = self._solutions[solution]['lattice']
                if (self._indxr_input_lattice is not None
                        and self._indxr_input_lattice != lattice):
                    continue
                if lattice in self._indxr_other_lattice_cell:
                    if self._indxr_other_lattice_cell[lattice]['metric'] < \
                      self._solutions[solution]['metric']:
                        continue

                self._indxr_other_lattice_cell[lattice] = {
                    'metric': self._solutions[solution]['metric'],
                    'cell': self._solutions[solution]['cell']
                }

            self._indxr_mosaic = self._solution['mosaic']

            experiment_list = load.experiment_list(
                self._solution['experiments_file'])
            self.set_indexer_experiment_list(experiment_list)

            # reindex the output experiments list to the reference setting
            # (from the best cell/conventional setting)
            cb_op_to_ref = experiment_list.crystals()[0].get_space_group().info()\
              .change_of_basis_op_to_reference_setting()
            reindex = self.Reindex()
            reindex.set_experiments_filename(
                self._solution['experiments_file'])
            reindex.set_cb_op(cb_op_to_ref)
            reindex.set_space_group(
                str(lattice_to_spacegroup_number(self._solution['lattice'])))
            reindex.run()
            experiments_file = reindex.get_reindexed_experiments_filename()
            experiment_list = load.experiment_list(experiments_file)
            self.set_indexer_experiment_list(experiment_list)
            self.set_indexer_payload("experiments_filename", experiments_file)

            # reindex the output reflection list to this solution
            reindex = self.Reindex()
            reindex.set_indexed_filename(indexed_file)
            reindex.set_cb_op(self._solution['cb_op'])
            reindex.set_space_group(
                str(lattice_to_spacegroup_number(self._solution['lattice'])))
            reindex.run()
            indexed_file = reindex.get_reindexed_reflections_filename()
            self.set_indexer_payload("indexed_filename", indexed_file)

        else:
            experiment_list = load.experiment_list(indexed_experiments)
            self.set_indexer_experiment_list(experiment_list)
            self.set_indexer_payload("experiments_filename",
                                     indexed_experiments)

            cryst = experiment_list.crystals()[0]
            lattice = str(
                bravais_types.bravais_lattice(group=cryst.get_space_group()))
            self._indxr_lattice = lattice
            self._solutions = {}
            self._solutions[0] = {
                'number': 0,
                'mosaic': 0.0,
                'metric': -1,
                'rmsd': -1,
                'nspots': -1,
                'lattice': lattice,
                'cell': cryst.get_unit_cell().parameters(),
                'experiments_file': indexed_experiments,
                'cb_op': 'a,b,c'
            }

            self._indxr_other_lattice_cell[lattice] = {
                'metric': self._solutions[0]['metric'],
                'cell': self._solutions[0]['cell']
            }

        return
Example #6
0
    def continue_from_error(self):
      # copy the LP file
      shutil.copyfile(os.path.join(self.get_working_directory(),
                                   'IDXREF.LP'),
                      os.path.join(self.get_working_directory(),
                                   '%d_IDXREF.LP' % self.get_xpid()))

      # parse the output
      lp = open(os.path.join(
          self.get_working_directory(), 'IDXREF.LP'), 'r').readlines()

      self._fraction_rmsd_rmsphi = _parse_idxref_lp_quality(lp)

      self._idxref_data = _parse_idxref_lp(lp)

      if not self._idxref_data:
        raise RuntimeError, 'indexing failed'

      st = _parse_idxref_lp_subtree(lp)

      if 2 in st:

        if st[2] > st[1] / 10.0:
          Debug.write('Look closely at autoindexing solution!')
          self._index_tree_problem = True
          for j in sorted(st):
            Debug.write('%2d: %5d' % (j, st[j]))

      # print out some (perhaps dire) warnings about the beam centre
      # if there is really any ambiguity...

      origins = _parse_idxref_index_origin(lp)

      assert((0, 0, 0) in origins)

      quality_0 = origins[(0, 0, 0)][0]

      alternatives = []

      for hkl in origins:
        if hkl == (0, 0, 0):
          continue
        if origins[hkl][0] < 4 * quality_0:
          quality, delta, beam_x, beam_y = origins[hkl]
          alternatives.append((hkl[0], hkl[1], hkl[2],
                               quality, beam_x, beam_y))

      if alternatives:
        Debug.write('Alternative indexing possible:')
        for alternative in alternatives:
          Debug.write('... %3d %3d %3d %4.1f %6.1f %6.1f' % \
                      alternative)

      # New algorithm in here - now use iotbx.lattice_symmetry with the
      # P1 indexing solution (solution #1) to determine the list of
      # allowable solutions - only consider those lattices in this
      # allowed list (unless we have user input)

      from xia2.Wrappers.Phenix.LatticeSymmetry import LatticeSymmetry
      ls = LatticeSymmetry()
      ls.set_lattice('aP')
      ls.set_cell(tuple(self._idxref_data[44]['cell']))
      ls.generate()

      allowed_lattices = ls.get_lattices()

      for j in range(1, 45):
        if j not in self._idxref_data:
          continue
        data = self._idxref_data[j]
        lattice = data['lattice']
        fit = data['fit']
        cell = data['cell']
        mosaic = data['mosaic']
        reidx = data['reidx']

        if self._symm and self._cell and \
               self._indxr_user_input_lattice:

          if self._compare_cell(self._cell, cell) and \
                 lattice_to_spacegroup_number(lattice) == self._symm:
            if lattice in self._indexing_solutions:
              if self._indexing_solutions[lattice][
                  'goodness'] < fit:
                continue

            self._indexing_solutions[lattice] = {
                'goodness':fit,
                'cell':cell}

        else:
          if lattice in allowed_lattices or \
              (self._symm and fit < 200.0):
            # bug 2417 - if we have an input lattice then we
            # don't want to include anything higher symmetry
            # in the results table...

            if self._symm:
              if lattice_to_spacegroup_number(lattice) \
                     > self._symm:
                Debug.write(
                    'Ignoring solution with lattice %s' % \
                    lattice)
                continue

            if lattice in self._indexing_solutions:
              if self._indexing_solutions[lattice][
                  'goodness'] < fit:
                continue

            self._indexing_solutions[lattice] = {
                'goodness':fit,
                'cell':cell}

      # postprocess this list, to remove lattice solutions which are
      # lower symmetry but higher penalty than the putative correct
      # one, if self._symm is set...

      if self._symm:
        assert len(self._indexing_solutions) > 0, "No remaining indexing solutions (%s, %s)" % (s2l(self._symm), self._symm);
      else:
        assert len(self._indexing_solutions) > 0, "No remaining indexing solutions"

#     print self._indexing_solutions
      if self._symm:
        max_p = 2.0 * self._indexing_solutions[
            s2l(self._symm)]['goodness']
        to_remove = []
        for lattice in self._indexing_solutions:
          if self._indexing_solutions[lattice]['goodness'] > max_p:
            to_remove.append(lattice)
        for lattice in to_remove:
          Debug.write('Ignoring solution with lattice %s' % \
                      lattice)
          del(self._indexing_solutions[lattice])


      # get the highest symmetry "acceptable" solution

      list = [(k, self._indexing_solutions[k]['cell']) for k in \
              self._indexing_solutions.keys()]

      # if there was a preassigned cell and symmetry return now
      # with everything done, else select the "top" solution and
      # reindex, resetting the input cell and symmetry.

      if self._cell:

        # select the solution which matches the input unit cell
        # actually after the changes above this should now be the
        # only solution in the table..

        Debug.write(
            'Target unit cell: %.2f %.2f %.2f %.2f %.2f %.2f' % \
            self._cell)

        for l in list:
          if lattice_to_spacegroup_number(l[0]) == self._symm:
            # this should be the correct solution...
            # check the unit cell...
            cell = l[1]

            if self._compare_cell(self._cell, cell) or True:

              cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % cell
              Debug.write(
              'Chosen unit cell: %s' % cell_str)

              self._indxr_lattice = l[0]
              self._indxr_cell = l[1]
              self._indxr_mosaic = mosaic

            else:

              cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % cell
              Debug.write(
              'Ignoring unit cell: %s' % cell_str)

      else:

        # select the top solution as the input cell and reset the
        # "indexing done" flag

        sorted_list = SortLattices(list)
#       print sorted_list

        self._symm = lattice_to_spacegroup_number(sorted_list[0][0])
        self._cell = sorted_list[0][1]

        return False

      # get the refined distance &c.

      beam, distance = _parse_idxref_lp_distance_etc(lp)

      self._refined_beam = beam
      self._refined_distance = distance

      # gather the output files

      for file in self._output_data_files_list:
        self._output_data_files[file] = os.path.join(
          self.get_working_directory(), file)

      return True
Example #7
0
    def _index(self):
        '''Actually index the diffraction pattern. Note well that
    this is not going to compute the matrix...'''

        # acknowledge this program

        Citations.cite('labelit')
        Citations.cite('distl')

        #self.reset()

        _images = []
        for i in self._indxr_images:
            for j in i:
                if not j in _images:
                    _images.append(j)

        _images.sort()

        images_str = '%d' % _images[0]
        for i in _images[1:]:
            images_str += ', %d' % i

        cell_str = None
        if self._indxr_input_cell:
            cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
                        self._indxr_input_cell

        if self._indxr_sweep_name:

            # then this is a proper autoindexing run - describe this
            # to the journal entry

            #if len(self._fp_directory) <= 50:
            #dirname = self._fp_directory
            #else:
            #dirname = '...%s' % self._fp_directory[-46:]
            dirname = os.path.dirname(self.get_imageset().get_template())

            Journal.block(
                'autoindexing', self._indxr_sweep_name, 'labelit', {
                    'images': images_str,
                    'target cell': cell_str,
                    'target lattice': self._indxr_input_lattice,
                    'template': self.get_imageset().get_template(),
                    'directory': dirname
                })

        if len(_images) > 4:
            raise RuntimeError('cannot use more than 4 images')

        from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex
        index = LabelitIndex()
        index.set_working_directory(self.get_working_directory())
        auto_logfiler(index)

        #task = 'Autoindex from images:'

        #for i in _images:
        #task += ' %s' % self.get_image_name(i)

        #self.set_task(task)

        Debug.write('Indexing from images:')
        for i in _images:
            index.add_image(self.get_image_name(i))
            Debug.write('%s' % self.get_image_name(i))

        xsweep = self.get_indexer_sweep()
        if xsweep is not None:
            if xsweep.get_distance() is not None:
                index.set_distance(xsweep.get_distance())
            #if self.get_wavelength_prov() == 'user':
            #index.set_wavelength(self.get_wavelength())
            if xsweep.get_beam_centre() is not None:
                index.set_beam_centre(xsweep.get_beam_centre())

        if self._refine_beam is False:
            index.set_refine_beam(False)
        else:
            index.set_refine_beam(True)
            index.set_beam_search_scope(self._beam_search_scope)

        if ((math.fabs(self.get_wavelength() - 1.54) < 0.01)
                or (math.fabs(self.get_wavelength() - 2.29) < 0.01)):
            index.set_Cu_KA_or_Cr_KA(True)

        #sweep = self.get_indexer_sweep_name()
        #FileHandler.record_log_file(
        #'%s INDEX' % (sweep), self.get_log_file())

        try:
            index.run()
        except RuntimeError as e:

            if self._refine_beam is False:
                raise e

            # can we improve the situation?

            if self._beam_search_scope < 4.0:
                self._beam_search_scope += 4.0

                # try repeating the indexing!

                self.set_indexer_done(False)
                return 'failed'

            # otherwise this is beyond redemption

            raise e

        self._solutions = index.get_solutions()

        # FIXME this needs to check the smilie status e.g.
        # ":)" or ";(" or "  ".

        # FIXME need to check the value of the RMSD and raise an
        # exception if the P1 solution has an RMSD > 1.0...

        # Change 27/FEB/08 to support user assigned spacegroups
        # (euugh!) have to "ignore" solutions with higher symmetry
        # otherwise the rest of xia will override us. Bummer.

        for i, solution in self._solutions.iteritems():
            if self._indxr_user_input_lattice:
                if (lattice_to_spacegroup(solution['lattice']) >
                        lattice_to_spacegroup(self._indxr_input_lattice)):
                    Debug.write('Ignoring solution: %s' % solution['lattice'])
                    del self._solutions[i]

        # check the RMSD from the triclinic unit cell
        if self._solutions[1]['rmsd'] > 1.0 and False:
            # don't know when this is useful - but I know when it is not!
            raise RuntimeError('high RMSD for triclinic solution')

        # configure the "right" solution
        self._solution = self.get_solution()

        # now store also all of the other solutions... keyed by the
        # lattice - however these should only be added if they
        # have a smiley in the appropriate record, perhaps?

        for solution in self._solutions.keys():
            lattice = self._solutions[solution]['lattice']
            if lattice in self._indxr_other_lattice_cell:
                if self._indxr_other_lattice_cell[lattice]['goodness'] < \
                   self._solutions[solution]['metric']:
                    continue

            self._indxr_other_lattice_cell[lattice] = {
                'goodness': self._solutions[solution]['metric'],
                'cell': self._solutions[solution]['cell']
            }

        self._indxr_lattice = self._solution['lattice']
        self._indxr_cell = tuple(self._solution['cell'])
        self._indxr_mosaic = self._solution['mosaic']

        lms = LabelitMosflmScript()
        lms.set_working_directory(self.get_working_directory())
        lms.set_solution(self._solution['number'])
        self._indxr_payload['mosflm_orientation_matrix'] = lms.calculate()

        # get the beam centre from the mosflm script - mosflm
        # may have inverted the beam centre and labelit will know
        # this!

        mosflm_beam_centre = lms.get_mosflm_beam()

        if mosflm_beam_centre:
            self._indxr_payload['mosflm_beam_centre'] = tuple(
                mosflm_beam_centre)

        import copy
        detector = copy.deepcopy(self.get_detector())
        beam = copy.deepcopy(self.get_beam())
        from dxtbx.model.detector_helpers import set_mosflm_beam_centre
        set_mosflm_beam_centre(detector, beam, mosflm_beam_centre)

        from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
        from scitbx import matrix
        from cctbx import sgtbx, uctbx
        from dxtbx.model import CrystalFactory
        mosflm_matrix = matrix.sqr([
            float(i) for line in lms.calculate()
            for i in line.replace("-", " -").split()
        ][:9])

        space_group = sgtbx.space_group_info(
            lattice_to_spacegroup_number(self._solution['lattice'])).group()
        crystal_model = CrystalFactory.from_mosflm_matrix(
            mosflm_matrix,
            unit_cell=uctbx.unit_cell(tuple(self._solution['cell'])),
            space_group=space_group)

        from dxtbx.model import Experiment, ExperimentList
        experiment = Experiment(
            beam=beam,
            detector=detector,
            goniometer=self.get_goniometer(),
            scan=self.get_scan(),
            crystal=crystal_model,
        )

        experiment_list = ExperimentList([experiment])
        self.set_indexer_experiment_list(experiment_list)

        # also get an estimate of the resolution limit from the
        # labelit.stats_distl output... FIXME the name is wrong!

        lsd = LabelitStats_distl()
        lsd.set_working_directory(self.get_working_directory())
        lsd.stats_distl()

        resolution = 1.0e6
        for i in _images:
            stats = lsd.get_statistics(self.get_image_name(i))

            resol = 0.5 * (stats['resol_one'] + stats['resol_two'])

            if resol < resolution:
                resolution = resol

        self._indxr_resolution_estimate = resolution

        return 'ok'
    def _index(self):
        """Actually index the diffraction pattern. Note well that
        this is not going to compute the matrix..."""

        # acknowledge this program

        if not self._indxr_images:
            raise RuntimeError("No good spots found on any images")

        Citations.cite("labelit")
        Citations.cite("distl")

        _images = []
        for i in self._indxr_images:
            for j in i:
                if not j in _images:
                    _images.append(j)

        _images.sort()

        images_str = "%d" % _images[0]
        for i in _images[1:]:
            images_str += ", %d" % i

        cell_str = None
        if self._indxr_input_cell:
            cell_str = "%.2f %.2f %.2f %.2f %.2f %.2f" % self._indxr_input_cell

        if self._indxr_sweep_name:

            # then this is a proper autoindexing run - describe this
            # to the journal entry

            if len(self._fp_directory) <= 50:
                dirname = self._fp_directory
            else:
                dirname = "...%s" % self._fp_directory[-46:]

            Journal.block(
                "autoindexing",
                self._indxr_sweep_name,
                "labelit",
                {
                    "images": images_str,
                    "target cell": cell_str,
                    "target lattice": self._indxr_input_lattice,
                    "template": self._fp_template,
                    "directory": dirname,
                },
            )

        # auto_logfiler(self)

        from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex

        index = LabelitIndex()
        index.set_working_directory(self.get_working_directory())
        auto_logfiler(index)

        # task = 'Autoindex from images:'

        # for i in _images:
        # task += ' %s' % self.get_image_name(i)

        # self.set_task(task)

        # self.add_command_line('--index_only')

        Debug.write("Indexing from images:")
        for i in _images:
            index.add_image(self.get_image_name(i))
            Debug.write("%s" % self.get_image_name(i))

        if self._primitive_unit_cell:
            index.set_primitive_unit_cell(self._primitive_unit_cell)

        if self._indxr_input_cell:
            index.set_max_cell(1.25 * max(self._indxr_input_cell[:3]))

        xsweep = self.get_indexer_sweep()
        if xsweep is not None:
            if xsweep.get_distance() is not None:
                index.set_distance(xsweep.get_distance())
            # if self.get_wavelength_prov() == 'user':
            # index.set_wavelength(self.get_wavelength())
            if xsweep.get_beam_centre() is not None:
                index.set_beam_centre(xsweep.get_beam_centre())

        if self._refine_beam is False:
            index.set_refine_beam(False)
        else:
            index.set_refine_beam(True)
            index.set_beam_search_scope(self._beam_search_scope)

        if (math.fabs(self.get_wavelength() - 1.54) <
                0.01) or (math.fabs(self.get_wavelength() - 2.29) < 0.01):
            index.set_Cu_KA_or_Cr_KA(True)

        try:
            index.run()
        except RuntimeError as e:

            if self._refine_beam is False:
                raise e

            # can we improve the situation?

            if self._beam_search_scope < 4.0:
                self._beam_search_scope += 4.0

                # try repeating the indexing!

                self.set_indexer_done(False)
                return "failed"

            # otherwise this is beyond redemption

            raise e

        self._solutions = index.get_solutions()

        # FIXME this needs to check the smilie status e.g.
        # ":)" or ";(" or "  ".

        # FIXME need to check the value of the RMSD and raise an
        # exception if the P1 solution has an RMSD > 1.0...

        # Change 27/FEB/08 to support user assigned spacegroups
        # (euugh!) have to "ignore" solutions with higher symmetry
        # otherwise the rest of xia will override us. Bummer.

        for i, solution in self._solutions.iteritems():
            if self._indxr_user_input_lattice:
                if lattice_to_spacegroup(
                        solution["lattice"]) > lattice_to_spacegroup(
                            self._indxr_input_lattice):
                    Debug.write("Ignoring solution: %s" % solution["lattice"])
                    del self._solutions[i]

        # configure the "right" solution
        self._solution = self.get_solution()

        # now store also all of the other solutions... keyed by the
        # lattice - however these should only be added if they
        # have a smiley in the appropriate record, perhaps?

        for solution in self._solutions.keys():
            lattice = self._solutions[solution]["lattice"]
            if lattice in self._indxr_other_lattice_cell:
                if (self._indxr_other_lattice_cell[lattice]["goodness"] <
                        self._solutions[solution]["metric"]):
                    continue

            self._indxr_other_lattice_cell[lattice] = {
                "goodness": self._solutions[solution]["metric"],
                "cell": self._solutions[solution]["cell"],
            }

        self._indxr_lattice = self._solution["lattice"]
        self._indxr_cell = tuple(self._solution["cell"])
        self._indxr_mosaic = self._solution["mosaic"]

        lms = LabelitMosflmMatrix()
        lms.set_working_directory(self.get_working_directory())
        lms.set_solution(self._solution["number"])
        self._indxr_payload["mosflm_orientation_matrix"] = lms.calculate()

        # get the beam centre from the mosflm script - mosflm
        # may have inverted the beam centre and labelit will know
        # this!

        mosflm_beam_centre = lms.get_mosflm_beam()

        if mosflm_beam_centre:
            self._indxr_payload["mosflm_beam_centre"] = tuple(
                mosflm_beam_centre)

        detector = copy.deepcopy(self.get_detector())
        beam = copy.deepcopy(self.get_beam())
        from dxtbx.model.detector_helpers import set_mosflm_beam_centre

        set_mosflm_beam_centre(detector, beam, mosflm_beam_centre)

        from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
        from scitbx import matrix
        from cctbx import sgtbx, uctbx
        from dxtbx.model import CrystalFactory

        mosflm_matrix = matrix.sqr([
            float(i) for line in lms.calculate()
            for i in line.replace("-", " -").split()
        ][:9])

        space_group = sgtbx.space_group_info(
            lattice_to_spacegroup_number(self._solution["lattice"])).group()
        crystal_model = CrystalFactory.from_mosflm_matrix(
            mosflm_matrix,
            unit_cell=uctbx.unit_cell(tuple(self._solution["cell"])),
            space_group=space_group,
        )

        from dxtbx.model import Experiment, ExperimentList

        experiment = Experiment(
            beam=beam,
            detector=detector,
            goniometer=self.get_goniometer(),
            scan=self.get_scan(),
            crystal=crystal_model,
        )

        experiment_list = ExperimentList([experiment])
        self.set_indexer_experiment_list(experiment_list)

        # also get an estimate of the resolution limit from the
        # labelit.stats_distl output... FIXME the name is wrong!

        lsd = LabelitStats_distl()
        lsd.set_working_directory(self.get_working_directory())
        lsd.stats_distl()

        resolution = 1.0e6
        for i in _images:
            stats = lsd.get_statistics(self.get_image_name(i))

            resol = 0.5 * (stats["resol_one"] + stats["resol_two"])

            if resol < resolution:
                resolution = resol

        self._indxr_resolution_estimate = resolution

        return "ok"
Example #9
0
    def run(self, ignore_errors = False):
      '''Run idxref.'''

      #image_header = self.get_header()

      ## crank through the header dictionary and replace incorrect
      ## information with updated values through the indexer
      ## interface if available...

      ## need to add distance, wavelength - that should be enough...

      #if self.get_distance():
        #image_header['distance'] = self.get_distance()

      #if self.get_wavelength():
        #image_header['wavelength'] = self.get_wavelength()

      #if self.get_two_theta():
        #image_header['two_theta'] = self.get_two_theta()

      header = imageset_to_xds(
        self.get_imageset(),
        refined_beam_vector=self._refined_beam_vector,
        refined_rotation_axis=self._refined_rotation_axis,
        refined_distance=self._refined_distance)

      xds_inp = open(os.path.join(self.get_working_directory(),
                                  'XDS.INP'), 'w')

      # what are we doing?
      xds_inp.write('JOB=IDXREF\n')
      xds_inp.write('MAXIMUM_NUMBER_OF_PROCESSORS=%d\n' % \
                    self._parallel)

      # FIXME this needs to be calculated from the beam centre...

      if self._refined_origin:
        xds_inp.write('ORGX=%f ORGY=%f\n' % \
                      tuple(self._refined_origin))
      else:
        xds_inp.write('ORGX=%f ORGY=%f\n' % \
                      tuple(self._org))

      # FIXME in here make sure sweep is wider than 5 degrees
      # before specifying AXIS: if <= 5 degrees replace AXIS with
      # nothing - base this on the maximum possible angular separation

      min_frame = self._spot_range[0][0]
      max_frame = self._spot_range[-1][1]

      refine_params = [p for p in self._params.refine]

      phi_width = self.get_phi_width()
      if ('AXIS' in refine_params
          and (max_frame - min_frame) * phi_width < 5.0):
        refine_params.remove('AXIS')

      xds_inp.write('REFINE(IDXREF)=%s\n' %
                    ' '.join(refine_params))

      if self._starting_frame and self._starting_angle:
        xds_inp.write('STARTING_FRAME=%d\n' % \
                      self._starting_frame)
        xds_inp.write('STARTING_ANGLE=%f\n' % \
                      self._starting_angle)

      # FIXME this looks like a potential bug - what will
      # happen if the input lattice has not been set??
      if self._indxr_input_cell:
        self._cell = self._indxr_input_cell
      if self._indxr_input_lattice:
        self._symm = lattice_to_spacegroup_number(
            self._indxr_input_lattice)

      if self._cell:
        xds_inp.write('SPACE_GROUP_NUMBER=%d\n' % self._symm)
        cell_format = '%6.2f %6.2f %6.2f %6.2f %6.2f %6.2f'
        xds_inp.write('UNIT_CELL_CONSTANTS=%s\n' % \
                      cell_format % self._cell)


      if self._a_axis:
        xds_inp.write('UNIT_CELL_A-AXIS=%.2f %.2f %.2f\n' %
                      tuple(self._a_axis))

      if self._b_axis:
        xds_inp.write('UNIT_CELL_B-AXIS=%.2f %.2f %.2f\n' %
                      tuple(self._b_axis))

      if self._c_axis:
        xds_inp.write('UNIT_CELL_C-AXIS=%.2f %.2f %.2f\n' %
                      tuple(self._c_axis))

      for record in header:
        xds_inp.write('%s\n' % record)

      name_template = template_to_xds(
        os.path.join(self.get_directory(), self.get_template()))

      record = 'NAME_TEMPLATE_OF_DATA_FRAMES=%s\n' % \
               name_template

      xds_inp.write(record)

      xds_inp.write('DATA_RANGE=%d %d\n' % self._data_range)
      for spot_range in self._spot_range:
        xds_inp.write('SPOT_RANGE=%d %d\n' % spot_range)
      xds_inp.write('BACKGROUND_RANGE=%d %d\n' % \
                    self._background_range)

      xds_inp.close()

      # copy the input file...
      shutil.copyfile(os.path.join(self.get_working_directory(),
                                   'XDS.INP'),
                      os.path.join(self.get_working_directory(),
                                   '%d_IDXREF.INP' % self.get_xpid()))

      # write the input data files...
      for file_name in self._input_data_files_list:
        src = self._input_data_files[file_name]
        dst = os.path.join(
            self.get_working_directory(), file_name)
        if src != dst:
          shutil.copyfile(src, dst)

      self.start()
      self.close_wait()

      xds_check_version_supported(self.get_all_output())
      if not ignore_errors:
        xds_check_error(self.get_all_output())

      # If xds_check_error detects any errors it will raise an exception
      # The caller can then continue using the run_continue_from_error()
      # function. If XDS does not throw any errors we just plow on.

      return self.continue_from_error()
Example #10
0
    def _index(self):
        if PhilIndex.params.dials.index.method in (libtbx.Auto, None):
            if self._indxr_input_cell is not None:
                indexer = self._do_indexing("real_space_grid_search")
            else:
                try:
                    indexer_fft3d = self._do_indexing(method="fft3d")
                    nref_3d, rmsd_3d = indexer_fft3d.get_nref_rmsds()
                except Exception as e:
                    nref_3d = None
                    rmsd_3d = None
                    indexing_failure = e
                try:
                    indexer_fft1d = self._do_indexing(method="fft1d")
                    nref_1d, rmsd_1d = indexer_fft1d.get_nref_rmsds()
                except Exception as e:
                    nref_1d = None
                    rmsd_1d = None
                    indexing_failure = e

                if (nref_1d is not None and nref_3d is None or
                    (nref_1d > nref_3d and rmsd_1d[0] < rmsd_3d[0]
                     and rmsd_1d[1] < rmsd_3d[1] and rmsd_1d[2] < rmsd_3d[2])):
                    indexer = indexer_fft1d
                elif nref_3d is not None:
                    indexer = indexer_fft3d
                else:
                    raise RuntimeError(indexing_failure)

        else:
            indexer = self._do_indexing(
                method=PhilIndex.params.dials.index.method)

        # not strictly the P1 cell, rather the cell that was used in indexing
        self._p1_cell = indexer._p1_cell
        self.set_indexer_payload("indexed_filename",
                                 indexer.get_indexed_filename())

        indexed_file = indexer.get_indexed_filename()
        indexed_experiments = indexer.get_experiments_filename()

        fast_mode = PhilIndex.params.dials.fast_mode
        trust_beam_centre = PhilIndex.params.xia2.settings.trust_beam_centre
        multi_sweep_indexing = PhilIndex.params.xia2.settings.multi_sweep_indexing
        check_indexing_symmetry = PhilIndex.params.dials.check_indexing_symmetry

        if check_indexing_symmetry and not (trust_beam_centre or fast_mode
                                            or multi_sweep_indexing):
            checksym = self.CheckIndexingSymmetry()
            checksym.set_experiments_filename(indexed_experiments)
            checksym.set_indexed_filename(indexed_file)
            checksym.set_grid_search_scope(1)
            checksym.run()
            hkl_offset = checksym.get_hkl_offset()
            logger.debug("hkl_offset: %s", str(hkl_offset))
            if hkl_offset is not None and hkl_offset != (0, 0, 0):
                reindex = self.Reindex()
                reindex.set_hkl_offset(hkl_offset)
                reindex.set_indexed_filename(indexed_file)
                reindex.run()
                indexed_file = reindex.get_reindexed_reflections_filename()

                # do some scan-static refinement - run twice, first without outlier
                # rejection as the model is too far from reality to do a sensible job of
                # outlier rejection
                refiner = self.Refine()
                refiner.set_experiments_filename(indexed_experiments)
                refiner.set_indexed_filename(
                    reindex.get_reindexed_reflections_filename())
                refiner.set_outlier_algorithm(None)
                refiner.run()
                indexed_experiments = refiner.get_refined_experiments_filename(
                )

                # now again with outlier rejection (possibly)
                refiner = self.Refine()
                refiner.set_experiments_filename(indexed_experiments)
                refiner.set_indexed_filename(indexed_file)
                refiner.run()
                indexed_experiments = refiner.get_refined_experiments_filename(
                )

        if self._indxr_input_lattice is None:

            # FIXME in here should respect the input unit cell and lattice if provided

            # FIXME from this (i) populate the helper table,
            # (ii) try to avoid re-running the indexing
            # step if we eliminate a solution as we have all of the refined results
            # already available.

            rbs = self.RefineBravaisSettings()
            rbs.set_experiments_filename(indexed_experiments)
            rbs.set_indexed_filename(indexed_file)
            if PhilIndex.params.dials.fix_geometry:
                rbs.set_detector_fix("all")
                rbs.set_beam_fix("all")
            elif PhilIndex.params.dials.fix_distance:
                rbs.set_detector_fix("distance")

            FileHandler.record_log_file(
                "%s LATTICE" % self.get_indexer_full_name(),
                rbs.get_log_file())
            rbs.run()

            for k in sorted(rbs.get_bravais_summary()):
                summary = rbs.get_bravais_summary()[k]

                # FIXME need to do this better - for the moment only accept lattices
                # where R.M.S. deviation is less than twice P1 R.M.S. deviation.

                if self._indxr_input_lattice is None:
                    if not summary["recommended"]:
                        continue

                experiments = load.experiment_list(summary["experiments_file"],
                                                   check_format=False)
                cryst = experiments.crystals()[0]
                cs = crystal.symmetry(unit_cell=cryst.get_unit_cell(),
                                      space_group=cryst.get_space_group())
                lattice = str(
                    bravais_types.bravais_lattice(group=cs.space_group()))
                cb_op = sgtbx.change_of_basis_op(str(summary["cb_op"]))

                self._solutions[k] = {
                    "number": k,
                    "mosaic": 0.0,
                    "metric": summary["max_angular_difference"],
                    "rmsd": summary["rmsd"],
                    "nspots": summary["nspots"],
                    "lattice": lattice,
                    "cell": cs.unit_cell().parameters(),
                    "experiments_file": summary["experiments_file"],
                    "cb_op": str(cb_op),
                }

            self._solution = self.get_solution()
            self._indxr_lattice = self._solution["lattice"]

            for solution in self._solutions:
                lattice = self._solutions[solution]["lattice"]
                if (self._indxr_input_lattice is not None
                        and self._indxr_input_lattice != lattice):
                    continue
                if lattice in self._indxr_other_lattice_cell:
                    if (self._indxr_other_lattice_cell[lattice]["metric"] <
                            self._solutions[solution]["metric"]):
                        continue

                self._indxr_other_lattice_cell[lattice] = {
                    "metric": self._solutions[solution]["metric"],
                    "cell": self._solutions[solution]["cell"],
                }

            self._indxr_mosaic = self._solution["mosaic"]

            experiments_file = self._solution["experiments_file"]
            experiment_list = load.experiment_list(experiments_file)
            self.set_indexer_experiment_list(experiment_list)

            self.set_indexer_payload("experiments_filename", experiments_file)

            # reindex the output reflection list to this solution
            reindex = self.Reindex()
            reindex.set_indexed_filename(indexed_file)
            reindex.set_cb_op(self._solution["cb_op"])
            reindex.set_space_group(
                str(lattice_to_spacegroup_number(self._solution["lattice"])))
            reindex.run()
            indexed_file = reindex.get_reindexed_reflections_filename()
            self.set_indexer_payload("indexed_filename", indexed_file)

        else:
            experiment_list = load.experiment_list(indexed_experiments)
            self.set_indexer_experiment_list(experiment_list)
            self.set_indexer_payload("experiments_filename",
                                     indexed_experiments)

            cryst = experiment_list.crystals()[0]
            lattice = str(
                bravais_types.bravais_lattice(group=cryst.get_space_group()))
            self._indxr_lattice = lattice
            self._solutions = {}
            self._solutions[0] = {
                "number": 0,
                "mosaic": 0.0,
                "metric": -1,
                "rmsd": -1,
                "nspots": -1,
                "lattice": lattice,
                "cell": cryst.get_unit_cell().parameters(),
                "experiments_file": indexed_experiments,
                "cb_op": "a,b,c",
            }

            self._indxr_other_lattice_cell[lattice] = {
                "metric": self._solutions[0]["metric"],
                "cell": self._solutions[0]["cell"],
            }
Example #11
0
    def _integrate_finish(self):
        """Finish off the integration by running correct."""

        # first run the postrefinement etc with spacegroup P1
        # and the current unit cell - this will be used to
        # obtain a benchmark rmsd in pixels / phi and also
        # cell deviations (this is working towards spotting bad
        # indexing solutions) - only do this if we have no
        # reindex matrix... and no postrefined cell...

        p1_deviations = None

        # fix for bug # 3264 -
        # if we have not run integration with refined parameters, make it so...
        # erm? shouldn't this therefore return if this is the principle, or
        # set the flag after we have tested the lattice?

        if ("GXPARM.XDS" not in self._xds_data_files
                and PhilIndex.params.xds.integrate.reintegrate):
            logger.debug(
                "Resetting integrater, to ensure refined orientation is used")
            self.set_integrater_done(False)

        if (not self.get_integrater_reindex_matrix() and not self._intgr_cell
                and PhilIndex.params.xia2.settings.lattice_rejection
                and not self.get_integrater_sweep().get_user_lattice()):
            correct = self.Correct()

            correct.set_data_range(
                self._intgr_wedge[0] + self.get_frame_offset(),
                self._intgr_wedge[1] + self.get_frame_offset(),
            )

            if self.get_polarization() > 0.0:
                correct.set_polarization(self.get_polarization())

            # FIXME should this be using the correctly transformed
            # cell or are the results ok without it?!

            correct.set_spacegroup_number(1)
            correct.set_cell(self._intgr_refiner_cell)

            correct.run()

            cell = correct.get_result("cell")
            cell_esd = correct.get_result("cell_esd")

            logger.debug("Postrefinement in P1 results:")
            logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" % tuple(cell))
            logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" %
                         tuple(cell_esd))
            logger.debug("Deviations: %.2f pixels %.2f degrees" %
                         (correct.get_result("rmsd_pixel"),
                          correct.get_result("rmsd_phi")))

            p1_deviations = (
                correct.get_result("rmsd_pixel"),
                correct.get_result("rmsd_phi"),
            )

        # next run the postrefinement etc with the given
        # cell / lattice - this will be the assumed result...

        integrate_hkl = os.path.join(self.get_working_directory(),
                                     "INTEGRATE.HKL")

        if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
            from dxtbx.serialize import load
            from dials.algorithms.shadowing.filter import filter_shadowed_reflections

            experiments_json = xparm_xds_to_experiments_json(
                os.path.join(self.get_working_directory(), "XPARM.XDS"),
                self.get_working_directory(),
            )
            experiments = load.experiment_list(experiments_json,
                                               check_format=True)
            imageset = experiments[0].imageset
            masker = (imageset.get_format_class().get_instance(
                imageset.paths()[0]).get_masker())
            if masker is not None:
                integrate_filename = integrate_hkl_to_reflection_file(
                    integrate_hkl, experiments_json,
                    self.get_working_directory())
                reflections = flex.reflection_table.from_file(
                    integrate_filename)

                t0 = time.time()
                sel = filter_shadowed_reflections(experiments, reflections)
                shadowed = reflections.select(sel)
                t1 = time.time()
                logger.debug("Filtered %i reflections in %.1f seconds" %
                             (sel.count(True), t1 - t0))

                filter_hkl = os.path.join(self.get_working_directory(),
                                          "FILTER.HKL")
                with open(filter_hkl, "wb") as f:
                    detector = experiments[0].detector
                    for ref in shadowed:
                        p = detector[ref["panel"]]
                        ox, oy = p.get_raw_image_offset()
                        h, k, l = ref["miller_index"]
                        x, y, z = ref["xyzcal.px"]
                        dx, dy, dz = (2, 2, 2)
                        print(
                            "%i %i %i %.1f %.1f %.1f %.1f %.1f %.1f" %
                            (h, k, l, x + ox, y + oy, z, dx, dy, dz),
                            file=f,
                        )
                t2 = time.time()
                logger.debug("Written FILTER.HKL in %.1f seconds" % (t2 - t1))

        correct = self.Correct()

        correct.set_data_range(
            self._intgr_wedge[0] + self.get_frame_offset(),
            self._intgr_wedge[1] + self.get_frame_offset(),
        )

        if self.get_polarization() > 0.0:
            correct.set_polarization(self.get_polarization())

        # BUG # 2695 probably comes from here - need to check...
        # if the pointless interface comes back with a different
        # crystal setting then the unit cell stored in self._intgr_cell
        # needs to be set to None...

        if self.get_integrater_spacegroup_number():
            correct.set_spacegroup_number(
                self.get_integrater_spacegroup_number())
            if not self._intgr_cell:
                raise RuntimeError("no unit cell to recycle")
            correct.set_cell(self._intgr_cell)

        # BUG # 3113 - new version of XDS will try and figure the
        # best spacegroup out from the intensities (and get it wrong!)
        # unless we set the spacegroup and cell explicitly

        if not self.get_integrater_spacegroup_number():
            cell = self._intgr_refiner_cell
            lattice = self._intgr_refiner.get_refiner_lattice()
            spacegroup_number = lattice_to_spacegroup_number(lattice)

            # this should not prevent the postrefinement from
            # working correctly, else what is above would not
            # work correctly (the postrefinement test)

            correct.set_spacegroup_number(spacegroup_number)
            correct.set_cell(cell)

            logger.debug("Setting spacegroup to: %d" % spacegroup_number)
            logger.debug("Setting cell to: %.2f %.2f %.2f %.2f %.2f %.2f" %
                         tuple(cell))

        if self.get_integrater_reindex_matrix():

            # bug! if the lattice is not primitive the values in this
            # reindex matrix need to be multiplied by a constant which
            # depends on the Bravais lattice centering.

            lattice = self._intgr_refiner.get_refiner_lattice()

            matrix = self.get_integrater_reindex_matrix()
            matrix = scitbx.matrix.sqr(matrix).transpose().elems
            matrix = r_to_rt(matrix)

            if lattice[1] == "P":
                mult = 1
            elif lattice[1] == "C" or lattice[1] == "I":
                mult = 2
            elif lattice[1] == "R":
                mult = 3
            elif lattice[1] == "F":
                mult = 4
            else:
                raise RuntimeError("unknown multiplier for lattice %s" %
                                   lattice)

            logger.debug("REIDX multiplier for lattice %s: %d" %
                         (lattice, mult))

            mult_matrix = [mult * m for m in matrix]

            logger.debug("REIDX set to %d %d %d %d %d %d %d %d %d %d %d %d" %
                         tuple(mult_matrix))
            correct.set_reindex_matrix(mult_matrix)

        correct.run()

        # record the log file -

        pname, xname, dname = self.get_integrater_project_info()
        sweep = self.get_integrater_sweep_name()
        FileHandler.record_log_file(
            f"{pname} {xname} {dname} {sweep} CORRECT",
            os.path.join(self.get_working_directory(), "CORRECT.LP"),
        )

        FileHandler.record_more_data_file(
            f"{pname} {xname} {dname} {sweep} CORRECT",
            os.path.join(self.get_working_directory(), "XDS_ASCII.HKL"),
        )

        # erm. just to be sure
        if self.get_integrater_reindex_matrix() and correct.get_reindex_used():
            raise RuntimeError("Reindex panic!")

        # get the reindex operation used, which may be useful if none was
        # set but XDS decided to apply one, e.g. #419.

        if not self.get_integrater_reindex_matrix(
        ) and correct.get_reindex_used():
            # convert this reindex operation to h, k, l form: n.b. this
            # will involve dividing through by the lattice centring multiplier

            matrix = rt_to_r(correct.get_reindex_used())

            matrix = scitbx.matrix.sqr(matrix).transpose().elems

            lattice = self._intgr_refiner.get_refiner_lattice()

            if lattice[1] == "P":
                mult = 1.0
            elif lattice[1] == "C" or lattice[1] == "I":
                mult = 2.0
            elif lattice[1] == "R":
                mult = 3.0
            elif lattice[1] == "F":
                mult = 4.0

            matrix = [m / mult for m in matrix]

            reindex_op = mat_to_symop(matrix)

            # assign this to self: will this reset?! make for a leaky
            # abstraction and just assign this...

            # self.set_integrater_reindex_operator(reindex)

            self._intgr_reindex_operator = reindex_op

        # record the log file -

        pname, xname, dname = self.get_integrater_project_info()
        sweep = self.get_integrater_sweep_name()
        FileHandler.record_log_file(
            f"{pname} {xname} {dname} {sweep} CORRECT",
            os.path.join(self.get_working_directory(), "CORRECT.LP"),
        )

        # should get some interesting stuff from the XDS correct file
        # here, for instance the resolution range to use in integration
        # (which should be fed back if not fast) and so on...

        self._intgr_corrected_hklout = os.path.join(
            self.get_working_directory(), "XDS_ASCII.HKL")

        # also record the batch range - needed for the analysis of the
        # radiation damage in chef...

        self._intgr_batches_out = (self._intgr_wedge[0], self._intgr_wedge[1])

        # FIXME perhaps I should also feedback the GXPARM file here??
        for file in ["GXPARM.XDS"]:
            self._xds_data_files[file] = correct.get_output_data_file(file)

        # record the postrefined cell parameters
        self._intgr_cell = correct.get_result("cell")
        self._intgr_n_ref = correct.get_result("n_ref")

        logger.debug('Postrefinement in "correct" spacegroup results:')
        logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" %
                     tuple(correct.get_result("cell")))
        logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" %
                     tuple(correct.get_result("cell_esd")))
        logger.debug(
            "Deviations: %.2f pixels %.2f degrees" %
            (correct.get_result("rmsd_pixel"), correct.get_result("rmsd_phi")))

        logger.debug("Error correction parameters: A=%.3f B=%.3f" %
                     correct.get_result("sdcorrection"))

        # compute misorientation of axes

        xparm_file = os.path.join(self.get_working_directory(), "GXPARM.XDS")

        handle = xparm.reader()
        handle.read_file(xparm_file)

        rotn = handle.rotation_axis
        beam = handle.beam_vector

        dot = sum(rotn[j] * beam[j] for j in range(3))
        r = math.sqrt(sum(rotn[j] * rotn[j] for j in range(3)))
        b = math.sqrt(sum(beam[j] * beam[j] for j in range(3)))

        rtod = 180.0 / math.pi

        angle = rtod * math.fabs(0.5 * math.pi - math.acos(dot / (r * b)))

        logger.debug("Axis misalignment %.2f degrees" % angle)

        correct_deviations = (
            correct.get_result("rmsd_pixel"),
            correct.get_result("rmsd_phi"),
        )

        if p1_deviations:
            # compare and reject if both > 50% higher - though adding a little
            # flexibility - 0.5 pixel / osc width slack.

            pixel = p1_deviations[0]
            phi = math.sqrt(0.05 * 0.05 + p1_deviations[1] * p1_deviations[1])

            threshold = PhilIndex.params.xia2.settings.lattice_rejection_threshold
            logger.debug("RMSD ratio: %.2f" % (correct_deviations[0] / pixel))
            logger.debug("RMSPhi ratio: %.2f" % (correct_deviations[1] / phi))

            if (correct_deviations[0] / pixel > threshold
                    and correct_deviations[1] / phi > threshold):

                logger.info(
                    "Eliminating this indexing solution as postrefinement")
                logger.info("deviations rather high relative to triclinic")
                raise BadLatticeError(
                    "high relative deviations in postrefinement")

        if (not PhilIndex.params.dials.fast_mode
                and not PhilIndex.params.xds.keep_outliers):
            # check for alien reflections and perhaps recycle - removing them
            correct_remove = correct.get_remove()
            if correct_remove:
                current_remove = set()
                final_remove = []

                # first ensure that there are no duplicate entries...
                if os.path.exists(
                        os.path.join(self.get_working_directory(),
                                     "REMOVE.HKL")):
                    with open(
                            os.path.join(self.get_working_directory(),
                                         "REMOVE.HKL")) as fh:
                        for line in fh.readlines():
                            h, k, l = list(map(int, line.split()[:3]))
                            z = float(line.split()[3])

                            if (h, k, l, z) not in current_remove:
                                current_remove.add((h, k, l, z))

                    for c in correct_remove:
                        if c in current_remove:
                            continue
                        final_remove.append(c)

                    logger.debug("%d alien reflections are already removed" %
                                 (len(correct_remove) - len(final_remove)))
                else:
                    # we want to remove all of the new dodgy reflections
                    final_remove = correct_remove

                z_min = PhilIndex.params.xds.z_min
                rejected = 0

                with open(
                        os.path.join(self.get_working_directory(),
                                     "REMOVE.HKL"), "w") as remove_hkl:

                    # write in the old reflections
                    for remove in current_remove:
                        z = remove[3]
                        if z >= z_min:
                            remove_hkl.write("%d %d %d %f\n" % remove)
                        else:
                            rejected += 1
                    logger.debug("Wrote %d old reflections to REMOVE.HKL" %
                                 (len(current_remove) - rejected))
                    logger.debug("Rejected %d as z < %f" % (rejected, z_min))

                    # and the new reflections
                    rejected = 0
                    used = 0
                    for remove in final_remove:
                        z = remove[3]
                        if z >= z_min:
                            used += 1
                            remove_hkl.write("%d %d %d %f\n" % remove)
                        else:
                            rejected += 1
                    logger.debug("Wrote %d new reflections to REMOVE.HKL" %
                                 (len(final_remove) - rejected))
                    logger.debug("Rejected %d as z < %f" % (rejected, z_min))

                # we want to rerun the finishing step so...
                # unless we have added no new reflections... or unless we
                # have not confirmed the point group (see SCI-398)

                if used and self.get_integrater_reindex_matrix():
                    self.set_integrater_finish_done(False)

        else:
            logger.debug(
                "Going quickly so not removing %d outlier reflections..." %
                len(correct.get_remove()))

        # Convert INTEGRATE.HKL to MTZ format and reapply any reindexing operations
        # spacegroup changes to allow use with CCP4 / Aimless for scaling

        hklout = os.path.splitext(integrate_hkl)[0] + ".mtz"
        self._factory.set_working_directory(self.get_working_directory())
        pointless = self._factory.Pointless()
        pointless.set_xdsin(integrate_hkl)
        pointless.set_hklout(hklout)
        pointless.xds_to_mtz()

        integrate_mtz = hklout

        if (self.get_integrater_reindex_operator()
                or self.get_integrater_spacegroup_number()):

            logger.debug("Reindexing things to MTZ")

            reindex = Reindex()
            reindex.set_working_directory(self.get_working_directory())
            auto_logfiler(reindex)

            if self.get_integrater_reindex_operator():
                reindex.set_operator(self.get_integrater_reindex_operator())

            if self.get_integrater_spacegroup_number():
                reindex.set_spacegroup(self.get_integrater_spacegroup_number())

            hklout = "%s_reindex.mtz" % os.path.splitext(integrate_mtz)[0]

            reindex.set_hklin(integrate_mtz)
            reindex.set_hklout(hklout)
            reindex.reindex()
            integrate_mtz = hklout

        experiments_json = xparm_xds_to_experiments_json(
            self._xds_data_files["GXPARM.XDS"], self.get_working_directory())
        pname, xname, dname = self.get_integrater_project_info()
        sweep = self.get_integrater_sweep_name()
        FileHandler.record_more_data_file(f"{pname} {xname} {dname} {sweep}",
                                          experiments_json)
        FileHandler.record_more_data_file(
            f"{pname} {xname} {dname} {sweep} INTEGRATE", integrate_mtz)

        self._intgr_experiments_filename = experiments_json

        return integrate_mtz
Example #12
0
File: Index.py Project: xia2/xia2
    def run(self, method):
      from xia2.Handlers.Streams import Debug
      Debug.write('Running dials.index')

      self.clear_command_line()
      for f in self._sweep_filenames:
        self.add_command_line(f)
      for f in self._spot_filenames:
        self.add_command_line(f)
      self.add_command_line('indexing.method=%s' % method)
      nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
      self.set_cpu_threads(nproc)
      self.add_command_line('indexing.nproc=%i' % nproc)
      if PhilIndex.params.xia2.settings.small_molecule == True:
        self.add_command_line('filter_ice=false')
      if self._reflections_per_degree is not None:
        self.add_command_line(
          'reflections_per_degree=%i' %self._reflections_per_degree)
      if self._fft3d_n_points is not None:
        self.add_command_line(
          'fft3d.reciprocal_space_grid.n_points=%i' %self._fft3d_n_points)
      if self._close_to_spindle_cutoff is not None:
        self.add_command_line(
          'close_to_spindle_cutoff=%f' %self._close_to_spindle_cutoff)
      if self._outlier_algorithm:
        self.add_command_line('outlier.algorithm=%s' % self._outlier_algorithm)
      if self._max_cell:
        self.add_command_line('max_cell=%d' % self._max_cell)
      if self._min_cell:
        self.add_command_line('min_cell=%d' % self._min_cell)
      if self._histogram_binning is not None:
        self.add_command_line('max_cell_estimation.histogram_binning=%s' %self._histogram_binning)
      if self._d_min_start:
        self.add_command_line('d_min_start=%f' % self._d_min_start)
      if self._indxr_input_lattice is not None:
        from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
        self._symm = lattice_to_spacegroup_number(
            self._indxr_input_lattice)
        self.add_command_line('known_symmetry.space_group=%s' % self._symm)
      if self._indxr_input_cell is not None:
        self.add_command_line(
          'known_symmetry.unit_cell="%s,%s,%s,%s,%s,%s"' %self._indxr_input_cell)
      if self._maximum_spot_error:
        self.add_command_line('maximum_spot_error=%.f' %
                              self._maximum_spot_error)
      if self._detector_fix:
        self.add_command_line('detector.fix=%s' % self._detector_fix)
      if self._beam_fix:
        self.add_command_line('beam.fix=%s' % self._beam_fix)
      if self._phil_file is not None:
        self.add_command_line("%s" %self._phil_file)

      self._experiment_filename = os.path.join(
        self.get_working_directory(), '%d_experiments.json' %self.get_xpid())
      self._indexed_filename = os.path.join(
        self.get_working_directory(), '%d_indexed.pickle' %self.get_xpid())
      self.add_command_line("output.experiments=%s" %self._experiment_filename)
      self.add_command_line("output.reflections=%s" %self._indexed_filename)

      self.start()
      self.close_wait()
      self.check_for_errors()

      from dials.array_family import flex
      from dxtbx.serialize import load
      self._experiment_list = load.experiment_list(self._experiment_filename)
      self._reflections = flex.reflection_table.from_pickle(
        self._indexed_filename)

      crystal = self._experiment_list.crystals()[0]
      self._p1_cell = crystal.get_unit_cell().parameters()

      refined_sel = self._reflections.get_flags(self._reflections.flags.used_in_refinement)
      refl = self._reflections.select(refined_sel)
      xc, yc, zc = refl['xyzcal.px'].parts()
      xo, yo, zo = refl['xyzobs.px.value'].parts()
      import math
      self._nref = refl.size()
      self._rmsd_x = math.sqrt(flex.mean(flex.pow2(xc - xo)))
      self._rmsd_y = math.sqrt(flex.mean(flex.pow2(yc - yo)))
      self._rmsd_z = math.sqrt(flex.mean(flex.pow2(zc - zo)))

      return
Example #13
0
File: Index.py Project: hainm/xia2
    def run(self, method):
      from xia2.Handlers.Streams import Debug
      Debug.write('Running dials.index')

      self.clear_command_line()
      for f in self._sweep_filenames:
        self.add_command_line(f)
      for f in self._spot_filenames:
        self.add_command_line(f)
      self.add_command_line('indexing.method=%s' % method)
      nproc = Flags.get_parallel()
      self.set_cpu_threads(nproc)
      self.add_command_line('indexing.nproc=%i' % nproc)
      if Flags.get_small_molecule():
        self.add_command_line('filter_ice=false')
      if self._reflections_per_degree is not None:
        self.add_command_line(
          'reflections_per_degree=%i' %self._reflections_per_degree)
      if self._fft3d_n_points is not None:
        self.add_command_line(
          'fft3d.reciprocal_space_grid.n_points=%i' %self._fft3d_n_points)
      if self._close_to_spindle_cutoff is not None:
        self.add_command_line(
          'close_to_spindle_cutoff=%f' %self._close_to_spindle_cutoff)
      if self._outlier_algorithm:
        self.add_command_line('outlier.algorithm=%s' % self._outlier_algorithm)
      if self._max_cell:
        self.add_command_line('max_cell=%d' % self._max_cell)
      if self._min_cell:
        self.add_command_line('min_cell=%d' % self._min_cell)
      if self._d_min_start:
        self.add_command_line('d_min_start=%f' % self._d_min_start)
      if self._indxr_input_lattice is not None:
        from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
        self._symm = lattice_to_spacegroup_number(
            self._indxr_input_lattice)
        self.add_command_line('known_symmetry.space_group=%s' % self._symm)
      if self._indxr_input_cell is not None:
        self.add_command_line(
          'known_symmetry.unit_cell="%s,%s,%s,%s,%s,%s"' %self._indxr_input_cell)
      if self._maximum_spot_error:
        self.add_command_line('maximum_spot_error=%.f' %
                              self._maximum_spot_error)
      if self._detector_fix:
        self.add_command_line('detector.fix=%s' % self._detector_fix)
      if self._beam_fix:
        self.add_command_line('beam.fix=%s' % self._beam_fix)
      if self._phil_file is not None:
        self.add_command_line("%s" %self._phil_file)

      self._experiment_filename = os.path.join(
        self.get_working_directory(), '%d_experiments.json' %self.get_xpid())
      self._indexed_filename = os.path.join(
        self.get_working_directory(), '%d_indexed.pickle' %self.get_xpid())
      self.add_command_line("output.experiments=%s" %self._experiment_filename)
      self.add_command_line("output.reflections=%s" %self._indexed_filename)

      self.start()
      self.close_wait()
      self.check_for_errors()

      records = self.get_all_output()

      for i, record in enumerate(records):
        if 'Unit cell:' in record:
          self._p1_cell = map(float, record.replace('(', '').replace(
            ')', '').replace(',', '').split()[-6:])

        if 'Final RMSDs by experiment' in record:
          values = records[i+6].strip().strip('|').split('|')
          if len(values):
            values = [float(v) for v in values]
            if values[0] == 0:
              self._nref = int(values[1])
              self._rmsd_x = values[2]
              self._rmsd_y = values[3]
              self._rmsd_z = values[4]

      return
Example #14
0
          'metric':self._solutions[solution]['metric'],
          'cell':self._solutions[solution]['cell']}

      self._indxr_mosaic = self._solution['mosaic']

      experiment_list = load.experiment_list(self._solution['experiments_file'])
      self.set_indexer_experiment_list(experiment_list)

      # reindex the output experiments list to the reference setting
      # (from the best cell/conventional setting)
      cb_op_to_ref = experiment_list.crystals()[0].get_space_group().info()\
        .change_of_basis_op_to_reference_setting()
      reindex = self.Reindex()
      reindex.set_experiments_filename(self._solution['experiments_file'])
      reindex.set_cb_op(cb_op_to_ref)
      reindex.set_space_group(str(lattice_to_spacegroup_number(
        self._solution['lattice'])))
      reindex.run()
      experiments_file = reindex.get_reindexed_experiments_filename()
      experiment_list = load.experiment_list(experiments_file)
      self.set_indexer_experiment_list(experiment_list)
      self.set_indexer_payload("experiments_filename", experiments_file)

      # reindex the output reflection list to this solution
      reindex = self.Reindex()
      reindex.set_indexed_filename(indexed_file)
      reindex.set_cb_op(self._solution['cb_op'])
      reindex.set_space_group(str(lattice_to_spacegroup_number(
        self._solution['lattice'])))
      reindex.run()
      indexed_file = reindex.get_reindexed_reflections_filename()
      self.set_indexer_payload("indexed_filename", indexed_file)