Esempio n. 1
0
    def _integrate_finish(self):
        '''Finish the integration - if necessary performing reindexing
    based on the pointgroup and the reindexing operator.'''

        if self._intgr_reindex_operator is None and \
           self._intgr_spacegroup_number == lattice_to_spacegroup(
            self.get_integrater_refiner().get_refiner_lattice()):
            return self._mosflm_hklout

        if self._intgr_reindex_operator is None and \
           self._intgr_spacegroup_number == 0:
            return self._mosflm_hklout

        Debug.write('Reindexing to spacegroup %d (%s)' % \
                    (self._intgr_spacegroup_number,
                     self._intgr_reindex_operator))

        hklin = self._mosflm_hklout
        reindex = Reindex()
        reindex.set_working_directory(self.get_working_directory())
        auto_logfiler(reindex)

        reindex.set_operator(self._intgr_reindex_operator)

        if self._intgr_spacegroup_number:
            reindex.set_spacegroup(self._intgr_spacegroup_number)

        hklout = '%s_reindex.mtz' % hklin[:-4]

        reindex.set_hklin(hklin)
        reindex.set_hklout(hklout)
        reindex.reindex()

        return hklout
Esempio n. 2
0
  def _integrate_finish(self):
    '''Finish the integration - if necessary performing reindexing
    based on the pointgroup and the reindexing operator.'''

    if self._intgr_reindex_operator is None and \
       self._intgr_spacegroup_number == lattice_to_spacegroup(
        self.get_integrater_refiner().get_refiner_lattice()):
      return self._mosflm_hklout

    if self._intgr_reindex_operator is None and \
       self._intgr_spacegroup_number == 0:
      return self._mosflm_hklout

    Debug.write('Reindexing to spacegroup %d (%s)' % \
                (self._intgr_spacegroup_number,
                 self._intgr_reindex_operator))

    hklin = self._mosflm_hklout
    reindex = Reindex()
    reindex.set_working_directory(self.get_working_directory())
    auto_logfiler(reindex)

    reindex.set_operator(self._intgr_reindex_operator)

    if self._intgr_spacegroup_number:
      reindex.set_spacegroup(self._intgr_spacegroup_number)

    hklout = '%s_reindex.mtz' % hklin[:-4]

    reindex.set_hklin(hklin)
    reindex.set_hklout(hklout)
    reindex.reindex()

    return hklout
Esempio n. 3
0
  def _integrate_finish(self):
    '''Finish off the integration by running dials.export.'''

    # FIXME - do we want to export every time we call this method
    # (the file will not have changed) and also (more important) do
    # we want a different exported MTZ file every time (I do not think
    # that we do; these can be very large) - was exporter.get_xpid() ->
    # now dials

    exporter = self.ExportMtz()
    exporter.set_reflections_filename(self._intgr_integrated_pickle)
    mtz_filename = os.path.join(
      self.get_working_directory(), '%s_integrated.mtz' % 'dials')
    exporter.set_mtz_filename(mtz_filename)
    exporter.run()
    self._intgr_integrated_filename = mtz_filename

    # record integrated MTZ file for e.g. BLEND.

    pname, xname, dname = self.get_integrater_project_info()
    sweep = self.get_integrater_sweep_name()
    FileHandler.record_more_data_file(
        '%s %s %s %s INTEGRATE' % (pname, xname, dname, sweep), mtz_filename)

    if not os.path.isfile(self._intgr_integrated_filename):
      raise RuntimeError("dials.export failed: %s does not exist."
                         % self._intgr_integrated_filename)

    if self._intgr_reindex_operator is None and \
      self._intgr_spacegroup_number == lattice_to_spacegroup(
        self.get_integrater_refiner().get_refiner_lattice()):
      Debug.write('Not reindexing to spacegroup %d (%s)' % \
                    (self._intgr_spacegroup_number,
                     self._intgr_reindex_operator))
      return mtz_filename

    if self._intgr_reindex_operator is None and \
      self._intgr_spacegroup_number == 0:
      Debug.write('Not reindexing to spacegroup %d (%s)' % \
                    (self._intgr_spacegroup_number,
                     self._intgr_reindex_operator))
      return mtz_filename

    Debug.write('Reindexing to spacegroup %d (%s)' % \
                (self._intgr_spacegroup_number,
                 self._intgr_reindex_operator))

    hklin = mtz_filename
    reindex = Reindex()
    reindex.set_working_directory(self.get_working_directory())
    auto_logfiler(reindex)

    reindex.set_operator(self._intgr_reindex_operator)

    if self._intgr_spacegroup_number:
      reindex.set_spacegroup(self._intgr_spacegroup_number)
    else:
      reindex.set_spacegroup(lattice_to_spacegroup(
        self.get_integrater_refiner().get_refiner_lattice()))

    hklout = '%s_reindex.mtz' % hklin[:-4]
    reindex.set_hklin(hklin)
    reindex.set_hklout(hklout)
    reindex.reindex()
    self._intgr_integrated_filename = hklout
    self._intgr_cell = reindex.get_cell()

    pname, xname, dname = self.get_integrater_project_info()
    sweep = self.get_integrater_sweep_name()
    FileHandler.record_more_data_file(
      '%s %s %s %s experiments' % (pname, xname, dname, sweep),
      self.get_integrated_experiments())

    from iotbx.reflection_file_reader import any_reflection_file
    miller_arrays = any_reflection_file(hklout).as_miller_arrays()
    # look for profile-fitted intensities
    intensities = [ma for ma in miller_arrays
                   if ma.info().labels == ['IPR', 'SIGIPR']]
    if len(intensities) == 0:
      # look instead for summation-integrated intensities
      intensities = [ma for ma in miller_arrays
                     if ma.info().labels == ['I', 'SIGI']]
      assert len(intensities)
    self._intgr_n_ref = intensities[0].size()

    return hklout
Esempio n. 4
0
  def _integrate_finish(self):
    '''Finish off the integration by running correct.'''

    # first run the postrefinement etc with spacegroup P1
    # and the current unit cell - this will be used to
    # obtain a benchmark rmsd in pixels / phi and also
    # cell deviations (this is working towards spotting bad
    # indexing solutions) - only do this if we have no
    # reindex matrix... and no postrefined cell...

    p1_deviations = None

    # fix for bug # 3264 -
    # if we have not run integration with refined parameters, make it so...
    # erm? shouldn't this therefore return if this is the principle, or
    # set the flag after we have tested the lattice?

    if not self._xds_data_files.has_key('GXPARM.XDS') and \
      PhilIndex.params.xds.integrate.reintegrate:
      Debug.write(
          'Resetting integrater, to ensure refined orientation is used')
      self.set_integrater_done(False)

    if not self.get_integrater_reindex_matrix() and not self._intgr_cell \
           and not Flags.get_no_lattice_test() and \
           not self.get_integrater_sweep().get_user_lattice():
      correct = self.Correct()

      correct.set_data_range(self._intgr_wedge[0],
                             self._intgr_wedge[1])

      if self.get_polarization() > 0.0:
        correct.set_polarization(self.get_polarization())

      # FIXME should this be using the correctly transformed
      # cell or are the results ok without it?!

      correct.set_spacegroup_number(1)
      correct.set_cell(self._intgr_refiner_cell)

      correct.run()

      # record the log file -

      pname, xname, dname = self.get_integrater_project_info()
      sweep = self.get_integrater_sweep_name()
      FileHandler.record_log_file('%s %s %s %s CORRECT' % \
                                  (pname, xname, dname, sweep),
                                  os.path.join(
          self.get_working_directory(),
          'CORRECT.LP'))

      FileHandler.record_more_data_file(
          '%s %s %s %s CORRECT' % (pname, xname, dname, sweep),
          os.path.join(self.get_working_directory(), 'XDS_ASCII.HKL'))

      cell = correct.get_result('cell')
      cell_esd = correct.get_result('cell_esd')

      Debug.write('Postrefinement in P1 results:')
      Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
                  tuple(cell))
      Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
                  tuple(cell_esd))
      Debug.write('Deviations: %.2f pixels %.2f degrees' % \
                  (correct.get_result('rmsd_pixel'),
                   correct.get_result('rmsd_phi')))

      p1_deviations = (correct.get_result('rmsd_pixel'),
                       correct.get_result('rmsd_phi'))

    # next run the postrefinement etc with the given
    # cell / lattice - this will be the assumed result...

    correct = self.Correct()

    correct.set_data_range(self._intgr_wedge[0],
                           self._intgr_wedge[1])

    if self.get_polarization() > 0.0:
      correct.set_polarization(self.get_polarization())

    # BUG # 2695 probably comes from here - need to check...
    # if the pointless interface comes back with a different
    # crystal setting then the unit cell stored in self._intgr_cell
    # needs to be set to None...

    if self.get_integrater_spacegroup_number():
      correct.set_spacegroup_number(
          self.get_integrater_spacegroup_number())
      if not self._intgr_cell:
        raise RuntimeError, 'no unit cell to recycle'
      correct.set_cell(self._intgr_cell)

    # BUG # 3113 - new version of XDS will try and figure the
    # best spacegroup out from the intensities (and get it wrong!)
    # unless we set the spacegroup and cell explicitly

    if not self.get_integrater_spacegroup_number():
      cell = self._intgr_refiner_cell
      lattice = self._intgr_refiner.get_refiner_lattice()
      spacegroup_number = lattice_to_spacegroup_number(lattice)

      # this should not prevent the postrefinement from
      # working correctly, else what is above would not
      # work correctly (the postrefinement test)

      correct.set_spacegroup_number(spacegroup_number)
      correct.set_cell(cell)

      Debug.write('Setting spacegroup to: %d' % spacegroup_number)
      Debug.write(
        'Setting cell to: %.2f %.2f %.2f %.2f %.2f %.2f' % tuple(cell))

    if self.get_integrater_reindex_matrix():

      # bug! if the lattice is not primitive the values in this
      # reindex matrix need to be multiplied by a constant which
      # depends on the Bravais lattice centering.

      lattice = self._intgr_refiner.get_refiner_lattice()

      import scitbx.matrix
      matrix = self.get_integrater_reindex_matrix()
      matrix = scitbx.matrix.sqr(matrix).transpose().elems
      matrix = r_to_rt(matrix)

      if lattice[1] == 'P':
        mult = 1
      elif lattice[1] == 'C' or lattice[1] == 'I':
        mult = 2
      elif lattice[1] == 'R':
        mult = 3
      elif lattice[1] == 'F':
        mult = 4
      else:
        raise RuntimeError, 'unknown multiplier for lattice %s' % \
              lattice

      Debug.write('REIDX multiplier for lattice %s: %d' % \
                  (lattice, mult))

      mult_matrix = [mult * m for m in matrix]

      Debug.write('REIDX set to %d %d %d %d %d %d %d %d %d %d %d %d' % \
                  tuple(mult_matrix))
      correct.set_reindex_matrix(mult_matrix)

    correct.run()

    # erm. just to be sure
    if self.get_integrater_reindex_matrix() and \
           correct.get_reindex_used():
      raise RuntimeError, 'Reindex panic!'

    # get the reindex operation used, which may be useful if none was
    # set but XDS decided to apply one, e.g. #419.

    if not self.get_integrater_reindex_matrix() and \
           correct.get_reindex_used():
      # convert this reindex operation to h, k, l form: n.b. this
      # will involve dividing through by the lattice centring multiplier

      matrix = rt_to_r(correct.get_reindex_used())
      import scitbx.matrix
      matrix = scitbx.matrix.sqr(matrix).transpose().elems

      lattice = self._intgr_refiner.get_refiner_lattice()

      if lattice[1] == 'P':
        mult = 1.0
      elif lattice[1] == 'C' or lattice[1] == 'I':
        mult = 2.0
      elif lattice[1] == 'R':
        mult = 3.0
      elif lattice[1] == 'F':
        mult = 4.0

      matrix = [m / mult for m in matrix]

      reindex_op = mat_to_symop(matrix)

      # assign this to self: will this reset?! make for a leaky
      # abstraction and just assign this...

      # self.set_integrater_reindex_operator(reindex)

      self._intgr_reindex_operator = reindex_op


    # record the log file -

    pname, xname, dname = self.get_integrater_project_info()
    sweep = self.get_integrater_sweep_name()
    FileHandler.record_log_file('%s %s %s %s CORRECT' % \
                                (pname, xname, dname, sweep),
                                os.path.join(self.get_working_directory(),
                                             'CORRECT.LP'))

    # should get some interesting stuff from the XDS correct file
    # here, for instance the resolution range to use in integration
    # (which should be fed back if not fast) and so on...

    self._intgr_corrected_hklout = os.path.join(self.get_working_directory(),
                                'XDS_ASCII.HKL')

    # also record the batch range - needed for the analysis of the
    # radiation damage in chef...

    self._intgr_batches_out = (self._intgr_wedge[0],
                               self._intgr_wedge[1])

    # FIXME perhaps I should also feedback the GXPARM file here??
    for file in ['GXPARM.XDS']:
      self._xds_data_files[file] = correct.get_output_data_file(file)

    # record the postrefined cell parameters
    self._intgr_cell = correct.get_result('cell')
    self._intgr_n_ref = correct.get_result('n_ref')

    Debug.write('Postrefinement in "correct" spacegroup results:')
    Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
                tuple(correct.get_result('cell')))
    Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
                tuple(correct.get_result('cell_esd')))
    Debug.write('Deviations: %.2f pixels %.2f degrees' % \
                (correct.get_result('rmsd_pixel'),
                 correct.get_result('rmsd_phi')))

    Debug.write('Error correction parameters: A=%.3f B=%.3f' % \
                correct.get_result('sdcorrection'))

    # compute misorientation of axes

    xparm_file = os.path.join(self.get_working_directory(), 'GXPARM.XDS')

    from iotbx.xds import xparm
    handle = xparm.reader()
    handle.read_file(xparm_file)

    rotn = handle.rotation_axis
    beam = handle.beam_vector

    dot = sum([rotn[j] * beam[j] for j in range(3)])
    r = math.sqrt(sum([rotn[j] * rotn[j] for j in range(3)]))
    b = math.sqrt(sum([beam[j] * beam[j] for j in range(3)]))

    rtod = 180.0 / math.pi

    angle = rtod * math.fabs(0.5 * math.pi - math.acos(dot / (r * b)))

    Debug.write('Axis misalignment %.2f degrees' % angle)

    correct_deviations = (correct.get_result('rmsd_pixel'),
                          correct.get_result('rmsd_phi'))

    if p1_deviations:
      # compare and reject if both > 50% higher - though adding a little
      # flexibility - 0.5 pixel / osc width slack.

      pixel = p1_deviations[0]
      phi = math.sqrt(0.05 * 0.05 + \
                      p1_deviations[1] * p1_deviations[1])

      threshold = Flags.get_rejection_threshold()

      Debug.write('RMSD ratio: %.2f' % (correct_deviations[0] / pixel))
      Debug.write('RMSPhi ratio: %.2f' % (correct_deviations[1] / phi))

      if correct_deviations[0] / pixel > threshold and \
             correct_deviations[1] / phi > threshold:

        Chatter.write(
        'Eliminating this indexing solution as postrefinement')
        Chatter.write(
        'deviations rather high relative to triclinic')
        raise BadLatticeError, \
              'high relative deviations in postrefinement'

    if not Flags.get_quick() and Flags.get_remove():
      # check for alien reflections and perhaps recycle - removing them
      if len(correct.get_remove()) > 0:

        correct_remove = correct.get_remove()
        current_remove = []
        final_remove = []

        # first ensure that there are no duplicate entries...
        if os.path.exists(os.path.join(
            self.get_working_directory(),
            'REMOVE.HKL')):
          for line in open(os.path.join(
              self.get_working_directory(),
              'REMOVE.HKL'), 'r').readlines():
            h, k, l = map(int, line.split()[:3])
            z = float(line.split()[3])

            if not (h, k, l, z) in current_remove:
              current_remove.append((h, k, l, z))

          for c in correct_remove:
            if c in current_remove:
              continue
            final_remove.append(c)

          Debug.write(
              '%d alien reflections are already removed' % \
              (len(correct_remove) - len(final_remove)))
        else:
          # we want to remove all of the new dodgy reflections
          final_remove = correct_remove

        remove_hkl = open(os.path.join(
            self.get_working_directory(),
            'REMOVE.HKL'), 'w')

        z_min = Flags.get_z_min()
        rejected = 0

        # write in the old reflections
        for remove in current_remove:
          z = remove[3]
          if z >= z_min:
            remove_hkl.write('%d %d %d %f\n' % remove)
          else:
            rejected += 1
        Debug.write('Wrote %d old reflections to REMOVE.HKL' % \
                    (len(current_remove) - rejected))
        Debug.write('Rejected %d as z < %f' % \
                    (rejected, z_min))

        # and the new reflections
        rejected = 0
        used = 0
        for remove in final_remove:
          z = remove[3]
          if z >= z_min:
            used += 1
            remove_hkl.write('%d %d %d %f\n' % remove)
          else:
            rejected += 1
        Debug.write('Wrote %d new reflections to REMOVE.HKL' % \
                    (len(final_remove) - rejected))
        Debug.write('Rejected %d as z < %f' % \
                    (rejected, z_min))

        remove_hkl.close()

        # we want to rerun the finishing step so...
        # unless we have added no new reflections... or unless we
        # have not confirmed the point group (see SCI-398)

        if used and self.get_integrater_reindex_matrix():
          self.set_integrater_finish_done(False)

    else:
      Debug.write(
          'Going quickly so not removing %d outlier reflections...' % \
          len(correct.get_remove()))

    # Convert INTEGRATE.HKL to MTZ format and reapply any reindexing operations
    # spacegroup changes to allow use with CCP4 / Aimless for scaling

    integrate_hkl = os.path.join(
      self.get_working_directory(), 'INTEGRATE.HKL')

    hklout = os.path.splitext(integrate_hkl)[0] + ".mtz"
    self._factory.set_working_directory(self.get_working_directory())
    pointless = self._factory.Pointless()
    pointless.set_xdsin(integrate_hkl)
    pointless.set_hklout(hklout)
    pointless.xds_to_mtz()

    integrate_mtz = hklout

    if self.get_integrater_reindex_operator() or \
       self.get_integrater_spacegroup_number():

      Debug.write('Reindexing things to MTZ')

      reindex = Reindex()
      reindex.set_working_directory(self.get_working_directory())
      auto_logfiler(reindex)

      if self.get_integrater_reindex_operator():
        reindex.set_operator(self.get_integrater_reindex_operator())

      if self.get_integrater_spacegroup_number():
        reindex.set_spacegroup(self.get_integrater_spacegroup_number())

      hklout = '%s_reindex.mtz' % os.path.splitext(integrate_mtz)[0]

      reindex.set_hklin(integrate_mtz)
      reindex.set_hklout(hklout)
      reindex.reindex()
      integrate_mtz = hklout

    return integrate_mtz
Esempio n. 5
0
    def _integrate_finish(self):
        """Finish off the integration by running correct."""

        # first run the postrefinement etc with spacegroup P1
        # and the current unit cell - this will be used to
        # obtain a benchmark rmsd in pixels / phi and also
        # cell deviations (this is working towards spotting bad
        # indexing solutions) - only do this if we have no
        # reindex matrix... and no postrefined cell...

        p1_deviations = None

        # fix for bug # 3264 -
        # if we have not run integration with refined parameters, make it so...
        # erm? shouldn't this therefore return if this is the principle, or
        # set the flag after we have tested the lattice?

        if ("GXPARM.XDS" not in self._xds_data_files
                and PhilIndex.params.xds.integrate.reintegrate):
            logger.debug(
                "Resetting integrater, to ensure refined orientation is used")
            self.set_integrater_done(False)

        if (not self.get_integrater_reindex_matrix() and not self._intgr_cell
                and PhilIndex.params.xia2.settings.lattice_rejection
                and not self.get_integrater_sweep().get_user_lattice()):
            correct = self.Correct()

            correct.set_data_range(
                self._intgr_wedge[0] + self.get_frame_offset(),
                self._intgr_wedge[1] + self.get_frame_offset(),
            )

            if self.get_polarization() > 0.0:
                correct.set_polarization(self.get_polarization())

            # FIXME should this be using the correctly transformed
            # cell or are the results ok without it?!

            correct.set_spacegroup_number(1)
            correct.set_cell(self._intgr_refiner_cell)

            correct.run()

            cell = correct.get_result("cell")
            cell_esd = correct.get_result("cell_esd")

            logger.debug("Postrefinement in P1 results:")
            logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" % tuple(cell))
            logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" %
                         tuple(cell_esd))
            logger.debug("Deviations: %.2f pixels %.2f degrees" %
                         (correct.get_result("rmsd_pixel"),
                          correct.get_result("rmsd_phi")))

            p1_deviations = (
                correct.get_result("rmsd_pixel"),
                correct.get_result("rmsd_phi"),
            )

        # next run the postrefinement etc with the given
        # cell / lattice - this will be the assumed result...

        integrate_hkl = os.path.join(self.get_working_directory(),
                                     "INTEGRATE.HKL")

        if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
            from dxtbx.serialize import load
            from dials.algorithms.shadowing.filter import filter_shadowed_reflections

            experiments_json = xparm_xds_to_experiments_json(
                os.path.join(self.get_working_directory(), "XPARM.XDS"),
                self.get_working_directory(),
            )
            experiments = load.experiment_list(experiments_json,
                                               check_format=True)
            imageset = experiments[0].imageset
            masker = (imageset.get_format_class().get_instance(
                imageset.paths()[0]).get_masker())
            if masker is not None:
                integrate_filename = integrate_hkl_to_reflection_file(
                    integrate_hkl, experiments_json,
                    self.get_working_directory())
                reflections = flex.reflection_table.from_file(
                    integrate_filename)

                t0 = time.time()
                sel = filter_shadowed_reflections(experiments, reflections)
                shadowed = reflections.select(sel)
                t1 = time.time()
                logger.debug("Filtered %i reflections in %.1f seconds" %
                             (sel.count(True), t1 - t0))

                filter_hkl = os.path.join(self.get_working_directory(),
                                          "FILTER.HKL")
                with open(filter_hkl, "wb") as f:
                    detector = experiments[0].detector
                    for ref in shadowed:
                        p = detector[ref["panel"]]
                        ox, oy = p.get_raw_image_offset()
                        h, k, l = ref["miller_index"]
                        x, y, z = ref["xyzcal.px"]
                        dx, dy, dz = (2, 2, 2)
                        print(
                            "%i %i %i %.1f %.1f %.1f %.1f %.1f %.1f" %
                            (h, k, l, x + ox, y + oy, z, dx, dy, dz),
                            file=f,
                        )
                t2 = time.time()
                logger.debug("Written FILTER.HKL in %.1f seconds" % (t2 - t1))

        correct = self.Correct()

        correct.set_data_range(
            self._intgr_wedge[0] + self.get_frame_offset(),
            self._intgr_wedge[1] + self.get_frame_offset(),
        )

        if self.get_polarization() > 0.0:
            correct.set_polarization(self.get_polarization())

        # BUG # 2695 probably comes from here - need to check...
        # if the pointless interface comes back with a different
        # crystal setting then the unit cell stored in self._intgr_cell
        # needs to be set to None...

        if self.get_integrater_spacegroup_number():
            correct.set_spacegroup_number(
                self.get_integrater_spacegroup_number())
            if not self._intgr_cell:
                raise RuntimeError("no unit cell to recycle")
            correct.set_cell(self._intgr_cell)

        # BUG # 3113 - new version of XDS will try and figure the
        # best spacegroup out from the intensities (and get it wrong!)
        # unless we set the spacegroup and cell explicitly

        if not self.get_integrater_spacegroup_number():
            cell = self._intgr_refiner_cell
            lattice = self._intgr_refiner.get_refiner_lattice()
            spacegroup_number = lattice_to_spacegroup_number(lattice)

            # this should not prevent the postrefinement from
            # working correctly, else what is above would not
            # work correctly (the postrefinement test)

            correct.set_spacegroup_number(spacegroup_number)
            correct.set_cell(cell)

            logger.debug("Setting spacegroup to: %d" % spacegroup_number)
            logger.debug("Setting cell to: %.2f %.2f %.2f %.2f %.2f %.2f" %
                         tuple(cell))

        if self.get_integrater_reindex_matrix():

            # bug! if the lattice is not primitive the values in this
            # reindex matrix need to be multiplied by a constant which
            # depends on the Bravais lattice centering.

            lattice = self._intgr_refiner.get_refiner_lattice()

            matrix = self.get_integrater_reindex_matrix()
            matrix = scitbx.matrix.sqr(matrix).transpose().elems
            matrix = r_to_rt(matrix)

            if lattice[1] == "P":
                mult = 1
            elif lattice[1] == "C" or lattice[1] == "I":
                mult = 2
            elif lattice[1] == "R":
                mult = 3
            elif lattice[1] == "F":
                mult = 4
            else:
                raise RuntimeError("unknown multiplier for lattice %s" %
                                   lattice)

            logger.debug("REIDX multiplier for lattice %s: %d" %
                         (lattice, mult))

            mult_matrix = [mult * m for m in matrix]

            logger.debug("REIDX set to %d %d %d %d %d %d %d %d %d %d %d %d" %
                         tuple(mult_matrix))
            correct.set_reindex_matrix(mult_matrix)

        correct.run()

        # record the log file -

        pname, xname, dname = self.get_integrater_project_info()
        sweep = self.get_integrater_sweep_name()
        FileHandler.record_log_file(
            f"{pname} {xname} {dname} {sweep} CORRECT",
            os.path.join(self.get_working_directory(), "CORRECT.LP"),
        )

        FileHandler.record_more_data_file(
            f"{pname} {xname} {dname} {sweep} CORRECT",
            os.path.join(self.get_working_directory(), "XDS_ASCII.HKL"),
        )

        # erm. just to be sure
        if self.get_integrater_reindex_matrix() and correct.get_reindex_used():
            raise RuntimeError("Reindex panic!")

        # get the reindex operation used, which may be useful if none was
        # set but XDS decided to apply one, e.g. #419.

        if not self.get_integrater_reindex_matrix(
        ) and correct.get_reindex_used():
            # convert this reindex operation to h, k, l form: n.b. this
            # will involve dividing through by the lattice centring multiplier

            matrix = rt_to_r(correct.get_reindex_used())

            matrix = scitbx.matrix.sqr(matrix).transpose().elems

            lattice = self._intgr_refiner.get_refiner_lattice()

            if lattice[1] == "P":
                mult = 1.0
            elif lattice[1] == "C" or lattice[1] == "I":
                mult = 2.0
            elif lattice[1] == "R":
                mult = 3.0
            elif lattice[1] == "F":
                mult = 4.0

            matrix = [m / mult for m in matrix]

            reindex_op = mat_to_symop(matrix)

            # assign this to self: will this reset?! make for a leaky
            # abstraction and just assign this...

            # self.set_integrater_reindex_operator(reindex)

            self._intgr_reindex_operator = reindex_op

        # record the log file -

        pname, xname, dname = self.get_integrater_project_info()
        sweep = self.get_integrater_sweep_name()
        FileHandler.record_log_file(
            f"{pname} {xname} {dname} {sweep} CORRECT",
            os.path.join(self.get_working_directory(), "CORRECT.LP"),
        )

        # should get some interesting stuff from the XDS correct file
        # here, for instance the resolution range to use in integration
        # (which should be fed back if not fast) and so on...

        self._intgr_corrected_hklout = os.path.join(
            self.get_working_directory(), "XDS_ASCII.HKL")

        # also record the batch range - needed for the analysis of the
        # radiation damage in chef...

        self._intgr_batches_out = (self._intgr_wedge[0], self._intgr_wedge[1])

        # FIXME perhaps I should also feedback the GXPARM file here??
        for file in ["GXPARM.XDS"]:
            self._xds_data_files[file] = correct.get_output_data_file(file)

        # record the postrefined cell parameters
        self._intgr_cell = correct.get_result("cell")
        self._intgr_n_ref = correct.get_result("n_ref")

        logger.debug('Postrefinement in "correct" spacegroup results:')
        logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" %
                     tuple(correct.get_result("cell")))
        logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" %
                     tuple(correct.get_result("cell_esd")))
        logger.debug(
            "Deviations: %.2f pixels %.2f degrees" %
            (correct.get_result("rmsd_pixel"), correct.get_result("rmsd_phi")))

        logger.debug("Error correction parameters: A=%.3f B=%.3f" %
                     correct.get_result("sdcorrection"))

        # compute misorientation of axes

        xparm_file = os.path.join(self.get_working_directory(), "GXPARM.XDS")

        handle = xparm.reader()
        handle.read_file(xparm_file)

        rotn = handle.rotation_axis
        beam = handle.beam_vector

        dot = sum(rotn[j] * beam[j] for j in range(3))
        r = math.sqrt(sum(rotn[j] * rotn[j] for j in range(3)))
        b = math.sqrt(sum(beam[j] * beam[j] for j in range(3)))

        rtod = 180.0 / math.pi

        angle = rtod * math.fabs(0.5 * math.pi - math.acos(dot / (r * b)))

        logger.debug("Axis misalignment %.2f degrees" % angle)

        correct_deviations = (
            correct.get_result("rmsd_pixel"),
            correct.get_result("rmsd_phi"),
        )

        if p1_deviations:
            # compare and reject if both > 50% higher - though adding a little
            # flexibility - 0.5 pixel / osc width slack.

            pixel = p1_deviations[0]
            phi = math.sqrt(0.05 * 0.05 + p1_deviations[1] * p1_deviations[1])

            threshold = PhilIndex.params.xia2.settings.lattice_rejection_threshold
            logger.debug("RMSD ratio: %.2f" % (correct_deviations[0] / pixel))
            logger.debug("RMSPhi ratio: %.2f" % (correct_deviations[1] / phi))

            if (correct_deviations[0] / pixel > threshold
                    and correct_deviations[1] / phi > threshold):

                logger.info(
                    "Eliminating this indexing solution as postrefinement")
                logger.info("deviations rather high relative to triclinic")
                raise BadLatticeError(
                    "high relative deviations in postrefinement")

        if (not PhilIndex.params.dials.fast_mode
                and not PhilIndex.params.xds.keep_outliers):
            # check for alien reflections and perhaps recycle - removing them
            correct_remove = correct.get_remove()
            if correct_remove:
                current_remove = set()
                final_remove = []

                # first ensure that there are no duplicate entries...
                if os.path.exists(
                        os.path.join(self.get_working_directory(),
                                     "REMOVE.HKL")):
                    with open(
                            os.path.join(self.get_working_directory(),
                                         "REMOVE.HKL")) as fh:
                        for line in fh.readlines():
                            h, k, l = list(map(int, line.split()[:3]))
                            z = float(line.split()[3])

                            if (h, k, l, z) not in current_remove:
                                current_remove.add((h, k, l, z))

                    for c in correct_remove:
                        if c in current_remove:
                            continue
                        final_remove.append(c)

                    logger.debug("%d alien reflections are already removed" %
                                 (len(correct_remove) - len(final_remove)))
                else:
                    # we want to remove all of the new dodgy reflections
                    final_remove = correct_remove

                z_min = PhilIndex.params.xds.z_min
                rejected = 0

                with open(
                        os.path.join(self.get_working_directory(),
                                     "REMOVE.HKL"), "w") as remove_hkl:

                    # write in the old reflections
                    for remove in current_remove:
                        z = remove[3]
                        if z >= z_min:
                            remove_hkl.write("%d %d %d %f\n" % remove)
                        else:
                            rejected += 1
                    logger.debug("Wrote %d old reflections to REMOVE.HKL" %
                                 (len(current_remove) - rejected))
                    logger.debug("Rejected %d as z < %f" % (rejected, z_min))

                    # and the new reflections
                    rejected = 0
                    used = 0
                    for remove in final_remove:
                        z = remove[3]
                        if z >= z_min:
                            used += 1
                            remove_hkl.write("%d %d %d %f\n" % remove)
                        else:
                            rejected += 1
                    logger.debug("Wrote %d new reflections to REMOVE.HKL" %
                                 (len(final_remove) - rejected))
                    logger.debug("Rejected %d as z < %f" % (rejected, z_min))

                # we want to rerun the finishing step so...
                # unless we have added no new reflections... or unless we
                # have not confirmed the point group (see SCI-398)

                if used and self.get_integrater_reindex_matrix():
                    self.set_integrater_finish_done(False)

        else:
            logger.debug(
                "Going quickly so not removing %d outlier reflections..." %
                len(correct.get_remove()))

        # Convert INTEGRATE.HKL to MTZ format and reapply any reindexing operations
        # spacegroup changes to allow use with CCP4 / Aimless for scaling

        hklout = os.path.splitext(integrate_hkl)[0] + ".mtz"
        self._factory.set_working_directory(self.get_working_directory())
        pointless = self._factory.Pointless()
        pointless.set_xdsin(integrate_hkl)
        pointless.set_hklout(hklout)
        pointless.xds_to_mtz()

        integrate_mtz = hklout

        if (self.get_integrater_reindex_operator()
                or self.get_integrater_spacegroup_number()):

            logger.debug("Reindexing things to MTZ")

            reindex = Reindex()
            reindex.set_working_directory(self.get_working_directory())
            auto_logfiler(reindex)

            if self.get_integrater_reindex_operator():
                reindex.set_operator(self.get_integrater_reindex_operator())

            if self.get_integrater_spacegroup_number():
                reindex.set_spacegroup(self.get_integrater_spacegroup_number())

            hklout = "%s_reindex.mtz" % os.path.splitext(integrate_mtz)[0]

            reindex.set_hklin(integrate_mtz)
            reindex.set_hklout(hklout)
            reindex.reindex()
            integrate_mtz = hklout

        experiments_json = xparm_xds_to_experiments_json(
            self._xds_data_files["GXPARM.XDS"], self.get_working_directory())
        pname, xname, dname = self.get_integrater_project_info()
        sweep = self.get_integrater_sweep_name()
        FileHandler.record_more_data_file(f"{pname} {xname} {dname} {sweep}",
                                          experiments_json)
        FileHandler.record_more_data_file(
            f"{pname} {xname} {dname} {sweep} INTEGRATE", integrate_mtz)

        self._intgr_experiments_filename = experiments_json

        return integrate_mtz
Esempio n. 6
0
    def _integrate_finish(self):
        '''Finish off the integration by running dials.export.'''

        # FIXME - do we want to export every time we call this method
        # (the file will not have changed) and also (more important) do
        # we want a different exported MTZ file every time (I do not think
        # that we do; these can be very large) - was exporter.get_xpid() ->
        # now dials

        exporter = self.ExportMtz()
        exporter.set_reflections_filename(self._intgr_integrated_pickle)
        mtz_filename = os.path.join(self.get_working_directory(),
                                    '%s_integrated.mtz' % 'dials')
        exporter.set_mtz_filename(mtz_filename)
        exporter.run()
        self._intgr_integrated_filename = mtz_filename

        # record integrated MTZ file for e.g. BLEND.

        pname, xname, dname = self.get_integrater_project_info()
        sweep = self.get_integrater_sweep_name()
        FileHandler.record_more_data_file(
            '%s %s %s %s INTEGRATE' % (pname, xname, dname, sweep),
            mtz_filename)

        from iotbx.reflection_file_reader import any_reflection_file
        miller_arrays = any_reflection_file(
            self._intgr_integrated_filename).as_miller_arrays()
        # look for profile-fitted intensities
        intensities = [
            ma for ma in miller_arrays
            if ma.info().labels == ['IPR', 'SIGIPR']
        ]
        if len(intensities) == 0:
            # look instead for summation-integrated intensities
            intensities = [
                ma for ma in miller_arrays
                if ma.info().labels == ['I', 'SIGI']
            ]
            assert len(intensities)
        self._intgr_n_ref = intensities[0].size()

        if not os.path.isfile(self._intgr_integrated_filename):
            raise RuntimeError("dials.export failed: %s does not exist." %
                               self._intgr_integrated_filename)

        if self._intgr_reindex_operator is None and \
          self._intgr_spacegroup_number == lattice_to_spacegroup(
            self.get_integrater_refiner().get_refiner_lattice()):
            Debug.write('Not reindexing to spacegroup %d (%s)' % \
                          (self._intgr_spacegroup_number,
                           self._intgr_reindex_operator))
            return mtz_filename

        if self._intgr_reindex_operator is None and \
          self._intgr_spacegroup_number == 0:
            Debug.write('Not reindexing to spacegroup %d (%s)' % \
                          (self._intgr_spacegroup_number,
                           self._intgr_reindex_operator))
            return mtz_filename

        Debug.write('Reindexing to spacegroup %d (%s)' % \
                    (self._intgr_spacegroup_number,
                     self._intgr_reindex_operator))

        hklin = mtz_filename
        reindex = Reindex()
        reindex.set_working_directory(self.get_working_directory())
        auto_logfiler(reindex)

        reindex.set_operator(self._intgr_reindex_operator)

        if self._intgr_spacegroup_number:
            reindex.set_spacegroup(self._intgr_spacegroup_number)
        else:
            reindex.set_spacegroup(
                lattice_to_spacegroup(
                    self.get_integrater_refiner().get_refiner_lattice()))

        hklout = '%s_reindex.mtz' % hklin[:-4]
        reindex.set_hklin(hklin)
        reindex.set_hklout(hklout)
        reindex.reindex()
        self._intgr_integrated_filename = hklout
        self._intgr_cell = reindex.get_cell()

        pname, xname, dname = self.get_integrater_project_info()
        sweep = self.get_integrater_sweep_name()
        FileHandler.record_more_data_file(
            '%s %s %s %s experiments' % (pname, xname, dname, sweep),
            self.get_integrated_experiments())

        return hklout
    def _integrate_finish(self):
        """Finish off the integration by running dials.export."""

        # FIXME - do we want to export every time we call this method
        # (the file will not have changed) and also (more important) do
        # we want a different exported MTZ file every time (I do not think
        # that we do; these can be very large) - was exporter.get_xpid() ->
        # now dials

        if self._output_format == "hkl":
            exporter = self.ExportMtz()
            exporter.set_reflections_filename(
                self._intgr_integrated_reflections)
            mtz_filename = os.path.join(self.get_working_directory(),
                                        "%s_integrated.mtz" % "dials")
            exporter.set_mtz_filename(mtz_filename)
            exporter.run()
            self._intgr_integrated_filename = mtz_filename

            # record integrated MTZ file for e.g. BLEND.

            pname, xname, dname = self.get_integrater_project_info()
            sweep = self.get_integrater_sweep_name()
            FileHandler.record_more_data_file(
                "%s %s %s %s INTEGRATE" % (pname, xname, dname, sweep),
                mtz_filename)

            from iotbx.reflection_file_reader import any_reflection_file

            miller_arrays = any_reflection_file(
                self._intgr_integrated_filename).as_miller_arrays()
            # look for profile-fitted intensities
            intensities = [
                ma for ma in miller_arrays
                if ma.info().labels == ["IPR", "SIGIPR"]
            ]
            if len(intensities) == 0:
                # look instead for summation-integrated intensities
                intensities = [
                    ma for ma in miller_arrays
                    if ma.info().labels == ["I", "SIGI"]
                ]
                assert len(intensities)
            self._intgr_n_ref = intensities[0].size()

            if not os.path.isfile(self._intgr_integrated_filename):
                raise RuntimeError("dials.export failed: %s does not exist." %
                                   self._intgr_integrated_filename)

            if (self._intgr_reindex_operator is None
                    and self._intgr_spacegroup_number == lattice_to_spacegroup(
                        self.get_integrater_refiner().get_refiner_lattice())):
                Debug.write("Not reindexing to spacegroup %d (%s)" %
                            (self._intgr_spacegroup_number,
                             self._intgr_reindex_operator))
                return mtz_filename

            if (self._intgr_reindex_operator is None
                    and self._intgr_spacegroup_number == 0):
                Debug.write("Not reindexing to spacegroup %d (%s)" %
                            (self._intgr_spacegroup_number,
                             self._intgr_reindex_operator))
                return mtz_filename

            Debug.write(
                "Reindexing to spacegroup %d (%s)" %
                (self._intgr_spacegroup_number, self._intgr_reindex_operator))

            hklin = mtz_filename
            from xia2.Wrappers.CCP4.Reindex import Reindex

            reindex = Reindex()
            reindex.set_working_directory(self.get_working_directory())
            auto_logfiler(reindex)

            reindex.set_operator(self._intgr_reindex_operator)

            if self._intgr_spacegroup_number:
                reindex.set_spacegroup(self._intgr_spacegroup_number)
            else:
                reindex.set_spacegroup(
                    lattice_to_spacegroup(
                        self.get_integrater_refiner().get_refiner_lattice()))

            hklout = "%s_reindex.mtz" % hklin[:-4]
            reindex.set_hklin(hklin)
            reindex.set_hklout(hklout)
            reindex.reindex()
            self._intgr_integrated_filename = hklout
            self._intgr_cell = reindex.get_cell()

            pname, xname, dname = self.get_integrater_project_info()
            sweep = self.get_integrater_sweep_name()
            FileHandler.record_more_data_file(
                "%s %s %s %s" % (pname, xname, dname, sweep),
                self.get_integrated_experiments(),
            )
            FileHandler.record_more_data_file(
                "%s %s %s %s" % (pname, xname, dname, sweep),
                self.get_integrated_reflections(),
            )

            return hklout

        elif self._output_format == "pickle":

            if (self._intgr_reindex_operator is None
                    and self._intgr_spacegroup_number == lattice_to_spacegroup(
                        self.get_integrater_refiner().get_refiner_lattice())):
                Debug.write("Not reindexing to spacegroup %d (%s)" %
                            (self._intgr_spacegroup_number,
                             self._intgr_reindex_operator))
                return self._intgr_integrated_reflections

            if (self._intgr_reindex_operator is None
                    and self._intgr_spacegroup_number == 0):
                Debug.write("Not reindexing to spacegroup %d (%s)" %
                            (self._intgr_spacegroup_number,
                             self._intgr_reindex_operator))
                return self._intgr_integrated_reflections

            Debug.write(
                "Reindexing to spacegroup %d (%s)" %
                (self._intgr_spacegroup_number, self._intgr_reindex_operator))
            from xia2.Wrappers.Dials.Reindex import Reindex

            reindex = Reindex()
            reindex.set_working_directory(self.get_working_directory())
            auto_logfiler(reindex)

            reindex.set_cb_op(self._intgr_reindex_operator)

            if self._intgr_spacegroup_number:
                reindex.set_space_group(self._intgr_spacegroup_number)
            else:
                reindex.set_space_group(
                    lattice_to_spacegroup(
                        self.get_integrater_refiner().get_refiner_lattice()))

            reindex.set_experiments_filename(self.get_integrated_experiments())
            reindex.set_indexed_filename(self.get_integrated_reflections())

            reindex.run()
            self._intgr_integrated_reflections = (
                reindex.get_reindexed_reflections_filename())
            self._intgr_integrated_filename = (
                reindex.get_reindexed_reflections_filename())
            self._intgr_experiments_filename = (
                reindex.get_reindexed_experiments_filename())

            pname, xname, dname = self.get_integrater_project_info()
            sweep = self.get_integrater_sweep_name()
            FileHandler.record_more_data_file(
                "%s %s %s %s" % (pname, xname, dname, sweep),
                self.get_integrated_experiments(),
            )
            FileHandler.record_more_data_file(
                "%s %s %s %s" % (pname, xname, dname, sweep),
                self.get_integrated_reflections(),
            )
            return None  # this will be set to intgr_hklout - better to cause failure