Ejemplo n.º 1
0
    def _integrate_finish(self):
        '''Finish the integration - if necessary performing reindexing
    based on the pointgroup and the reindexing operator.'''

        if self._intgr_reindex_operator is None and \
           self._intgr_spacegroup_number == lattice_to_spacegroup(
            self.get_integrater_refiner().get_refiner_lattice()):
            return self._mosflm_hklout

        if self._intgr_reindex_operator is None and \
           self._intgr_spacegroup_number == 0:
            return self._mosflm_hklout

        Debug.write('Reindexing to spacegroup %d (%s)' % \
                    (self._intgr_spacegroup_number,
                     self._intgr_reindex_operator))

        hklin = self._mosflm_hklout
        reindex = Reindex()
        reindex.set_working_directory(self.get_working_directory())
        auto_logfiler(reindex)

        reindex.set_operator(self._intgr_reindex_operator)

        if self._intgr_spacegroup_number:
            reindex.set_spacegroup(self._intgr_spacegroup_number)

        hklout = '%s_reindex.mtz' % hklin[:-4]

        reindex.set_hklin(hklin)
        reindex.set_hklout(hklout)
        reindex.reindex()

        return hklout
Ejemplo n.º 2
0
  def _integrate_finish(self):
    '''Finish the integration - if necessary performing reindexing
    based on the pointgroup and the reindexing operator.'''

    if self._intgr_reindex_operator is None and \
       self._intgr_spacegroup_number == lattice_to_spacegroup(
        self.get_integrater_refiner().get_refiner_lattice()):
      return self._mosflm_hklout

    if self._intgr_reindex_operator is None and \
       self._intgr_spacegroup_number == 0:
      return self._mosflm_hklout

    Debug.write('Reindexing to spacegroup %d (%s)' % \
                (self._intgr_spacegroup_number,
                 self._intgr_reindex_operator))

    hklin = self._mosflm_hklout
    reindex = Reindex()
    reindex.set_working_directory(self.get_working_directory())
    auto_logfiler(reindex)

    reindex.set_operator(self._intgr_reindex_operator)

    if self._intgr_spacegroup_number:
      reindex.set_spacegroup(self._intgr_spacegroup_number)

    hklout = '%s_reindex.mtz' % hklin[:-4]

    reindex.set_hklin(hklin)
    reindex.set_hklout(hklout)
    reindex.reindex()

    return hklout
Ejemplo n.º 3
0
    def _index(self):
        '''Implement the indexer interface.'''

        Citations.cite('mosflm')

        indexer = MosflmIndex()
        indexer.set_working_directory(self.get_working_directory())
        auto_logfiler(indexer)

        from xia2.lib.bits import unique_elements
        _images = unique_elements(self._indxr_images)
        indexer.set_images(_images)
        images_str = ', '.join(map(str, _images))

        cell_str = None
        if self._indxr_input_cell:
            cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
                        self._indxr_input_cell

        if self._indxr_sweep_name:

            #if len(self._fp_directory) <= 50:
            #dirname = self._fp_directory
            #else:
            #dirname = '...%s' % self._fp_directory[-46:]
            dirname = os.path.dirname(self.get_imageset().get_template())

            Journal.block(
                'autoindexing', self._indxr_sweep_name, 'mosflm', {
                    'images': images_str,
                    'target cell': self._indxr_input_cell,
                    'target lattice': self._indxr_input_lattice,
                    'template': self.get_imageset().get_template(),
                    'directory': dirname
                })

        #task = 'Autoindex from images:'

        #for i in _images:
        #task += ' %s' % self.get_image_name(i)

        #self.set_task(task)

        indexer.set_template(os.path.basename(self.get_template()))
        indexer.set_directory(self.get_directory())

        xsweep = self.get_indexer_sweep()
        if xsweep is not None:
            if xsweep.get_distance() is not None:
                indexer.set_distance(xsweep.get_distance())
            #if self.get_wavelength_prov() == 'user':
            #index.set_wavelength(self.get_wavelength())
            if xsweep.get_beam_centre() is not None:
                indexer.set_beam_centre(xsweep.get_beam_centre())

        if self._indxr_input_cell:
            indexer.set_unit_cell(self._indxr_input_cell)

        if self._indxr_input_lattice is not None:
            spacegroup_number = lattice_to_spacegroup(
                self._indxr_input_lattice)
            indexer.set_space_group_number(spacegroup_number)

        if not self._mosflm_autoindex_thresh:

            try:

                min_peaks = 200

                Debug.write('Aiming for at least %d spots...' % min_peaks)

                thresholds = []

                for i in _images:

                    p = Printpeaks()
                    p.set_working_directory(self.get_working_directory())
                    auto_logfiler(p)
                    p.set_image(self.get_image_name(i))
                    thresh = p.threshold(min_peaks)

                    Debug.write('Autoindex threshold for image %d: %d' % \
                                (i, thresh))

                    thresholds.append(thresh)

                thresh = min(thresholds)
                self._mosflm_autoindex_thresh = thresh

            except Exception as e:
                print str(e)  #XXX this should disappear!
                Debug.write('Error computing threshold: %s' % str(e))
                Debug.write('Using default of 20.0')
                thresh = 20.0

        else:
            thresh = self._mosflm_autoindex_thresh

        Debug.write('Using autoindex threshold: %d' % thresh)

        if self._mosflm_autoindex_sol:
            indexer.set_solution_number(self._mosflm_autoindex_sol)
        indexer.set_threshold(thresh)

        # now forget this to prevent weird things happening later on
        if self._mosflm_autoindex_sol:
            self._mosflm_autoindex_sol = 0

        indexer.run()

        indxr_cell = indexer.get_refined_unit_cell()
        self._indxr_lattice = indexer.get_lattice()
        space_group_number = indexer.get_indexed_space_group_number()
        detector_distance = indexer.get_refined_distance()
        beam_centre = indexer.get_refined_beam_centre()
        mosaic_spreads = indexer.get_mosaic_spreads()

        if min(list(indxr_cell)) < 10.0 and \
           indxr_cell[2] / indxr_cell[0] > 6:

            Debug.write('Unrealistic autoindexing solution: ' +
                        '%.2f %.2f %.2f %.2f %.2f %.2f' % indxr_cell)

            # tweak some parameters and try again...
            self._mosflm_autoindex_thresh *= 1.5
            self.set_indexer_done(False)

            return

        intgr_params = {}

        # look up other possible indexing solutions (not well - in
        # standard settings only!) This is moved earlier as it could
        # result in returning if Mosflm has selected the wrong
        # solution!

        try:
            self._indxr_other_lattice_cell = indexer.get_solutions()

            # Change 27/FEB/08 to support user assigned spacegroups
            if self._indxr_user_input_lattice:
                lattice_to_spacegroup_dict = {
                    'aP': 1,
                    'mP': 3,
                    'mC': 5,
                    'oP': 16,
                    'oC': 20,
                    'oF': 22,
                    'oI': 23,
                    'tP': 75,
                    'tI': 79,
                    'hP': 143,
                    'hR': 146,
                    'cP': 195,
                    'cF': 196,
                    'cI': 197
                }
                for k in self._indxr_other_lattice_cell.keys():
                    if lattice_to_spacegroup_dict[k] > \
                           lattice_to_spacegroup_dict[
                        self._indxr_input_lattice]:
                        del (self._indxr_other_lattice_cell[k])

            # check that the selected unit cell matches - and if
            # not raise a "horrible" exception

            if self._indxr_input_cell:
                assert indxr_cell is not None
                for j in range(6):
                    if math.fabs(self._indxr_input_cell[j] -
                                 indxr_cell[j]) > 2.0:
                        Chatter.write('Mosflm autoindexing did not select ' +
                                      'correct (target) unit cell')
                        raise RuntimeError(
                            'something horrible happened in indexing')

        except RuntimeError as e:
            # check if mosflm rejected a solution we have it
            if 'horribl' in str(e):
                # ok it did - time to break out the big guns...
                if not self._indxr_input_cell:
                    raise RuntimeError(
                        'error in solution selection when not preset')

                # XXX FIXME
                self._mosflm_autoindex_sol = _get_indexing_solution_number(
                    indexer.get_all_output(), self._indxr_input_cell,
                    self._indxr_input_lattice)

                # set the fact that we are not done...
                self.set_indexer_done(False)

                # and return - hopefully this will restart everything
                return
            else:
                raise e

        if len(mosaic_spreads) == 0:
            # then consider setting it do a default value...
            # equal to the oscillation width (a good guess)
            phi_width = self.get_phi_width()
            Chatter.write(
                'Mosaic estimation failed, so guessing at %4.2f' % \
                phi_width)
            # only consider this if we have thus far no idea on the
            # mosaic spread...
            mosaic_spreads.append(phi_width)

        intgr_params['raster'] = indexer.get_raster()

        intgr_params['separation'] = indexer.get_separation()

        self._indxr_resolution_estimate = indexer.get_resolution_estimate()

        # compute mosaic as mean(mosaic_spreads)

        self._indxr_mosaic = sum(mosaic_spreads) / len(mosaic_spreads)

        self._indxr_payload['mosflm_integration_parameters'] = intgr_params

        self._indxr_payload['mosflm_orientation_matrix'] = open(
            os.path.join(self.get_working_directory(), 'xiaindex.mat'),
            'r').readlines()

        import copy
        from dxtbx.model.detector_helpers import set_mosflm_beam_centre
        from xia2.Wrappers.Mosflm.AutoindexHelpers import set_distance
        from xia2.Wrappers.Mosflm.AutoindexHelpers import crystal_model_from_mosflm_mat
        from cctbx import sgtbx, uctbx

        # update the beam centre (i.e. shift the origin of the detector)
        detector = copy.deepcopy(self.get_detector())
        beam = copy.deepcopy(self.get_beam())
        set_mosflm_beam_centre(detector, beam, beam_centre)
        if detector_distance is not None:
            set_distance(detector, detector_distance)

        # make a dxtbx crystal_model object from the mosflm matrix
        space_group = sgtbx.space_group_info(number=space_group_number).group()
        crystal_model = crystal_model_from_mosflm_mat(
            self._indxr_payload['mosflm_orientation_matrix'],
            unit_cell=uctbx.unit_cell(tuple(indxr_cell)),
            space_group=space_group)

        # construct an experiment_list
        from dxtbx.model import Experiment, ExperimentList
        experiment = Experiment(beam=beam,
                                detector=detector,
                                goniometer=self.get_goniometer(),
                                scan=self.get_scan(),
                                crystal=crystal_model)

        experiment_list = ExperimentList([experiment])
        self.set_indexer_experiment_list(experiment_list)
Ejemplo n.º 4
0
  def _integrate_finish(self):
    '''Finish off the integration by running dials.export.'''

    # FIXME - do we want to export every time we call this method
    # (the file will not have changed) and also (more important) do
    # we want a different exported MTZ file every time (I do not think
    # that we do; these can be very large) - was exporter.get_xpid() ->
    # now dials

    exporter = self.ExportMtz()
    exporter.set_reflections_filename(self._intgr_integrated_pickle)
    mtz_filename = os.path.join(
      self.get_working_directory(), '%s_integrated.mtz' % 'dials')
    exporter.set_mtz_filename(mtz_filename)
    exporter.run()
    self._intgr_integrated_filename = mtz_filename

    # record integrated MTZ file for e.g. BLEND.

    pname, xname, dname = self.get_integrater_project_info()
    sweep = self.get_integrater_sweep_name()
    FileHandler.record_more_data_file(
        '%s %s %s %s INTEGRATE' % (pname, xname, dname, sweep), mtz_filename)

    if not os.path.isfile(self._intgr_integrated_filename):
      raise RuntimeError("dials.export failed: %s does not exist."
                         % self._intgr_integrated_filename)

    if self._intgr_reindex_operator is None and \
      self._intgr_spacegroup_number == lattice_to_spacegroup(
        self.get_integrater_refiner().get_refiner_lattice()):
      Debug.write('Not reindexing to spacegroup %d (%s)' % \
                    (self._intgr_spacegroup_number,
                     self._intgr_reindex_operator))
      return mtz_filename

    if self._intgr_reindex_operator is None and \
      self._intgr_spacegroup_number == 0:
      Debug.write('Not reindexing to spacegroup %d (%s)' % \
                    (self._intgr_spacegroup_number,
                     self._intgr_reindex_operator))
      return mtz_filename

    Debug.write('Reindexing to spacegroup %d (%s)' % \
                (self._intgr_spacegroup_number,
                 self._intgr_reindex_operator))

    hklin = mtz_filename
    reindex = Reindex()
    reindex.set_working_directory(self.get_working_directory())
    auto_logfiler(reindex)

    reindex.set_operator(self._intgr_reindex_operator)

    if self._intgr_spacegroup_number:
      reindex.set_spacegroup(self._intgr_spacegroup_number)
    else:
      reindex.set_spacegroup(lattice_to_spacegroup(
        self.get_integrater_refiner().get_refiner_lattice()))

    hklout = '%s_reindex.mtz' % hklin[:-4]
    reindex.set_hklin(hklin)
    reindex.set_hklout(hklout)
    reindex.reindex()
    self._intgr_integrated_filename = hklout
    self._intgr_cell = reindex.get_cell()

    pname, xname, dname = self.get_integrater_project_info()
    sweep = self.get_integrater_sweep_name()
    FileHandler.record_more_data_file(
      '%s %s %s %s experiments' % (pname, xname, dname, sweep),
      self.get_integrated_experiments())

    from iotbx.reflection_file_reader import any_reflection_file
    miller_arrays = any_reflection_file(hklout).as_miller_arrays()
    # look for profile-fitted intensities
    intensities = [ma for ma in miller_arrays
                   if ma.info().labels == ['IPR', 'SIGIPR']]
    if len(intensities) == 0:
      # look instead for summation-integrated intensities
      intensities = [ma for ma in miller_arrays
                     if ma.info().labels == ['I', 'SIGI']]
      assert len(intensities)
    self._intgr_n_ref = intensities[0].size()

    return hklout
Ejemplo n.º 5
0
    def _index(self):
        '''Actually index the diffraction pattern. Note well that
    this is not going to compute the matrix...'''

        # acknowledge this program

        Citations.cite('labelit')
        Citations.cite('distl')

        #self.reset()

        _images = []
        for i in self._indxr_images:
            for j in i:
                if not j in _images:
                    _images.append(j)

        _images.sort()

        images_str = '%d' % _images[0]
        for i in _images[1:]:
            images_str += ', %d' % i

        cell_str = None
        if self._indxr_input_cell:
            cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
                        self._indxr_input_cell

        if self._indxr_sweep_name:

            # then this is a proper autoindexing run - describe this
            # to the journal entry

            #if len(self._fp_directory) <= 50:
            #dirname = self._fp_directory
            #else:
            #dirname = '...%s' % self._fp_directory[-46:]
            dirname = os.path.dirname(self.get_imageset().get_template())

            Journal.block(
                'autoindexing', self._indxr_sweep_name, 'labelit', {
                    'images': images_str,
                    'target cell': cell_str,
                    'target lattice': self._indxr_input_lattice,
                    'template': self.get_imageset().get_template(),
                    'directory': dirname
                })

        if len(_images) > 4:
            raise RuntimeError('cannot use more than 4 images')

        from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex
        index = LabelitIndex()
        index.set_working_directory(self.get_working_directory())
        auto_logfiler(index)

        #task = 'Autoindex from images:'

        #for i in _images:
        #task += ' %s' % self.get_image_name(i)

        #self.set_task(task)

        Debug.write('Indexing from images:')
        for i in _images:
            index.add_image(self.get_image_name(i))
            Debug.write('%s' % self.get_image_name(i))

        xsweep = self.get_indexer_sweep()
        if xsweep is not None:
            if xsweep.get_distance() is not None:
                index.set_distance(xsweep.get_distance())
            #if self.get_wavelength_prov() == 'user':
            #index.set_wavelength(self.get_wavelength())
            if xsweep.get_beam_centre() is not None:
                index.set_beam_centre(xsweep.get_beam_centre())

        if self._refine_beam is False:
            index.set_refine_beam(False)
        else:
            index.set_refine_beam(True)
            index.set_beam_search_scope(self._beam_search_scope)

        if ((math.fabs(self.get_wavelength() - 1.54) < 0.01)
                or (math.fabs(self.get_wavelength() - 2.29) < 0.01)):
            index.set_Cu_KA_or_Cr_KA(True)

        #sweep = self.get_indexer_sweep_name()
        #FileHandler.record_log_file(
        #'%s INDEX' % (sweep), self.get_log_file())

        try:
            index.run()
        except RuntimeError as e:

            if self._refine_beam is False:
                raise e

            # can we improve the situation?

            if self._beam_search_scope < 4.0:
                self._beam_search_scope += 4.0

                # try repeating the indexing!

                self.set_indexer_done(False)
                return 'failed'

            # otherwise this is beyond redemption

            raise e

        self._solutions = index.get_solutions()

        # FIXME this needs to check the smilie status e.g.
        # ":)" or ";(" or "  ".

        # FIXME need to check the value of the RMSD and raise an
        # exception if the P1 solution has an RMSD > 1.0...

        # Change 27/FEB/08 to support user assigned spacegroups
        # (euugh!) have to "ignore" solutions with higher symmetry
        # otherwise the rest of xia will override us. Bummer.

        for i, solution in self._solutions.iteritems():
            if self._indxr_user_input_lattice:
                if (lattice_to_spacegroup(solution['lattice']) >
                        lattice_to_spacegroup(self._indxr_input_lattice)):
                    Debug.write('Ignoring solution: %s' % solution['lattice'])
                    del self._solutions[i]

        # check the RMSD from the triclinic unit cell
        if self._solutions[1]['rmsd'] > 1.0 and False:
            # don't know when this is useful - but I know when it is not!
            raise RuntimeError('high RMSD for triclinic solution')

        # configure the "right" solution
        self._solution = self.get_solution()

        # now store also all of the other solutions... keyed by the
        # lattice - however these should only be added if they
        # have a smiley in the appropriate record, perhaps?

        for solution in self._solutions.keys():
            lattice = self._solutions[solution]['lattice']
            if lattice in self._indxr_other_lattice_cell:
                if self._indxr_other_lattice_cell[lattice]['goodness'] < \
                   self._solutions[solution]['metric']:
                    continue

            self._indxr_other_lattice_cell[lattice] = {
                'goodness': self._solutions[solution]['metric'],
                'cell': self._solutions[solution]['cell']
            }

        self._indxr_lattice = self._solution['lattice']
        self._indxr_cell = tuple(self._solution['cell'])
        self._indxr_mosaic = self._solution['mosaic']

        lms = LabelitMosflmScript()
        lms.set_working_directory(self.get_working_directory())
        lms.set_solution(self._solution['number'])
        self._indxr_payload['mosflm_orientation_matrix'] = lms.calculate()

        # get the beam centre from the mosflm script - mosflm
        # may have inverted the beam centre and labelit will know
        # this!

        mosflm_beam_centre = lms.get_mosflm_beam()

        if mosflm_beam_centre:
            self._indxr_payload['mosflm_beam_centre'] = tuple(
                mosflm_beam_centre)

        import copy
        detector = copy.deepcopy(self.get_detector())
        beam = copy.deepcopy(self.get_beam())
        from dxtbx.model.detector_helpers import set_mosflm_beam_centre
        set_mosflm_beam_centre(detector, beam, mosflm_beam_centre)

        from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
        from scitbx import matrix
        from cctbx import sgtbx, uctbx
        from dxtbx.model import CrystalFactory
        mosflm_matrix = matrix.sqr([
            float(i) for line in lms.calculate()
            for i in line.replace("-", " -").split()
        ][:9])

        space_group = sgtbx.space_group_info(
            lattice_to_spacegroup_number(self._solution['lattice'])).group()
        crystal_model = CrystalFactory.from_mosflm_matrix(
            mosflm_matrix,
            unit_cell=uctbx.unit_cell(tuple(self._solution['cell'])),
            space_group=space_group)

        from dxtbx.model import Experiment, ExperimentList
        experiment = Experiment(
            beam=beam,
            detector=detector,
            goniometer=self.get_goniometer(),
            scan=self.get_scan(),
            crystal=crystal_model,
        )

        experiment_list = ExperimentList([experiment])
        self.set_indexer_experiment_list(experiment_list)

        # also get an estimate of the resolution limit from the
        # labelit.stats_distl output... FIXME the name is wrong!

        lsd = LabelitStats_distl()
        lsd.set_working_directory(self.get_working_directory())
        lsd.stats_distl()

        resolution = 1.0e6
        for i in _images:
            stats = lsd.get_statistics(self.get_image_name(i))

            resol = 0.5 * (stats['resol_one'] + stats['resol_two'])

            if resol < resolution:
                resolution = resol

        self._indxr_resolution_estimate = resolution

        return 'ok'
    def _index(self):
        """Actually index the diffraction pattern. Note well that
        this is not going to compute the matrix..."""

        # acknowledge this program

        if not self._indxr_images:
            raise RuntimeError("No good spots found on any images")

        Citations.cite("labelit")
        Citations.cite("distl")

        _images = []
        for i in self._indxr_images:
            for j in i:
                if not j in _images:
                    _images.append(j)

        _images.sort()

        images_str = "%d" % _images[0]
        for i in _images[1:]:
            images_str += ", %d" % i

        cell_str = None
        if self._indxr_input_cell:
            cell_str = "%.2f %.2f %.2f %.2f %.2f %.2f" % self._indxr_input_cell

        if self._indxr_sweep_name:

            # then this is a proper autoindexing run - describe this
            # to the journal entry

            if len(self._fp_directory) <= 50:
                dirname = self._fp_directory
            else:
                dirname = "...%s" % self._fp_directory[-46:]

            Journal.block(
                "autoindexing",
                self._indxr_sweep_name,
                "labelit",
                {
                    "images": images_str,
                    "target cell": cell_str,
                    "target lattice": self._indxr_input_lattice,
                    "template": self._fp_template,
                    "directory": dirname,
                },
            )

        # auto_logfiler(self)

        from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex

        index = LabelitIndex()
        index.set_working_directory(self.get_working_directory())
        auto_logfiler(index)

        # task = 'Autoindex from images:'

        # for i in _images:
        # task += ' %s' % self.get_image_name(i)

        # self.set_task(task)

        # self.add_command_line('--index_only')

        Debug.write("Indexing from images:")
        for i in _images:
            index.add_image(self.get_image_name(i))
            Debug.write("%s" % self.get_image_name(i))

        if self._primitive_unit_cell:
            index.set_primitive_unit_cell(self._primitive_unit_cell)

        if self._indxr_input_cell:
            index.set_max_cell(1.25 * max(self._indxr_input_cell[:3]))

        xsweep = self.get_indexer_sweep()
        if xsweep is not None:
            if xsweep.get_distance() is not None:
                index.set_distance(xsweep.get_distance())
            # if self.get_wavelength_prov() == 'user':
            # index.set_wavelength(self.get_wavelength())
            if xsweep.get_beam_centre() is not None:
                index.set_beam_centre(xsweep.get_beam_centre())

        if self._refine_beam is False:
            index.set_refine_beam(False)
        else:
            index.set_refine_beam(True)
            index.set_beam_search_scope(self._beam_search_scope)

        if (math.fabs(self.get_wavelength() - 1.54) <
                0.01) or (math.fabs(self.get_wavelength() - 2.29) < 0.01):
            index.set_Cu_KA_or_Cr_KA(True)

        try:
            index.run()
        except RuntimeError as e:

            if self._refine_beam is False:
                raise e

            # can we improve the situation?

            if self._beam_search_scope < 4.0:
                self._beam_search_scope += 4.0

                # try repeating the indexing!

                self.set_indexer_done(False)
                return "failed"

            # otherwise this is beyond redemption

            raise e

        self._solutions = index.get_solutions()

        # FIXME this needs to check the smilie status e.g.
        # ":)" or ";(" or "  ".

        # FIXME need to check the value of the RMSD and raise an
        # exception if the P1 solution has an RMSD > 1.0...

        # Change 27/FEB/08 to support user assigned spacegroups
        # (euugh!) have to "ignore" solutions with higher symmetry
        # otherwise the rest of xia will override us. Bummer.

        for i, solution in self._solutions.iteritems():
            if self._indxr_user_input_lattice:
                if lattice_to_spacegroup(
                        solution["lattice"]) > lattice_to_spacegroup(
                            self._indxr_input_lattice):
                    Debug.write("Ignoring solution: %s" % solution["lattice"])
                    del self._solutions[i]

        # configure the "right" solution
        self._solution = self.get_solution()

        # now store also all of the other solutions... keyed by the
        # lattice - however these should only be added if they
        # have a smiley in the appropriate record, perhaps?

        for solution in self._solutions.keys():
            lattice = self._solutions[solution]["lattice"]
            if lattice in self._indxr_other_lattice_cell:
                if (self._indxr_other_lattice_cell[lattice]["goodness"] <
                        self._solutions[solution]["metric"]):
                    continue

            self._indxr_other_lattice_cell[lattice] = {
                "goodness": self._solutions[solution]["metric"],
                "cell": self._solutions[solution]["cell"],
            }

        self._indxr_lattice = self._solution["lattice"]
        self._indxr_cell = tuple(self._solution["cell"])
        self._indxr_mosaic = self._solution["mosaic"]

        lms = LabelitMosflmMatrix()
        lms.set_working_directory(self.get_working_directory())
        lms.set_solution(self._solution["number"])
        self._indxr_payload["mosflm_orientation_matrix"] = lms.calculate()

        # get the beam centre from the mosflm script - mosflm
        # may have inverted the beam centre and labelit will know
        # this!

        mosflm_beam_centre = lms.get_mosflm_beam()

        if mosflm_beam_centre:
            self._indxr_payload["mosflm_beam_centre"] = tuple(
                mosflm_beam_centre)

        detector = copy.deepcopy(self.get_detector())
        beam = copy.deepcopy(self.get_beam())
        from dxtbx.model.detector_helpers import set_mosflm_beam_centre

        set_mosflm_beam_centre(detector, beam, mosflm_beam_centre)

        from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
        from scitbx import matrix
        from cctbx import sgtbx, uctbx
        from dxtbx.model import CrystalFactory

        mosflm_matrix = matrix.sqr([
            float(i) for line in lms.calculate()
            for i in line.replace("-", " -").split()
        ][:9])

        space_group = sgtbx.space_group_info(
            lattice_to_spacegroup_number(self._solution["lattice"])).group()
        crystal_model = CrystalFactory.from_mosflm_matrix(
            mosflm_matrix,
            unit_cell=uctbx.unit_cell(tuple(self._solution["cell"])),
            space_group=space_group,
        )

        from dxtbx.model import Experiment, ExperimentList

        experiment = Experiment(
            beam=beam,
            detector=detector,
            goniometer=self.get_goniometer(),
            scan=self.get_scan(),
            crystal=crystal_model,
        )

        experiment_list = ExperimentList([experiment])
        self.set_indexer_experiment_list(experiment_list)

        # also get an estimate of the resolution limit from the
        # labelit.stats_distl output... FIXME the name is wrong!

        lsd = LabelitStats_distl()
        lsd.set_working_directory(self.get_working_directory())
        lsd.stats_distl()

        resolution = 1.0e6
        for i in _images:
            stats = lsd.get_statistics(self.get_image_name(i))

            resol = 0.5 * (stats["resol_one"] + stats["resol_two"])

            if resol < resolution:
                resolution = resol

        self._indxr_resolution_estimate = resolution

        return "ok"
Ejemplo n.º 7
0
    def _mosflm_refine_cell(self, idxr, set_spacegroup=None):
        '''Perform the refinement of the unit cell. This will populate
    all of the information needed to perform the integration.'''

        # FIXME this will die after #1285

        #if not self.get_integrater_indexer():
        #Debug.write('Replacing indexer of %s with self at %d' % \
        #(str(self.get_integrater_indexer()), __line__))
        #self.set_integrater_indexer(self)

        #idxr = self.get_integrater_indexer()

        if not idxr.get_indexer_payload('mosflm_orientation_matrix'):
            raise RuntimeError('unexpected situation in indexing')

        lattice = idxr.get_indexer_lattice()
        mosaic = idxr.get_indexer_mosaic()
        cell = idxr.get_indexer_cell()
        beam_centre = idxr.get_indexer_beam_centre()

        # bug # 3174 - if mosaic is very small (here defined to be
        # 0.25 x osc_width) then set to this minimum value.

        phi_width = idxr.get_phi_width()
        if mosaic < 0.25 * phi_width:
            mosaic = 0.25 * phi_width

        if idxr.get_indexer_payload('mosflm_beam_centre'):
            beam_centre = idxr.get_indexer_payload('mosflm_beam_centre')

        distance = idxr.get_indexer_distance()
        matrix = idxr.get_indexer_payload('mosflm_orientation_matrix')

        integration_params = idxr.get_indexer_payload(
            'mosflm_integration_parameters')

        if integration_params is None:
            integration_params = {}

        if integration_params:
            if 'separation' in integration_params:
                self.set_refiner_parameter(
                    'mosflm', 'separation',
                    '%f %f' % tuple(integration_params['separation']))
            if 'raster' in integration_params:
                self.set_refiner_parameter(
                    'mosflm', 'raster',
                    '%d %d %d %d %d' % tuple(integration_params['raster']))

        idxr.set_indexer_payload('mosflm_integration_parameters', None)

        spacegroup_number = lattice_to_spacegroup(lattice)

        # copy these into myself for later reference, if indexer
        # is not myself - everything else is copied via the
        # cell refinement process...

        from cctbx import sgtbx
        from dxtbx.model import Crystal
        from dxtbx.model.detector_helpers import set_mosflm_beam_centre

        experiment = idxr.get_indexer_experiment_list()[0]
        set_mosflm_beam_centre(experiment.detector, experiment.beam,
                               beam_centre)
        space_group = sgtbx.space_group_info(number=spacegroup_number).group()
        a, b, c = experiment.crystal.get_real_space_vectors()
        experiment.crystal = Crystal(a, b, c, space_group=space_group)

        # FIXME surely these have been assigned further up?!

        if not self._mosflm_cell_ref_images:
            self._mosflm_cell_ref_images = self._refine_select_images(mosaic)

        f = open(
            os.path.join(self.get_working_directory(),
                         'xiaindex-%s.mat' % lattice), 'w')
        for m in matrix:
            f.write(m)
        f.close()

        # then start the cell refinement

        refiner = MosflmRefineCell()
        refiner.set_working_directory(self.get_working_directory())
        auto_logfiler(refiner)

        if self._mosflm_gain:
            refiner.set_gain(self._mosflm_gain)

        refiner.set_template(os.path.basename(idxr.get_template()))
        refiner.set_directory(idxr.get_directory())
        refiner.set_input_mat_file('xiaindex-%s.mat' % lattice)
        refiner.set_output_mat_file('xiarefine.mat')
        refiner.set_beam_centre(beam_centre)
        refiner.set_unit_cell(cell)
        refiner.set_distance(distance)
        if set_spacegroup:
            refiner.set_space_group_number(set_spacegroup)
        else:
            refiner.set_space_group_number(spacegroup_number)

        # FIXME 18/JUN/08 - it may help to have an overestimate
        # of the mosaic spread in here as it *may* refine down
        # better than up... - this is not a good idea as it may
        # also not refine at all! - 12972 # integration failed

        # Bug # 3103
        if self._mosflm_cell_ref_double_mosaic:
            mosaic *= 2.0
        refiner.set_mosaic(mosaic)

        # if set, use the resolution for cell refinement - see
        # bug # 2078...

        if self._mosflm_cell_ref_resolution:
            refiner.set_resolution(self._mosflm_cell_ref_resolution)

        refiner.set_fix_mosaic(self._mosflm_postref_fix_mosaic)

        # note well that the beam centre is coming from indexing so
        # should be already properly handled

        #if idxr.get_wavelength_prov() == 'user':
        #refiner.set_wavelength(idxr.get_wavelength())

        # belt + braces mode - only to be used when considering failover,
        # will run an additional step of autoindexing prior to cell
        # refinement, to be used only after proving that not going it
        # will result in cell refinement failure - will use the first
        # wedge... N.B. this is only useful if the indexer is Labelit
        # not Mosflm...

        refiner.set_add_autoindex(self._mosflm_cell_ref_add_autoindex)

        # get all of the stored parameter values
        parameters = self.get_refiner_parameters('mosflm')
        refiner.update_parameters(parameters)

        detector = idxr.get_detector()
        detector_width, detector_height = detector[0].get_image_size_mm()

        lim_x = 0.5 * detector_width
        lim_y = 0.5 * detector_height

        Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
        refiner.set_limits(lim_x, lim_y)
        refiner.set_images(self._mosflm_cell_ref_images)

        failover = PhilIndex.params.xia2.settings.failover

        if failover and not self._mosflm_cell_ref_add_autoindex:
            refiner.set_ignore_cell_refinement_failure(True)

        refiner.run()

        # then look to see if the cell refinement worked ok - if it
        # didn't then this may indicate that the lattice was wrongly
        # selected.

        cell_refinement_ok = refiner.cell_refinement_ok()

        if not cell_refinement_ok:
            Debug.write('Repeating cell refinement...')
            self.set_integrater_prepare_done(False)
            self._mosflm_cell_ref_add_autoindex = True
            return [0.0], [0.0]

        rms_values = refiner.get_rms_values()
        background_residual = refiner.get_background_residual()
        self._refinr_cell = refiner.get_refined_unit_cell()
        distance = refiner.get_refined_distance2()
        experiment = idxr.get_indexer_experiment_list()[0]
        from xia2.Wrappers.Mosflm.AutoindexHelpers import set_distance
        set_distance(experiment.detector, distance)

        self.set_refiner_parameter('mosflm', 'distortion yscale',
                                   refiner.get_refined_distortion_yscale())

        self.set_refiner_parameter('mosflm', 'raster', refiner.get_raster())

        #integration_params['distortion yscale'] \
        #= refiner.get_refined_distortion_yscale()
        #integration_params['raster'] = refiner.get_raster()

        separation = refiner.get_separation()
        if separation is not None:
            self.set_refiner_parameter('mosflm', 'separation',
                                       '%s %s' % refiner.get_separation())
            #integration_params['separation'] = refiner.get_separation()

        self.set_refiner_parameter('mosflm', 'beam',
                                   '%s %s' % refiner.get_refined_beam_centre())
        self.set_refiner_parameter('mosflm', 'distance',
                                   refiner.get_refined_distance())
        self.set_refiner_parameter('mosflm', 'distortion tilt',
                                   refiner.get_refined_distortion_tilt())
        self.set_refiner_parameter('mosflm', 'distortion twist',
                                   refiner.get_refined_distortion_twist())

        integration_params['beam'] = tuple(
            float(b) for b in refiner.get_refined_beam_centre())
        integration_params['distance'] = refiner.get_refined_distance()
        integration_params[
            'distortion tilt'] = refiner.get_refined_distortion_tilt()
        integration_params[
            'distortion twist'] = refiner.get_refined_distortion_twist()

        idxr._indxr_mosaic = refiner.get_refined_mosaic()

        idxr.set_indexer_payload(
            'mosflm_orientation_matrix',
            open(os.path.join(self.get_working_directory(), 'xiarefine.mat'),
                 'r').readlines())
        self.set_refiner_payload(
            'mosflm_orientation_matrix',
            idxr.get_indexer_payload('mosflm_orientation_matrix'))
        self.set_refiner_payload('mosaic', refiner.get_refined_mosaic())
        self.set_refiner_payload('beam', integration_params['beam'])
        self.set_refiner_payload('distance', integration_params['distance'])

        from xia2.Wrappers.Mosflm.AutoindexHelpers import crystal_model_from_mosflm_mat
        # make a dxtbx crystal_model object from the mosflm matrix
        experiment = idxr.get_indexer_experiment_list()[0]
        crystal_model = crystal_model_from_mosflm_mat(
            idxr._indxr_payload['mosflm_orientation_matrix'],
            unit_cell=refiner.get_refined_unit_cell(),
            space_group=experiment.crystal.get_space_group())
        experiment.crystal = crystal_model

        #self.set_refiner_payload(
        #'mosflm_integration_parameters', integration_params)

        self._refinr_refined_experiment_list = ExperimentList([experiment])

        return rms_values, background_residual
Ejemplo n.º 8
0
    def _mosflm_test_refine_cell(self, idxr, test_lattice):
        '''Test performing cell refinement in with a different
    lattice to the one which was selected by the autoindex
    procedure. This should not change anything in the class.'''

        #idxr = self.get_integrater_indexer()

        lattice = idxr.get_indexer_lattice()
        mosaic = idxr.get_indexer_mosaic()
        beam_centre = idxr.get_indexer_beam_centre()
        distance = idxr.get_indexer_distance()
        matrix = idxr.get_indexer_payload('mosflm_orientation_matrix')

        phi_width = idxr.get_phi_width()
        if mosaic < 0.25 * phi_width:
            mosaic = 0.25 * phi_width

        input_matrix = ''
        for m in matrix:
            input_matrix += '%s\n' % m

        new_matrix = transmogrify_matrix(lattice, input_matrix, test_lattice,
                                         idxr.get_wavelength(),
                                         self.get_working_directory())

        spacegroup_number = lattice_to_spacegroup(test_lattice)

        if not self._mosflm_cell_ref_images:
            raise RuntimeError('wedges must be assigned already')

        open(
            os.path.join(self.get_working_directory(),
                         'test-xiaindex-%s.mat' % lattice),
            'w').write(new_matrix)

        refiner = MosflmRefineCell()
        refiner.set_working_directory(self.get_working_directory())
        auto_logfiler(refiner)

        if self._mosflm_gain:
            refiner.set_gain(self._mosflm_gain)

        refiner.set_template(os.path.basename(idxr.get_template()))
        refiner.set_directory(idxr.get_directory())
        refiner.set_input_mat_file('test-xiaindex-%s.mat' % lattice)
        refiner.set_output_mat_file('test-xiarefine.mat')
        refiner.set_beam_centre(beam_centre)
        refiner.set_distance(distance)
        refiner.set_space_group_number(spacegroup_number)

        # FIXME 18/JUN/08 - it may help to have an overestimate
        # of the mosaic spread in here as it *may* refine down
        # better than up... - this is not a good idea as it may
        # also not refine at all! - 12972 # integration failed

        # Bug # 3103
        if self._mosflm_cell_ref_double_mosaic:
            mosaic *= 2.0
        refiner.set_mosaic(mosaic)

        # if set, use the resolution for cell refinement - see
        # bug # 2078...

        if self._mosflm_cell_ref_resolution:
            refiner.set_resolution(self._mosflm_cell_ref_resolution)

        refiner.set_fix_mosaic(self._mosflm_postref_fix_mosaic)

        # note well that the beam centre is coming from indexing so
        # should be already properly handled

        #if idxr.get_wavelength_prov() == 'user':
        #refiner.set_wavelength(idxr.get_wavelength())

        # belt + braces mode - only to be used when considering failover,
        # will run an additional step of autoindexing prior to cell
        # refinement, to be used only after proving that not going it
        # will result in cell refinement failure - will use the first
        # wedge... N.B. this is only useful if the indexer is Labelit
        # not Mosflm...

        refiner.set_add_autoindex(self._mosflm_cell_ref_add_autoindex)

        # XXX
        # get all of the stored parameter values
        parameters = self.get_refiner_parameters('mosflm')
        refiner.update_parameters(parameters)

        detector = idxr.get_detector()
        detector_width, detector_height = detector[0].get_image_size_mm()

        lim_x = 0.5 * detector_width
        lim_y = 0.5 * detector_height

        Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
        refiner.set_limits(lim_x, lim_y)
        refiner.set_images(self._mosflm_cell_ref_images)

        refiner.run()

        rms_values = refiner.get_rms_values()
        background_residual = refiner.get_background_residual()

        return rms_values, background_residual
Ejemplo n.º 9
0
    def _mosflm_parallel_integrate(self):
        '''Perform the integration as before, but this time as a
    number of parallel Mosflm jobs (hence, in separate directories)
    and including a step of pre-refinement of the mosaic spread and
    missets. This will all be kind of explicit and hence probably
    messy!'''

        refinr = self.get_integrater_refiner()

        lattice = refinr.get_refiner_lattice()
        spacegroup_number = lattice_to_spacegroup(lattice)
        mosaic = refinr.get_refiner_payload('mosaic')
        beam = refinr.get_refiner_payload('beam')
        distance = refinr.get_refiner_payload('distance')
        matrix = refinr.get_refiner_payload('mosflm_orientation_matrix')

        integration_params = refinr.get_refiner_payload(
            'mosflm_integration_parameters')

        if integration_params:
            if 'separation' in integration_params:
                self.set_integrater_parameter(
                    'mosflm', 'separation',
                    '%s %s' % tuple(integration_params['separation']))
            if 'raster' in integration_params:
                self.set_integrater_parameter(
                    'mosflm', 'raster',
                    '%d %d %d %d %d' % tuple(integration_params['raster']))

        refinr.set_refiner_payload('mosflm_integration_parameters', None)
        pname, xname, dname = self.get_integrater_project_info()

        # what follows below should (i) be run in separate directories
        # and (ii) be repeated N=parallel times.

        nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
        parallel = nproc

        # FIXME this is something of a kludge - if too few frames refinement
        # and integration does not work well... ideally want at least 15
        # frames / chunk (say)
        nframes = self._intgr_wedge[1] - self._intgr_wedge[0] + 1

        if parallel > nframes / 15:
            parallel = nframes // 15

        if not parallel:
            raise RuntimeError('parallel not set')
        if parallel < 2:
            raise RuntimeError('parallel not parallel: %s' % parallel)

        jobs = []
        hklouts = []
        nref = 0

        # calculate the chunks to use
        offset = self.get_frame_offset()
        start = self._intgr_wedge[0] - offset
        end = self._intgr_wedge[1] - offset

        left_images = 1 + end - start
        left_chunks = parallel
        chunks = []

        while left_images > 0:
            size = left_images // left_chunks
            chunks.append((start, start + size - 1))
            start += size
            left_images -= size
            left_chunks -= 1

        summary_files = []

        for j in range(parallel):

            # make some working directories, as necessary - chunk-(0:N-1)
            wd = os.path.join(self.get_working_directory(), 'chunk-%d' % j)
            if not os.path.exists(wd):
                os.makedirs(wd)

            job = MosflmIntegrate()
            job.set_working_directory(wd)

            auto_logfiler(job)

            l = refinr.get_refiner_lattice()

            # create the starting point
            f = open(os.path.join(wd, 'xiaintegrate-%s.mat' % l), 'w')
            for m in matrix:
                f.write(m)
            f.close()

            spacegroup_number = lattice_to_spacegroup(lattice)

            job.set_refine_profiles(self._mosflm_refine_profiles)

            # N.B. for harvesting need to append N to dname.

            if pname is not None and xname is not None and dname is not None:
                Debug.write('Harvesting: %s/%s/%s' % (pname, xname, dname))
                harvest_dir = self.get_working_directory()
                temp_dname = '%s_%s' % \
                             (dname, self.get_integrater_sweep_name())
                job.set_pname_xname_dname(pname, xname, temp_dname)

            job.set_template(os.path.basename(self.get_template()))
            job.set_directory(self.get_directory())

            # check for ice - and if so, exclude (ranges taken from
            # XDS documentation)
            if self.get_integrater_ice() != 0:
                Debug.write('Excluding ice rings')
                job.set_exclude_ice(True)

            # exclude specified resolution ranges
            if len(self.get_integrater_excluded_regions()) != 0:
                regions = self.get_integrater_excluded_regions()
                Debug.write('Excluding regions: %s' % repr(regions))
                job.set_exclude_regions(regions)

            mask = standard_mask(self.get_detector())
            for m in mask:
                job.add_instruction(m)

            job.set_input_mat_file('xiaintegrate-%s.mat' % l)

            job.set_beam_centre(beam)
            job.set_distance(distance)
            job.set_space_group_number(spacegroup_number)
            job.set_mosaic(mosaic)

            if self.get_wavelength_prov() == 'user':
                job.set_wavelength(self.get_wavelength())

            parameters = self.get_integrater_parameters('mosflm')
            job.update_parameters(parameters)

            if self._mosflm_gain:
                job.set_gain(self._mosflm_gain)

            # check for resolution limits
            if self._intgr_reso_high > 0.0:
                job.set_d_min(self._intgr_reso_high)
            if self._intgr_reso_low:
                job.set_d_max(self._intgr_reso_low)

            if PhilIndex.params.general.backstop_mask:
                from xia2.Toolkit.BackstopMask import BackstopMask
                mask = BackstopMask(PhilIndex.params.general.backstop_mask)
                mask = mask.calculate_mask_mosflm(self.get_header())
                job.set_mask(mask)

            detector = self.get_detector()
            detector_width, detector_height = detector[0].get_image_size_mm()

            lim_x = 0.5 * detector_width
            lim_y = 0.5 * detector_height

            Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
            job.set_limits(lim_x, lim_y)

            job.set_fix_mosaic(self._mosflm_postref_fix_mosaic)

            job.set_pre_refinement(True)
            job.set_image_range(chunks[j])

            # these are now running so ...

            jobs.append(job)

            continue

        # ok, at this stage I need to ...
        #
        # (i) accumulate the statistics as a function of batch
        # (ii) mong them into a single block
        #
        # This is likely to be a pain in the arse!

        first_integrated_batch = 1.0e6
        last_integrated_batch = -1.0e6

        all_residuals = []

        threads = []

        for j in range(parallel):
            job = jobs[j]

            # now wait for them to finish - first wait will really be the
            # first one, then all should be finished...

            thread = Background(job, 'run')
            thread.start()
            threads.append(thread)

        mosaics = []
        postref_result = {}

        integrated_images_first = 1.0e6
        integrated_images_last = -1.0e6
        self._intgr_per_image_statistics = {}

        for j in range(parallel):
            thread = threads[j]
            thread.stop()
            job = jobs[j]

            # get the log file
            output = job.get_all_output()

            # record a copy of it, perhaps - though not if parallel
            if self.get_integrater_sweep_name() and False:
                pname, xname, dname = self.get_integrater_project_info()
                FileHandler.record_log_file(
                    '%s %s %s %s mosflm integrate' % \
                    (self.get_integrater_sweep_name(),
                     pname, xname, '%s_%d' % (dname, j)),
                    job.get_log_file())

            # look for things that we want to know...
            # that is, the output reflection file name, the updated
            # value for the gain (if present,) any warnings, errors,
            # or just interesting facts.

            batches = job.get_batches_out()
            integrated_images_first = min(batches[0], integrated_images_first)
            integrated_images_last = max(batches[1], integrated_images_last)

            mosaics.extend(job.get_mosaic_spreads())

            if min(mosaics) < 0:
                raise IntegrationError('negative mosaic spread: %s' %
                                       min(mosaic))

            if (job.get_detector_gain_error()
                    and not (self.get_imageset().get_detector()[0].get_type()
                             == 'SENSOR_PAD')):
                gain = job.get_suggested_gain()
                if gain is not None:
                    self.set_integrater_parameter('mosflm', 'gain', gain)
                    self.set_integrater_export_parameter(
                        'mosflm', 'gain', gain)
                    if self._mosflm_gain:
                        Debug.write('GAIN updated to %f' % gain)
                    else:
                        Debug.write('GAIN found to be %f' % gain)

                    self._mosflm_gain = gain
                    self._mosflm_rerun_integration = True

            hklout = job.get_hklout()
            Debug.write('Integration output: %s' % hklout)
            hklouts.append(hklout)

            nref += job.get_nref()

            # if a BGSIG error happened try not refining the
            # profile and running again...

            if job.get_bgsig_too_large():
                if not self._mosflm_refine_profiles:
                    raise RuntimeError('BGSIG error with profiles fixed')

                Debug.write('BGSIG error detected - try fixing profile...')

                self._mosflm_refine_profiles = False
                self.set_integrater_done(False)

                return

            if job.get_getprof_error():
                Debug.write('GETPROF error detected - try fixing profile...')
                self._mosflm_refine_profiles = False
                self.set_integrater_done(False)

                return

            # here
            # write the report for each image as .*-#$ to Chatter -
            # detailed report will be written automagically to science...

            self._intgr_per_image_statistics.update(
                job.get_per_image_statistics())
            postref_result.update(job.get_postref_result())

            # inspect the output for e.g. very high weighted residuals

            all_residuals.extend(job.get_residuals())

        self._intgr_batches_out = (integrated_images_first,
                                   integrated_images_last)

        if mosaics and len(mosaics) > 0:
            self.set_integrater_mosaic_min_mean_max(
                min(mosaics),
                sum(mosaics) / len(mosaics), max(mosaics))
        else:
            m = indxr.get_indexer_mosaic()
            self.set_integrater_mosaic_min_mean_max(m, m, m)

        Chatter.write(self.show_per_image_statistics())

        Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
                      self.get_integrater_mosaic_min_mean_max())

        # gather the statistics from the postrefinement for all sweeps
        # now write this to a postrefinement log

        postref_log = os.path.join(self.get_working_directory(),
                                   'postrefinement.log')

        fout = open(postref_log, 'w')

        fout.write('$TABLE: Postrefinement for %s:\n' % \
                   self._intgr_sweep_name)
        fout.write('$GRAPHS: Missetting angles:A:1, 2, 3, 4: $$\n')
        fout.write('Batch PhiX PhiY PhiZ $$ Batch PhiX PhiY PhiZ $$\n')

        for image in sorted(postref_result):
            phix = postref_result[image].get('phix', 0.0)
            phiy = postref_result[image].get('phiy', 0.0)
            phiz = postref_result[image].get('phiz', 0.0)

            fout.write('%d %5.2f %5.2f %5.2f\n' % \
                       (image, phix, phiy, phiz))

        fout.write('$$\n')
        fout.close()

        if self.get_integrater_sweep_name():
            pname, xname, dname = self.get_integrater_project_info()
            FileHandler.record_log_file('%s %s %s %s postrefinement' % \
                                        (self.get_integrater_sweep_name(),
                                         pname, xname, dname),
                                        postref_log)

        hklouts.sort()

        hklout = os.path.join(self.get_working_directory(),
                              os.path.split(hklouts[0])[-1])

        Debug.write('Sorting data to %s' % hklout)
        for hklin in hklouts:
            Debug.write('<= %s' % hklin)

        sortmtz = Sortmtz()
        sortmtz.set_hklout(hklout)
        for hklin in hklouts:
            sortmtz.add_hklin(hklin)

        sortmtz.sort()

        self._mosflm_hklout = hklout

        return self._mosflm_hklout
Ejemplo n.º 10
0
    def _mosflm_integrate(self):
        '''Perform the actual integration, based on the results of the
    cell refinement or indexing (they have the equivalent form.)'''

        refinr = self.get_integrater_refiner()

        if not refinr.get_refiner_payload('mosflm_orientation_matrix'):
            raise RuntimeError('unexpected situation in indexing')

        lattice = refinr.get_refiner_lattice()
        spacegroup_number = lattice_to_spacegroup(lattice)
        mosaic = refinr.get_refiner_payload('mosaic')
        beam = refinr.get_refiner_payload('beam')
        distance = refinr.get_refiner_payload('distance')
        matrix = refinr.get_refiner_payload('mosflm_orientation_matrix')

        integration_params = refinr.get_refiner_payload(
            'mosflm_integration_parameters')

        if integration_params:
            if 'separation' in integration_params:
                self.set_integrater_parameter(
                    'mosflm', 'separation',
                    '%s %s' % tuple(integration_params['separation']))
            if 'raster' in integration_params:
                self.set_integrater_parameter(
                    'mosflm', 'raster',
                    '%d %d %d %d %d' % tuple(integration_params['raster']))

        refinr.set_refiner_payload('mosflm_integration_parameters', None)

        f = open(
            os.path.join(self.get_working_directory(), 'xiaintegrate.mat'),
            'w')
        for m in matrix:
            f.write(m)
        f.close()

        # then start the integration
        integrater = MosflmIntegrate()
        integrater.set_working_directory(self.get_working_directory())
        auto_logfiler(integrater)

        integrater.set_refine_profiles(self._mosflm_refine_profiles)

        pname, xname, dname = self.get_integrater_project_info()

        if pname is not None and xname is not None and dname is not None:
            Debug.write('Harvesting: %s/%s/%s' % (pname, xname, dname))
            harvest_dir = self.get_working_directory()
            # harvest file name will be %s.mosflm_run_start_end % dname
            temp_dname = '%s_%s' % \
                         (dname, self.get_integrater_sweep_name())
            integrater.set_pname_xname_dname(pname, xname, temp_dname)

        integrater.set_template(os.path.basename(self.get_template()))
        integrater.set_directory(self.get_directory())

        # check for ice - and if so, exclude (ranges taken from
        # XDS documentation)
        if self.get_integrater_ice() != 0:
            Debug.write('Excluding ice rings')
            integrater.set_exclude_ice(True)

        # exclude specified resolution ranges
        if len(self.get_integrater_excluded_regions()) != 0:
            regions = self.get_integrater_excluded_regions()
            Debug.write('Excluding regions: %s' % repr(regions))
            integrater.set_exclude_regions(regions)

        mask = standard_mask(self.get_detector())
        for m in mask:
            integrater.add_instruction(m)

        integrater.set_input_mat_file('xiaintegrate.mat')

        integrater.set_beam_centre(beam)
        integrater.set_distance(distance)
        integrater.set_space_group_number(spacegroup_number)
        integrater.set_mosaic(mosaic)

        if self.get_wavelength_prov() == 'user':
            integrater.set_wavelength(self.get_wavelength())

        parameters = self.get_integrater_parameters('mosflm')
        integrater.update_parameters(parameters)

        if self._mosflm_gain:
            integrater.set_gain(self._mosflm_gain)

        # check for resolution limits
        if self._intgr_reso_high > 0.0:
            integrater.set_d_min(self._intgr_reso_high)
        if self._intgr_reso_low:
            integrater.set_d_max(self._intgr_reso_low)

        if PhilIndex.params.general.backstop_mask:
            from xia2.Toolkit.BackstopMask import BackstopMask
            mask = BackstopMask(PhilIndex.params.general.backstop_mask)
            mask = mask.calculate_mask_mosflm(self.get_header())
            integrater.set_mask(mask)

        detector = self.get_detector()
        detector_width, detector_height = detector[0].get_image_size_mm()

        lim_x = 0.5 * detector_width
        lim_y = 0.5 * detector_height

        Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
        integrater.set_limits(lim_x, lim_y)

        integrater.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
        offset = self.get_frame_offset()

        integrater.set_image_range(
            (self._intgr_wedge[0] - offset, self._intgr_wedge[1] - offset))

        try:
            integrater.run()
        except RuntimeError as e:
            if 'integration failed: reason unknown' in str(e):
                Chatter.write('Mosflm has failed in integration')
                message = 'The input was:\n\n'
                for input in integrater.get_all_input():
                    message += '  %s' % input
                Chatter.write(message)
            raise

        FileHandler.record_log_file(
            '%s %s %s %s mosflm integrate' % \
            (self.get_integrater_sweep_name(),
             pname, xname, dname),
            integrater.get_log_file())

        self._intgr_per_image_statistics = integrater.get_per_image_statistics(
        )

        self._mosflm_hklout = integrater.get_hklout()
        Debug.write('Integration output: %s' % self._mosflm_hklout)

        self._intgr_n_ref = integrater.get_nref()

        # if a BGSIG error happened try not refining the
        # profile and running again...

        if integrater.get_bgsig_too_large():
            if not self._mosflm_refine_profiles:
                raise RuntimeError('BGSIG error with profiles fixed')

            Debug.write('BGSIG error detected - try fixing profile...')

            self._mosflm_refine_profiles = False
            self.set_integrater_done(False)

            return

        if integrater.get_getprof_error():
            Debug.write('GETPROF error detected - try fixing profile...')
            self._mosflm_refine_profiles = False
            self.set_integrater_done(False)

            return

        if (integrater.get_detector_gain_error()
                and not (self.get_imageset().get_detector()[0].get_type()
                         == 'SENSOR_PAD')):
            gain = integrater.get_suggested_gain()
            if gain is not None:
                self.set_integrater_parameter('mosflm', 'gain', gain)
                self.set_integrater_export_parameter('mosflm', 'gain', gain)
                if self._mosflm_gain:
                    Debug.write('GAIN updated to %f' % gain)
                else:
                    Debug.write('GAIN found to be %f' % gain)

                self._mosflm_gain = gain
                self._mosflm_rerun_integration = True

        if not self._mosflm_hklout:
            raise RuntimeError('processing abandoned')

        self._intgr_batches_out = integrater.get_batches_out()

        mosaics = integrater.get_mosaic_spreads()
        if mosaics and len(mosaics) > 0:
            self.set_integrater_mosaic_min_mean_max(
                min(mosaics),
                sum(mosaics) / len(mosaics), max(mosaics))
        else:
            m = indxr.get_indexer_mosaic()
            self.set_integrater_mosaic_min_mean_max(m, m, m)

        # write the report for each image as .*-#$ to Chatter -
        # detailed report will be written automagically to science...

        Chatter.write(self.show_per_image_statistics())

        Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
                      self.get_integrater_mosaic_min_mean_max())

        # gather the statistics from the postrefinement
        postref_result = integrater.get_postref_result()

        # now write this to a postrefinement log
        postref_log = os.path.join(self.get_working_directory(),
                                   'postrefinement.log')

        fout = open(postref_log, 'w')

        fout.write('$TABLE: Postrefinement for %s:\n' % \
                   self._intgr_sweep_name)
        fout.write('$GRAPHS: Missetting angles:A:1, 2, 3, 4: $$\n')
        fout.write('Batch PhiX PhiY PhiZ $$ Batch PhiX PhiY PhiZ $$\n')

        for image in sorted(postref_result):
            phix = postref_result[image].get('phix', 0.0)
            phiy = postref_result[image].get('phiy', 0.0)
            phiz = postref_result[image].get('phiz', 0.0)

            fout.write('%d %5.2f %5.2f %5.2f\n' % \
                       (image, phix, phiy, phiz))

        fout.write('$$\n')
        fout.close()

        if self.get_integrater_sweep_name():
            pname, xname, dname = self.get_integrater_project_info()
            FileHandler.record_log_file('%s %s %s %s postrefinement' % \
                                        (self.get_integrater_sweep_name(),
                                         pname, xname, dname),
                                        postref_log)

        return self._mosflm_hklout
Ejemplo n.º 11
0
  def _mosflm_parallel_integrate(self):
    '''Perform the integration as before, but this time as a
    number of parallel Mosflm jobs (hence, in separate directories)
    and including a step of pre-refinement of the mosaic spread and
    missets. This will all be kind of explicit and hence probably
    messy!'''

    refinr = self.get_integrater_refiner()

    lattice = refinr.get_refiner_lattice()
    spacegroup_number = lattice_to_spacegroup(lattice)
    mosaic = refinr.get_refiner_payload('mosaic')
    beam = refinr.get_refiner_payload('beam')
    distance = refinr.get_refiner_payload('distance')
    matrix = refinr.get_refiner_payload('mosflm_orientation_matrix')

    integration_params = refinr.get_refiner_payload(
      'mosflm_integration_parameters')

    if integration_params:
      if 'separation' in integration_params:
        self.set_integrater_parameter(
            'mosflm', 'separation',
            '%s %s' % tuple(integration_params['separation']))
      if 'raster' in integration_params:
        self.set_integrater_parameter(
            'mosflm', 'raster',
            '%d %d %d %d %d' % tuple(integration_params['raster']))

    refinr.set_refiner_payload('mosflm_integration_parameters', None)
    pname, xname, dname = self.get_integrater_project_info()

    # what follows below should (i) be run in separate directories
    # and (ii) be repeated N=parallel times.

    nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
    parallel = nproc

    # FIXME this is something of a kludge - if too few frames refinement
    # and integration does not work well... ideally want at least 15
    # frames / chunk (say)
    nframes = self._intgr_wedge[1] - self._intgr_wedge[0] + 1

    if parallel > nframes / 15:
      parallel = nframes // 15

    if not parallel:
      raise RuntimeError, 'parallel not set'
    if parallel < 2:
      raise RuntimeError, 'parallel not parallel: %s' % parallel

    jobs = []
    hklouts = []
    nref = 0

    # calculate the chunks to use
    offset = self.get_frame_offset()
    start = self._intgr_wedge[0] - offset
    end = self._intgr_wedge[1] - offset

    left_images = 1 + end - start
    left_chunks = parallel
    chunks = []

    while left_images > 0:
      size = left_images // left_chunks
      chunks.append((start, start + size - 1))
      start += size
      left_images -= size
      left_chunks -= 1

    summary_files = []

    for j in range(parallel):

      # make some working directories, as necessary - chunk-(0:N-1)
      wd = os.path.join(self.get_working_directory(),
                        'chunk-%d' % j)
      if not os.path.exists(wd):
        os.makedirs(wd)

      job = MosflmIntegrate()
      job.set_working_directory(wd)

      auto_logfiler(job)

      l = refinr.get_refiner_lattice()

      # create the starting point
      f = open(os.path.join(wd, 'xiaintegrate-%s.mat' % l), 'w')
      for m in matrix:
        f.write(m)
      f.close()

      spacegroup_number = lattice_to_spacegroup(lattice)

      job.set_refine_profiles(self._mosflm_refine_profiles)

      # N.B. for harvesting need to append N to dname.

      if pname is not None and xname is not None and dname is not None:
        Debug.write('Harvesting: %s/%s/%s' %
                    (pname, xname, dname))
        harvest_dir = self.get_working_directory()
        temp_dname = '%s_%s' % \
                     (dname, self.get_integrater_sweep_name())
        job.set_pname_xname_dname(pname, xname, temp_dname)

      job.set_template(os.path.basename(self.get_template()))
      job.set_directory(self.get_directory())

      # check for ice - and if so, exclude (ranges taken from
      # XDS documentation)
      if self.get_integrater_ice() != 0:
        Debug.write('Excluding ice rings')
        job.set_exclude_ice(True)

      # exclude specified resolution ranges
      if len(self.get_integrater_excluded_regions()) != 0:
        regions = self.get_integrater_excluded_regions()
        Debug.write('Excluding regions: %s' % `regions`)
        job.set_exclude_regions(regions)

      mask = standard_mask(self.get_detector())
      for m in mask:
        job.add_instruction(m)

      job.set_input_mat_file('xiaintegrate-%s.mat' % l)

      job.set_beam_centre(beam)
      job.set_distance(distance)
      job.set_space_group_number(spacegroup_number)
      job.set_mosaic(mosaic)

      if self.get_wavelength_prov() == 'user':
        job.set_wavelength(self.get_wavelength())

      parameters = self.get_integrater_parameters('mosflm')
      job.update_parameters(parameters)

      if self._mosflm_gain:
        job.set_gain(self._mosflm_gain)

      # check for resolution limits
      if self._intgr_reso_high > 0.0:
        job.set_d_min(self._intgr_reso_high)
      if self._intgr_reso_low:
        job.set_d_max(self._intgr_reso_low)

      if PhilIndex.params.general.backstop_mask:
        from xia2.Toolkit.BackstopMask import BackstopMask
        mask = BackstopMask(PhilIndex.params.general.backstop_mask)
        mask = mask.calculate_mask_mosflm(self.get_header())
        job.set_mask(mask)

      detector = self.get_detector()
      detector_width, detector_height = detector[0].get_image_size_mm()

      lim_x = 0.5 * detector_width
      lim_y = 0.5 * detector_height

      Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
      job.set_limits(lim_x, lim_y)

      job.set_fix_mosaic(self._mosflm_postref_fix_mosaic)

      job.set_pre_refinement(True)
      job.set_image_range(chunks[j])


      # these are now running so ...

      jobs.append(job)

      continue

    # ok, at this stage I need to ...
    #
    # (i) accumulate the statistics as a function of batch
    # (ii) mong them into a single block
    #
    # This is likely to be a pain in the arse!

    first_integrated_batch = 1.0e6
    last_integrated_batch = -1.0e6

    all_residuals = []

    threads = []

    for j in range(parallel):
      job = jobs[j]

      # now wait for them to finish - first wait will really be the
      # first one, then all should be finished...

      thread = Background(job, 'run')
      thread.start()
      threads.append(thread)

    mosaics = []
    postref_result = { }

    integrated_images_first = 1.0e6
    integrated_images_last = -1.0e6
    self._intgr_per_image_statistics = {}

    for j in range(parallel):
      thread = threads[j]
      thread.stop()
      job = jobs[j]

      # get the log file
      output = job.get_all_output()

      # record a copy of it, perhaps - though not if parallel
      if self.get_integrater_sweep_name() and False:
        pname, xname, dname = self.get_integrater_project_info()
        FileHandler.record_log_file(
            '%s %s %s %s mosflm integrate' % \
            (self.get_integrater_sweep_name(),
             pname, xname, '%s_%d' % (dname, j)),
            job.get_log_file())

      # look for things that we want to know...
      # that is, the output reflection file name, the updated
      # value for the gain (if present,) any warnings, errors,
      # or just interesting facts.

      batches = job.get_batches_out()
      integrated_images_first = min(batches[0], integrated_images_first)
      integrated_images_last = max(batches[1], integrated_images_last)

      mosaics.extend(job.get_mosaic_spreads())

      if min(mosaics) < 0:
        raise IntegrationError, 'negative mosaic spread: %s' % min(mosaic)

      if (job.get_detector_gain_error() and not
          (self.get_imageset().get_detector()[0].get_type() == 'SENSOR_PAD')):
        gain = job.get_suggested_gain()
        if gain is not None:
          self.set_integrater_parameter('mosflm', 'gain', gain)
          self.set_integrater_export_parameter('mosflm', 'gain', gain)
          if self._mosflm_gain:
            Debug.write('GAIN updated to %f' % gain)
          else:
            Debug.write('GAIN found to be %f' % gain)

          self._mosflm_gain = gain
          self._mosflm_rerun_integration = True

      hklout = job.get_hklout()
      Debug.write('Integration output: %s' % hklout)
      hklouts.append(hklout)

      nref += job.get_nref()

      # if a BGSIG error happened try not refining the
      # profile and running again...

      if job.get_bgsig_too_large():
        if not self._mosflm_refine_profiles:
          raise RuntimeError, 'BGSIG error with profiles fixed'

        Debug.write(
            'BGSIG error detected - try fixing profile...')

        self._mosflm_refine_profiles = False
        self.set_integrater_done(False)

        return

      if job.get_getprof_error():
        Debug.write(
            'GETPROF error detected - try fixing profile...')
        self._mosflm_refine_profiles = False
        self.set_integrater_done(False)

        return

      # here
      # write the report for each image as .*-#$ to Chatter -
      # detailed report will be written automagically to science...

      self._intgr_per_image_statistics.update(job.get_per_image_statistics())
      postref_result.update(job.get_postref_result())

      # inspect the output for e.g. very high weighted residuals

      all_residuals.extend(job.get_residuals())

    self._intgr_batches_out = (integrated_images_first,
                               integrated_images_last)

    if mosaics and len(mosaics) > 0:
      self.set_integrater_mosaic_min_mean_max(
          min(mosaics), sum(mosaics) / len(mosaics), max(mosaics))
    else:
      m = indxr.get_indexer_mosaic()
      self.set_integrater_mosaic_min_mean_max(m, m, m)

    Chatter.write(self.show_per_image_statistics())

    Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
                  self.get_integrater_mosaic_min_mean_max())

    # gather the statistics from the postrefinement for all sweeps
    # now write this to a postrefinement log

    postref_log = os.path.join(self.get_working_directory(),
                               'postrefinement.log')

    fout = open(postref_log, 'w')

    fout.write('$TABLE: Postrefinement for %s:\n' % \
               self._intgr_sweep_name)
    fout.write('$GRAPHS: Missetting angles:A:1, 2, 3, 4: $$\n')
    fout.write('Batch PhiX PhiY PhiZ $$ Batch PhiX PhiY PhiZ $$\n')

    for image in sorted(postref_result):
      phix = postref_result[image].get('phix', 0.0)
      phiy = postref_result[image].get('phiy', 0.0)
      phiz = postref_result[image].get('phiz', 0.0)

      fout.write('%d %5.2f %5.2f %5.2f\n' % \
                 (image, phix, phiy, phiz))

    fout.write('$$\n')
    fout.close()

    if self.get_integrater_sweep_name():
      pname, xname, dname = self.get_integrater_project_info()
      FileHandler.record_log_file('%s %s %s %s postrefinement' % \
                                  (self.get_integrater_sweep_name(),
                                   pname, xname, dname),
                                  postref_log)

    hklouts.sort()

    hklout = os.path.join(self.get_working_directory(),
                          os.path.split(hklouts[0])[-1])

    Debug.write('Sorting data to %s' % hklout)
    for hklin in hklouts:
      Debug.write('<= %s' % hklin)

    sortmtz = Sortmtz()
    sortmtz.set_hklout(hklout)
    for hklin in hklouts:
      sortmtz.add_hklin(hklin)

    sortmtz.sort()

    self._mosflm_hklout = hklout

    return self._mosflm_hklout
Ejemplo n.º 12
0
  def _mosflm_integrate(self):
    '''Perform the actual integration, based on the results of the
    cell refinement or indexing (they have the equivalent form.)'''

    refinr = self.get_integrater_refiner()

    if not refinr.get_refiner_payload('mosflm_orientation_matrix'):
      raise RuntimeError, 'unexpected situation in indexing'

    lattice = refinr.get_refiner_lattice()
    spacegroup_number = lattice_to_spacegroup(lattice)
    mosaic = refinr.get_refiner_payload('mosaic')
    beam = refinr.get_refiner_payload('beam')
    distance = refinr.get_refiner_payload('distance')
    matrix = refinr.get_refiner_payload('mosflm_orientation_matrix')

    integration_params = refinr.get_refiner_payload(
      'mosflm_integration_parameters')

    if integration_params:
      if 'separation' in integration_params:
        self.set_integrater_parameter(
          'mosflm', 'separation',
          '%s %s' % tuple(integration_params['separation']))
      if 'raster' in integration_params:
        self.set_integrater_parameter(
          'mosflm', 'raster',
          '%d %d %d %d %d' % tuple(integration_params['raster']))

    refinr.set_refiner_payload('mosflm_integration_parameters', None)

    f = open(os.path.join(self.get_working_directory(),
                          'xiaintegrate.mat'), 'w')
    for m in matrix:
      f.write(m)
    f.close()

    # then start the integration
    integrater = MosflmIntegrate()
    integrater.set_working_directory(self.get_working_directory())
    auto_logfiler(integrater)

    integrater.set_refine_profiles(self._mosflm_refine_profiles)

    pname, xname, dname = self.get_integrater_project_info()

    if pname is not None and xname is not None and dname is not None:
      Debug.write('Harvesting: %s/%s/%s' % (pname, xname, dname))
      harvest_dir = self.get_working_directory()
      # harvest file name will be %s.mosflm_run_start_end % dname
      temp_dname = '%s_%s' % \
                   (dname, self.get_integrater_sweep_name())
      integrater.set_pname_xname_dname(pname, xname, temp_dname)

    integrater.set_template(os.path.basename(self.get_template()))
    integrater.set_directory(self.get_directory())

    # check for ice - and if so, exclude (ranges taken from
    # XDS documentation)
    if self.get_integrater_ice() != 0:
      Debug.write('Excluding ice rings')
      integrater.set_exclude_ice(True)

    # exclude specified resolution ranges
    if len(self.get_integrater_excluded_regions()) != 0:
      regions = self.get_integrater_excluded_regions()
      Debug.write('Excluding regions: %s' % `regions`)
      integrater.set_exclude_regions(regions)

    mask = standard_mask(self.get_detector())
    for m in mask:
      integrater.add_instruction(m)

    integrater.set_input_mat_file('xiaintegrate.mat')

    integrater.set_beam_centre(beam)
    integrater.set_distance(distance)
    integrater.set_space_group_number(spacegroup_number)
    integrater.set_mosaic(mosaic)

    if self.get_wavelength_prov() == 'user':
      integrater.set_wavelength(self.get_wavelength())

    parameters = self.get_integrater_parameters('mosflm')
    integrater.update_parameters(parameters)

    if self._mosflm_gain:
      integrater.set_gain(self._mosflm_gain)

    # check for resolution limits
    if self._intgr_reso_high > 0.0:
      integrater.set_d_min(self._intgr_reso_high)
    if self._intgr_reso_low:
      integrater.set_d_max(self._intgr_reso_low)

    if PhilIndex.params.general.backstop_mask:
      from xia2.Toolkit.BackstopMask import BackstopMask
      mask = BackstopMask(PhilIndex.params.general.backstop_mask)
      mask = mask.calculate_mask_mosflm(self.get_header())
      integrater.set_mask(mask)

    detector = self.get_detector()
    detector_width, detector_height = detector[0].get_image_size_mm()

    lim_x = 0.5 * detector_width
    lim_y = 0.5 * detector_height

    Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
    integrater.set_limits(lim_x, lim_y)

    integrater.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
    offset = self.get_frame_offset()

    integrater.set_image_range(
      (self._intgr_wedge[0] - offset, self._intgr_wedge[1] - offset))

    try:
      integrater.run()
    except RuntimeError, e:
      if 'integration failed: reason unknown' in str(e):
        Chatter.write('Mosflm has failed in integration')
        message = 'The input was:\n\n'
        for input in integrater.get_all_input():
          message += '  %s' % input
        Chatter.write(message)
      raise
Ejemplo n.º 13
0
  def _index(self):
    '''Actually index the diffraction pattern. Note well that
    this is not going to compute the matrix...'''

    # acknowledge this program

    if not self._indxr_images:
      raise RuntimeError, 'No good spots found on any images'

    Citations.cite('labelit')
    Citations.cite('distl')

    _images = []
    for i in self._indxr_images:
      for j in i:
        if not j in _images:
          _images.append(j)

    _images.sort()

    images_str = '%d' % _images[0]
    for i in _images[1:]:
      images_str += ', %d' % i

    cell_str = None
    if self._indxr_input_cell:
      cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
                  self._indxr_input_cell

    if self._indxr_sweep_name:

      # then this is a proper autoindexing run - describe this
      # to the journal entry

      if len(self._fp_directory) <= 50:
        dirname = self._fp_directory
      else:
        dirname = '...%s' % self._fp_directory[-46:]

      Journal.block(
          'autoindexing', self._indxr_sweep_name, 'labelit',
          {'images':images_str,
           'target cell':cell_str,
           'target lattice':self._indxr_input_lattice,
           'template':self._fp_template,
           'directory':dirname})

    #auto_logfiler(self)

    from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex
    index = LabelitIndex()
    index.set_working_directory(self.get_working_directory())
    auto_logfiler(index)

    #task = 'Autoindex from images:'

    #for i in _images:
      #task += ' %s' % self.get_image_name(i)

    #self.set_task(task)

    #self.add_command_line('--index_only')

    Debug.write('Indexing from images:')
    for i in _images:
      index.add_image(self.get_image_name(i))
      Debug.write('%s' % self.get_image_name(i))

    if self._indxr_input_lattice and False:
      index.set_space_group_number(
        lattice_to_spacegroup(self._indxr_input_lattice))

    if self._primitive_unit_cell:
      index.set_primitive_unit_cell(self._primitive_unit_cell)

    if self._indxr_input_cell:
      index.set_max_cell(1.25 * max(self._indxr_input_cell[:3]))

    xsweep = self.get_indexer_sweep()
    if xsweep is not None:
      if xsweep.get_distance() is not None:
        index.set_distance(xsweep.get_distance())
      #if self.get_wavelength_prov() == 'user':
        #index.set_wavelength(self.get_wavelength())
      if xsweep.get_beam_centre() is not None:
        index.set_beam_centre(xsweep.get_beam_centre())

    if self._refine_beam is False:
      index.set_refine_beam(False)
    else:
      index.set_refine_beam(True)
      index.set_beam_search_scope(self._beam_search_scope)

    if ((math.fabs(self.get_wavelength() - 1.54) < 0.01) or
        (math.fabs(self.get_wavelength() - 2.29) < 0.01)):
      index.set_Cu_KA_or_Cr_KA(True)

    try:
      index.run()
    except RuntimeError, e:

      if self._refine_beam is False:
        raise e

      # can we improve the situation?

      if self._beam_search_scope < 4.0:
        self._beam_search_scope += 4.0

        # try repeating the indexing!

        self.set_indexer_done(False)
        return 'failed'

      # otherwise this is beyond redemption

      raise e
Ejemplo n.º 14
0
    def _integrate_finish(self):
        '''Finish off the integration by running dials.export.'''

        # FIXME - do we want to export every time we call this method
        # (the file will not have changed) and also (more important) do
        # we want a different exported MTZ file every time (I do not think
        # that we do; these can be very large) - was exporter.get_xpid() ->
        # now dials

        exporter = self.ExportMtz()
        exporter.set_reflections_filename(self._intgr_integrated_pickle)
        mtz_filename = os.path.join(self.get_working_directory(),
                                    '%s_integrated.mtz' % 'dials')
        exporter.set_mtz_filename(mtz_filename)
        exporter.run()
        self._intgr_integrated_filename = mtz_filename

        # record integrated MTZ file for e.g. BLEND.

        pname, xname, dname = self.get_integrater_project_info()
        sweep = self.get_integrater_sweep_name()
        FileHandler.record_more_data_file(
            '%s %s %s %s INTEGRATE' % (pname, xname, dname, sweep),
            mtz_filename)

        from iotbx.reflection_file_reader import any_reflection_file
        miller_arrays = any_reflection_file(
            self._intgr_integrated_filename).as_miller_arrays()
        # look for profile-fitted intensities
        intensities = [
            ma for ma in miller_arrays
            if ma.info().labels == ['IPR', 'SIGIPR']
        ]
        if len(intensities) == 0:
            # look instead for summation-integrated intensities
            intensities = [
                ma for ma in miller_arrays
                if ma.info().labels == ['I', 'SIGI']
            ]
            assert len(intensities)
        self._intgr_n_ref = intensities[0].size()

        if not os.path.isfile(self._intgr_integrated_filename):
            raise RuntimeError("dials.export failed: %s does not exist." %
                               self._intgr_integrated_filename)

        if self._intgr_reindex_operator is None and \
          self._intgr_spacegroup_number == lattice_to_spacegroup(
            self.get_integrater_refiner().get_refiner_lattice()):
            Debug.write('Not reindexing to spacegroup %d (%s)' % \
                          (self._intgr_spacegroup_number,
                           self._intgr_reindex_operator))
            return mtz_filename

        if self._intgr_reindex_operator is None and \
          self._intgr_spacegroup_number == 0:
            Debug.write('Not reindexing to spacegroup %d (%s)' % \
                          (self._intgr_spacegroup_number,
                           self._intgr_reindex_operator))
            return mtz_filename

        Debug.write('Reindexing to spacegroup %d (%s)' % \
                    (self._intgr_spacegroup_number,
                     self._intgr_reindex_operator))

        hklin = mtz_filename
        reindex = Reindex()
        reindex.set_working_directory(self.get_working_directory())
        auto_logfiler(reindex)

        reindex.set_operator(self._intgr_reindex_operator)

        if self._intgr_spacegroup_number:
            reindex.set_spacegroup(self._intgr_spacegroup_number)
        else:
            reindex.set_spacegroup(
                lattice_to_spacegroup(
                    self.get_integrater_refiner().get_refiner_lattice()))

        hklout = '%s_reindex.mtz' % hklin[:-4]
        reindex.set_hklin(hklin)
        reindex.set_hklout(hklout)
        reindex.reindex()
        self._intgr_integrated_filename = hklout
        self._intgr_cell = reindex.get_cell()

        pname, xname, dname = self.get_integrater_project_info()
        sweep = self.get_integrater_sweep_name()
        FileHandler.record_more_data_file(
            '%s %s %s %s experiments' % (pname, xname, dname, sweep),
            self.get_integrated_experiments())

        return hklout
Ejemplo n.º 15
0
  def _index(self):
    '''Implement the indexer interface.'''

    Citations.cite('mosflm')

    indexer = MosflmIndex()
    indexer.set_working_directory(self.get_working_directory())
    auto_logfiler(indexer)

    from xia2.lib.bits import unique_elements
    _images = unique_elements(self._indxr_images)
    indexer.set_images(_images)
    images_str = ', '.join(map(str, _images))

    cell_str = None
    if self._indxr_input_cell:
      cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
                  self._indxr_input_cell

    if self._indxr_sweep_name:

      #if len(self._fp_directory) <= 50:
        #dirname = self._fp_directory
      #else:
        #dirname = '...%s' % self._fp_directory[-46:]
      dirname = os.path.dirname(self.get_imageset().get_template())

      Journal.block(
          'autoindexing', self._indxr_sweep_name, 'mosflm',
          {'images':images_str,
           'target cell':self._indxr_input_cell,
           'target lattice':self._indxr_input_lattice,
           'template':self.get_imageset().get_template(),
           'directory':dirname})

    #task = 'Autoindex from images:'

    #for i in _images:
      #task += ' %s' % self.get_image_name(i)

    #self.set_task(task)

    indexer.set_template(os.path.basename(self.get_template()))
    indexer.set_directory(self.get_directory())

    xsweep = self.get_indexer_sweep()
    if xsweep is not None:
      if xsweep.get_distance() is not None:
        indexer.set_distance(xsweep.get_distance())
      #if self.get_wavelength_prov() == 'user':
        #index.set_wavelength(self.get_wavelength())
      if xsweep.get_beam_centre() is not None:
        indexer.set_beam_centre(xsweep.get_beam_centre())

    if self._indxr_input_cell:
      indexer.set_unit_cell(self._indxr_input_cell)

    if self._indxr_input_lattice != None:
      spacegroup_number = lattice_to_spacegroup(
          self._indxr_input_lattice)
      indexer.set_space_group_number(spacegroup_number)

    if not self._mosflm_autoindex_thresh:

      try:

        min_peaks = 200

        Debug.write('Aiming for at least %d spots...' % min_peaks)

        thresholds = []

        for i in _images:

          p = Printpeaks()
          p.set_working_directory(self.get_working_directory())
          auto_logfiler(p)
          p.set_image(self.get_image_name(i))
          thresh = p.threshold(min_peaks)

          Debug.write('Autoindex threshold for image %d: %d' % \
                      (i, thresh))

          thresholds.append(thresh)

        thresh = min(thresholds)
        self._mosflm_autoindex_thresh = thresh

      except Exception as e:
        print str(e) #XXX this should disappear!
        Debug.write('Error computing threshold: %s' % str(e))
        Debug.write('Using default of 20.0')
        thresh = 20.0

    else:
      thresh = self._mosflm_autoindex_thresh

    Debug.write('Using autoindex threshold: %d' % thresh)

    if self._mosflm_autoindex_sol:
      indexer.set_solution_number(self._mosflm_autoindex_sol)
    indexer.set_threshold(thresh)

    # now forget this to prevent weird things happening later on
    if self._mosflm_autoindex_sol:
      self._mosflm_autoindex_sol = 0

    indexer.run()

    #sweep = self.get_indexer_sweep_name()
    #FileHandler.record_log_file(
        #'%s INDEX' % (sweep), self.get_log_file())

    indxr_cell = indexer.get_refined_unit_cell()
    self._indxr_lattice = indexer.get_lattice()
    space_group_number = indexer.get_indexed_space_group_number()
    detector_distance = indexer.get_refined_distance()
    beam_centre = indexer.get_refined_beam_centre()
    mosaic_spreads = indexer.get_mosaic_spreads()

    if min(list(indxr_cell)) < 10.0 and \
       indxr_cell[2] / indxr_cell[0] > 6:

      Debug.write(
          'Unrealistic autoindexing solution: ' +
          '%.2f %.2f %.2f %.2f %.2f %.2f' % indxr_cell)

      # tweak some parameters and try again...
      self._mosflm_autoindex_thresh *= 1.5
      self.set_indexer_done(False)

      return

    intgr_params = { }

    # look up other possible indexing solutions (not well - in
    # standard settings only!) This is moved earlier as it could
    # result in returning if Mosflm has selected the wrong
    # solution!

    try:
      self._indxr_other_lattice_cell = indexer.get_solutions()

      # Change 27/FEB/08 to support user assigned spacegroups
      if self._indxr_user_input_lattice:
        lattice_to_spacegroup_dict = {
            'aP':1, 'mP':3, 'mC':5, 'oP':16, 'oC':20, 'oF':22,
            'oI':23, 'tP':75, 'tI':79, 'hP':143, 'hR':146,
            'cP':195, 'cF':196, 'cI':197}
        for k in self._indxr_other_lattice_cell.keys():
          if lattice_to_spacegroup_dict[k] > \
                 lattice_to_spacegroup_dict[
              self._indxr_input_lattice]:
            del(self._indxr_other_lattice_cell[k])

      # check that the selected unit cell matches - and if
      # not raise a "horrible" exception

      if self._indxr_input_cell:
        assert indxr_cell is not None
        for j in range(6):
          if math.fabs(self._indxr_input_cell[j] - indxr_cell[j]) > 2.0:
            Chatter.write(
                'Mosflm autoindexing did not select ' +
                'correct (target) unit cell')
            raise RuntimeError, \
                  'something horrible happened in indexing'

    except RuntimeError, e:
      # check if mosflm rejected a solution we have it
      if 'horribl' in str(e):
        # ok it did - time to break out the big guns...
        if not self._indxr_input_cell:
          raise RuntimeError, \
                'error in solution selection when not preset'

        # XXX FIXME
        self._mosflm_autoindex_sol = _get_indexing_solution_number(
          indexer.get_all_output(),
          self._indxr_input_cell,
          self._indxr_input_lattice)

        # set the fact that we are not done...
        self.set_indexer_done(False)

        # and return - hopefully this will restart everything
        return
      else:
        raise e
    def _integrate_finish(self):
        """Finish off the integration by running dials.export."""

        # FIXME - do we want to export every time we call this method
        # (the file will not have changed) and also (more important) do
        # we want a different exported MTZ file every time (I do not think
        # that we do; these can be very large) - was exporter.get_xpid() ->
        # now dials

        if self._output_format == "hkl":
            exporter = self.ExportMtz()
            exporter.set_reflections_filename(
                self._intgr_integrated_reflections)
            mtz_filename = os.path.join(self.get_working_directory(),
                                        "%s_integrated.mtz" % "dials")
            exporter.set_mtz_filename(mtz_filename)
            exporter.run()
            self._intgr_integrated_filename = mtz_filename

            # record integrated MTZ file for e.g. BLEND.

            pname, xname, dname = self.get_integrater_project_info()
            sweep = self.get_integrater_sweep_name()
            FileHandler.record_more_data_file(
                "%s %s %s %s INTEGRATE" % (pname, xname, dname, sweep),
                mtz_filename)

            from iotbx.reflection_file_reader import any_reflection_file

            miller_arrays = any_reflection_file(
                self._intgr_integrated_filename).as_miller_arrays()
            # look for profile-fitted intensities
            intensities = [
                ma for ma in miller_arrays
                if ma.info().labels == ["IPR", "SIGIPR"]
            ]
            if len(intensities) == 0:
                # look instead for summation-integrated intensities
                intensities = [
                    ma for ma in miller_arrays
                    if ma.info().labels == ["I", "SIGI"]
                ]
                assert len(intensities)
            self._intgr_n_ref = intensities[0].size()

            if not os.path.isfile(self._intgr_integrated_filename):
                raise RuntimeError("dials.export failed: %s does not exist." %
                                   self._intgr_integrated_filename)

            if (self._intgr_reindex_operator is None
                    and self._intgr_spacegroup_number == lattice_to_spacegroup(
                        self.get_integrater_refiner().get_refiner_lattice())):
                Debug.write("Not reindexing to spacegroup %d (%s)" %
                            (self._intgr_spacegroup_number,
                             self._intgr_reindex_operator))
                return mtz_filename

            if (self._intgr_reindex_operator is None
                    and self._intgr_spacegroup_number == 0):
                Debug.write("Not reindexing to spacegroup %d (%s)" %
                            (self._intgr_spacegroup_number,
                             self._intgr_reindex_operator))
                return mtz_filename

            Debug.write(
                "Reindexing to spacegroup %d (%s)" %
                (self._intgr_spacegroup_number, self._intgr_reindex_operator))

            hklin = mtz_filename
            from xia2.Wrappers.CCP4.Reindex import Reindex

            reindex = Reindex()
            reindex.set_working_directory(self.get_working_directory())
            auto_logfiler(reindex)

            reindex.set_operator(self._intgr_reindex_operator)

            if self._intgr_spacegroup_number:
                reindex.set_spacegroup(self._intgr_spacegroup_number)
            else:
                reindex.set_spacegroup(
                    lattice_to_spacegroup(
                        self.get_integrater_refiner().get_refiner_lattice()))

            hklout = "%s_reindex.mtz" % hklin[:-4]
            reindex.set_hklin(hklin)
            reindex.set_hklout(hklout)
            reindex.reindex()
            self._intgr_integrated_filename = hklout
            self._intgr_cell = reindex.get_cell()

            pname, xname, dname = self.get_integrater_project_info()
            sweep = self.get_integrater_sweep_name()
            FileHandler.record_more_data_file(
                "%s %s %s %s" % (pname, xname, dname, sweep),
                self.get_integrated_experiments(),
            )
            FileHandler.record_more_data_file(
                "%s %s %s %s" % (pname, xname, dname, sweep),
                self.get_integrated_reflections(),
            )

            return hklout

        elif self._output_format == "pickle":

            if (self._intgr_reindex_operator is None
                    and self._intgr_spacegroup_number == lattice_to_spacegroup(
                        self.get_integrater_refiner().get_refiner_lattice())):
                Debug.write("Not reindexing to spacegroup %d (%s)" %
                            (self._intgr_spacegroup_number,
                             self._intgr_reindex_operator))
                return self._intgr_integrated_reflections

            if (self._intgr_reindex_operator is None
                    and self._intgr_spacegroup_number == 0):
                Debug.write("Not reindexing to spacegroup %d (%s)" %
                            (self._intgr_spacegroup_number,
                             self._intgr_reindex_operator))
                return self._intgr_integrated_reflections

            Debug.write(
                "Reindexing to spacegroup %d (%s)" %
                (self._intgr_spacegroup_number, self._intgr_reindex_operator))
            from xia2.Wrappers.Dials.Reindex import Reindex

            reindex = Reindex()
            reindex.set_working_directory(self.get_working_directory())
            auto_logfiler(reindex)

            reindex.set_cb_op(self._intgr_reindex_operator)

            if self._intgr_spacegroup_number:
                reindex.set_space_group(self._intgr_spacegroup_number)
            else:
                reindex.set_space_group(
                    lattice_to_spacegroup(
                        self.get_integrater_refiner().get_refiner_lattice()))

            reindex.set_experiments_filename(self.get_integrated_experiments())
            reindex.set_indexed_filename(self.get_integrated_reflections())

            reindex.run()
            self._intgr_integrated_reflections = (
                reindex.get_reindexed_reflections_filename())
            self._intgr_integrated_filename = (
                reindex.get_reindexed_reflections_filename())
            self._intgr_experiments_filename = (
                reindex.get_reindexed_experiments_filename())

            pname, xname, dname = self.get_integrater_project_info()
            sweep = self.get_integrater_sweep_name()
            FileHandler.record_more_data_file(
                "%s %s %s %s" % (pname, xname, dname, sweep),
                self.get_integrated_experiments(),
            )
            FileHandler.record_more_data_file(
                "%s %s %s %s" % (pname, xname, dname, sweep),
                self.get_integrated_reflections(),
            )
            return None  # this will be set to intgr_hklout - better to cause failure