Beispiel #1
0
def run():
  try:
    check_environment()
    check()
  except exceptions.Exception, e:
    traceback.print_exc(file = open('xia2.error', 'w'))
    Chatter.write('Status: error "%s"' % str(e))
def run():
    if os.path.exists('xia2-working.phil'):
        sys.argv.append('xia2-working.phil')
    try:
        check_environment()
    except Exception as e:
        traceback.print_exc(file=open('xia2.error', 'w'))
        Chatter.write('Status: error "%s"' % str(e))

    if len(sys.argv) < 2 or '-help' in sys.argv:
        help()
        sys.exit()

    wd = os.getcwd()

    try:
        multi_crystal_analysis()
        Chatter.write('Status: normal termination')

    except Exception as e:
        traceback.print_exc(file=open(os.path.join(wd, 'xia2.error'), 'w'))
        Chatter.write('Status: error "%s"' % str(e))
        Chatter.write(
            'Please send the contents of xia2.txt, xia2.error and xia2-debug.txt to:'
        )
        Chatter.write('*****@*****.**')
        sys.exit(1)
Beispiel #3
0
def load_reference_geometries(geometry_file_list):
    from dxtbx.serialize import load

    reference_components = []
    for file in geometry_file_list:
        try:
            experiments = load.experiment_list(file, check_format=False)
            assert len(experiments.detectors()) == 1
            assert len(experiments.beams()) == 1
            reference_detector = experiments.detectors()[0]
            reference_beam = experiments.beams()[0]
        except Exception:
            datablock = load.datablock(file)
            assert len(datablock) == 1
            imageset = datablock[0].extract_imagesets()[0]
            reference_detector = imageset.get_detector()
            reference_beam = imageset.get_beam()
        reference_components.append({
            'detector': reference_detector,
            'beam': reference_beam,
            'file': file
        })

    import itertools
    for combination in itertools.combinations(reference_components, 2):
        if compare_geometries(combination[0]['detector'],
                              combination[1]['detector']):
            from xia2.Handlers.Streams import Chatter
            Chatter.write(
                'Reference geometries given in %s and %s are too similar' %
                (combination[0]['file'], combination[1]['file']))
            raise Exception('Reference geometries too similar')
    return reference_components
Beispiel #4
0
    def _scale_finish_chunk_8_raddam(self):
        crd = CCP4InterRadiationDamageDetector()

        crd.set_working_directory(self.get_working_directory())

        crd.set_hklin(self._scalr_scaled_reflection_files['mtz'])

        if self.get_scaler_anomalous():
            crd.set_anomalous(True)

        hklout = os.path.join(self.get_working_directory(), 'temp.mtz')
        FileHandler.record_temporary_file(hklout)

        crd.set_hklout(hklout)

        status = crd.detect()

        if status:
            Chatter.write('')
            Chatter.banner('Local Scaling %s' % self._scalr_xname)
            for s in status:
                Chatter.write('%s %s' % s)
            Chatter.banner('')
        else:
            Debug.write('Local scaling failed')
def run():
    assert os.path.exists("xia2.json")
    from xia2.Schema.XProject import XProject

    xinfo = XProject.from_json(filename="xia2.json")
    Chatter.write(xinfo.get_output())
    write_citations()
Beispiel #6
0
        def run(self):
            """Run dials.space_group"""
            self.clear_command_line()

            assert self._experiments_filename
            assert self._reflections_filename
            self.add_command_line(self._reflections_filename)
            self.add_command_line(self._experiments_filename)

            if not self._symmetrized_experiments:
                self._symmetrized_experiments = os.path.join(
                    self.get_working_directory(),
                    "%i_symmetrized.expt" % self.get_xpid(),
                )
            self.add_command_line(
                "output.experiments=%s" % self._symmetrized_experiments
            )

            self.start()
            self.close_wait()

            # check for errors

            try:
                self.check_for_errors()
            except Exception:
                Chatter.write(
                    "dials.space_group failed, see log file for more details:\n  %s"
                    % self.get_log_file()
                )
                raise

            Debug.write("dials.space_group status: OK")
Beispiel #7
0
    def run_one_sweep(args):

      from xia2.Handlers.Streams import Debug

      assert len(args) == 3
      s, failover, job_type = args

      if job_type:
        DriverFactory.set_driver_type(job_type)

      Chatter.cache()
      Debug.cache()

      try:
        s.get_integrater_intensities()
      except Exception as e:
        if failover:
          Chatter.write('Processing sweep %s failed: %s' % \
                        (s.get_name(), str(e)))
          s = None
        else:
          raise
      finally:
        Chatter.uncache()
        Debug.uncache()
        return s
def load_reference_geometries(geometry_file_list):
    from dxtbx.serialize import load

    reference_components = []
    for file in geometry_file_list:
        try:
            experiments = load.experiment_list(file, check_format=False)
            assert len(experiments.detectors()) == 1
            assert len(experiments.beams()) == 1
            reference_detector = experiments.detectors()[0]
            reference_beam = experiments.beams()[0]
        except Exception:
            experiments = load.experiment_list(file)
            imageset = experiments.imagesets()[0]
            reference_detector = imageset.get_detector()
            reference_beam = imageset.get_beam()
        reference_components.append(
            {"detector": reference_detector, "beam": reference_beam, "file": file}
        )

    import itertools

    for combination in itertools.combinations(reference_components, 2):
        if compare_geometries(combination[0]["detector"], combination[1]["detector"]):
            from xia2.Handlers.Streams import Chatter

            Chatter.write(
                "Reference geometries given in %s and %s are too similar"
                % (combination[0]["file"], combination[1]["file"])
            )
            raise Exception("Reference geometries too similar")
    return reference_components
Beispiel #9
0
    def _scale_finish_chunk_7_twinning(self):
        hklout = self._scalr_scaled_reflection_files['mtz']

        m = mtz.object(hklout)
        # FIXME in here should be able to just drop down to the lowest symmetry
        # space group with the rotational elements for this calculation? I.e.
        # P422 for P4/mmm?
        if not m.space_group().is_centric():
            from xia2.Toolkit.E4 import E4_mtz
            E4s = E4_mtz(hklout, native=True)
            self._scalr_twinning_score = E4s.items()[0][1]

            if self._scalr_twinning_score > 1.9:
                self._scalr_twinning_conclusion = 'Your data do not appear twinned'
            elif self._scalr_twinning_score < 1.6:
                self._scalr_twinning_conclusion = 'Your data appear to be twinned'
            else:
                self._scalr_twinning_conclusion = 'Ambiguous score (1.6 < score < 1.9)'

        else:
            self._scalr_twinning_conclusion = 'Data are centric'
            self._scalr_twinning_score = 0

        Chatter.write('Overall twinning score: %4.2f' %
                      self._scalr_twinning_score)
        Chatter.write(self._scalr_twinning_conclusion)
Beispiel #10
0
 def get_new_scales_file(self):
   '''Get the file to which the scales have been written.'''
   if self._new_scales_file:
     if not os.path.isfile(os.path.join(self.get_working_directory(), self._new_scales_file)):
       Chatter.write(
         "Aimless did not scale the data, see log file for more details:\n  %s" %self.get_log_file())
       raise RuntimeError, 'data not scaled'
   return os.path.join(self.get_working_directory(), self._new_scales_file)
Beispiel #11
0
def run():
  if os.path.exists('xia2-working.phil'):
    sys.argv.append('xia2-working.phil')
  try:
    check_environment()
  except exceptions.Exception, e:
    traceback.print_exc(file = open('xia2.error', 'w'))
    Chatter.write('Status: error "%s"' % str(e))
Beispiel #12
0
def which(pgm, debug=False):
  path = os.getenv('PATH')
  for p in path.split(os.path.pathsep):
    p = os.path.join(p, pgm)
    if debug:
      Chatter.write('Seeking %s' % p)
    if os.path.exists(p) and os.access(p, os.X_OK):
      return p
Beispiel #13
0
 def get_new_scales_file(self):
   '''Get the file to which the scales have been written.'''
   if self._new_scales_file:
     if not os.path.isfile(os.path.join(self.get_working_directory(), self._new_scales_file)):
       Chatter.write(
         "Aimless did not scale the data, see log file for more details:\n  %s" %self.get_log_file())
       raise RuntimeError('data not scaled')
   return os.path.join(self.get_working_directory(), self._new_scales_file)
Beispiel #14
0
  def _integrate(self):
    '''Implement the integrater interface.'''

    # cite the program
    Citations.cite('mosflm')

    images_str = '%d to %d' % tuple(self._intgr_wedge)
    cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % tuple(self._intgr_cell)

    if len(self._fp_directory) <= 50:
      dirname = self._fp_directory
    else:
      dirname = '...%s' % self._fp_directory[-46:]

    Journal.block(
        'integrating', self._intgr_sweep_name, 'mosflm',
        {'images':images_str,
         'cell':cell_str,
         'lattice':self.get_integrater_refiner().get_refiner_lattice(),
         'template':self._fp_template,
         'directory':dirname,
         'resolution':'%.2f' % self._intgr_reso_high})

    self._mosflm_rerun_integration = False

    wd = self.get_working_directory()

    try:

      if self.get_integrater_sweep_name():
        pname, xname, dname = self.get_integrater_project_info()

      nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
      if nproc > 1:
        Debug.write('Parallel integration: %d jobs' %nproc)
        self._mosflm_hklout = self._mosflm_parallel_integrate()
      else:
        self._mosflm_hklout = self._mosflm_integrate()

      # record integration output for e.g. BLEND.

      sweep = self.get_integrater_sweep_name()
      if sweep:
        FileHandler.record_more_data_file(
            '%s %s %s %s INTEGRATE' % (pname, xname, dname, sweep),
            self._mosflm_hklout)

    except IntegrationError, e:
      if 'negative mosaic spread' in str(e):
        if self._mosflm_postref_fix_mosaic:
          Chatter.write(
              'Negative mosaic spread - stopping integration')
          raise BadLatticeError, 'negative mosaic spread'

        Chatter.write(
            'Negative mosaic spread - rerunning integration')
        self.set_integrater_done(False)
        self._mosflm_postref_fix_mosaic = True
Beispiel #15
0
  def integrate(self):
    '''Actually perform integration until we think we are done...'''

    while not self.get_integrater_finish_done():
      while not self.get_integrater_done():
        while not self.get_integrater_prepare_done():

          Debug.write('Preparing to do some integration...')
          self.set_integrater_prepare_done(True)

          # if this raises an exception, perhaps the autoindexing
          # solution has too high symmetry. if this the case, then
          # perform a self._intgr_indexer.eliminate() - this should
          # reset the indexing system

          try:
            self._integrate_prepare()

          except BadLatticeError, e:

            Journal.banner('eliminated this lattice', size = 80)

            Chatter.write('Rejecting bad lattice %s' % str(e))
            self._intgr_refiner.eliminate()
            self._integrater_reset()

        # FIXME x1698 - may be the case that _integrate() returns the
        # raw intensities, _integrate_finish() returns intensities
        # which may have been adjusted or corrected. See #1698 below.

        Debug.write('Doing some integration...')

        self.set_integrater_done(True)

        template = self.get_integrater_sweep().get_template()

        if self._intgr_sweep_name:
          if PhilIndex.params.xia2.settings.show_template:
            Chatter.banner('Integrating %s (%s)' % \
                           (self._intgr_sweep_name, template))
          else:
            Chatter.banner('Integrating %s' % \
                           (self._intgr_sweep_name))
        try:

          #1698
          self._intgr_hklout_raw = self._integrate()

        except BadLatticeError, e:
          Chatter.write('Rejecting bad lattice %s' % str(e))

          Journal.banner('eliminated this lattice', size = 80)

          self._intgr_refiner.eliminate()
          self._integrater_reset()
Beispiel #16
0
    def _index_prepare(self):
        Chatter.banner('Spotfinding %s' % self.get_indexer_sweep_name())
        super(XDSIndexerII, self)._index_prepare()

        from dials.array_family import flex
        from dials.util.ascii_art import spot_counts_per_image_plot
        reflection_pickle = spot_xds_to_reflection_pickle(
            self._indxr_payload['SPOT.XDS'],
            working_directory=self.get_working_directory())
        refl = flex.reflection_table.from_pickle(reflection_pickle)
        Chatter.write(spot_counts_per_image_plot(refl), strip=False)
Beispiel #17
0
def run():
  from libtbx.utils import Sorry
  if len(sys.argv) < 2 or '-help' in sys.argv or '--help' in sys.argv:
    help()
    sys.exit()

  try:
    check_environment()
  except exceptions.Exception, e:
    traceback.print_exc(file = open('xia2.error', 'w'))
    Chatter.write('Status: error "%s"' % str(e))
Beispiel #18
0
  def _index_prepare(self):
    Chatter.banner('Spotfinding %s' %self.get_indexer_sweep_name())
    super(XDSIndexerII, self)._index_prepare()

    from dials.array_family import flex
    from dials.util.ascii_art import spot_counts_per_image_plot
    reflection_pickle = spot_xds_to_reflection_pickle(
      self._indxr_payload['SPOT.XDS'],
      working_directory=self.get_working_directory())
    refl = flex.reflection_table.from_pickle(reflection_pickle)
    Chatter.write(spot_counts_per_image_plot(refl), strip=False)
Beispiel #19
0
  def eliminate(self, indxr_print = True):
    '''Eliminate the highest currently allowed lattice.'''

    if len(self._sorted_list) <= 1:
      raise RuntimeError, 'cannot eliminate only solution'

    if indxr_print:
      Chatter.write('Eliminating indexing solution:')
      Chatter.write(self.repr()[0])

    self._sorted_list = self._sorted_list[1:]
    def eliminate(self, indxr_print=True):
        """Eliminate the highest currently allowed lattice."""

        if len(self._sorted_list) <= 1:
            raise RuntimeError("cannot eliminate only solution")

        if indxr_print:
            Chatter.write("Eliminating indexing solution:")
            Chatter.write(self.repr()[0])

        self._sorted_list = self._sorted_list[1:]
Beispiel #21
0
def which(pgm, debug=False):
  # python equivalent to the 'which' command
  # http://stackoverflow.com/questions/9877462/is-there-a-python-equivalent-to-the-which-command
  # FIXME this will not work on Windows as you need to check that there is a
  # .bat or a .exe extension
  # FIXME also this is implemented in Driver/DriverHelper.py:executable_exists
  path = os.getenv('PATH')
  for p in path.split(os.path.pathsep):
    p = os.path.join(p, pgm)
    if debug:
      Chatter.write('Seeking %s' % p)
    if os.path.exists(p) and os.access(p, os.X_OK):
      return p
Beispiel #22
0
    def run(self):
      from xia2.Handlers.Streams import Chatter, Debug
      Debug.write('Running dials.refine')

      self.clear_command_line()
      self.add_command_line(self._experiments_filename)
      self.add_command_line(self._indexed_filename)
      self.add_command_line('scan_varying=%s' % self._scan_varying)
      if self._close_to_spindle_cutoff is not None:
        self.add_command_line(
          'close_to_spindle_cutoff=%f' %self._close_to_spindle_cutoff)
      if self._outlier_algorithm:
        self.add_command_line('outlier.algorithm=%s' % self._outlier_algorithm)
      self._refined_experiments_filename = os.path.join(
        self.get_working_directory(),
        '%s_refined_experiments.json' % self.get_xpid())
      self.add_command_line(
        'output.experiments=%s' % self._refined_experiments_filename)
      self._refined_filename = os.path.join(
        self.get_working_directory(), '%s_refined.pickle' % self.get_xpid())
      self.add_command_line('output.reflections=%s' % self._refined_filename)
      if self._reflections_per_degree is not None:
        self.add_command_line(
          'reflections_per_degree=%i' %self._reflections_per_degree)
      if self._interval_width_degrees is not None:
        self.add_command_line(
          'unit_cell.smoother.interval_width_degrees=%i' % self._interval_width_degrees)
        self.add_command_line(
          'orientation.smoother.interval_width_degrees=%i' % self._interval_width_degrees)
      if self._detector_fix:
        self.add_command_line('detector.fix=%s' % self._detector_fix)
      if self._beam_fix:
        self.add_command_line('beam.fix=%s' % self._beam_fix)
      if self._phil_file is not None:
        self.add_command_line('%s' %self._phil_file)

      self.start()
      self.close_wait()

      if not os.path.isfile(self._refined_filename) or \
         not os.path.isfile(self._refined_experiments_filename):
        Chatter.write(
          "DIALS did not refine the data, see log file for more details:\n  %s" %self.get_log_file())
        raise RuntimeError, 'data not refined'
      for record in self.get_all_output():
        if 'Sorry: Too few reflections to' in record:
          raise RuntimeError, record.strip()

      self.check_for_errors()
      return
Beispiel #23
0
def make_logfile_html(logfile):
    tables = extract_loggraph_tables(logfile)
    if not tables:
        return

    rst = []
    ## local files in xia2 distro
    #c3css = os.path.join(xia2_root_dir, 'c3', 'c3.css')
    #c3js = os.path.join(xia2_root_dir, 'c3', 'c3.min.js')
    #d3js = os.path.join(xia2_root_dir, 'd3', 'd3.min.js')

    # webhosted files
    c3css = 'https://cdnjs.cloudflare.com/ajax/libs/c3/0.4.10/c3.css'
    c3js = 'https://cdnjs.cloudflare.com/ajax/libs/c3/0.4.10/c3.min.js'
    d3js = 'https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js'

    rst.append('.. raw:: html')
    rst.append('\n    '.join([
        '', '<!-- Load c3.css -->',
        '<link href="%s" rel="stylesheet" type="text/css">' % c3css,
        '<!-- Load d3.js and c3.js -->',
        '<script src="%s" charset="utf-8"></script>' % d3js,
        '<script src="%s"></script>' % c3js, ''
    ]))

    for table in tables:
        try:
            #for graph_name, html in table_to_google_charts(table).iteritems():
            for graph_name, html in table_to_c3js_charts(table).iteritems():
                #rst.append('.. __%s:\n' %graph_name)
                rst.append('.. raw:: html')
                rst.append('\n    '.join(html.split('\n')))
        except Exception as e:
            Chatter.write('=' * 80)
            Chatter.write('Error (%s) while processing table' % str(e))
            Chatter.write("  '%s'" % table.title)
            Chatter.write('in %s' % logfile)
            Chatter.write('=' * 80)
            Debug.write(
                'Exception raised while processing log file %s, table %s' %
                (logfile, table.title))
            Debug.write(traceback.format_exc())

    rst = '\n'.join(rst)

    html_file = '%s.html' % (os.path.splitext(logfile)[0])
    with open(html_file, 'wb') as f:
        f.write(rst2html(rst))
    return html_file
def run():
    if os.path.exists("xia2-working.phil"):
        sys.argv.append("xia2-working.phil")
    try:
        check_environment()
    except Exception as e:
        traceback.print_exc(file=open("xia2.error", "w"))
        Chatter.write('Status: error "%s"' % str(e))

    # print the version
    Chatter.write(Version)
    Citations.cite("xia2")

    start_time = time.time()

    assert os.path.exists("xia2.json")
    from xia2.Schema.XProject import XProject

    xinfo = XProject.from_json(filename="xia2.json")

    crystals = xinfo.get_crystals()
    for crystal_id, crystal in crystals.iteritems():
        # cwd = os.path.abspath(os.curdir)
        from libtbx import Auto

        scale_dir = PhilIndex.params.xia2.settings.scale.directory
        if scale_dir is Auto:
            scale_dir = "scale"
            i = 0
            while os.path.exists(os.path.join(crystal.get_name(), scale_dir)):
                i += 1
                scale_dir = "scale%i" % i
            PhilIndex.params.xia2.settings.scale.directory = scale_dir
        working_directory = Environment.generate_directory(
            [crystal.get_name(), scale_dir])
        # os.chdir(working_directory)

        crystals[crystal_id]._scaler = None  # reset scaler

        scaler = crystal._get_scaler()
        Chatter.write(xinfo.get_output())
        crystal.serialize()

    duration = time.time() - start_time

    # write out the time taken in a human readable way
    Chatter.write("Processing took %s" %
                  time.strftime("%Hh %Mm %Ss", time.gmtime(duration)))

    # delete all of the temporary mtz files...
    cleanup()

    write_citations()

    xinfo.as_json(filename="xia2.json")

    Environment.cleanup()
Beispiel #25
0
        def run(self):
            """Run dials.merge"""
            self.clear_command_line()

            assert self._experiments_filename
            assert self._reflections_filename
            self.add_command_line(self._reflections_filename)
            self.add_command_line(self._experiments_filename)

            self.add_command_line("truncate=%s" % self._truncate)

            if self._mtz_filename:
                self.add_command_line("output.mtz=%s" % self._mtz_filename)

            if self._project_name:
                self.add_command_line("output.project_name=%s" %
                                      self._project_name)
            if self._crystal_names:
                self.add_command_line("output.crystal_names=%s" %
                                      self._crystal_names)
            if self._dataset_names:
                self.add_command_line("output.dataset_names=%s" %
                                      self._dataset_names)
            if self._partiality_threshold:
                self.add_command_line("partiality_threshold=%s" %
                                      self._partiality_threshold)

            self.start()
            self.close_wait()

            # check for errors

            try:
                self.check_for_errors()
            except Exception:
                Chatter.write(
                    "dials.merge failed, see log file for more details:\n  %s"
                    % self.get_log_file())
                raise

            Debug.write("dials.merge status: OK")
Beispiel #26
0
def run():
    try:
        check_environment()
    except Exception as e:
        with open("xia2.error", "w") as fh:
            traceback.print_exc(file=fh)
        Chatter.write('Status: error "%s"' % str(e))
        sys.exit(1)

    if len(sys.argv) < 2 or "-help" in sys.argv:
        help()
        sys.exit()

    wd = os.getcwd()

    try:
        from xia2.command_line.xia2_main import xia2_main

        xia2_main(stop_after="integrate")
        Chatter.write("Status: normal termination")

    except Exception as e:
        with open(os.path.join(wd, "xia2.error"), "w") as fh:
            traceback.print_exc(file=fh)
        Chatter.write('Status: error "%s"' % str(e))
        sys.exit(1)
Beispiel #27
0
    def run_one_sweep(args):

      from xia2.Handlers.Streams import Debug

      assert len(args) == 3
      s, failover, job_type = args

      if job_type:
        DriverFactory.set_driver_type(job_type)

      Chatter.cache()
      Debug.cache()

      try:
        s.get_integrater_intensities()
      except Exception, e:
        if failover:
          Chatter.write('Processing sweep %s failed: %s' % \
                        (s.get_name(), str(e)))
          s = None
        else:
          raise
Beispiel #28
0
Datei: html.py Projekt: xia2/xia2
def make_logfile_html(logfile):
    tables = extract_loggraph_tables(logfile)
    if not tables:
      return

    rst = []

    for table in tables:
      try:
        #for graph_name, html in table_to_google_charts(table).iteritems():
        for graph_name, html in table_to_c3js_charts(table).iteritems():
          #rst.append('.. __%s:\n' %graph_name)
          rst.append('.. raw:: html')
          rst.append('\n    '.join(html.split('\n')))
      except StandardError, e:
        Chatter.write('=' * 80)
        Chatter.write('Error (%s) while processing table' % str(e))
        Chatter.write("  '%s'" % table.title)
        Chatter.write('in %s' % logfile)
        Chatter.write('=' * 80)
        Debug.write('Exception raised while processing log file %s, table %s' % (logfile, table.title))
        Debug.write(traceback.format_exc())
def write_citations():
    # tell the user which programs were used...
    Chatter.write("XIA2 used... %s" % ", ".join(Citations.get_programs()))
    Chatter.write(
        "Here are the appropriate citations (BIBTeX in xia2-citations.bib.)")

    for citation in Citations.get_citations_acta():
        Chatter.write(citation)

    # and write the bibtex versions
    out = open("xia2-citations.bib", "w")

    for citation in Citations.get_citations():
        out.write("%s\n" % citation)

    out.close()
Beispiel #30
0
def write_citations():
    # tell the user which programs were used...
    Chatter.write('XIA2 used... %s' % \
        ', '.join(Citations.get_programs()))
    Chatter.write(
        'Here are the appropriate citations (BIBTeX in xia2-citations.bib.)')

    for citation in Citations.get_citations_acta():
        Chatter.write(citation)

    # and write the bibtex versions
    out = open('xia2-citations.bib', 'w')

    for citation in Citations.get_citations():
        out.write('%s\n' % citation)

    out.close()
Beispiel #31
0
def run():
  try:
    check_environment()
  except Exception as e:
    traceback.print_exc(file = open('xia2.error', 'w'))
    Chatter.write('Status: error "%s"' % str(e))

  if len(sys.argv) < 2 or '-help' in sys.argv:
    help()
    sys.exit()

  wd = os.getcwd()

  try:
    from xia2.command_line.xia2_main import xia2_main
    xia2_main(stop_after='integrate')
    Chatter.write('Status: normal termination')

  except Exception as e:
    traceback.print_exc(file = open(os.path.join(wd, 'xia2.error'), 'w'))
    Chatter.write('Status: error "%s"' % str(e))
Beispiel #32
0
  def _integrate_finish(self):
    '''Finish off the integration by running correct.'''

    # first run the postrefinement etc with spacegroup P1
    # and the current unit cell - this will be used to
    # obtain a benchmark rmsd in pixels / phi and also
    # cell deviations (this is working towards spotting bad
    # indexing solutions) - only do this if we have no
    # reindex matrix... and no postrefined cell...

    p1_deviations = None

    # fix for bug # 3264 -
    # if we have not run integration with refined parameters, make it so...
    # erm? shouldn't this therefore return if this is the principle, or
    # set the flag after we have tested the lattice?

    if not self._xds_data_files.has_key('GXPARM.XDS') and \
      PhilIndex.params.xds.integrate.reintegrate:
      Debug.write(
          'Resetting integrater, to ensure refined orientation is used')
      self.set_integrater_done(False)

    if not self.get_integrater_reindex_matrix() and not self._intgr_cell \
           and not Flags.get_no_lattice_test() and \
           not self.get_integrater_sweep().get_user_lattice():
      correct = self.Correct()

      correct.set_data_range(self._intgr_wedge[0],
                             self._intgr_wedge[1])

      if self.get_polarization() > 0.0:
        correct.set_polarization(self.get_polarization())

      # FIXME should this be using the correctly transformed
      # cell or are the results ok without it?!

      correct.set_spacegroup_number(1)
      correct.set_cell(self._intgr_refiner_cell)

      correct.run()

      # record the log file -

      pname, xname, dname = self.get_integrater_project_info()
      sweep = self.get_integrater_sweep_name()
      FileHandler.record_log_file('%s %s %s %s CORRECT' % \
                                  (pname, xname, dname, sweep),
                                  os.path.join(
          self.get_working_directory(),
          'CORRECT.LP'))

      FileHandler.record_more_data_file(
          '%s %s %s %s CORRECT' % (pname, xname, dname, sweep),
          os.path.join(self.get_working_directory(), 'XDS_ASCII.HKL'))

      cell = correct.get_result('cell')
      cell_esd = correct.get_result('cell_esd')

      Debug.write('Postrefinement in P1 results:')
      Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
                  tuple(cell))
      Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
                  tuple(cell_esd))
      Debug.write('Deviations: %.2f pixels %.2f degrees' % \
                  (correct.get_result('rmsd_pixel'),
                   correct.get_result('rmsd_phi')))

      p1_deviations = (correct.get_result('rmsd_pixel'),
                       correct.get_result('rmsd_phi'))

    # next run the postrefinement etc with the given
    # cell / lattice - this will be the assumed result...

    correct = self.Correct()

    correct.set_data_range(self._intgr_wedge[0],
                           self._intgr_wedge[1])

    if self.get_polarization() > 0.0:
      correct.set_polarization(self.get_polarization())

    # BUG # 2695 probably comes from here - need to check...
    # if the pointless interface comes back with a different
    # crystal setting then the unit cell stored in self._intgr_cell
    # needs to be set to None...

    if self.get_integrater_spacegroup_number():
      correct.set_spacegroup_number(
          self.get_integrater_spacegroup_number())
      if not self._intgr_cell:
        raise RuntimeError, 'no unit cell to recycle'
      correct.set_cell(self._intgr_cell)

    # BUG # 3113 - new version of XDS will try and figure the
    # best spacegroup out from the intensities (and get it wrong!)
    # unless we set the spacegroup and cell explicitly

    if not self.get_integrater_spacegroup_number():
      cell = self._intgr_refiner_cell
      lattice = self._intgr_refiner.get_refiner_lattice()
      spacegroup_number = lattice_to_spacegroup_number(lattice)

      # this should not prevent the postrefinement from
      # working correctly, else what is above would not
      # work correctly (the postrefinement test)

      correct.set_spacegroup_number(spacegroup_number)
      correct.set_cell(cell)

      Debug.write('Setting spacegroup to: %d' % spacegroup_number)
      Debug.write(
        'Setting cell to: %.2f %.2f %.2f %.2f %.2f %.2f' % tuple(cell))

    if self.get_integrater_reindex_matrix():

      # bug! if the lattice is not primitive the values in this
      # reindex matrix need to be multiplied by a constant which
      # depends on the Bravais lattice centering.

      lattice = self._intgr_refiner.get_refiner_lattice()

      import scitbx.matrix
      matrix = self.get_integrater_reindex_matrix()
      matrix = scitbx.matrix.sqr(matrix).transpose().elems
      matrix = r_to_rt(matrix)

      if lattice[1] == 'P':
        mult = 1
      elif lattice[1] == 'C' or lattice[1] == 'I':
        mult = 2
      elif lattice[1] == 'R':
        mult = 3
      elif lattice[1] == 'F':
        mult = 4
      else:
        raise RuntimeError, 'unknown multiplier for lattice %s' % \
              lattice

      Debug.write('REIDX multiplier for lattice %s: %d' % \
                  (lattice, mult))

      mult_matrix = [mult * m for m in matrix]

      Debug.write('REIDX set to %d %d %d %d %d %d %d %d %d %d %d %d' % \
                  tuple(mult_matrix))
      correct.set_reindex_matrix(mult_matrix)

    correct.run()

    # erm. just to be sure
    if self.get_integrater_reindex_matrix() and \
           correct.get_reindex_used():
      raise RuntimeError, 'Reindex panic!'

    # get the reindex operation used, which may be useful if none was
    # set but XDS decided to apply one, e.g. #419.

    if not self.get_integrater_reindex_matrix() and \
           correct.get_reindex_used():
      # convert this reindex operation to h, k, l form: n.b. this
      # will involve dividing through by the lattice centring multiplier

      matrix = rt_to_r(correct.get_reindex_used())
      import scitbx.matrix
      matrix = scitbx.matrix.sqr(matrix).transpose().elems

      lattice = self._intgr_refiner.get_refiner_lattice()

      if lattice[1] == 'P':
        mult = 1.0
      elif lattice[1] == 'C' or lattice[1] == 'I':
        mult = 2.0
      elif lattice[1] == 'R':
        mult = 3.0
      elif lattice[1] == 'F':
        mult = 4.0

      matrix = [m / mult for m in matrix]

      reindex_op = mat_to_symop(matrix)

      # assign this to self: will this reset?! make for a leaky
      # abstraction and just assign this...

      # self.set_integrater_reindex_operator(reindex)

      self._intgr_reindex_operator = reindex_op


    # record the log file -

    pname, xname, dname = self.get_integrater_project_info()
    sweep = self.get_integrater_sweep_name()
    FileHandler.record_log_file('%s %s %s %s CORRECT' % \
                                (pname, xname, dname, sweep),
                                os.path.join(self.get_working_directory(),
                                             'CORRECT.LP'))

    # should get some interesting stuff from the XDS correct file
    # here, for instance the resolution range to use in integration
    # (which should be fed back if not fast) and so on...

    self._intgr_corrected_hklout = os.path.join(self.get_working_directory(),
                                'XDS_ASCII.HKL')

    # also record the batch range - needed for the analysis of the
    # radiation damage in chef...

    self._intgr_batches_out = (self._intgr_wedge[0],
                               self._intgr_wedge[1])

    # FIXME perhaps I should also feedback the GXPARM file here??
    for file in ['GXPARM.XDS']:
      self._xds_data_files[file] = correct.get_output_data_file(file)

    # record the postrefined cell parameters
    self._intgr_cell = correct.get_result('cell')
    self._intgr_n_ref = correct.get_result('n_ref')

    Debug.write('Postrefinement in "correct" spacegroup results:')
    Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
                tuple(correct.get_result('cell')))
    Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
                tuple(correct.get_result('cell_esd')))
    Debug.write('Deviations: %.2f pixels %.2f degrees' % \
                (correct.get_result('rmsd_pixel'),
                 correct.get_result('rmsd_phi')))

    Debug.write('Error correction parameters: A=%.3f B=%.3f' % \
                correct.get_result('sdcorrection'))

    # compute misorientation of axes

    xparm_file = os.path.join(self.get_working_directory(), 'GXPARM.XDS')

    from iotbx.xds import xparm
    handle = xparm.reader()
    handle.read_file(xparm_file)

    rotn = handle.rotation_axis
    beam = handle.beam_vector

    dot = sum([rotn[j] * beam[j] for j in range(3)])
    r = math.sqrt(sum([rotn[j] * rotn[j] for j in range(3)]))
    b = math.sqrt(sum([beam[j] * beam[j] for j in range(3)]))

    rtod = 180.0 / math.pi

    angle = rtod * math.fabs(0.5 * math.pi - math.acos(dot / (r * b)))

    Debug.write('Axis misalignment %.2f degrees' % angle)

    correct_deviations = (correct.get_result('rmsd_pixel'),
                          correct.get_result('rmsd_phi'))

    if p1_deviations:
      # compare and reject if both > 50% higher - though adding a little
      # flexibility - 0.5 pixel / osc width slack.

      pixel = p1_deviations[0]
      phi = math.sqrt(0.05 * 0.05 + \
                      p1_deviations[1] * p1_deviations[1])

      threshold = Flags.get_rejection_threshold()

      Debug.write('RMSD ratio: %.2f' % (correct_deviations[0] / pixel))
      Debug.write('RMSPhi ratio: %.2f' % (correct_deviations[1] / phi))

      if correct_deviations[0] / pixel > threshold and \
             correct_deviations[1] / phi > threshold:

        Chatter.write(
        'Eliminating this indexing solution as postrefinement')
        Chatter.write(
        'deviations rather high relative to triclinic')
        raise BadLatticeError, \
              'high relative deviations in postrefinement'

    if not Flags.get_quick() and Flags.get_remove():
      # check for alien reflections and perhaps recycle - removing them
      if len(correct.get_remove()) > 0:

        correct_remove = correct.get_remove()
        current_remove = []
        final_remove = []

        # first ensure that there are no duplicate entries...
        if os.path.exists(os.path.join(
            self.get_working_directory(),
            'REMOVE.HKL')):
          for line in open(os.path.join(
              self.get_working_directory(),
              'REMOVE.HKL'), 'r').readlines():
            h, k, l = map(int, line.split()[:3])
            z = float(line.split()[3])

            if not (h, k, l, z) in current_remove:
              current_remove.append((h, k, l, z))

          for c in correct_remove:
            if c in current_remove:
              continue
            final_remove.append(c)

          Debug.write(
              '%d alien reflections are already removed' % \
              (len(correct_remove) - len(final_remove)))
        else:
          # we want to remove all of the new dodgy reflections
          final_remove = correct_remove

        remove_hkl = open(os.path.join(
            self.get_working_directory(),
            'REMOVE.HKL'), 'w')

        z_min = Flags.get_z_min()
        rejected = 0

        # write in the old reflections
        for remove in current_remove:
          z = remove[3]
          if z >= z_min:
            remove_hkl.write('%d %d %d %f\n' % remove)
          else:
            rejected += 1
        Debug.write('Wrote %d old reflections to REMOVE.HKL' % \
                    (len(current_remove) - rejected))
        Debug.write('Rejected %d as z < %f' % \
                    (rejected, z_min))

        # and the new reflections
        rejected = 0
        used = 0
        for remove in final_remove:
          z = remove[3]
          if z >= z_min:
            used += 1
            remove_hkl.write('%d %d %d %f\n' % remove)
          else:
            rejected += 1
        Debug.write('Wrote %d new reflections to REMOVE.HKL' % \
                    (len(final_remove) - rejected))
        Debug.write('Rejected %d as z < %f' % \
                    (rejected, z_min))

        remove_hkl.close()

        # we want to rerun the finishing step so...
        # unless we have added no new reflections... or unless we
        # have not confirmed the point group (see SCI-398)

        if used and self.get_integrater_reindex_matrix():
          self.set_integrater_finish_done(False)

    else:
      Debug.write(
          'Going quickly so not removing %d outlier reflections...' % \
          len(correct.get_remove()))

    # Convert INTEGRATE.HKL to MTZ format and reapply any reindexing operations
    # spacegroup changes to allow use with CCP4 / Aimless for scaling

    integrate_hkl = os.path.join(
      self.get_working_directory(), 'INTEGRATE.HKL')

    hklout = os.path.splitext(integrate_hkl)[0] + ".mtz"
    self._factory.set_working_directory(self.get_working_directory())
    pointless = self._factory.Pointless()
    pointless.set_xdsin(integrate_hkl)
    pointless.set_hklout(hklout)
    pointless.xds_to_mtz()

    integrate_mtz = hklout

    if self.get_integrater_reindex_operator() or \
       self.get_integrater_spacegroup_number():

      Debug.write('Reindexing things to MTZ')

      reindex = Reindex()
      reindex.set_working_directory(self.get_working_directory())
      auto_logfiler(reindex)

      if self.get_integrater_reindex_operator():
        reindex.set_operator(self.get_integrater_reindex_operator())

      if self.get_integrater_spacegroup_number():
        reindex.set_spacegroup(self.get_integrater_spacegroup_number())

      hklout = '%s_reindex.mtz' % os.path.splitext(integrate_mtz)[0]

      reindex.set_hklin(integrate_mtz)
      reindex.set_hklout(hklout)
      reindex.reindex()
      integrate_mtz = hklout

    return integrate_mtz
    def index(self):

        if not self.get_indexer_finish_done():
            f = inspect.currentframe().f_back.f_back
            m = f.f_code.co_filename
            l = f.f_lineno

            Debug.write(
                "Index in %s called from %s %d" % (self.__class__.__name__, m, l)
            )

        while not self.get_indexer_finish_done():
            while not self.get_indexer_done():
                while not self.get_indexer_prepare_done():

                    # --------------
                    # call prepare()
                    # --------------

                    self.set_indexer_prepare_done(True)
                    self._index_prepare()

                # --------------------------------------------
                # then do the proper indexing - using the best
                # solution already stored if available (c/f
                # eliminate above)
                # --------------------------------------------

                self.set_indexer_done(True)

                if self.get_indexer_sweeps():
                    xsweeps = [s.get_name() for s in self.get_indexer_sweeps()]
                    if len(xsweeps) > 1:
                        # find "SWEEPn, SWEEP(n+1), (..), SWEEPm" and aggregate to "SWEEPS n-m"
                        xsweeps = map(
                            lambda x: (int(x[5:]), int(x[5:]))
                            if x.startswith("SWEEP")
                            else x,
                            xsweeps,
                        )
                        xsweeps[0] = [xsweeps[0]]

                        def compress(seen, nxt):
                            if (
                                isinstance(seen[-1], tuple)
                                and isinstance(nxt, tuple)
                                and (seen[-1][1] + 1 == nxt[0])
                            ):
                                seen[-1] = (seen[-1][0], nxt[1])
                            else:
                                seen.append(nxt)
                            return seen

                        xsweeps = reduce(compress, xsweeps)
                        xsweeps = map(
                            lambda x: (
                                "SWEEP%d" % x[0]
                                if x[0] == x[1]
                                else "SWEEPS %d to %d" % (x[0], x[1])
                            )
                            if isinstance(x, tuple)
                            else x,
                            xsweeps,
                        )
                    if len(xsweeps) > 1:
                        sweep_names = ", ".join(xsweeps[:-1])
                        sweep_names += " & " + xsweeps[-1]
                    else:
                        sweep_names = xsweeps[0]

                    if PhilIndex.params.xia2.settings.show_template:
                        template = self.get_indexer_sweep().get_template()
                        Chatter.banner("Autoindexing %s (%s)" % (sweep_names, template))
                    else:
                        Chatter.banner("Autoindexing %s" % sweep_names)

                if not self._indxr_helper:

                    result = self._index()

                    if not self._indxr_done:
                        Debug.write("Looks like indexing failed - try again!")
                        continue

                    solutions = {}
                    for k in self._indxr_other_lattice_cell.keys():
                        solutions[k] = self._indxr_other_lattice_cell[k]["cell"]

                    # create a helper for the indexer to manage solutions
                    self._indxr_helper = _IndexerHelper(solutions)

                    solution = self._indxr_helper.get()

                    # compare these against the final solution, if different
                    # reject solution and return - correct solution will
                    # be used next cycle

                    if (
                        self._indxr_lattice != solution[0]
                        and not self._indxr_input_cell
                        and not PhilIndex.params.xia2.settings.integrate_p1
                    ):
                        Chatter.write(
                            "Rerunning indexing lattice %s to %s"
                            % (self._indxr_lattice, solution[0])
                        )
                        Debug.write(
                            "Rerunning indexing with target lattice %s" % solution[0]
                        )
                        self.set_indexer_done(False)

                else:
                    # rerun autoindexing with the best known current solution

                    solution = self._indxr_helper.get()
                    self._indxr_input_lattice = solution[0]
                    self._indxr_input_cell = solution[1]
                    result = self._index()

            # next finish up...

            self.set_indexer_finish_done(True)
            self._index_finish()

            if self._indxr_print:
                Chatter.write(self.show_indexer_solutions())
Beispiel #34
0
    def _index_prepare(self):

        from xia2.Handlers.Citations import Citations
        Citations.cite('dials')

        #all_images = self.get_matching_images()
        #first = min(all_images)
        #last = max(all_images)

        spot_lists = []
        datablocks = []

        for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps):

            Chatter.banner('Spotfinding %s' % xsweep.get_name())

            first, last = imageset.get_scan().get_image_range()

            # at this stage, break out to run the DIALS code: this sets itself up
            # now cheat and pass in some information... save re-reading all of the
            # image headers

            # FIXME need to adjust this to allow (say) three chunks of images

            from dxtbx.serialize import dump
            from dxtbx.datablock import DataBlock
            sweep_filename = os.path.join(
                self.get_working_directory(),
                '%s_datablock.json' % xsweep.get_name())
            dump.datablock(DataBlock([imageset]), sweep_filename)

            genmask = self.GenerateMask()
            genmask.set_input_datablock(sweep_filename)
            genmask.set_output_datablock(
                os.path.join(
                    self.get_working_directory(), '%s_%s_datablock.json' %
                    (genmask.get_xpid(), xsweep.get_name())))
            genmask.set_params(PhilIndex.params.dials.masking)
            sweep_filename, mask_pickle = genmask.run()
            Debug.write('Generated mask for %s: %s' %
                        (xsweep.get_name(), mask_pickle))

            gain = PhilIndex.params.xia2.settings.input.gain
            if gain is libtbx.Auto:
                gain_estimater = self.EstimateGain()
                gain_estimater.set_sweep_filename(sweep_filename)
                gain_estimater.run()
                gain = gain_estimater.get_gain()
                Chatter.write('Estimated gain: %.2f' % gain)
                PhilIndex.params.xia2.settings.input.gain = gain

            # FIXME this should really use the assigned spot finding regions
            #offset = self.get_frame_offset()
            spotfinder = self.Spotfinder()
            if last - first > 10:
                spotfinder.set_write_hot_mask(True)
            spotfinder.set_input_sweep_filename(sweep_filename)
            spotfinder.set_output_sweep_filename(
                '%s_%s_datablock.json' %
                (spotfinder.get_xpid(), xsweep.get_name()))
            spotfinder.set_input_spot_filename(
                '%s_%s_strong.pickle' %
                (spotfinder.get_xpid(), xsweep.get_name()))
            if PhilIndex.params.dials.fast_mode:
                wedges = self._index_select_images_i(imageset)
                spotfinder.set_scan_ranges(wedges)
            else:
                spotfinder.set_scan_ranges([(first, last)])
            if PhilIndex.params.dials.find_spots.phil_file is not None:
                spotfinder.set_phil_file(
                    PhilIndex.params.dials.find_spots.phil_file)
            min_spot_size = PhilIndex.params.dials.find_spots.min_spot_size
            if min_spot_size is libtbx.Auto:
                if imageset.get_detector()[0].get_type() == 'SENSOR_PAD':
                    min_spot_size = 3
                else:
                    min_spot_size = None
            if min_spot_size is not None:
                spotfinder.set_min_spot_size(min_spot_size)
            min_local = PhilIndex.params.dials.find_spots.min_local
            if min_local is not None:
                spotfinder.set_min_local(min_local)
            sigma_strong = PhilIndex.params.dials.find_spots.sigma_strong
            if sigma_strong:
                spotfinder.set_sigma_strong(sigma_strong)
            gain = PhilIndex.params.xia2.settings.input.gain
            if gain:
                spotfinder.set_gain(gain)
            filter_ice_rings = PhilIndex.params.dials.find_spots.filter_ice_rings
            if filter_ice_rings:
                spotfinder.set_filter_ice_rings(filter_ice_rings)
            kernel_size = PhilIndex.params.dials.find_spots.kernel_size
            if kernel_size:
                spotfinder.set_kernel_size(kernel_size)
            global_threshold = PhilIndex.params.dials.find_spots.global_threshold
            if global_threshold is not None:
                spotfinder.set_global_threshold(global_threshold)
            spotfinder.run()

            spot_filename = spotfinder.get_spot_filename()
            if not os.path.exists(spot_filename):
                raise RuntimeError("Spotfinding failed: %s does not exist." %
                                   os.path.basename(spot_filename))

            spot_lists.append(spot_filename)
            datablocks.append(spotfinder.get_output_sweep_filename())

            from libtbx import easy_pickle
            from dials.util.ascii_art import spot_counts_per_image_plot
            refl = easy_pickle.load(spot_filename)
            if not len(refl):
                raise RuntimeError('No spots found in sweep %s' %
                                   xsweep.get_name())
            Chatter.write(spot_counts_per_image_plot(refl), strip=False)

            if not PhilIndex.params.dials.fast_mode:
                detectblanks = self.DetectBlanks()
                detectblanks.set_sweep_filename(datablocks[-1])
                detectblanks.set_reflections_filename(spot_filename)
                detectblanks.run()
                json = detectblanks.get_results()
                offset = imageset.get_scan().get_image_range()[0]
                blank_regions = json['strong']['blank_regions']
                if len(blank_regions):
                    blank_regions = [(int(s), int(e))
                                     for s, e in blank_regions]
                    for blank_start, blank_end in blank_regions:
                        Chatter.write(
                            'WARNING: Potential blank images: %i -> %i' %
                            (blank_start + 1, blank_end))

                    if PhilIndex.params.xia2.settings.remove_blanks:
                        non_blanks = []
                        start, end = imageset.get_array_range()
                        last_blank_end = start
                        for blank_start, blank_end in blank_regions:
                            if blank_start > start:
                                non_blanks.append(
                                    (last_blank_end, blank_start))
                            last_blank_end = blank_end

                        if last_blank_end + 1 < end:
                            non_blanks.append((last_blank_end, end))

                        xsweep = self.get_indexer_sweep()
                        xwav = xsweep.get_wavelength()
                        xsample = xsweep.get_xsample()

                        sweep_name = xsweep.get_name()
                        import string
                        for i, (nb_start, nb_end) in enumerate(non_blanks):
                            assert i < 26
                            if i == 0:
                                sub_imageset = imageset[nb_start -
                                                        start:nb_end - start]
                                xsweep._frames_to_process = (nb_start + 1,
                                                             nb_end + 1)
                                self.set_indexer_prepare_done(done=False)
                                self._indxr_imagesets[
                                    self._indxr_imagesets.index(
                                        imageset)] = sub_imageset
                                xsweep._integrater._setup_from_imageset(
                                    sub_imageset)
                            else:
                                new_name = '_'.join(
                                    (sweep_name, string.ascii_lowercase[i]))
                                new_sweep = xwav.add_sweep(
                                    new_name,
                                    xsample,
                                    directory=os.path.join(
                                        os.path.basename(
                                            xsweep.get_directory()), new_name),
                                    image=imageset.get_path(nb_start - start),
                                    frames_to_process=(nb_start + 1, nb_end),
                                )
                                Chatter.write(
                                    "Generating new sweep: %s (%s:%i:%i)" %
                                    (new_sweep.get_name(),
                                     new_sweep.get_image(),
                                     new_sweep.get_frames_to_process()[0],
                                     new_sweep.get_frames_to_process()[1]))
                        return

            if not PhilIndex.params.xia2.settings.trust_beam_centre:
                discovery = self.DiscoverBetterExperimentalModel()
                discovery.set_sweep_filename(datablocks[-1])
                discovery.set_spot_filename(spot_filename)
                #wedges = self._index_select_images_i(imageset)
                #discovery.set_scan_ranges(wedges)
                #discovery.set_scan_ranges([(first + offset, last + offset)])
                try:
                    discovery.run()
                except Exception as e:
                    Debug.write('DIALS beam centre search failed: %s' % str(e))
                else:
                    # overwrite datablock.json in datablocks list
                    datablocks[
                        -1] = discovery.get_optimized_datablock_filename()

        self.set_indexer_payload("spot_lists", spot_lists)
        self.set_indexer_payload("datablocks", datablocks)

        return
Beispiel #35
0
def run():
    from libtbx.utils import Sorry
    if len(sys.argv) < 2 or '-help' in sys.argv or '--help' in sys.argv:
        help()
        sys.exit()

    if '-version' in sys.argv or '--version' in sys.argv:
        import xia2.XIA2Version
        print xia2.XIA2Version.Version
        print dials_version()
        ccp4_version = get_ccp4_version()
        if ccp4_version is not None:
            print 'CCP4 %s' % ccp4_version
        sys.exit()

    try:
        check_environment()
    except Exception as e:
        traceback.print_exc(file=open('xia2.error', 'w'))
        Chatter.write('Error setting up xia2 environment: %s' % str(e))
        Chatter.write(
            'Please send the contents of xia2.txt, xia2.error and xia2-debug.txt to:'
        )
        Chatter.write('*****@*****.**')
        sys.exit(1)

    wd = os.getcwd()

    try:
        xinfo = xia2_main()
        Chatter.write('Status: normal termination')

        Debug.write('\n------\nTiming summary:')
        import xia2.Driver.DefaultDriver
        xia2.Driver.DefaultDriver.output_timing_information()
        return xinfo
    except Sorry as s:
        Chatter.write('Error: %s' % str(s))
        sys.exit(1)
    except Exception as e:
        traceback.print_exc(file=open(os.path.join(wd, 'xia2.error'), 'w'))
        Chatter.write('Error: %s' % str(e))
        Chatter.write(
            'Please send the contents of xia2.txt, xia2.error and xia2-debug.txt to:'
        )
        Chatter.write('*****@*****.**')
        sys.exit(1)
Beispiel #36
0
        def run(self):
            '''Run labelit.index'''

            assert len(self._images) > 0
            self._images.sort()

            if self._max_cell is None and len(self._images) > 4:
                raise RuntimeError('cannot use more than 4 images')

            #task = 'Autoindex from images:'

            #for i in _images:
            #task += ' %s' % self.get_image_name(i)

            #self.set_task(task)

            self.add_command_line('--index_only')

            for image in self._images:
                self.add_command_line(image)

            if self._space_group_number is not None:
                self.add_command_line('known_symmetry=%d' %
                                      self._space_group_number)

            if self._primitive_unit_cell is not None:
                self.add_command_line('target_cell=%f,%f,%f,%f,%f,%f' %
                                      tuple(self._primitive_unit_cell))

            if self._max_cell is not None:
                self.add_command_line('codecamp.maxcell=%f' % self._max_cell)

            self._write_dataset_preferences(len(self._images))

            shutil.copyfile(os.path.join(self.get_working_directory(),
                                         'dataset_preferences.py'),
                            os.path.join(self.get_working_directory(),
                                         '%d_dataset_preferences.py' % \
                                         self.get_xpid()))

            self.start()
            self.close_wait()

            #sweep = self.get_indexer_sweep_name()
            #FileHandler.record_log_file(
            #'%s INDEX' % (sweep), self.get_log_file())

            # check for errors
            self.check_for_errors()

            # check for labelit errors - if something went wrong, then
            # try to address it by e.g. extending the beam search area...

            self.check_labelit_errors()

            # ok now we're done, let's look through for some useful stuff
            output = self.get_all_output()

            counter = 0

            # FIXME 03/NOV/06 something to do with the new centre search...

            # example output:

            # Beam center is not immediately clear; rigorously retesting \
            #                                             2 solutions
            # Beam x 109.0 y 105.1, initial score 538; refined rmsd: 0.1969
            # Beam x 108.8 y 106.1, initial score 354; refined rmsd: 0.1792

            # in here want to parse the beam centre search if it was done,
            # and check that the highest scoring solution was declared
            # the "best" - though should also have a check on the
            # R.M.S. deviation of that solution...

            # do this first!

            for j in range(len(output)):
                o = output[j]
                if 'Beam centre is not immediately clear' in o:
                    # read the solutions that it has found and parse the
                    # information

                    centres = []
                    scores = []
                    rmsds = []

                    num_solutions = int(o.split()[-2])

                    for n in range(num_solutions):
                        record = output[j + n + 1].replace(',', ' ').replace(
                            ';', ' ').split()
                        x, y = float(record[2]), \
                               float(record[4])

                        centres.append((x, y))
                        scores.append(int(record[7]))
                        rmsds.append(float(record[-1]))

                    # next perform some analysis and perhaps assert the
                    # correct solution - for the moment just raise a warning
                    # if it looks like wrong solution may have been picked

                    best_beam_score = (0.0, 0.0, 0)
                    best_beam_rms = (0.0, 0.0, 1.0e8)

                    for n in range(num_solutions):
                        beam = centres[n]
                        score = scores[n]
                        rmsd = rmsds[n]

                        if score > best_beam_score[2]:
                            best_beam_score = (beam[0], beam[1], score)

                        if rmsd < best_beam_rmsd[2]:
                            best_beam_rmsd = (beam[0], beam[1], rmsd)

                    # allow a difference of 0.1mm in either direction...
                    if math.fabs(
                        best_beam_score[0] -
                        best_beam_rmsd[0]) > 0.1 or \
                        math.fabs(best_beam_score[1] -
                                  best_beam_rmsd[1]) > 0.1:
                        Chatter.write(
                            'Labelit may have picked the wrong beam centre')

                        # FIXME as soon as I get the indexing loop
                        # structure set up, this should reset the
                        # indexing done flag, set the search range to
                        # 0, correct beam and then return...

                        # should also allow for the possibility that
                        # labelit has selected the best solution - so this
                        # will need to remember the stats for this solution,
                        # then compare them against the stats (one day) from
                        # running with the other solution - eventually the
                        # correct solution will result...

            for o in output:
                l = o.split()

                if l[:3] == ['Beam', 'center', 'x']:
                    x = float(l[3].replace('mm,', ''))
                    y = float(l[5].replace('mm,', ''))

                    self._mosflm_beam_centre = (x, y)
                    self._mosflm_detector_distance = float(l[7].replace(
                        'mm', ''))
                    #self.set_indexer_beam_centre((x, y))
                    #self.set_indexer_distance(float(l[7].replace('mm', '')))

                    self._mosaic = float(l[10].replace('mosaicity=', ''))

                if l[:3] == ['Solution', 'Metric', 'fit']:
                    break

                counter += 1

            # if we've just broken out (counter < len(output)) then
            # we need to gather the output

            if counter >= len(output):
                raise RuntimeError('error in indexing')

            # FIXME this needs to check the smilie status e.g.
            # ":)" or ";(" or "  ".

            for i in range(counter + 1, len(output)):
                o = output[i][3:]
                smiley = output[i][:3]
                l = o.split()
                if l:

                    self._solutions[int(l[0])] = {
                        'number': int(l[0]),
                        'mosaic': self._mosaic,
                        'metric': float(l[1]),
                        'rmsd': float(l[3]),
                        'nspots': int(l[4]),
                        'lattice': l[6],
                        'cell': map(float, l[7:13]),
                        'volume': int(l[-1]),
                        'smiley': smiley
                    }

            # remove clearly incorrect solutions i.e. rmsd >> rmsd for P1 i.e. factor
            # of 4.0 or more...

            for s in sorted(self._solutions):
                if self._solutions[s][
                        'rmsd'] > 4.0 * self._solutions[1]['rmsd']:
                    del (self._solutions[s])

            return 'ok'
Beispiel #37
0
  def index(self):

    if not self.get_indexer_finish_done():
      f = inspect.currentframe().f_back.f_back
      m = f.f_code.co_filename
      l = f.f_lineno

      Debug.write('Index in %s called from %s %d' %
                  (self.__class__.__name__, m, l))

    while not self.get_indexer_finish_done():
      while not self.get_indexer_done():
        while not self.get_indexer_prepare_done():

          # --------------
          # call prepare()
          # --------------

          self.set_indexer_prepare_done(True)
          self._index_prepare()

        # --------------------------------------------
        # then do the proper indexing - using the best
        # solution already stored if available (c/f
        # eliminate above)
        # --------------------------------------------

        self.set_indexer_done(True)

        if self.get_indexer_sweeps():
          xsweeps = [ s.get_name() for s in self.get_indexer_sweeps() ]
          if len(xsweeps) > 1:
            # find "SWEEPn, SWEEP(n+1), (..), SWEEPm" and aggregate to "SWEEPS n-m"
            xsweeps = map(lambda x: (int(x[5:]), int(x[5:])) if x.startswith('SWEEP') else x, xsweeps)
            xsweeps[0] = [xsweeps[0]]
            def compress(seen, nxt):
              if isinstance(seen[-1], tuple) and isinstance(nxt, tuple) and (seen[-1][1] + 1 == nxt[0]):
                seen[-1] = (seen[-1][0], nxt[1])
              else:
                seen.append(nxt)
              return seen
            xsweeps = reduce(compress, xsweeps)
            xsweeps = map(lambda x: ('SWEEP%d' % x[0] if x[0] == x[1] else
                                     'SWEEPS %d to %d' % (x[0], x[1])) if isinstance(x, tuple)
                                     else x, xsweeps)
          if len(xsweeps) > 1:
            sweep_names = ', '.join(xsweeps[:-1])
            sweep_names += ' & ' + xsweeps[-1]
          else:
            sweep_names = xsweeps[0]

          if PhilIndex.params.xia2.settings.show_template:
            template = self.get_indexer_sweep().get_template()
            Chatter.banner(
              'Autoindexing %s (%s)' %(sweep_names, template))
          else:
            Chatter.banner('Autoindexing %s' %sweep_names)


        if not self._indxr_helper:

          result = self._index()

          if not self._indxr_done:
            Debug.write(
              'Looks like indexing failed - try again!')
            continue

          solutions = { }
          for k in self._indxr_other_lattice_cell.keys():
            solutions[k] = self._indxr_other_lattice_cell[k][
              'cell']

          # create a helper for the indexer to manage solutions
          self._indxr_helper = _IndexerHelper(solutions)

          solution = self._indxr_helper.get()

          # compare these against the final solution, if different
          # reject solution and return - correct solution will
          # be used next cycle

          if self._indxr_lattice != solution[0] and \
             not self._indxr_input_cell and \
             not PhilIndex.params.xia2.settings.integrate_p1:
            Chatter.write('Rerunning indexing lattice %s to %s' %
                          (self._indxr_lattice, solution[0]))
            Debug.write(
              'Rerunning indexing with target lattice %s' % \
              solution[0])
            self.set_indexer_done(False)

        else:
          # rerun autoindexing with the best known current solution

          solution = self._indxr_helper.get()
          self._indxr_input_lattice = solution[0]
          self._indxr_input_cell = solution[1]
          result = self._index()

      # next finish up...

      self.set_indexer_finish_done(True)
      self._index_finish()

      if self._indxr_print:
        Chatter.write(self.show_indexer_solutions())
Beispiel #38
0
from xia2.Applications.xia2 import check, check_cctbx_version, check_environment
from xia2.Applications.xia2 import get_command_line, write_citations, help


def run():
  if os.path.exists('xia2-working.phil'):
    sys.argv.append('xia2-working.phil')
  try:
    check_environment()
    check()
  except exceptions.Exception, e:
    traceback.print_exc(file = open('xia2.error', 'w'))
    Chatter.write('Status: error "%s"' % str(e))

  # print the version
  Chatter.write(Version)
  Citations.cite('xia2')

  start_time = time.time()

  assert os.path.exists('xia2.json')
  from xia2.Schema.XProject import XProject
  xinfo = XProject.from_json(filename='xia2.json')

  crystals = xinfo.get_crystals()
  for crystal_id, crystal in crystals.iteritems():
    #cwd = os.path.abspath(os.curdir)
    from libtbx import Auto
    scale_dir = PhilIndex.params.xia2.settings.scale.directory
    if scale_dir is Auto:
      scale_dir = 'scale'
Beispiel #39
0
def run():
  assert os.path.exists('xia2.json')
  from xia2.Schema.XProject import XProject
  xinfo = XProject.from_json(filename='xia2.json')
  Chatter.write(xinfo.get_output())
  write_citations()
Beispiel #40
0
 def print_command_line(self):
     cl = self.get_command_line()
     Chatter.write("Command line: %s" % cl)
Beispiel #41
0
    def setup(self):
        """Set everything up..."""

        # check arguments are all ascii

        Debug.write("Start parsing command line: " + str(sys.argv))

        for token in sys.argv:
            try:
                token.encode("ascii")
            except UnicodeDecodeError:
                raise RuntimeError("non-ascii characters in input")

        self._argv = copy.deepcopy(sys.argv)

        replacements = {
            "-2d": "pipeline=2d",
            "-2di": "pipeline=2di",
            "-3d": "pipeline=3d",
            "-3di": "pipeline=3di",
            "-3dii": "pipeline=3dii",
            "-3dd": "pipeline=3dd",
            "-dials": "pipeline=dials",
            "-quick": "dials.fast_mode=true",
            "-failover": "failover=true",
            "-small_molecule": "small_molecule=true",
        }
        for k, v in replacements.iteritems():
            if k in self._argv:
                print(
                    "***\nCommand line option %s is deprecated.\nPlease use %s instead\n***"
                    % (k, v))
                self._argv[self._argv.index(k)] = v
        if "-atom" in self._argv:
            idx = self._argv.index("-atom")
            element = self._argv[idx + 1]
            self._argv[idx:idx + 2] = ["atom=%s" % element]
            print(
                "***\nCommand line option -atom %s is deprecated.\nPlease use atom=%s instead\n***"
                % (element, element))

        # first of all try to interpret arguments as phil parameters/files

        from xia2.Handlers.Phil import master_phil
        from libtbx.phil import command_line

        cmd_line = command_line.argument_interpreter(master_phil=master_phil)
        working_phil, self._argv = cmd_line.process_and_fetch(
            args=self._argv, custom_processor="collect_remaining")

        PhilIndex.merge_phil(working_phil)
        try:
            params = PhilIndex.get_python_object()
        except RuntimeError as e:
            raise Sorry(e)

        # sanity check / interpret Auto in input
        from libtbx import Auto

        if params.xia2.settings.input.atom is None:
            if params.xia2.settings.input.anomalous is Auto:
                PhilIndex.update("xia2.settings.input.anomalous=false")
        else:
            if params.xia2.settings.input.anomalous is False:
                raise Sorry(
                    "Setting anomalous=false and atom type inconsistent")
            params.xia2.settings.input.anomalous = True
            PhilIndex.update("xia2.settings.input.anomalous=true")

        if params.xia2.settings.resolution.keep_all_reflections is Auto:
            if (params.xia2.settings.small_molecule is True
                    and params.xia2.settings.resolution.d_min is None
                    and params.xia2.settings.resolution.d_max is None):
                PhilIndex.update(
                    "xia2.settings.resolution.keep_all_reflections=true")
            else:
                PhilIndex.update(
                    "xia2.settings.resolution.keep_all_reflections=false")

        if params.xia2.settings.small_molecule is True:
            Debug.write("Small molecule selected")
            if params.xia2.settings.symmetry.chirality is None:
                PhilIndex.update("xia2.settings.symmetry.chirality=nonchiral")
            params = PhilIndex.get_python_object()

        # pipeline options
        self._read_pipeline()

        for (parameter, value) in (
            ("project", params.xia2.settings.project),
            ("crystal", params.xia2.settings.crystal),
        ):
            validate_project_crystal_name(parameter, value)

        Debug.write("Project: %s" % params.xia2.settings.project)
        Debug.write("Crystal: %s" % params.xia2.settings.crystal)

        # FIXME add some consistency checks in here e.g. that there are
        # images assigned, there is a lattice assigned if cell constants
        # are given and so on

        params = PhilIndex.get_python_object()
        mp_params = params.xia2.settings.multiprocessing
        from xia2.Handlers.Environment import get_number_cpus

        if mp_params.mode == "parallel":
            if mp_params.type == "qsub":
                if which("qsub") is None:
                    raise Sorry("qsub not available")
            if mp_params.njob is Auto:
                mp_params.njob = get_number_cpus()
                if mp_params.nproc is Auto:
                    mp_params.nproc = 1
            elif mp_params.nproc is Auto:
                mp_params.nproc = get_number_cpus()
        elif mp_params.mode == "serial":
            if mp_params.type == "qsub":
                if which("qsub") is None:
                    raise Sorry("qsub not available")
            if mp_params.njob is Auto:
                mp_params.njob = 1
            if mp_params.nproc is Auto:
                mp_params.nproc = get_number_cpus()

        PhilIndex.update("xia2.settings.multiprocessing.njob=%d" %
                         mp_params.njob)
        PhilIndex.update("xia2.settings.multiprocessing.nproc=%d" %
                         mp_params.nproc)
        params = PhilIndex.get_python_object()
        mp_params = params.xia2.settings.multiprocessing

        if mp_params.nproc > 1 and os.name == "nt":
            raise Sorry("nproc > 1 is not supported on Windows.")  # #191

        if params.xia2.settings.indexer is not None:
            add_preference("indexer", params.xia2.settings.indexer)
        if params.xia2.settings.refiner is not None:
            add_preference("refiner", params.xia2.settings.refiner)
        if params.xia2.settings.integrater is not None:
            add_preference("integrater", params.xia2.settings.integrater)
        if params.xia2.settings.scaler is not None:
            add_preference("scaler", params.xia2.settings.scaler)

        if params.xia2.settings.multi_sweep_indexing is Auto:
            if (params.xia2.settings.small_molecule is True
                    and "dials" == params.xia2.settings.indexer):
                PhilIndex.update("xia2.settings.multi_sweep_indexing=True")
            else:
                PhilIndex.update("xia2.settings.multi_sweep_indexing=False")
        if (params.xia2.settings.multi_sweep_indexing is True
                and params.xia2.settings.multiprocessing.mode == "parallel"):
            Chatter.write(
                "Multi sweep indexing disabled:\nMSI is not available for parallel processing."
            )
            PhilIndex.update("xia2.settings.multi_sweep_indexing=False")

        input_json = params.xia2.settings.input.json
        if input_json is not None and len(input_json):
            for json_file in input_json:
                assert os.path.isfile(json_file)
                load_experiments(json_file)

        reference_geometry = params.xia2.settings.input.reference_geometry
        if reference_geometry is not None and len(reference_geometry) > 0:
            reference_geometries = "\n".join([
                "xia2.settings.input.reference_geometry=%s" %
                os.path.abspath(g)
                for g in params.xia2.settings.input.reference_geometry
            ])
            Debug.write(reference_geometries)
            PhilIndex.update(reference_geometries)
            Debug.write("xia2.settings.trust_beam_centre=true")
            PhilIndex.update("xia2.settings.trust_beam_centre=true")
            params = PhilIndex.get_python_object()

        params = PhilIndex.get_python_object()
        if params.xia2.settings.input.xinfo is not None:
            xinfo_file = os.path.abspath(params.xia2.settings.input.xinfo)
            PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file)
            params = PhilIndex.get_python_object()
            self.set_xinfo(xinfo_file)

            # issue #55 if not set ATOM in xinfo but anomalous=true or atom= set
            # on commandline, set here, should be idempotent

            if params.xia2.settings.input.anomalous is True:
                crystals = self._xinfo.get_crystals()
                for xname in crystals:
                    xtal = crystals[xname]
                    Debug.write("Setting anomalous for crystal %s" % xname)
                    xtal.set_anomalous(True)
        else:
            xinfo_file = "%s/automatic.xinfo" % os.path.abspath(os.curdir)
            PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file)
            params = PhilIndex.get_python_object()

        if params.dials.find_spots.phil_file is not None:
            PhilIndex.update(
                "dials.find_spots.phil_file=%s" %
                os.path.abspath(params.dials.find_spots.phil_file))
        if params.dials.index.phil_file is not None:
            PhilIndex.update("dials.index.phil_file=%s" %
                             os.path.abspath(params.dials.index.phil_file))
        if params.dials.refine.phil_file is not None:
            PhilIndex.update("dials.refine.phil_file=%s" %
                             os.path.abspath(params.dials.refine.phil_file))
        if params.dials.integrate.phil_file is not None:
            PhilIndex.update("dials.integrate.phil_file=%s" %
                             os.path.abspath(params.dials.integrate.phil_file))
        if params.xds.index.xparm is not None:
            Flags.set_xparm(params.xds.index.xparm)
        if params.xds.index.xparm_ub is not None:
            Flags.set_xparm_ub(params.xds.index.xparm_ub)

        if params.xia2.settings.scale.freer_file is not None:
            freer_file = os.path.abspath(params.xia2.settings.scale.freer_file)
            if not os.path.exists(freer_file):
                raise RuntimeError("%s does not exist" % freer_file)
            from xia2.Modules.FindFreeFlag import FindFreeFlag

            column = FindFreeFlag(freer_file)
            Debug.write("FreeR_flag column in %s found: %s" %
                        (freer_file, column))
            PhilIndex.update("xia2.settings.scale.freer_file=%s" % freer_file)

        if params.xia2.settings.scale.reference_reflection_file is not None:
            reference_reflection_file = os.path.abspath(
                params.xia2.settings.scale.reference_reflection_file)
            if not os.path.exists(reference_reflection_file):
                raise RuntimeError("%s does not exist" %
                                   reference_reflection_file)
            PhilIndex.update(
                "xia2.settings.scale.reference_reflection_file=%s" %
                reference_reflection_file)

        params = PhilIndex.get_python_object()

        datasets = unroll_datasets(PhilIndex.params.xia2.settings.input.image)

        for dataset in datasets:

            start_end = None

            # here we only care about ':' which are later than C:\
            if ":" in dataset[3:]:
                tokens = dataset.split(":")
                # cope with windows drives i.e. C:\data\blah\thing_0001.cbf:1:100
                if len(tokens[0]) == 1:
                    tokens = ["%s:%s" % (tokens[0], tokens[1])] + tokens[2:]
                if len(tokens) != 3:
                    raise RuntimeError("/path/to/image_0001.cbf:start:end")

                dataset = tokens[0]
                start_end = int(tokens[1]), int(tokens[2])

            from xia2.Applications.xia2setup import is_hd5f_name

            if os.path.exists(os.path.abspath(dataset)):
                dataset = os.path.abspath(dataset)
            else:
                directories = [os.getcwd()] + self._argv[1:]
                found = False
                for d in directories:
                    if os.path.exists(os.path.join(d, dataset)):
                        dataset = os.path.join(d, dataset)
                        found = True
                        break
                if not found:
                    raise Sorry("Could not find %s in %s" %
                                (dataset, " ".join(directories)))

            if is_hd5f_name(dataset):
                self._hdf5_master_files.append(dataset)
                if start_end:
                    Debug.write("Image range: %d %d" % start_end)
                    if dataset not in self._default_start_end:
                        self._default_start_end[dataset] = []
                    self._default_start_end[dataset].append(start_end)
                else:
                    Debug.write("No image range specified")

            else:
                template, directory = image2template_directory(
                    os.path.abspath(dataset))

                self._default_template.append(os.path.join(
                    directory, template))
                self._default_directory.append(directory)

                Debug.write("Interpreted from image %s:" % dataset)
                Debug.write("Template %s" % template)
                Debug.write("Directory %s" % directory)

                if start_end:
                    Debug.write("Image range: %d %d" % start_end)
                    key = os.path.join(directory, template)
                    if key not in self._default_start_end:
                        self._default_start_end[key] = []
                    self._default_start_end[key].append(start_end)
                else:
                    Debug.write("No image range specified")

        # finally, check that all arguments were read and raise an exception
        # if any of them were nonsense.

        with open("xia2-working.phil", "wb") as f:
            f.write(PhilIndex.working_phil.as_str())
            f.write(
                os.linesep
            )  # temporarily required for https://github.com/dials/dials/issues/522
        with open("xia2-diff.phil", "wb") as f:
            f.write(PhilIndex.get_diff().as_str())
            f.write(
                os.linesep
            )  # temporarily required for https://github.com/dials/dials/issues/522

        Debug.write("\nDifference PHIL:")
        Debug.write(PhilIndex.get_diff().as_str(), strip=False)

        Debug.write("Working PHIL:")
        Debug.write(PhilIndex.working_phil.as_str(), strip=False)

        nonsense = "Unknown command-line options:"
        was_nonsense = False

        for j, argv in enumerate(self._argv):
            if j == 0:
                continue
            if argv[0] != "-" and "=" not in argv:
                continue
            if j not in self._understood:
                nonsense += " %s" % argv
                was_nonsense = True

        if was_nonsense:
            raise RuntimeError(nonsense)
Beispiel #42
0
def compute_nmol_from_volume(volume, mass, resolution):

    file = open(nmolparams, "r")

    while True:
        line = file.readline()
        if not line[0] == "#":
            break

    resolutions = map(float, line.split(","))[:13]
    P0_list = map(float, file.readline().split(","))[:13]
    Vm_bar_list = map(float, file.readline().split(","))[:13]
    w_list = map(float, file.readline().split(","))[:13]
    A_list = map(float, file.readline().split(","))[:13]
    s_list = map(float, file.readline().split(","))[:13]
    file.close()

    if resolution > resolutions[-1]:
        Chatter.write(
            "Resolution lower than %s -> computing for %f"
            % (resolutions[-1], resolution)
        )
        resolution = resolutions[-1]
    if resolution < resolutions[0]:
        Chatter.write(
            "Resolution higher than peak %f -> %f" % (resolution, resolutions[0])
        )
        resolution = resolutions[0]

    start = 0

    for start in range(12):
        if resolution > resolutions[start] and resolution < resolutions[start + 1]:
            break

    # can start at start - interpolate

    diff = (resolution - resolutions[start]) / (
        resolutions[start + 1] - resolutions[start]
    )

    P0 = P0_list[start] + diff * (P0_list[start + 1] - P0_list[start])
    Vm_bar = Vm_bar_list[start] + diff * (Vm_bar_list[start + 1] - Vm_bar_list[start])
    w = w_list[start] + diff * (w_list[start + 1] - w_list[start])
    A = A_list[start] + diff * (A_list[start + 1] - A_list[start])
    s = s_list[start] + diff * (s_list[start + 1] - s_list[start])

    nmols = []
    pdfs = []

    nmol = 1
    Vm = volume / (mass * nmol)
    while True:
        z = (Vm - Vm_bar) / w
        p = P0 + A * math.exp(-math.exp(-z) - z * s + 1)
        nmols.append(nmol)
        pdfs.append(p)
        nmol += 1
        Vm = volume / (mass * nmol)
        if Vm < 1.0:
            break

    return nmols, pdfs
Beispiel #43
0
    def integrate(self):
        '''Actually perform integration until we think we are done...'''

        while not self.get_integrater_finish_done():
            while not self.get_integrater_done():
                while not self.get_integrater_prepare_done():

                    Debug.write('Preparing to do some integration...')
                    self.set_integrater_prepare_done(True)

                    # if this raises an exception, perhaps the autoindexing
                    # solution has too high symmetry. if this the case, then
                    # perform a self._intgr_indexer.eliminate() - this should
                    # reset the indexing system

                    try:
                        self._integrate_prepare()

                    except BadLatticeError as e:

                        Journal.banner('eliminated this lattice', size=80)

                        Chatter.write('Rejecting bad lattice %s' % str(e))
                        self._intgr_refiner.eliminate()
                        self._integrater_reset()

                # FIXME x1698 - may be the case that _integrate() returns the
                # raw intensities, _integrate_finish() returns intensities
                # which may have been adjusted or corrected. See #1698 below.

                Debug.write('Doing some integration...')

                self.set_integrater_done(True)

                template = self.get_integrater_sweep().get_template()

                if self._intgr_sweep_name:
                    if PhilIndex.params.xia2.settings.show_template:
                        Chatter.banner('Integrating %s (%s)' % \
                                       (self._intgr_sweep_name, template))
                    else:
                        Chatter.banner('Integrating %s' % \
                                       (self._intgr_sweep_name))
                try:

                    #1698
                    self._intgr_hklout_raw = self._integrate()

                except BadLatticeError as e:
                    Chatter.write('Rejecting bad lattice %s' % str(e))

                    Journal.banner('eliminated this lattice', size=80)

                    self._intgr_refiner.eliminate()
                    self._integrater_reset()

            self.set_integrater_finish_done(True)

            try:
                # allow for the fact that postrefinement may be used
                # to reject the lattice...

                self._intgr_hklout = self._integrate_finish()

            except BadLatticeError as e:
                Chatter.write('Bad Lattice Error: %s' % str(e))
                self._intgr_refiner.eliminate()
                self._integrater_reset()

        return self._intgr_hklout
Beispiel #44
0
      raise RuntimeError, 'processing abandoned'

    self._intgr_batches_out = integrater.get_batches_out()

    mosaics = integrater.get_mosaic_spreads()
    if mosaics and len(mosaics) > 0:
      self.set_integrater_mosaic_min_mean_max(
          min(mosaics), sum(mosaics) / len(mosaics), max(mosaics))
    else:
      m = indxr.get_indexer_mosaic()
      self.set_integrater_mosaic_min_mean_max(m, m, m)

    # write the report for each image as .*-#$ to Chatter -
    # detailed report will be written automagically to science...

    Chatter.write(self.show_per_image_statistics())

    Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
                  self.get_integrater_mosaic_min_mean_max())

    # gather the statistics from the postrefinement
    postref_result = integrater.get_postref_result()

    # now write this to a postrefinement log
    postref_log = os.path.join(self.get_working_directory(),
                               'postrefinement.log')

    fout = open(postref_log, 'w')

    fout.write('$TABLE: Postrefinement for %s:\n' % \
               self._intgr_sweep_name)
Beispiel #45
0
  def get_output(self):
    result = 'Wavelength name: %s\n' % self._name
    result += 'Wavelength %7.5f\n' % self._wavelength
    if self._f_pr != 0.0 and self._f_prpr != 0.0:
      result += 'F\', F\'\' = (%5.2f, %5.2f)\n' % (self._f_pr,
                                                   self._f_prpr)

    result += 'Sweeps:\n'

    from xia2.Driver.DriverFactory import DriverFactory

    def run_one_sweep(args):

      from xia2.Handlers.Streams import Debug

      assert len(args) == 3
      s, failover, job_type = args

      if job_type:
        DriverFactory.set_driver_type(job_type)

      Chatter.cache()
      Debug.cache()

      try:
        s.get_integrater_intensities()
      except Exception as e:
        if failover:
          Chatter.write('Processing sweep %s failed: %s' % \
                        (s.get_name(), str(e)))
          s = None
        else:
          raise
      finally:
        Chatter.uncache()
        Debug.uncache()
        return s

    remove = []

    from xia2.Handlers.Phil import PhilIndex
    params = PhilIndex.get_python_object()
    failover = params.xia2.settings.failover

    for s in self._sweeps:

      # would be nice to put this somewhere else in the hierarchy - not
      # sure how to do that though (should be handled in Interfaces?)

      try:
        result += '%s\n' % s.get_output()
      except Exception as e:
        if failover:
          Chatter.write('Processing sweep %s failed: %s' % \
                        (s.get_name(), str(e)))
          remove.append(s)
        else:
          raise

    for s in remove:
      self._sweeps.remove(s)

    return result[:-1]
Beispiel #46
0
  def _mosflm_integrate(self):
    '''Perform the actual integration, based on the results of the
    cell refinement or indexing (they have the equivalent form.)'''

    refinr = self.get_integrater_refiner()

    if not refinr.get_refiner_payload('mosflm_orientation_matrix'):
      raise RuntimeError, 'unexpected situation in indexing'

    lattice = refinr.get_refiner_lattice()
    spacegroup_number = lattice_to_spacegroup(lattice)
    mosaic = refinr.get_refiner_payload('mosaic')
    beam = refinr.get_refiner_payload('beam')
    distance = refinr.get_refiner_payload('distance')
    matrix = refinr.get_refiner_payload('mosflm_orientation_matrix')

    integration_params = refinr.get_refiner_payload(
      'mosflm_integration_parameters')

    if integration_params:
      if 'separation' in integration_params:
        self.set_integrater_parameter(
          'mosflm', 'separation',
          '%s %s' % tuple(integration_params['separation']))
      if 'raster' in integration_params:
        self.set_integrater_parameter(
          'mosflm', 'raster',
          '%d %d %d %d %d' % tuple(integration_params['raster']))

    refinr.set_refiner_payload('mosflm_integration_parameters', None)

    f = open(os.path.join(self.get_working_directory(),
                          'xiaintegrate.mat'), 'w')
    for m in matrix:
      f.write(m)
    f.close()

    # then start the integration
    integrater = MosflmIntegrate()
    integrater.set_working_directory(self.get_working_directory())
    auto_logfiler(integrater)

    integrater.set_refine_profiles(self._mosflm_refine_profiles)

    pname, xname, dname = self.get_integrater_project_info()

    if pname is not None and xname is not None and dname is not None:
      Debug.write('Harvesting: %s/%s/%s' % (pname, xname, dname))
      harvest_dir = self.get_working_directory()
      # harvest file name will be %s.mosflm_run_start_end % dname
      temp_dname = '%s_%s' % \
                   (dname, self.get_integrater_sweep_name())
      integrater.set_pname_xname_dname(pname, xname, temp_dname)

    integrater.set_template(os.path.basename(self.get_template()))
    integrater.set_directory(self.get_directory())

    # check for ice - and if so, exclude (ranges taken from
    # XDS documentation)
    if self.get_integrater_ice() != 0:
      Debug.write('Excluding ice rings')
      integrater.set_exclude_ice(True)

    # exclude specified resolution ranges
    if len(self.get_integrater_excluded_regions()) != 0:
      regions = self.get_integrater_excluded_regions()
      Debug.write('Excluding regions: %s' % `regions`)
      integrater.set_exclude_regions(regions)

    mask = standard_mask(self.get_detector())
    for m in mask:
      integrater.add_instruction(m)

    integrater.set_input_mat_file('xiaintegrate.mat')

    integrater.set_beam_centre(beam)
    integrater.set_distance(distance)
    integrater.set_space_group_number(spacegroup_number)
    integrater.set_mosaic(mosaic)

    if self.get_wavelength_prov() == 'user':
      integrater.set_wavelength(self.get_wavelength())

    parameters = self.get_integrater_parameters('mosflm')
    integrater.update_parameters(parameters)

    if self._mosflm_gain:
      integrater.set_gain(self._mosflm_gain)

    # check for resolution limits
    if self._intgr_reso_high > 0.0:
      integrater.set_d_min(self._intgr_reso_high)
    if self._intgr_reso_low:
      integrater.set_d_max(self._intgr_reso_low)

    if PhilIndex.params.general.backstop_mask:
      from xia2.Toolkit.BackstopMask import BackstopMask
      mask = BackstopMask(PhilIndex.params.general.backstop_mask)
      mask = mask.calculate_mask_mosflm(self.get_header())
      integrater.set_mask(mask)

    detector = self.get_detector()
    detector_width, detector_height = detector[0].get_image_size_mm()

    lim_x = 0.5 * detector_width
    lim_y = 0.5 * detector_height

    Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
    integrater.set_limits(lim_x, lim_y)

    integrater.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
    offset = self.get_frame_offset()

    integrater.set_image_range(
      (self._intgr_wedge[0] - offset, self._intgr_wedge[1] - offset))

    try:
      integrater.run()
    except RuntimeError, e:
      if 'integration failed: reason unknown' in str(e):
        Chatter.write('Mosflm has failed in integration')
        message = 'The input was:\n\n'
        for input in integrater.get_all_input():
          message += '  %s' % input
        Chatter.write(message)
      raise
Beispiel #47
0
      if bl_info:
        from xia2.Handlers.CIF import CIF, mmCIF
        cifblock, mmcifblock = bl_info.CIF_block(), bl_info.mmCIF_block()
        if cifblock:
          CIF.set_block(bl_info.get_block_name(), cifblock)
        if mmcifblock:
          mmCIF.set_block(bl_info.get_block_name(), mmcifblock)

if __name__ == '__main__':
  directory = os.path.join('z:', 'data', '12287')

  image = '12287_1_E1_001.img'

  xs = XSweep('DEMO', None, directory, image)

  xs_descr = str(xs)

  Chatter.write('.')
  for record in xs_descr.split('\n'):
    Chatter.write(record.strip())

  Chatter.write('.')

  Chatter.write('Refined beam is: %6.2f %6.2f' % xs.get_indexer_beam_centre())
  Chatter.write('Distance:        %6.2f' % xs.get_indexer_distance())
  Chatter.write('Cell: %6.2f %6.2f %6.2f %6.2f %6.2f %6.2f' % \
                xs.get_indexer_cell())
  Chatter.write('Lattice: %s' % xs.get_indexer_lattice())
  Chatter.write('Mosaic: %6.2f' % xs.get_indexer_mosaic())
  Chatter.write('Hklout: %s' % xs.get_integrater_intensities())
Beispiel #48
0
      raise RuntimeError, 'no resolution info'

    for epoch in epochs:

      si = self._sweep_handler.get_sweep_information(epoch)
      pname, xname, dname = si.get_project_info()
      sname = si.get_sweep_name()
      intgr = si.get_integrater()
      start, end = si.get_batch_range()

      if (dname, sname) in user_resolution_limits:
        resolution = user_resolution_limits[(dname, sname)]
        self._scalr_resolution_limits[(dname, sname)] = resolution
        if resolution < highest_resolution:
          highest_resolution = resolution
        Chatter.write('Resolution limit for %s: %5.2f' % \
                      (dname, resolution))
        continue

      hklin = sc.get_unmerged_reflection_file()
      resolution = self._estimate_resolution_limit(
        hklin, batch_range=(start, end))
      Debug.write('Resolution for sweep %s: %.2f' % \
                  (sname, resolution))

      if not (dname, sname) in self._scalr_resolution_limits:
        self._scalr_resolution_limits[(dname, sname)] = resolution
        self.set_scaler_done(False)

      if resolution < highest_resolution:
        highest_resolution = resolution
Beispiel #49
0
  def index(self):

    if not self.get_indexer_finish_done():
      f = inspect.currentframe().f_back.f_back
      m = f.f_code.co_filename
      l = f.f_lineno

      Debug.write('Index in %s called from %s %d' %
                  (self.__class__.__name__, m, l))

    while not self.get_indexer_finish_done():
      while not self.get_indexer_done():
        while not self.get_indexer_prepare_done():

          # --------------
          # call prepare()
          # --------------

          self.set_indexer_prepare_done(True)
          self._index_prepare()

        # --------------------------------------------
        # then do the proper indexing - using the best
        # solution already stored if available (c/f
        # eliminate above)
        # --------------------------------------------

        self.set_indexer_done(True)

        if self.get_indexer_sweeps():
          xsweeps = self.get_indexer_sweeps()
          if len(xsweeps) > 1:
            sweep_names = ', '.join(x.get_name() for x in xsweeps[:-1])
            sweep_names += ' & ' + xsweeps[-1].get_name()
          else:
            sweep_names = xsweeps[0].get_name()

          if PhilIndex.params.xia2.settings.show_template:
            template = self.get_indexer_sweep().get_template()
            Chatter.banner(
              'Autoindexing %s (%s)' %(sweep_names, template))
          else:
            Chatter.banner('Autoindexing %s' %sweep_names)


        if not self._indxr_helper:

          result = self._index()

          if not self._indxr_done:
            Debug.write(
              'Looks like indexing failed - try again!')
            continue

          solutions = { }
          for k in self._indxr_other_lattice_cell.keys():
            solutions[k] = self._indxr_other_lattice_cell[k][
              'cell']

          # create a helper for the indexer to manage solutions
          self._indxr_helper = _IndexerHelper(solutions)

          solution = self._indxr_helper.get()

          # compare these against the final solution, if different
          # reject solution and return - correct solution will
          # be used next cycle

          if self._indxr_lattice != solution[0] and \
             not self._indxr_input_cell and \
             not PhilIndex.params.xia2.settings.integrate_p1:
            Chatter.write('Rerunning indexing lattice %s to %s' %
                          (self._indxr_lattice, solution[0]))
            Debug.write(
              'Rerunning indexing with target lattice %s' % \
              solution[0])
            self.set_indexer_done(False)

        else:
          # rerun autoindexing with the best known current solution

          solution = self._indxr_helper.get()
          self._indxr_input_lattice = solution[0]
          self._indxr_input_cell = solution[1]
          result = self._index()

      # next finish up...

      self.set_indexer_finish_done(True)
      self._index_finish()

      if self._indxr_print:
        Chatter.write(self.show_indexer_solutions())

      # FIXED 23/OCT/06 at this stage I need to look at the list of
      # reasonable solutions and try to figure out if the indexing
      # program has picked the highest - if not, then constrain the
      # unit cell (need to implement this somewhere, sure it's
      # around!) then rerun the autoindexing (perhaps?) with this
      # new target - this means that we are always working from the
      # top downwards with these things. Once we decide somewhere
      # else (e.g. in cell refinement) that this cell isn't good
      # then we can eliminate it from the list, select the next
      # lower symmetry solution and continue. This solution is a
      # general one, so may be implemented in the general indexer
      # interface rather than in specific code...

    return
Beispiel #50
0
  def _scale_prepare(self):
    '''Perform all of the preparation required to deliver the scaled
    data. This should sort together the reflection files, ensure that
    they are correctly indexed (via pointless) and generally tidy
    things up.'''

    # acknowledge all of the programs we are about to use...

    Citations.cite('pointless')
    Citations.cite('aimless')
    Citations.cite('ccp4')

    # ---------- GATHER ----------

    self._sweep_handler = SweepInformationHandler(self._scalr_integraters)

    Journal.block(
        'gathering', self.get_scaler_xcrystal().get_name(), 'CCP4',
        {'working directory':self.get_working_directory()})

    for epoch in self._sweep_handler.get_epochs():
      si = self._sweep_handler.get_sweep_information(epoch)
      pname, xname, dname = si.get_project_info()
      sname = si.get_sweep_name()

      exclude_sweep = False

      for sweep in PhilIndex.params.xia2.settings.sweep:
        if sweep.id == sname and sweep.exclude:
          exclude_sweep = True
          break

      if exclude_sweep:
        self._sweep_handler.remove_epoch(epoch)
        Debug.write('Excluding sweep %s' %sname)
      else:
        Journal.entry({'adding data from':'%s/%s/%s' % \
                       (xname, dname, sname)})

    # gather data for all images which belonged to the parent
    # crystal - allowing for the fact that things could go wrong
    # e.g. epoch information not available, exposure times not in
    # headers etc...

    for e in self._sweep_handler.get_epochs():
      si = self._sweep_handler.get_sweep_information(e)
      assert is_mtz_file(si.get_reflections())

    p, x = self._sweep_handler.get_project_info()
    self._scalr_pname = p
    self._scalr_xname = x

    # verify that the lattices are consistent, calling eliminate if
    # they are not N.B. there could be corner cases here

    need_to_return = False

    multi_sweep_indexing = \
      PhilIndex.params.xia2.settings.developmental.multi_sweep_indexing


    if len(self._sweep_handler.get_epochs()) > 1:

      if multi_sweep_indexing and not self._scalr_input_pointgroup:
        pointless_hklins = []

        max_batches = 0
        for epoch in self._sweep_handler.get_epochs():
          si = self._sweep_handler.get_sweep_information(epoch)
          hklin = si.get_reflections()

          md = self._factory.Mtzdump()
          md.set_hklin(hklin)
          md.dump()

          batches = md.get_batches()
          if 1 + max(batches) - min(batches) > max_batches:
            max_batches = max(batches) - min(batches) + 1

          datasets = md.get_datasets()

          Debug.write('In reflection file %s found:' % hklin)
          for d in datasets:
            Debug.write('... %s' % d)

          dataset_info = md.get_dataset_info(datasets[0])

        from xia2.lib.bits import nifty_power_of_ten
        Debug.write('Biggest sweep has %d batches' % max_batches)
        max_batches = nifty_power_of_ten(max_batches)

        counter = 0

        for epoch in self._sweep_handler.get_epochs():
          si = self._sweep_handler.get_sweep_information(epoch)
          hklin = si.get_reflections()
          integrater = si.get_integrater()
          refiner = integrater.get_integrater_refiner()

          hklin = self._prepare_pointless_hklin(
            hklin, si.get_integrater().get_phi_width())

          rb = self._factory.Rebatch()

          hklout = os.path.join(self.get_working_directory(),
                                '%s_%s_%s_%s_prepointless.mtz' % \
                                (pname, xname, dname, si.get_sweep_name()))

          # we will want to delete this one exit
          FileHandler.record_temporary_file(hklout)

          first_batch = min(si.get_batches())
          si.set_batch_offset(counter * max_batches - first_batch + 1)

          rb.set_hklin(hklin)
          rb.set_first_batch(counter * max_batches + 1)
          rb.set_project_info(pname, xname, dname)
          rb.set_hklout(hklout)

          new_batches = rb.rebatch()

          pointless_hklins.append(hklout)

          # update the counter & recycle
          counter += 1

        s = self._factory.Sortmtz()

        pointless_hklin = os.path.join(self.get_working_directory(),
                              '%s_%s_prepointless_sorted.mtz' % \
                              (self._scalr_pname, self._scalr_xname))

        s.set_hklout(pointless_hklin)

        for hklin in pointless_hklins:
          s.add_hklin(hklin)

        s.sort()

        pointgroup, reindex_op, ntr, pt = \
                    self._pointless_indexer_jiffy(
            pointless_hklin, refiner)

        Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))

        lattices = [Syminfo.get_lattice(pointgroup)]

        for epoch in self._sweep_handler.get_epochs():
          si = self._sweep_handler.get_sweep_information(epoch)
          intgr = si.get_integrater()
          hklin = si.get_reflections()
          refiner = intgr.get_integrater_refiner()

          if ntr:
            intgr.integrater_reset_reindex_operator()
            need_to_return = True

      else:
        lattices = []

        for epoch in self._sweep_handler.get_epochs():

          si = self._sweep_handler.get_sweep_information(epoch)
          intgr = si.get_integrater()
          hklin = si.get_reflections()
          refiner = intgr.get_integrater_refiner()

          if self._scalr_input_pointgroup:
            pointgroup = self._scalr_input_pointgroup
            reindex_op = 'h,k,l'
            ntr = False

          else:
            pointless_hklin = self._prepare_pointless_hklin(
              hklin, si.get_integrater().get_phi_width())

            pointgroup, reindex_op, ntr, pt = \
                        self._pointless_indexer_jiffy(
                pointless_hklin, refiner)

            Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))

          lattice = Syminfo.get_lattice(pointgroup)

          if not lattice in lattices:
            lattices.append(lattice)

          if ntr:

            intgr.integrater_reset_reindex_operator()
            need_to_return = True

      if len(lattices) > 1:

        # why not using pointless indexer jiffy??!

        correct_lattice = sort_lattices(lattices)[0]

        Chatter.write('Correct lattice asserted to be %s' % \
                      correct_lattice)

        # transfer this information back to the indexers
        for epoch in self._sweep_handler.get_epochs():

          si = self._sweep_handler.get_sweep_information(epoch)
          refiner = si.get_integrater().get_integrater_refiner()
          sname = si.get_sweep_name()

          state = refiner.set_refiner_asserted_lattice(
              correct_lattice)

          if state == refiner.LATTICE_CORRECT:
            Chatter.write('Lattice %s ok for sweep %s' % \
                          (correct_lattice, sname))
          elif state == refiner.LATTICE_IMPOSSIBLE:
            raise RuntimeError, 'Lattice %s impossible for %s' \
                  % (correct_lattice, sname)
          elif state == refiner.LATTICE_POSSIBLE:
            Chatter.write('Lattice %s assigned for sweep %s' % \
                          (correct_lattice, sname))
            need_to_return = True

    # if one or more of them was not in the lowest lattice,
    # need to return here to allow reprocessing

    if need_to_return:
      self.set_scaler_done(False)
      self.set_scaler_prepare_done(False)
      return

    # ---------- REINDEX ALL DATA TO CORRECT POINTGROUP ----------

    # all should share the same pointgroup, unless twinned... in which
    # case force them to be...

    pointgroups = { }
    reindex_ops = { }
    probably_twinned = False

    need_to_return = False

    multi_sweep_indexing = \
      PhilIndex.params.xia2.settings.developmental.multi_sweep_indexing

    if multi_sweep_indexing and not self._scalr_input_pointgroup:
      pointless_hklins = []

      max_batches = 0
      for epoch in self._sweep_handler.get_epochs():
        si = self._sweep_handler.get_sweep_information(epoch)
        hklin = si.get_reflections()

        md = self._factory.Mtzdump()
        md.set_hklin(hklin)
        md.dump()

        batches = md.get_batches()
        if 1 + max(batches) - min(batches) > max_batches:
          max_batches = max(batches) - min(batches) + 1

        datasets = md.get_datasets()

        Debug.write('In reflection file %s found:' % hklin)
        for d in datasets:
          Debug.write('... %s' % d)

        dataset_info = md.get_dataset_info(datasets[0])

      from xia2.lib.bits import nifty_power_of_ten
      Debug.write('Biggest sweep has %d batches' % max_batches)
      max_batches = nifty_power_of_ten(max_batches)

      counter = 0

      for epoch in self._sweep_handler.get_epochs():
        si = self._sweep_handler.get_sweep_information(epoch)
        hklin = si.get_reflections()
        integrater = si.get_integrater()
        refiner = integrater.get_integrater_refiner()

        hklin = self._prepare_pointless_hklin(
            hklin, si.get_integrater().get_phi_width())

        rb = self._factory.Rebatch()

        hklout = os.path.join(self.get_working_directory(),
                              '%s_%s_%s_%s_prepointless.mtz' % \
                              (pname, xname, dname, si.get_sweep_name()))

        # we will want to delete this one exit
        FileHandler.record_temporary_file(hklout)

        first_batch = min(si.get_batches())
        si.set_batch_offset(counter * max_batches - first_batch + 1)

        rb.set_hklin(hklin)
        rb.set_first_batch(counter * max_batches + 1)
        rb.set_project_info(pname, xname, dname)
        rb.set_hklout(hklout)

        new_batches = rb.rebatch()

        pointless_hklins.append(hklout)

        # update the counter & recycle
        counter += 1

      s = self._factory.Sortmtz()

      pointless_hklin = os.path.join(self.get_working_directory(),
                            '%s_%s_prepointless_sorted.mtz' % \
                            (self._scalr_pname, self._scalr_xname))

      s.set_hklout(pointless_hklin)

      for hklin in pointless_hklins:
        s.add_hklin(hklin)

      s.sort()

      pointgroup, reindex_op, ntr, pt = \
                  self._pointless_indexer_jiffy(
          pointless_hklin, refiner)

      for epoch in self._sweep_handler.get_epochs():
        pointgroups[epoch] = pointgroup
        reindex_ops[epoch] = reindex_op

    else:
      for epoch in self._sweep_handler.get_epochs():
        si = self._sweep_handler.get_sweep_information(epoch)

        hklin = si.get_reflections()
        #hklout = os.path.join(
            #self.get_working_directory(),
            #os.path.split(hklin)[-1].replace('.mtz', '_rdx.mtz'))

        #FileHandler.record_temporary_file(hklout)

        integrater = si.get_integrater()
        refiner = integrater.get_integrater_refiner()

        if self._scalr_input_pointgroup:
          Debug.write('Using input pointgroup: %s' % \
                      self._scalr_input_pointgroup)
          pointgroup = self._scalr_input_pointgroup
          reindex_op = 'h,k,l'
          pt = False

        else:

          pointless_hklin = self._prepare_pointless_hklin(
              hklin, si.get_integrater().get_phi_width())

          pointgroup, reindex_op, ntr, pt = \
                      self._pointless_indexer_jiffy(
              pointless_hklin, refiner)

          Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))

          if ntr:

            integrater.integrater_reset_reindex_operator()
            need_to_return = True

        if pt and not probably_twinned:
          probably_twinned = True

        Debug.write('Pointgroup: %s (%s)' % (pointgroup, reindex_op))

        pointgroups[epoch] = pointgroup
        reindex_ops[epoch] = reindex_op

    overall_pointgroup = None

    pointgroup_set = set([pointgroups[e] for e in pointgroups])

    if len(pointgroup_set) > 1 and \
       not probably_twinned:
      raise RuntimeError, 'non uniform pointgroups'

    if len(pointgroup_set) > 1:
      Debug.write('Probably twinned, pointgroups: %s' % \
                  ' '.join([p.replace(' ', '') for p in \
                            list(pointgroup_set)]))
      numbers = [Syminfo.spacegroup_name_to_number(s) for s in \
                 pointgroup_set]
      overall_pointgroup = Syminfo.spacegroup_number_to_name(
          min(numbers))
      self._scalr_input_pointgroup = overall_pointgroup

      Chatter.write('Twinning detected, assume pointgroup %s' % \
                    overall_pointgroup)

      need_to_return = True

    else:
      overall_pointgroup = pointgroup_set.pop()

    for epoch in self._sweep_handler.get_epochs():
      si = self._sweep_handler.get_sweep_information(epoch)

      integrater = si.get_integrater()

      integrater.set_integrater_spacegroup_number(
          Syminfo.spacegroup_name_to_number(overall_pointgroup))
      integrater.set_integrater_reindex_operator(
          reindex_ops[epoch], reason='setting point group')
      # This will give us the reflections in the correct point group
      si.set_reflections(integrater.get_integrater_intensities())

    if need_to_return:
      self.set_scaler_done(False)
      self.set_scaler_prepare_done(False)
      return

    # in here now optinally work through the data files which should be
    # indexed with a consistent point group, and transform the orientation
    # matrices by the lattice symmetry operations (if possible) to get a
    # consistent definition of U matrix modulo fixed rotations

    if PhilIndex.params.xia2.settings.unify_setting:

      from scitbx.matrix import sqr
      reference_U = None
      i3 = sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))

      for epoch in self._sweep_handler.get_epochs():
        si = self._sweep_handler.get_sweep_information(epoch)
        intgr = si.get_integrater()
        fixed = sqr(intgr.get_goniometer().get_fixed_rotation())
        u, b, s = get_umat_bmat_lattice_symmetry_from_mtz(si.get_reflections())
        U = fixed.inverse() * sqr(u).transpose()
        B = sqr(b)

        if reference_U is None:
          reference_U = U
          continue

        results = []
        for op in s.all_ops():
          R = B * sqr(op.r().as_double()).transpose() * B.inverse()
          nearly_i3 = (U * R).inverse() * reference_U
          score = sum([abs(_n - _i) for (_n, _i) in zip(nearly_i3, i3)])
          results.append((score, op.r().as_hkl(), op))

        results.sort()
        best = results[0]
        Debug.write('Best reindex: %s %.3f' % (best[1], best[0]))
        intgr.set_integrater_reindex_operator(best[2].r().inverse().as_hkl(),
                                              reason='unifying [U] setting')
        si.set_reflections(intgr.get_integrater_intensities())

        # recalculate to verify
        u, b, s = get_umat_bmat_lattice_symmetry_from_mtz(si.get_reflections())
        U = fixed.inverse() * sqr(u).transpose()
        Debug.write('New reindex: %s' % (U.inverse() * reference_U))

        # FIXME I should probably raise an exception at this stage if this
        # is not about I3...

    if self.get_scaler_reference_reflection_file():
      self._reference = self.get_scaler_reference_reflection_file()
      Debug.write('Using HKLREF %s' % self._reference)

    elif Flags.get_reference_reflection_file():
      self._reference = Flags.get_reference_reflection_file()
      Debug.write('Using HKLREF %s' % self._reference)

    params = PhilIndex.params
    use_brehm_diederichs = params.xia2.settings.use_brehm_diederichs
    if len(self._sweep_handler.get_epochs()) > 1 and use_brehm_diederichs:

      brehm_diederichs_files_in = []
      for epoch in self._sweep_handler.get_epochs():

        si = self._sweep_handler.get_sweep_information(epoch)
        hklin = si.get_reflections()
        brehm_diederichs_files_in.append(hklin)

      # now run cctbx.brehm_diederichs to figure out the indexing hand for
      # each sweep
      from xia2.Wrappers.Cctbx.BrehmDiederichs import BrehmDiederichs
      from xia2.lib.bits import auto_logfiler
      brehm_diederichs = BrehmDiederichs()
      brehm_diederichs.set_working_directory(self.get_working_directory())
      auto_logfiler(brehm_diederichs)
      brehm_diederichs.set_input_filenames(brehm_diederichs_files_in)
      # 1 or 3? 1 seems to work better?
      brehm_diederichs.set_asymmetric(1)
      brehm_diederichs.run()
      reindexing_dict = brehm_diederichs.get_reindexing_dict()

      for epoch in self._sweep_handler.get_epochs():

        si = self._sweep_handler.get_sweep_information(epoch)
        intgr = si.get_integrater()
        hklin = si.get_reflections()

        reindex_op = reindexing_dict.get(os.path.abspath(hklin))
        assert reindex_op is not None

        if 1 or reindex_op != 'h,k,l':
          # apply the reindexing operator
          intgr.set_integrater_reindex_operator(
            reindex_op, reason='match reference')
          si.set_reflections(intgr.get_integrater_intensities())

    elif len(self._sweep_handler.get_epochs()) > 1 and \
           not self._reference:

      first = self._sweep_handler.get_epochs()[0]
      si = self._sweep_handler.get_sweep_information(first)
      self._reference = si.get_reflections()

    if self._reference:

      md = self._factory.Mtzdump()
      md.set_hklin(self._reference)
      md.dump()

      if md.get_batches() and False:
        raise RuntimeError, 'reference reflection file %s unmerged' % \
              self._reference

      datasets = md.get_datasets()

      if len(datasets) > 1 and False:
        raise RuntimeError, 'more than one dataset in %s' % \
              self._reference

      # then get the unit cell, lattice etc.

      reference_lattice = Syminfo.get_lattice(md.get_spacegroup())
      reference_cell = md.get_dataset_info(datasets[0])['cell']

      # then compute the pointgroup from this...

      # ---------- REINDEX TO CORRECT (REFERENCE) SETTING ----------

      for epoch in self._sweep_handler.get_epochs():
        pl = self._factory.Pointless()

        si = self._sweep_handler.get_sweep_information(epoch)
        hklin = si.get_reflections()

        pl.set_hklin(self._prepare_pointless_hklin(
            hklin, si.get_integrater().get_phi_width()))

        hklout = os.path.join(
            self.get_working_directory(),
            '%s_rdx2.mtz' % os.path.split(hklin)[-1][:-4])

        # we will want to delete this one exit
        FileHandler.record_temporary_file(hklout)

        # now set the initial reflection set as a reference...

        pl.set_hklref(self._reference)

        # write a pointless log file...
        pl.decide_pointgroup()

        Debug.write('Reindexing analysis of %s' % pl.get_hklin())

        pointgroup = pl.get_pointgroup()
        reindex_op = pl.get_reindex_operator()

        Debug.write('Operator: %s' % reindex_op)

        # apply this...

        integrater = si.get_integrater()

        integrater.set_integrater_reindex_operator(reindex_op,
                                                   reason='match reference')
        integrater.set_integrater_spacegroup_number(
            Syminfo.spacegroup_name_to_number(pointgroup))
        si.set_reflections(integrater.get_integrater_intensities())

        md = self._factory.Mtzdump()
        md.set_hklin(si.get_reflections())
        md.dump()

        datasets = md.get_datasets()

        if len(datasets) > 1:
          raise RuntimeError, 'more than one dataset in %s' % \
                si.get_reflections()

        # then get the unit cell, lattice etc.

        lattice = Syminfo.get_lattice(md.get_spacegroup())
        cell = md.get_dataset_info(datasets[0])['cell']

        if lattice != reference_lattice:
          raise RuntimeError, 'lattices differ in %s and %s' % \
                (self._reference, si.get_reflections())

        for j in range(6):
          if math.fabs((cell[j] - reference_cell[j]) /
                       reference_cell[j]) > 0.1:
            raise RuntimeError, \
                  'unit cell parameters differ in %s and %s' % \
                  (self._reference, si.get_reflections())

    # ---------- SORT TOGETHER DATA ----------

    self._sort_together_data_ccp4()

    self._scalr_resolution_limits = { }

    # store central resolution limit estimates

    batch_ranges = [self._sweep_handler.get_sweep_information(
        epoch).get_batch_range() for epoch in
                    self._sweep_handler.get_epochs()]

    self._resolution_limit_estimates = erzatz_resolution(
        self._prepared_reflections, batch_ranges)


    return
Beispiel #51
0
  def _scale(self):
    '''Actually scale all of the data together.'''

    from xia2.Handlers.Environment import debug_memory_usage
    debug_memory_usage()

    Journal.block(
        'scaling', self.get_scaler_xcrystal().get_name(), 'XSCALE',
        {'scaling model':'default (all)'})

    epochs = self._sweep_information.keys()
    epochs.sort()

    xscale = self.XScale()

    xscale.set_spacegroup_number(self._xds_spacegroup)
    xscale.set_cell(self._scalr_cell)

    Debug.write('Set CELL: %.2f %.2f %.2f %.2f %.2f %.2f' % \
                tuple(self._scalr_cell))
    Debug.write('Set SPACEGROUP_NUMBER: %d' % \
                self._xds_spacegroup)

    Debug.write('Gathering measurements for scaling')

    for epoch in epochs:

      # get the prepared reflections
      reflections = self._sweep_information[epoch][
          'prepared_reflections']

      # and the get wavelength that this belongs to
      dname = self._sweep_information[epoch]['dname']
      sname = self._sweep_information[epoch]['sname']

      # and the resolution range for the reflections
      intgr = self._sweep_information[epoch]['integrater']
      Debug.write('Epoch: %d' % epoch)
      Debug.write('HKL: %s (%s/%s)' % (reflections, dname, sname))

      resolution_low = intgr.get_integrater_low_resolution()
      resolution_high, _ = self._scalr_resolution_limits.get((dname, sname), (0.0, None))

      resolution = (resolution_high, resolution_low)

      xscale.add_reflection_file(reflections, dname, resolution)

    # set the global properties of the sample
    xscale.set_crystal(self._scalr_xname)
    xscale.set_anomalous(self._scalr_anomalous)

    debug_memory_usage()
    xscale.run()

    scale_factor = xscale.get_scale_factor()

    Debug.write('XSCALE scale factor found to be: %e' % scale_factor)

    # record the log file

    pname = self._scalr_pname
    xname = self._scalr_xname

    FileHandler.record_log_file('%s %s XSCALE' % \
                                (pname, xname),
                                os.path.join(self.get_working_directory(),
                                             'XSCALE.LP'))

    # check for outlier reflections and if a number are found
    # then iterate (that is, rerun XSCALE, rejecting these outliers)

    if not PhilIndex.params.dials.fast_mode and not PhilIndex.params.xds.keep_outliers:
      xscale_remove = xscale.get_remove()
      if xscale_remove:
        current_remove = []
        final_remove = []

        # first ensure that there are no duplicate entries...
        if os.path.exists(os.path.join(
            self.get_working_directory(),
            'REMOVE.HKL')):
          for line in open(os.path.join(
              self.get_working_directory(),
              'REMOVE.HKL'), 'r').readlines():
            h, k, l = map(int, line.split()[:3])
            z = float(line.split()[3])

            if not (h, k, l, z) in current_remove:
              current_remove.append((h, k, l, z))

          for c in xscale_remove:
            if c in current_remove:
              continue
            final_remove.append(c)

          Debug.write(
              '%d alien reflections are already removed' % \
              (len(xscale_remove) - len(final_remove)))

        else:
          # we want to remove all of the new dodgy reflections
          final_remove = xscale_remove

        remove_hkl = open(os.path.join(
            self.get_working_directory(),
            'REMOVE.HKL'), 'w')

        z_min = PhilIndex.params.xds.z_min
        rejected = 0

        # write in the old reflections
        for remove in current_remove:
          z = remove[3]
          if z >= z_min:
            remove_hkl.write('%d %d %d %f\n' % remove)
          else:
            rejected += 1
        Debug.write('Wrote %d old reflections to REMOVE.HKL' % \
                    (len(current_remove) - rejected))
        Debug.write('Rejected %d as z < %f' % \
                    (rejected, z_min))

        # and the new reflections
        rejected = 0
        used = 0
        for remove in final_remove:
          z = remove[3]
          if z >= z_min:
            used += 1
            remove_hkl.write('%d %d %d %f\n' % remove)
          else:
            rejected += 1
        Debug.write('Wrote %d new reflections to REMOVE.HKL' % \
                    (len(final_remove) - rejected))
        Debug.write('Rejected %d as z < %f' % \
                    (rejected, z_min))

        remove_hkl.close()

        # we want to rerun the finishing step so...
        # unless we have added no new reflections
        if used:
          self.set_scaler_done(False)

    if not self.get_scaler_done():
      Chatter.write('Excluding outlier reflections Z > %.2f' %
                    PhilIndex.params.xds.z_min)
      return

    debug_memory_usage()

    # now get the reflection files out and merge them with aimless

    output_files = xscale.get_output_reflection_files()
    wavelength_names = output_files.keys()

    # these are per wavelength - also allow for user defined resolution
    # limits a la bug # 3183. No longer...

    for epoch in self._sweep_information.keys():

      input = self._sweep_information[epoch]

      intgr = input['integrater']

      rkey = input['dname'], input['sname']

      if intgr.get_integrater_user_resolution():
        dmin = intgr.get_integrater_high_resolution()

        if rkey not in self._user_resolution_limits:
          self._scalr_resolution_limits[rkey] = (dmin, None)
          self._user_resolution_limits[rkey] = dmin
        elif dmin < self._user_resolution_limits[rkey]:
          self._scalr_resolution_limits[rkey] = (dmin, None)
          self._user_resolution_limits[rkey] = dmin

    self._scalr_scaled_refl_files = { }

    self._scalr_statistics = { }

    max_batches = 0
    mtz_dict = { }

    project_info = { }
    for epoch in self._sweep_information.keys():
      pname = self._scalr_pname
      xname = self._scalr_xname
      dname = self._sweep_information[epoch]['dname']
      reflections = os.path.split(
          self._sweep_information[epoch]['prepared_reflections'])[-1]
      project_info[reflections] = (pname, xname, dname)

    for epoch in self._sweep_information.keys():
      self._sweep_information[epoch]['scaled_reflections'] = None

    debug_memory_usage()

    for wavelength in wavelength_names:
      hklin = output_files[wavelength]

      xsh = XDSScalerHelper()
      xsh.set_working_directory(self.get_working_directory())

      ref = xsh.split_and_convert_xscale_output(
          hklin, 'SCALED_', project_info, 1.0 / scale_factor)

      for hklout in ref.keys():
        for epoch in self._sweep_information.keys():
          if os.path.split(self._sweep_information[epoch][
              'prepared_reflections'])[-1] == \
              os.path.split(hklout)[-1]:
            if self._sweep_information[epoch][
                'scaled_reflections'] is not None:
              raise RuntimeError, 'duplicate entries'
            self._sweep_information[epoch][
                'scaled_reflections'] = ref[hklout]

      del(xsh)

    debug_memory_usage()

    for epoch in self._sweep_information.keys():
      hklin = self._sweep_information[epoch]['scaled_reflections']
      dname = self._sweep_information[epoch]['dname']
      sname = self._sweep_information[epoch]['sname']

      hkl_copy = os.path.join(self.get_working_directory(),
                              'R_%s' % os.path.split(hklin)[-1])

      if not os.path.exists(hkl_copy):
        shutil.copyfile(hklin, hkl_copy)

      # let's properly listen to the user's resolution limit needs...

      if self._user_resolution_limits.get((dname, sname), False):
        resolution = self._user_resolution_limits[(dname, sname)]

      else:
        if PhilIndex.params.xia2.settings.resolution.keep_all_reflections == True:
          try:
            resolution = intgr.get_detector().get_max_resolution(intgr.get_beam_obj().get_s0())
            Debug.write('keep_all_reflections set, using detector limits')
          except Exception:
            resolution = self._estimate_resolution_limit(hklin)
        else:
          resolution = self._estimate_resolution_limit(hklin)

      Chatter.write('Resolution for sweep %s/%s: %.2f' % \
                    (dname, sname, resolution))

      if (dname, sname) not in self._scalr_resolution_limits:
        self._scalr_resolution_limits[(dname, sname)] = (resolution, None)
        self.set_scaler_done(False)
      else:
        if resolution < self._scalr_resolution_limits[(dname, sname)][0]:
          self._scalr_resolution_limits[(dname, sname)] = (resolution, None)
          self.set_scaler_done(False)

    debug_memory_usage()

    if not self.get_scaler_done():
      Debug.write('Returning as scaling not finished...')
      return

    self._sort_together_data_xds()

    highest_resolution = min(limit for limit, _ in self._scalr_resolution_limits.values())

    self._scalr_highest_resolution = highest_resolution

    Debug.write('Scaler highest resolution set to %5.2f' % \
                highest_resolution)

    if not self.get_scaler_done():
      Debug.write('Returning as scaling not finished...')
      return

    sdadd_full = 0.0
    sdb_full = 0.0

    # ---------- FINAL MERGING ----------

    sc = self._factory.Aimless()

    FileHandler.record_log_file('%s %s aimless' % (self._scalr_pname,
                                                   self._scalr_xname),
                                sc.get_log_file())

    sc.set_resolution(highest_resolution)
    sc.set_hklin(self._prepared_reflections)
    sc.set_new_scales_file('%s_final.scales' % self._scalr_xname)

    if sdadd_full == 0.0 and sdb_full == 0.0:
      pass
    else:
      sc.add_sd_correction('both', 1.0, sdadd_full, sdb_full)

    for epoch in epochs:
      input = self._sweep_information[epoch]
      start, end = (min(input['batches']), max(input['batches']))

      rkey = input['dname'], input['sname']
      run_resolution_limit, _ = self._scalr_resolution_limits[rkey]

      sc.add_run(start, end, exclude = False,
                 resolution = run_resolution_limit,
                 name = input['sname'])

    sc.set_hklout(os.path.join(self.get_working_directory(),
                               '%s_%s_scaled.mtz' % \
                               (self._scalr_pname, self._scalr_xname)))

    if self.get_scaler_anomalous():
      sc.set_anomalous()

    sc.multi_merge()

    FileHandler.record_xml_file('%s %s aimless xml' % (self._scalr_pname,
                                                       self._scalr_xname),
                                sc.get_xmlout())
    data = sc.get_summary()

    loggraph = sc.parse_ccp4_loggraph()

    standard_deviation_info = { }

    for key in loggraph.keys():
      if 'standard deviation v. Intensity' in key:
        dataset = key.split(',')[-1].strip()
        standard_deviation_info[dataset] = transpose_loggraph(
            loggraph[key])

    resolution_info = { }

    for key in loggraph.keys():
      if 'Analysis against resolution' in key:
        dataset = key.split(',')[-1].strip()
        resolution_info[dataset] = transpose_loggraph(
            loggraph[key])

    # and also radiation damage stuff...

    batch_info = { }

    for key in loggraph.keys():
      if 'Analysis against Batch' in key:
        dataset = key.split(',')[-1].strip()
        batch_info[dataset] = transpose_loggraph(
            loggraph[key])


    # finally put all of the results "somewhere useful"

    self._scalr_statistics = data

    self._scalr_scaled_refl_files = copy.deepcopy(
        sc.get_scaled_reflection_files())

    self._scalr_scaled_reflection_files = { }

    # also output the unmerged scalepack format files...

    sc = self._factory.Aimless()
    sc.set_resolution(highest_resolution)
    sc.set_hklin(self._prepared_reflections)
    sc.set_scalepack()

    for epoch in epochs:
      input = self._sweep_information[epoch]
      start, end = (min(input['batches']), max(input['batches']))

      rkey = input['dname'], input['sname']
      run_resolution_limit, _ = self._scalr_resolution_limits[rkey]

      sc.add_run(start, end, exclude = False,
                 resolution = run_resolution_limit,
                 name = input['sname'])

    sc.set_hklout(os.path.join(self.get_working_directory(),
                               '%s_%s_scaled.mtz' % \
                               (self._scalr_pname,
                                self._scalr_xname)))

    if self.get_scaler_anomalous():
      sc.set_anomalous()

    sc.multi_merge()

    self._scalr_scaled_reflection_files['sca_unmerged'] = { }
    self._scalr_scaled_reflection_files['mtz_unmerged'] = { }

    for dataset in sc.get_scaled_reflection_files().keys():
      hklout = sc.get_scaled_reflection_files()[dataset]

      # then mark the scalepack files for copying...

      scalepack = os.path.join(os.path.split(hklout)[0],
                               os.path.split(hklout)[1].replace(
          '_scaled', '_scaled_unmerged').replace('.mtz', '.sca'))
      self._scalr_scaled_reflection_files['sca_unmerged'][
          dataset] = scalepack
      FileHandler.record_data_file(scalepack)
      mtz_unmerged = os.path.splitext(scalepack)[0] + '.mtz'
      self._scalr_scaled_reflection_files['mtz_unmerged'][dataset] = mtz_unmerged
      FileHandler.record_data_file(mtz_unmerged)

    if PhilIndex.params.xia2.settings.merging_statistics.source == 'cctbx':
      for key in self._scalr_scaled_refl_files:
        stats = self._compute_scaler_statistics(
          self._scalr_scaled_reflection_files['mtz_unmerged'][key], wave=key)
        self._scalr_statistics[
          (self._scalr_pname, self._scalr_xname, key)] = stats

    # convert reflection files to .sca format - use mtz2various for this

    self._scalr_scaled_reflection_files['sca'] = { }
    self._scalr_scaled_reflection_files['hkl'] = { }

    for key in self._scalr_scaled_refl_files:

      f = self._scalr_scaled_refl_files[key]
      scaout = '%s.sca' % f[:-4]

      m2v = self._factory.Mtz2various()
      m2v.set_hklin(f)
      m2v.set_hklout(scaout)
      m2v.convert()

      self._scalr_scaled_reflection_files['sca'][key] = scaout
      FileHandler.record_data_file(scaout)

      if PhilIndex.params.xia2.settings.small_molecule == True:
        hklout = '%s.hkl' % f[:-4]

        m2v = self._factory.Mtz2various()
        m2v.set_hklin(f)
        m2v.set_hklout(hklout)
        m2v.convert_shelx()

        self._scalr_scaled_reflection_files['hkl'][key] = hklout
        FileHandler.record_data_file(hklout)
Beispiel #52
0
    def run(self):
      '''Run defpix.'''

      #image_header = self.get_header()

      ## crank through the header dictionary and replace incorrect
      ## information with updated values through the indexer
      ## interface if available...

      ## need to add distance, wavelength - that should be enough...

      #if self.get_distance():
        #image_header['distance'] = self.get_distance()

      #if self.get_wavelength():
        #image_header['wavelength'] = self.get_wavelength()

      #if self.get_two_theta():
        #image_header['two_theta'] = self.get_two_theta()

      header = imageset_to_xds(self.get_imageset())

      xds_inp = open(os.path.join(self.get_working_directory(),
                                  'XDS.INP'), 'w')

      # what are we doing?
      xds_inp.write('JOB=DEFPIX\n')

      for record in header:
        xds_inp.write('%s\n' % record)

      name_template = os.path.join(self.get_directory(),
                                   self.get_template().replace('#', '?'))

      record = 'NAME_TEMPLATE_OF_DATA_FRAMES=%s\n' % \
               name_template

      xds_inp.write(record)

      xds_inp.write('DATA_RANGE=%d %d\n' % self._data_range)

      # include the resolution range, perhaps
      if self._resolution_high > 0.0 or self._resolution_low > 0.0:
        xds_inp.write('INCLUDE_RESOLUTION_RANGE=%.2f %.2f\n' % \
                      (self._resolution_low, self._resolution_high))

      xds_inp.close()


      # copy the input file...
      shutil.copyfile(os.path.join(self.get_working_directory(),
                                   'XDS.INP'),
                      os.path.join(self.get_working_directory(),
                                   '%d_DEFPIX.INP' % self.get_xpid()))

      # write the input data files...

      for file_name in self._input_data_files_list:
        src = self._input_data_files[file_name]
        dst = os.path.join(
            self.get_working_directory(), file_name)
        if src != dst:
          shutil.copyfile(src, dst)

      self.start()
      self.close_wait()

      xds_check_version_supported(self.get_all_output())

      # copy the LP file
      shutil.copyfile(os.path.join(self.get_working_directory(),
                                   'DEFPIX.LP'),
                      os.path.join(self.get_working_directory(),
                                   '%d_DEFPIX.LP' % self.get_xpid()))

      # check the resolution asked for is achievable (if set)
      for record in open(os.path.join(self.get_working_directory(),
                                      'DEFPIX.LP')):
        if 'RESOLUTION RANGE RECORDED BY DETECTOR' in record:
          real_high = float(record.split()[-1])
          if self._resolution_high:
            if real_high > self._resolution_high + 0.01:
              Chatter.write(
                  'Warning: resolution limited to %.2f' % \
                  real_high)


      # gather the output files

      for file in self._output_data_files_list:
        self._output_data_files[file] = os.path.join(
          self.get_working_directory(), file)

      return
Beispiel #53
0
        def run(self):
            """Run labelit.index"""

            assert len(self._images) > 0
            self._images.sort()

            if self._max_cell is None and len(self._images) > 4:
                raise RuntimeError, "cannot use more than 4 images"

            # task = 'Autoindex from images:'

            # for i in _images:
            # task += ' %s' % self.get_image_name(i)

            # self.set_task(task)

            self.add_command_line("--index_only")

            for image in self._images:
                self.add_command_line(image)

            if self._space_group_number is not None:
                self.add_command_line("known_symmetry=%d" % self._space_group_number)

            if self._primitive_unit_cell is not None:
                self.add_command_line("target_cell=%f,%f,%f,%f,%f,%f" % tuple(self._primitive_unit_cell))

            if self._max_cell is not None:
                self.add_command_line("codecamp.maxcell=%f" % self._max_cell)

            self._write_dataset_preferences(len(self._images))

            shutil.copyfile(
                os.path.join(self.get_working_directory(), "dataset_preferences.py"),
                os.path.join(self.get_working_directory(), "%d_dataset_preferences.py" % self.get_xpid()),
            )

            self.start()
            self.close_wait()

            # sweep = self.get_indexer_sweep_name()
            # FileHandler.record_log_file(
            #'%s INDEX' % (sweep), self.get_log_file())

            # check for errors
            self.check_for_errors()

            # check for labelit errors - if something went wrong, then
            # try to address it by e.g. extending the beam search area...

            self.check_labelit_errors()

            # ok now we're done, let's look through for some useful stuff
            output = self.get_all_output()

            counter = 0

            # FIXME 03/NOV/06 something to do with the new centre search...

            # example output:

            # Beam center is not immediately clear; rigorously retesting \
            #                                             2 solutions
            # Beam x 109.0 y 105.1, initial score 538; refined rmsd: 0.1969
            # Beam x 108.8 y 106.1, initial score 354; refined rmsd: 0.1792

            # in here want to parse the beam centre search if it was done,
            # and check that the highest scoring solution was declared
            # the "best" - though should also have a check on the
            # R.M.S. deviation of that solution...

            # do this first!

            for j in range(len(output)):
                o = output[j]
                if "Beam centre is not immediately clear" in o:
                    # read the solutions that it has found and parse the
                    # information

                    centres = []
                    scores = []
                    rmsds = []

                    num_solutions = int(o.split()[-2])

                    for n in range(num_solutions):
                        record = output[j + n + 1].replace(",", " ").replace(";", " ").split()
                        x, y = float(record[2]), float(record[4])

                        centres.append((x, y))
                        scores.append(int(record[7]))
                        rmsds.append(float(record[-1]))

                    # next perform some analysis and perhaps assert the
                    # correct solution - for the moment just raise a warning
                    # if it looks like wrong solution may have been picked

                    best_beam_score = (0.0, 0.0, 0)
                    best_beam_rms = (0.0, 0.0, 1.0e8)

                    for n in range(num_solutions):
                        beam = centres[n]
                        score = scores[n]
                        rmsd = rmsds[n]

                        if score > best_beam_score[2]:
                            best_beam_score = (beam[0], beam[1], score)

                        if rmsd < best_beam_rmsd[2]:
                            best_beam_rmsd = (beam[0], beam[1], rmsd)

                    # allow a difference of 0.1mm in either direction...
                    if (
                        math.fabs(best_beam_score[0] - best_beam_rmsd[0]) > 0.1
                        or math.fabs(best_beam_score[1] - best_beam_rmsd[1]) > 0.1
                    ):
                        Chatter.write("Labelit may have picked the wrong beam centre")

                        # FIXME as soon as I get the indexing loop
                        # structure set up, this should reset the
                        # indexing done flag, set the search range to
                        # 0, correct beam and then return...

                        # should also allow for the possibility that
                        # labelit has selected the best solution - so this
                        # will need to remember the stats for this solution,
                        # then compare them against the stats (one day) from
                        # running with the other solution - eventually the
                        # correct solution will result...

            for o in output:
                l = o.split()

                if l[:3] == ["Beam", "center", "x"]:
                    x = float(l[3].replace("mm,", ""))
                    y = float(l[5].replace("mm,", ""))

                    self._mosflm_beam_centre = (x, y)
                    self._mosflm_detector_distance = float(l[7].replace("mm", ""))
                    # self.set_indexer_beam_centre((x, y))
                    # self.set_indexer_distance(float(l[7].replace('mm', '')))

                    self._mosaic = float(l[10].replace("mosaicity=", ""))

                if l[:3] == ["Solution", "Metric", "fit"]:
                    break

                counter += 1

            # if we've just broken out (counter < len(output)) then
            # we need to gather the output

            if counter >= len(output):
                raise RuntimeError, "error in indexing"

            # FIXME this needs to check the smilie status e.g.
            # ":)" or ";(" or "  ".

            for i in range(counter + 1, len(output)):
                o = output[i][3:]
                smiley = output[i][:3]
                l = o.split()
                if l:

                    self._solutions[int(l[0])] = {
                        "number": int(l[0]),
                        "mosaic": self._mosaic,
                        "metric": float(l[1]),
                        "rmsd": float(l[3]),
                        "nspots": int(l[4]),
                        "lattice": l[6],
                        "cell": map(float, l[7:13]),
                        "volume": int(l[-1]),
                        "smiley": smiley,
                    }

            # remove clearly incorrect solutions i.e. rmsd >> rmsd for P1 i.e. factor
            # of 4.0 or more...

            for s in sorted(self._solutions):
                if self._solutions[s]["rmsd"] > 4.0 * self._solutions[1]["rmsd"]:
                    del (self._solutions[s])

            return "ok"
Beispiel #54
0
  def __init__(self, name,
               wavelength,
               sample,
               directory = None,
               image = None,
               beam = None,
               reversephi = False,
               distance = None,
               gain = 0.0,
               dmin = 0.0,
               dmax = 0.0,
               polarization = 0.0,
               frames_to_process = None,
               user_lattice = None,
               user_cell = None,
               epoch = 0,
               ice = False,
               excluded_regions = None):
    '''Create a new sweep named name, belonging to XWavelength object
    wavelength, representing the images in directory starting with image,
    with beam centre optionally defined.'''
    if excluded_regions is None:
      excluded_regions = []

    # + check the wavelength is an XWavelength object
    #   raise an exception if not... or not...

    if not wavelength.__class__.__name__ == 'XWavelength':
      pass

    # FIXME bug 2221 if DIRECTORY starts with ~/ or ~graeme (say) need to
    # interpret this properly - e.g. map it to a full PATH.

    directory = expand_path(directory)

    self._name = name
    self._wavelength = wavelength
    self._sample = sample
    self._directory = directory
    self._image = image
    self._reversephi = reversephi
    self._epoch = epoch
    self._user_lattice = user_lattice
    self._user_cell = user_cell
    self._header = { }
    self._resolution_high = dmin
    self._resolution_low = dmax
    self._ice = ice
    self._excluded_regions = excluded_regions
    self._imageset = None

    # FIXME in here also need to be able to accumulate the total
    # dose from all experimental measurements (complex) and provide
    # a _epoch_to_dose dictionary or some such... may be fiddly as
    # this will need to parse across multiple templates. c/f Bug # 2798

    self._epoch_to_image = { }
    self._image_to_epoch = { }

    # to allow first, last image for processing to be
    # set... c/f integrater interface
    self._frames_to_process = frames_to_process

    # + derive template, list of images

    params = PhilIndex.get_python_object()
    if directory and image:
      self._template, self._directory = \
                      image2template_directory(os.path.join(directory,
                                                            image))

      from xia2.Schema import load_imagesets
      imagesets = load_imagesets(
        self._template, self._directory, image_range=self._frames_to_process,
        reversephi=(params.xia2.settings.input.reverse_phi or self._reversephi))

      assert len(imagesets) == 1, "one imageset expected, %d found" % \
          len(imagesets)
      self._imageset = copy.deepcopy(imagesets[0])
      start, end = self._imageset.get_array_range()
      self._images = list(range(start+1, end+1))

      # FIXME in here check that (1) the list of images is continuous
      # and (2) that all of the images are readable. This should also
      # take into account frames_to_process if set.

      if self._frames_to_process is None:
        self._frames_to_process = min(self._images), max(self._images)

      start, end = self._frames_to_process

      error = False

      if params.general.check_image_files_readable:
        for j in range(start, end + 1):
          image_name = self.get_imageset().get_path(j-start)
          if not j in self._images:
            Debug.write('image %s missing' %image_name)
            error = True
            continue
          if not os.access(image_name, os.R_OK):
            Debug.write('image %s unreadable' %image_name)
            error = True
            continue

        if error:
          raise RuntimeError, 'problem with sweep %s' % self._name

      # + read the image header information into here?
      #   or don't I need it? it would be useful for checking
      #   against wavelength.getWavelength() I guess to make
      #   sure that the plumbing is all sound.

      # check that they match by closer than 0.0001A, if wavelength
      # is not None

      beam_ = self._imageset.get_beam()
      scan = self._imageset.get_scan()
      if wavelength is not None:

        # FIXME 29/NOV/06 if the wavelength wavelength value
        # is 0.0 then first set it to the header value - note
        # that this assumes that the header value is correct
        # (a reasonable assumption)

        if wavelength.get_wavelength() == 0.0:
          wavelength.set_wavelength(beam_.get_wavelength())

        # FIXME 08/DEC/06 in here need to allow for the fact
        # that the wavelength in the image header could be wrong and
        # in fact it should be replaced with the input value -
        # through the user will need to be warned of this and
        # also everything using the FrameProcessor interface
        # will also have to respect this!

        if math.fabs(beam_.get_wavelength() -
                     wavelength.get_wavelength()) > 0.0001:
          # format = 'wavelength for sweep %s does not ' + \
          # 'match wavelength %s'
          # raise RuntimeError, format  % \
          # (name, wavelength.get_name())

          format = 'Header wavelength for sweep %s different' + \
                   ' to assigned value (%4.2f vs. %4.2f)'

          Chatter.write(format % (name, beam_.get_wavelength(),
                                  wavelength.get_wavelength()))


      # also in here look at the image headers to see if we can
      # construct a mapping between exposure epoch and image ...

      images = []

      if self._frames_to_process:
        start, end = self._frames_to_process
        for j in self._images:
          if j >= start and j <= end:
            images.append(j)
      else:
        images = self._images

      for j in images:
        epoch = scan.get_image_epoch(j)
        if epoch == 0.0:
          epoch = float(os.stat(self._imageset.get_path(j-images[0])).st_mtime)
        self._epoch_to_image[epoch] = j
        self._image_to_epoch[j] = epoch

      epochs = self._epoch_to_image.keys()

      Debug.write('Exposure epoch for sweep %s: %d %d' % \
                  (self._template, min(epochs), max(epochs)))

    self._input_imageset = copy.deepcopy(self._imageset)

    # + get the lattice - can this be a pointer, so that when
    #   this object updates lattice it is globally-for-this-crystal
    #   updated? The lattice included directly in here includes an
    #   exact unit cell for data reduction, the crystal lattice
    #   contains an approximate unit cell which should be
    #   from the unit cells from all sweeps contained in the
    #   XCrystal. FIXME should I be using a LatticeInfo object
    #   in here? See what the Indexer interface produces. ALT:
    #   just provide an Indexer implementation "hook".
    #   See Headnote 001 above. See also _get_indexer,
    #   _get_integrater below.

    self._indexer = None
    self._refiner = None
    self._integrater = None

    # I don't need this - it is equivalent to self.getWavelength(
    # ).getCrystal().getLattice()
    # self._crystal_lattice = None

    # this means that this module will have to present largely the
    # same interface as Indexer and Integrater so that the calls
    # can be appropriately forwarded.

    # finally configure the beam if set

    if beam is not None:
      from dxtbx.model.detector_helpers import set_mosflm_beam_centre
      try:
        set_mosflm_beam_centre(self.get_imageset().get_detector(),
                               self.get_imageset().get_beam(),
                               beam)
      except AssertionError, e:
        Debug.write('Error setting mosflm beam centre: %s' % e)
Beispiel #55
0
def xia2_main(stop_after=None):
    '''Actually process something...'''
    Citations.cite('xia2')

    # print versions of related software
    Chatter.write(dials_version())

    ccp4_version = get_ccp4_version()
    if ccp4_version is not None:
        Chatter.write('CCP4 %s' % ccp4_version)

    start_time = time.time()

    CommandLine = get_command_line()
    start_dir = Flags.get_starting_directory()

    # check that something useful has been assigned for processing...
    xtals = CommandLine.get_xinfo().get_crystals()

    no_images = True

    for name in xtals.keys():
        xtal = xtals[name]

        if not xtal.get_all_image_names():

            Chatter.write('-----------------------------------' + \
                          '-' * len(name))
            Chatter.write('| No images assigned for crystal %s |' % name)
            Chatter.write('-----------------------------------' + '-' \
                          * len(name))
        else:
            no_images = False

    args = []

    from xia2.Handlers.Phil import PhilIndex
    params = PhilIndex.get_python_object()
    mp_params = params.xia2.settings.multiprocessing
    njob = mp_params.njob

    from libtbx import group_args

    xinfo = CommandLine.get_xinfo()

    if os.path.exists('xia2.json'):
        from xia2.Schema.XProject import XProject
        xinfo_new = xinfo
        xinfo = XProject.from_json(filename='xia2.json')

        crystals = xinfo.get_crystals()
        crystals_new = xinfo_new.get_crystals()
        for crystal_id in crystals_new.keys():
            if crystal_id not in crystals:
                crystals[crystal_id] = crystals_new[crystal_id]
                continue
            crystals[crystal_id]._scaler = None  # reset scaler
            for wavelength_id in crystals_new[crystal_id].get_wavelength_names(
            ):
                wavelength_new = crystals_new[crystal_id].get_xwavelength(
                    wavelength_id)
                if wavelength_id not in crystals[
                        crystal_id].get_wavelength_names():
                    crystals[crystal_id].add_wavelength(
                        crystals_new[crystal_id].get_xwavelength(
                            wavelength_new))
                    continue
                wavelength = crystals[crystal_id].get_xwavelength(
                    wavelength_id)
                sweeps_new = wavelength_new.get_sweeps()
                sweeps = wavelength.get_sweeps()
                sweep_names = [s.get_name() for s in sweeps]
                sweep_keys = [(s.get_directory(), s.get_template(),
                               s.get_image_range()) for s in sweeps]
                for sweep in sweeps_new:
                    if ((sweep.get_directory(), sweep.get_template(),
                         sweep.get_image_range()) not in sweep_keys):
                        if sweep.get_name() in sweep_names:
                            i = 1
                            while 'SWEEEP%i' % i in sweep_names:
                                i += 1
                            sweep._name = 'SWEEP%i' % i
                            break
                        wavelength.add_sweep(
                            name=sweep.get_name(),
                            sample=sweep.get_xsample(),
                            directory=sweep.get_directory(),
                            image=sweep.get_image(),
                            beam=sweep.get_beam_centre(),
                            reversephi=sweep.get_reversephi(),
                            distance=sweep.get_distance(),
                            gain=sweep.get_gain(),
                            dmin=sweep.get_resolution_high(),
                            dmax=sweep.get_resolution_low(),
                            polarization=sweep.get_polarization(),
                            frames_to_process=sweep.get_frames_to_process(),
                            user_lattice=sweep.get_user_lattice(),
                            user_cell=sweep.get_user_cell(),
                            epoch=sweep._epoch,
                            ice=sweep._ice,
                            excluded_regions=sweep._excluded_regions,
                        )
                        sweep_names.append(sweep.get_name())

    crystals = xinfo.get_crystals()

    failover = params.xia2.settings.failover

    if mp_params.mode == 'parallel' and njob > 1:
        driver_type = mp_params.type
        command_line_args = CommandLine.get_argv()[1:]
        for crystal_id in crystals.keys():
            for wavelength_id in crystals[crystal_id].get_wavelength_names():
                wavelength = crystals[crystal_id].get_xwavelength(
                    wavelength_id)
                sweeps = wavelength.get_sweeps()
                for sweep in sweeps:
                    sweep._get_indexer()
                    sweep._get_refiner()
                    sweep._get_integrater()
                    args.append((group_args(
                        driver_type=driver_type,
                        stop_after=stop_after,
                        failover=failover,
                        command_line_args=command_line_args,
                        nproc=mp_params.nproc,
                        crystal_id=crystal_id,
                        wavelength_id=wavelength_id,
                        sweep_id=sweep.get_name(),
                    ), ))

        from xia2.Driver.DriverFactory import DriverFactory
        default_driver_type = DriverFactory.get_driver_type()

        # run every nth job on the current computer (no need to submit to qsub)
        for i_job, arg in enumerate(args):
            if (i_job % njob) == 0:
                arg[0].driver_type = default_driver_type

        if mp_params.type == "qsub":
            method = "sge"
        else:
            method = "multiprocessing"
        nproc = mp_params.nproc
        qsub_command = mp_params.qsub_command
        if not qsub_command:
            qsub_command = 'qsub'
        qsub_command = '%s -V -cwd -pe smp %d' % (qsub_command, nproc)

        from libtbx import easy_mp
        results = easy_mp.parallel_map(
            process_one_sweep,
            args,
            processes=njob,
            #method=method,
            method="multiprocessing",
            qsub_command=qsub_command,
            preserve_order=True,
            preserve_exception_message=True)

        # Hack to update sweep with the serialized indexers/refiners/integraters
        i_sweep = 0
        for crystal_id in crystals.keys():
            for wavelength_id in crystals[crystal_id].get_wavelength_names():
                wavelength = crystals[crystal_id].get_xwavelength(
                    wavelength_id)
                remove_sweeps = []
                sweeps = wavelength.get_sweeps()
                for sweep in sweeps:
                    success, output, xsweep_dict = results[i_sweep]
                    if output is not None:
                        Chatter.write(output)
                    if not success:
                        Chatter.write('Sweep failed: removing %s' %
                                      sweep.get_name())
                        remove_sweeps.append(sweep)
                    else:
                        assert xsweep_dict is not None
                        Chatter.write('Loading sweep: %s' % sweep.get_name())
                        from xia2.Schema.XSweep import XSweep
                        new_sweep = XSweep.from_dict(xsweep_dict)
                        sweep._indexer = new_sweep._indexer
                        sweep._refiner = new_sweep._refiner
                        sweep._integrater = new_sweep._integrater
                    i_sweep += 1
                for sweep in remove_sweeps:
                    wavelength.remove_sweep(sweep)
                    sample = sweep.get_xsample()
                    sample.remove_sweep(sweep)

    else:
        for crystal_id in crystals.keys():
            for wavelength_id in crystals[crystal_id].get_wavelength_names():
                wavelength = crystals[crystal_id].get_xwavelength(
                    wavelength_id)
                remove_sweeps = []
                sweeps = wavelength.get_sweeps()
                for sweep in sweeps:
                    from dials.command_line.show import show_datablocks
                    from dxtbx.datablock import DataBlock
                    Debug.write(sweep.get_name())
                    Debug.write(
                        show_datablocks([DataBlock([sweep.get_imageset()])]))
                    try:
                        if stop_after == 'index':
                            sweep.get_indexer_cell()
                        else:
                            sweep.get_integrater_intensities()
                        sweep.serialize()
                    except Exception as e:
                        if failover:
                            Chatter.write('Processing sweep %s failed: %s' % \
                                          (sweep.get_name(), str(e)))
                            remove_sweeps.append(sweep)
                        else:
                            raise
                for sweep in remove_sweeps:
                    wavelength.remove_sweep(sweep)
                    sample = sweep.get_xsample()
                    sample.remove_sweep(sweep)

    # save intermediate xia2.json file in case scaling step fails
    xinfo.as_json(filename='xia2.json')

    if stop_after not in ('index', 'integrate'):
        Chatter.write(xinfo.get_output(), strip=False)

    for crystal in crystals.values():
        crystal.serialize()

    # save final xia2.json file in case report generation fails
    xinfo.as_json(filename='xia2.json')

    duration = time.time() - start_time

    # write out the time taken in a human readable way
    Chatter.write('Processing took %s' % \
                  time.strftime("%Hh %Mm %Ss", time.gmtime(duration)))

    if stop_after not in ('index', 'integrate'):
        # and the summary file
        with open('xia2-summary.dat', 'w') as fh:
            for record in xinfo.summarise():
                fh.write('%s\n' % record)

        # looks like this import overwrites the initial command line
        # Phil overrides so... for https://github.com/xia2/xia2/issues/150
        from xia2.command_line.html import generate_xia2_html

        if params.xia2.settings.small_molecule == True:
            params.xia2.settings.report.xtriage_analysis = False
            params.xia2.settings.report.include_radiation_damage = False

        generate_xia2_html(xinfo,
                           filename='xia2.html',
                           params=params.xia2.settings.report)

    write_citations()

    # delete all of the temporary mtz files...
    cleanup()
    Environment.cleanup()
Beispiel #56
0
  def _integrate(self):
    '''Actually do the integration - in XDS terms this will mean running
    DEFPIX and INTEGRATE to measure all the reflections.'''

    experiment = self._intgr_refiner.get_refined_experiment_list(
      self.get_integrater_epoch())[0]
    crystal_model = experiment.crystal
    self._intgr_refiner_cell = crystal_model.get_unit_cell().parameters()

    images_str = '%d to %d' % tuple(self._intgr_wedge)
    cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' %tuple(self._intgr_refiner_cell)

    if len(self._fp_directory) <= 50:
      dirname = self._fp_directory
    else:
      dirname = '...%s' % self._fp_directory[-46:]

    Journal.block(
        'integrating', self._intgr_sweep_name, 'XDS',
        {'images':images_str,
         'cell':cell_str,
         'lattice':self._intgr_refiner.get_refiner_lattice(),
         'template':self._fp_template,
         'directory':dirname,
         'resolution':'%.2f' % self._intgr_reso_high})

    first_image_in_wedge = self.get_image_name(self._intgr_wedge[0])

    defpix = self.Defpix()

    # pass in the correct data

    for file in ['X-CORRECTIONS.cbf',
                 'Y-CORRECTIONS.cbf',
                 'BKGINIT.cbf',
                 'XPARM.XDS']:
      defpix.set_input_data_file(file, self._xds_data_files[file])

    defpix.set_data_range(self._intgr_wedge[0],
                          self._intgr_wedge[1])

    if self.get_integrater_high_resolution() > 0.0 and \
           self.get_integrater_user_resolution():
      Debug.write('Setting resolution limit in DEFPIX to %.2f' % \
                  self.get_integrater_high_resolution())
      defpix.set_resolution_high(self.get_integrater_high_resolution())
      defpix.set_resolution_low(self.get_integrater_low_resolution())

    elif self.get_integrater_low_resolution():
      Debug.write('Setting low resolution limit in DEFPIX to %.2f' % \
                  self.get_integrater_low_resolution())
      defpix.set_resolution_high(0.0)
      defpix.set_resolution_low(self.get_integrater_low_resolution())

    defpix.run()

    # and gather the result files
    for file in ['BKGPIX.cbf',
                 'ABS.cbf']:
      self._xds_data_files[file] = defpix.get_output_data_file(file)

    integrate = self.Integrate()

    if self._xds_integrate_parameters:
      integrate.set_updates(self._xds_integrate_parameters)

    # decide what images we are going to process, if not already
    # specified

    if not self._intgr_wedge:
      images = self.get_matching_images()
      self.set_integrater_wedge(min(images),
                                max(images))

    first_image_in_wedge = self.get_image_name(self._intgr_wedge[0])

    integrate.set_data_range(self._intgr_wedge[0],
                             self._intgr_wedge[1])

    for file in ['X-CORRECTIONS.cbf',
                 'Y-CORRECTIONS.cbf',
                 'BLANK.cbf',
                 'BKGPIX.cbf',
                 'GAIN.cbf']:
      integrate.set_input_data_file(file, self._xds_data_files[file])

    if self._xds_data_files.has_key('GXPARM.XDS'):
      Debug.write('Using globally refined parameters')
      integrate.set_input_data_file(
          'XPARM.XDS', self._xds_data_files['GXPARM.XDS'])
      integrate.set_refined_xparm()
    else:
      integrate.set_input_data_file(
          'XPARM.XDS', self._xds_data_files['XPARM.XDS'])

    integrate.run()

    self._intgr_per_image_statistics = integrate.get_per_image_statistics()
    Chatter.write(self.show_per_image_statistics())

    # record the log file -

    pname, xname, dname = self.get_integrater_project_info()
    sweep = self.get_integrater_sweep_name()
    FileHandler.record_log_file('%s %s %s %s INTEGRATE' % \
                                (pname, xname, dname, sweep),
                                os.path.join(self.get_working_directory(),
                                             'INTEGRATE.LP'))

    # and copy the first pass INTEGRATE.HKL...

    lattice = self._intgr_refiner.get_refiner_lattice()
    if not os.path.exists(os.path.join(
        self.get_working_directory(),
        'INTEGRATE-%s.HKL' % lattice)):
      here = self.get_working_directory()
      shutil.copyfile(os.path.join(here, 'INTEGRATE.HKL'),
                      os.path.join(here, 'INTEGRATE-%s.HKL' % lattice))

    # record INTEGRATE.HKL for e.g. BLEND.

    FileHandler.record_more_data_file(
        '%s %s %s %s INTEGRATE' % (pname, xname, dname, sweep),
        os.path.join(self.get_working_directory(), 'INTEGRATE.HKL'))

    # should the existence of these require that I rerun the
    # integration or can we assume that the application of a
    # sensible resolution limit will achieve this??

    self._xds_integrate_parameters = integrate.get_updates()

    # record the mosaic spread &c.

    m_min, m_mean, m_max = integrate.get_mosaic()
    self.set_integrater_mosaic_min_mean_max(m_min, m_mean, m_max)

    Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
                  self.get_integrater_mosaic_min_mean_max())

    return os.path.join(self.get_working_directory(), 'INTEGRATE.HKL')
Beispiel #57
0
  def _scale(self):
    '''Perform all of the operations required to deliver the scaled
    data.'''

    epochs = self._sweep_handler.get_epochs()

    if PhilIndex.params.xia2.settings.optimize_scaling:
      self._determine_best_scale_model_8way()
    else:
      self._scalr_corrections = True
      self._scalr_correct_absorption = True
      self._scalr_correct_partiality = False
      self._scalr_correct_decay = True

    if self._scalr_corrections:
      Journal.block(
          'scaling', self.get_scaler_xcrystal().get_name(), 'CCP4',
          {'scaling model':'automatic',
           'absorption':self._scalr_correct_absorption,
           'tails':self._scalr_correct_partiality,
           'decay':self._scalr_correct_decay
           })

    else:
      Journal.block(
          'scaling', self.get_scaler_xcrystal().get_name(), 'CCP4',
          {'scaling model':'default'})

    sc = self._updated_aimless()
    sc.set_hklin(self._prepared_reflections)
    sc.set_intensities(PhilIndex.params.ccp4.aimless.intensities)

    sc.set_chef_unmerged(True)

    sc.set_new_scales_file('%s.scales' % self._scalr_xname)

    user_resolution_limits = { }

    for epoch in epochs:

      si = self._sweep_handler.get_sweep_information(epoch)
      pname, xname, dname = si.get_project_info()
      sname = si.get_sweep_name()
      intgr = si.get_integrater()

      if intgr.get_integrater_user_resolution():
        dmin = intgr.get_integrater_high_resolution()

        if not user_resolution_limits.has_key((dname, sname)):
          user_resolution_limits[(dname, sname)] = dmin
        elif dmin < user_resolution_limits[(dname, sname)]:
          user_resolution_limits[(dname, sname)] = dmin

      start, end = si.get_batch_range()

      if (dname, sname) in self._scalr_resolution_limits:
        resolution = self._scalr_resolution_limits[(dname, sname)]
        sc.add_run(start, end, exclude = False,
                   resolution = resolution, name = sname)
      else:
        sc.add_run(start, end, name = sname)

    sc.set_hklout(os.path.join(self.get_working_directory(),
                               '%s_%s_scaled_test.mtz' % \
                               (self._scalr_pname, self._scalr_xname)))

    if self.get_scaler_anomalous():
      sc.set_anomalous()

    # what follows, sucks

    if Flags.get_failover():

      try:
        sc.scale()
      except RuntimeError, e:

        es = str(e)

        if 'bad batch' in es or \
               'negative scales run' in es or \
               'no observations' in es:

          # first ID the sweep from the batch no

          batch = int(es.split()[-1])
          epoch = self._identify_sweep_epoch(batch)
          sweep = self._scalr_integraters[
              epoch].get_integrater_sweep()

          # then remove it from my parent xcrystal

          self.get_scaler_xcrystal().remove_sweep(sweep)

          # then remove it from the scaler list of intergraters
          # - this should really be a scaler interface method

          del(self._scalr_integraters[epoch])

          # then tell the user what is happening

          Chatter.write(
              'Sweep %s gave negative scales - removing' % \
              sweep.get_name())

          # then reset the prepare, do, finish flags

          self.set_scaler_prepare_done(False)
          self.set_scaler_done(False)
          self.set_scaler_finish_done(False)

          # and return

          return

        else:

          raise e
Beispiel #58
0
def compute_nmol_from_volume(volume, mass, resolution):

  file = open(nmolparams, 'r')

  while 1:
    line = file.readline()
    if not line[0] == '#':
      break

  resolutions = map(float, line.split(','))[:13]
  P0_list = map(float, file.readline().split(','))[:13]
  Vm_bar_list = map(float, file.readline().split(','))[:13]
  w_list = map(float, file.readline().split(','))[:13]
  A_list = map(float, file.readline().split(','))[:13]
  s_list = map(float, file.readline().split(','))[:13]
  file.close()

  if resolution > resolutions[-1]:
    # raise RuntimeError, 'Resolution outside useful range'
    Chatter.write('Resolution lower than %s -> computing for %f' % \
                  (resolutions[-1], resolution))
    resolution = resolutions[-1]
  if resolution < resolutions[0]:
    Chatter.write('Resolution higher than peak %f -> %f' % \
                  (resolution, resolutions[0]))
    resolution = resolutions[0]


  start = 0

  for start in range(12):
    if resolution > resolutions[start] and \
       resolution < resolutions[start + 1]:
      break

  # can start at start - interpolate

  diff = (resolution - resolutions[start]) / \
         (resolutions[start + 1] - resolutions[start])

  P0 = P0_list[start] + diff * (P0_list[start + 1] - P0_list[start])
  Vm_bar = Vm_bar_list[start] + \
           diff * (Vm_bar_list[start + 1] - Vm_bar_list[start])
  w = w_list[start] + diff * (w_list[start + 1] - w_list[start])
  A = A_list[start] + diff * (A_list[start + 1] - A_list[start])
  s = s_list[start] + diff * (s_list[start + 1] - s_list[start])

  nmols = []
  pdfs = []

  nmol = 1
  Vm = volume / (mass * nmol)
  while 1:
    z = (Vm - Vm_bar) / w
    p = P0 + A * math.exp(- math.exp(-z) - z * s + 1)
    nmols.append(nmol)
    pdfs.append(p)
    nmol += 1
    Vm = volume / (mass * nmol)
    if Vm < 1.0:
      break

  return nmols, pdfs
Beispiel #59
0
    def _index(self):
        '''Implement the indexer interface.'''

        Citations.cite('mosflm')

        indexer = MosflmIndex()
        indexer.set_working_directory(self.get_working_directory())
        auto_logfiler(indexer)

        from xia2.lib.bits import unique_elements
        _images = unique_elements(self._indxr_images)
        indexer.set_images(_images)
        images_str = ', '.join(map(str, _images))

        cell_str = None
        if self._indxr_input_cell:
            cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
                        self._indxr_input_cell

        if self._indxr_sweep_name:

            #if len(self._fp_directory) <= 50:
            #dirname = self._fp_directory
            #else:
            #dirname = '...%s' % self._fp_directory[-46:]
            dirname = os.path.dirname(self.get_imageset().get_template())

            Journal.block(
                'autoindexing', self._indxr_sweep_name, 'mosflm', {
                    'images': images_str,
                    'target cell': self._indxr_input_cell,
                    'target lattice': self._indxr_input_lattice,
                    'template': self.get_imageset().get_template(),
                    'directory': dirname
                })

        #task = 'Autoindex from images:'

        #for i in _images:
        #task += ' %s' % self.get_image_name(i)

        #self.set_task(task)

        indexer.set_template(os.path.basename(self.get_template()))
        indexer.set_directory(self.get_directory())

        xsweep = self.get_indexer_sweep()
        if xsweep is not None:
            if xsweep.get_distance() is not None:
                indexer.set_distance(xsweep.get_distance())
            #if self.get_wavelength_prov() == 'user':
            #index.set_wavelength(self.get_wavelength())
            if xsweep.get_beam_centre() is not None:
                indexer.set_beam_centre(xsweep.get_beam_centre())

        if self._indxr_input_cell:
            indexer.set_unit_cell(self._indxr_input_cell)

        if self._indxr_input_lattice is not None:
            spacegroup_number = lattice_to_spacegroup(
                self._indxr_input_lattice)
            indexer.set_space_group_number(spacegroup_number)

        if not self._mosflm_autoindex_thresh:

            try:

                min_peaks = 200

                Debug.write('Aiming for at least %d spots...' % min_peaks)

                thresholds = []

                for i in _images:

                    p = Printpeaks()
                    p.set_working_directory(self.get_working_directory())
                    auto_logfiler(p)
                    p.set_image(self.get_image_name(i))
                    thresh = p.threshold(min_peaks)

                    Debug.write('Autoindex threshold for image %d: %d' % \
                                (i, thresh))

                    thresholds.append(thresh)

                thresh = min(thresholds)
                self._mosflm_autoindex_thresh = thresh

            except Exception as e:
                print str(e)  #XXX this should disappear!
                Debug.write('Error computing threshold: %s' % str(e))
                Debug.write('Using default of 20.0')
                thresh = 20.0

        else:
            thresh = self._mosflm_autoindex_thresh

        Debug.write('Using autoindex threshold: %d' % thresh)

        if self._mosflm_autoindex_sol:
            indexer.set_solution_number(self._mosflm_autoindex_sol)
        indexer.set_threshold(thresh)

        # now forget this to prevent weird things happening later on
        if self._mosflm_autoindex_sol:
            self._mosflm_autoindex_sol = 0

        indexer.run()

        indxr_cell = indexer.get_refined_unit_cell()
        self._indxr_lattice = indexer.get_lattice()
        space_group_number = indexer.get_indexed_space_group_number()
        detector_distance = indexer.get_refined_distance()
        beam_centre = indexer.get_refined_beam_centre()
        mosaic_spreads = indexer.get_mosaic_spreads()

        if min(list(indxr_cell)) < 10.0 and \
           indxr_cell[2] / indxr_cell[0] > 6:

            Debug.write('Unrealistic autoindexing solution: ' +
                        '%.2f %.2f %.2f %.2f %.2f %.2f' % indxr_cell)

            # tweak some parameters and try again...
            self._mosflm_autoindex_thresh *= 1.5
            self.set_indexer_done(False)

            return

        intgr_params = {}

        # look up other possible indexing solutions (not well - in
        # standard settings only!) This is moved earlier as it could
        # result in returning if Mosflm has selected the wrong
        # solution!

        try:
            self._indxr_other_lattice_cell = indexer.get_solutions()

            # Change 27/FEB/08 to support user assigned spacegroups
            if self._indxr_user_input_lattice:
                lattice_to_spacegroup_dict = {
                    'aP': 1,
                    'mP': 3,
                    'mC': 5,
                    'oP': 16,
                    'oC': 20,
                    'oF': 22,
                    'oI': 23,
                    'tP': 75,
                    'tI': 79,
                    'hP': 143,
                    'hR': 146,
                    'cP': 195,
                    'cF': 196,
                    'cI': 197
                }
                for k in self._indxr_other_lattice_cell.keys():
                    if lattice_to_spacegroup_dict[k] > \
                           lattice_to_spacegroup_dict[
                        self._indxr_input_lattice]:
                        del (self._indxr_other_lattice_cell[k])

            # check that the selected unit cell matches - and if
            # not raise a "horrible" exception

            if self._indxr_input_cell:
                assert indxr_cell is not None
                for j in range(6):
                    if math.fabs(self._indxr_input_cell[j] -
                                 indxr_cell[j]) > 2.0:
                        Chatter.write('Mosflm autoindexing did not select ' +
                                      'correct (target) unit cell')
                        raise RuntimeError(
                            'something horrible happened in indexing')

        except RuntimeError as e:
            # check if mosflm rejected a solution we have it
            if 'horribl' in str(e):
                # ok it did - time to break out the big guns...
                if not self._indxr_input_cell:
                    raise RuntimeError(
                        'error in solution selection when not preset')

                # XXX FIXME
                self._mosflm_autoindex_sol = _get_indexing_solution_number(
                    indexer.get_all_output(), self._indxr_input_cell,
                    self._indxr_input_lattice)

                # set the fact that we are not done...
                self.set_indexer_done(False)

                # and return - hopefully this will restart everything
                return
            else:
                raise e

        if len(mosaic_spreads) == 0:
            # then consider setting it do a default value...
            # equal to the oscillation width (a good guess)
            phi_width = self.get_phi_width()
            Chatter.write(
                'Mosaic estimation failed, so guessing at %4.2f' % \
                phi_width)
            # only consider this if we have thus far no idea on the
            # mosaic spread...
            mosaic_spreads.append(phi_width)

        intgr_params['raster'] = indexer.get_raster()

        intgr_params['separation'] = indexer.get_separation()

        self._indxr_resolution_estimate = indexer.get_resolution_estimate()

        # compute mosaic as mean(mosaic_spreads)

        self._indxr_mosaic = sum(mosaic_spreads) / len(mosaic_spreads)

        self._indxr_payload['mosflm_integration_parameters'] = intgr_params

        self._indxr_payload['mosflm_orientation_matrix'] = open(
            os.path.join(self.get_working_directory(), 'xiaindex.mat'),
            'r').readlines()

        import copy
        from dxtbx.model.detector_helpers import set_mosflm_beam_centre
        from xia2.Wrappers.Mosflm.AutoindexHelpers import set_distance
        from xia2.Wrappers.Mosflm.AutoindexHelpers import crystal_model_from_mosflm_mat
        from cctbx import sgtbx, uctbx

        # update the beam centre (i.e. shift the origin of the detector)
        detector = copy.deepcopy(self.get_detector())
        beam = copy.deepcopy(self.get_beam())
        set_mosflm_beam_centre(detector, beam, beam_centre)
        if detector_distance is not None:
            set_distance(detector, detector_distance)

        # make a dxtbx crystal_model object from the mosflm matrix
        space_group = sgtbx.space_group_info(number=space_group_number).group()
        crystal_model = crystal_model_from_mosflm_mat(
            self._indxr_payload['mosflm_orientation_matrix'],
            unit_cell=uctbx.unit_cell(tuple(indxr_cell)),
            space_group=space_group)

        # construct an experiment_list
        from dxtbx.model import Experiment, ExperimentList
        experiment = Experiment(beam=beam,
                                detector=detector,
                                goniometer=self.get_goniometer(),
                                scan=self.get_scan(),
                                crystal=crystal_model)

        experiment_list = ExperimentList([experiment])
        self.set_indexer_experiment_list(experiment_list)
Beispiel #60
0
  def _mosflm_parallel_integrate(self):
    '''Perform the integration as before, but this time as a
    number of parallel Mosflm jobs (hence, in separate directories)
    and including a step of pre-refinement of the mosaic spread and
    missets. This will all be kind of explicit and hence probably
    messy!'''

    refinr = self.get_integrater_refiner()

    lattice = refinr.get_refiner_lattice()
    spacegroup_number = lattice_to_spacegroup(lattice)
    mosaic = refinr.get_refiner_payload('mosaic')
    beam = refinr.get_refiner_payload('beam')
    distance = refinr.get_refiner_payload('distance')
    matrix = refinr.get_refiner_payload('mosflm_orientation_matrix')

    integration_params = refinr.get_refiner_payload(
      'mosflm_integration_parameters')

    if integration_params:
      if 'separation' in integration_params:
        self.set_integrater_parameter(
            'mosflm', 'separation',
            '%s %s' % tuple(integration_params['separation']))
      if 'raster' in integration_params:
        self.set_integrater_parameter(
            'mosflm', 'raster',
            '%d %d %d %d %d' % tuple(integration_params['raster']))

    refinr.set_refiner_payload('mosflm_integration_parameters', None)
    pname, xname, dname = self.get_integrater_project_info()

    # what follows below should (i) be run in separate directories
    # and (ii) be repeated N=parallel times.

    nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
    parallel = nproc

    # FIXME this is something of a kludge - if too few frames refinement
    # and integration does not work well... ideally want at least 15
    # frames / chunk (say)
    nframes = self._intgr_wedge[1] - self._intgr_wedge[0] + 1

    if parallel > nframes / 15:
      parallel = nframes // 15

    if not parallel:
      raise RuntimeError, 'parallel not set'
    if parallel < 2:
      raise RuntimeError, 'parallel not parallel: %s' % parallel

    jobs = []
    hklouts = []
    nref = 0

    # calculate the chunks to use
    offset = self.get_frame_offset()
    start = self._intgr_wedge[0] - offset
    end = self._intgr_wedge[1] - offset

    left_images = 1 + end - start
    left_chunks = parallel
    chunks = []

    while left_images > 0:
      size = left_images // left_chunks
      chunks.append((start, start + size - 1))
      start += size
      left_images -= size
      left_chunks -= 1

    summary_files = []

    for j in range(parallel):

      # make some working directories, as necessary - chunk-(0:N-1)
      wd = os.path.join(self.get_working_directory(),
                        'chunk-%d' % j)
      if not os.path.exists(wd):
        os.makedirs(wd)

      job = MosflmIntegrate()
      job.set_working_directory(wd)

      auto_logfiler(job)

      l = refinr.get_refiner_lattice()

      # create the starting point
      f = open(os.path.join(wd, 'xiaintegrate-%s.mat' % l), 'w')
      for m in matrix:
        f.write(m)
      f.close()

      spacegroup_number = lattice_to_spacegroup(lattice)

      job.set_refine_profiles(self._mosflm_refine_profiles)

      # N.B. for harvesting need to append N to dname.

      if pname is not None and xname is not None and dname is not None:
        Debug.write('Harvesting: %s/%s/%s' %
                    (pname, xname, dname))
        harvest_dir = self.get_working_directory()
        temp_dname = '%s_%s' % \
                     (dname, self.get_integrater_sweep_name())
        job.set_pname_xname_dname(pname, xname, temp_dname)

      job.set_template(os.path.basename(self.get_template()))
      job.set_directory(self.get_directory())

      # check for ice - and if so, exclude (ranges taken from
      # XDS documentation)
      if self.get_integrater_ice() != 0:
        Debug.write('Excluding ice rings')
        job.set_exclude_ice(True)

      # exclude specified resolution ranges
      if len(self.get_integrater_excluded_regions()) != 0:
        regions = self.get_integrater_excluded_regions()
        Debug.write('Excluding regions: %s' % `regions`)
        job.set_exclude_regions(regions)

      mask = standard_mask(self.get_detector())
      for m in mask:
        job.add_instruction(m)

      job.set_input_mat_file('xiaintegrate-%s.mat' % l)

      job.set_beam_centre(beam)
      job.set_distance(distance)
      job.set_space_group_number(spacegroup_number)
      job.set_mosaic(mosaic)

      if self.get_wavelength_prov() == 'user':
        job.set_wavelength(self.get_wavelength())

      parameters = self.get_integrater_parameters('mosflm')
      job.update_parameters(parameters)

      if self._mosflm_gain:
        job.set_gain(self._mosflm_gain)

      # check for resolution limits
      if self._intgr_reso_high > 0.0:
        job.set_d_min(self._intgr_reso_high)
      if self._intgr_reso_low:
        job.set_d_max(self._intgr_reso_low)

      if PhilIndex.params.general.backstop_mask:
        from xia2.Toolkit.BackstopMask import BackstopMask
        mask = BackstopMask(PhilIndex.params.general.backstop_mask)
        mask = mask.calculate_mask_mosflm(self.get_header())
        job.set_mask(mask)

      detector = self.get_detector()
      detector_width, detector_height = detector[0].get_image_size_mm()

      lim_x = 0.5 * detector_width
      lim_y = 0.5 * detector_height

      Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
      job.set_limits(lim_x, lim_y)

      job.set_fix_mosaic(self._mosflm_postref_fix_mosaic)

      job.set_pre_refinement(True)
      job.set_image_range(chunks[j])


      # these are now running so ...

      jobs.append(job)

      continue

    # ok, at this stage I need to ...
    #
    # (i) accumulate the statistics as a function of batch
    # (ii) mong them into a single block
    #
    # This is likely to be a pain in the arse!

    first_integrated_batch = 1.0e6
    last_integrated_batch = -1.0e6

    all_residuals = []

    threads = []

    for j in range(parallel):
      job = jobs[j]

      # now wait for them to finish - first wait will really be the
      # first one, then all should be finished...

      thread = Background(job, 'run')
      thread.start()
      threads.append(thread)

    mosaics = []
    postref_result = { }

    integrated_images_first = 1.0e6
    integrated_images_last = -1.0e6
    self._intgr_per_image_statistics = {}

    for j in range(parallel):
      thread = threads[j]
      thread.stop()
      job = jobs[j]

      # get the log file
      output = job.get_all_output()

      # record a copy of it, perhaps - though not if parallel
      if self.get_integrater_sweep_name() and False:
        pname, xname, dname = self.get_integrater_project_info()
        FileHandler.record_log_file(
            '%s %s %s %s mosflm integrate' % \
            (self.get_integrater_sweep_name(),
             pname, xname, '%s_%d' % (dname, j)),
            job.get_log_file())

      # look for things that we want to know...
      # that is, the output reflection file name, the updated
      # value for the gain (if present,) any warnings, errors,
      # or just interesting facts.

      batches = job.get_batches_out()
      integrated_images_first = min(batches[0], integrated_images_first)
      integrated_images_last = max(batches[1], integrated_images_last)

      mosaics.extend(job.get_mosaic_spreads())

      if min(mosaics) < 0:
        raise IntegrationError, 'negative mosaic spread: %s' % min(mosaic)

      if (job.get_detector_gain_error() and not
          (self.get_imageset().get_detector()[0].get_type() == 'SENSOR_PAD')):
        gain = job.get_suggested_gain()
        if gain is not None:
          self.set_integrater_parameter('mosflm', 'gain', gain)
          self.set_integrater_export_parameter('mosflm', 'gain', gain)
          if self._mosflm_gain:
            Debug.write('GAIN updated to %f' % gain)
          else:
            Debug.write('GAIN found to be %f' % gain)

          self._mosflm_gain = gain
          self._mosflm_rerun_integration = True

      hklout = job.get_hklout()
      Debug.write('Integration output: %s' % hklout)
      hklouts.append(hklout)

      nref += job.get_nref()

      # if a BGSIG error happened try not refining the
      # profile and running again...

      if job.get_bgsig_too_large():
        if not self._mosflm_refine_profiles:
          raise RuntimeError, 'BGSIG error with profiles fixed'

        Debug.write(
            'BGSIG error detected - try fixing profile...')

        self._mosflm_refine_profiles = False
        self.set_integrater_done(False)

        return

      if job.get_getprof_error():
        Debug.write(
            'GETPROF error detected - try fixing profile...')
        self._mosflm_refine_profiles = False
        self.set_integrater_done(False)

        return

      # here
      # write the report for each image as .*-#$ to Chatter -
      # detailed report will be written automagically to science...

      self._intgr_per_image_statistics.update(job.get_per_image_statistics())
      postref_result.update(job.get_postref_result())

      # inspect the output for e.g. very high weighted residuals

      all_residuals.extend(job.get_residuals())

    self._intgr_batches_out = (integrated_images_first,
                               integrated_images_last)

    if mosaics and len(mosaics) > 0:
      self.set_integrater_mosaic_min_mean_max(
          min(mosaics), sum(mosaics) / len(mosaics), max(mosaics))
    else:
      m = indxr.get_indexer_mosaic()
      self.set_integrater_mosaic_min_mean_max(m, m, m)

    Chatter.write(self.show_per_image_statistics())

    Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
                  self.get_integrater_mosaic_min_mean_max())

    # gather the statistics from the postrefinement for all sweeps
    # now write this to a postrefinement log

    postref_log = os.path.join(self.get_working_directory(),
                               'postrefinement.log')

    fout = open(postref_log, 'w')

    fout.write('$TABLE: Postrefinement for %s:\n' % \
               self._intgr_sweep_name)
    fout.write('$GRAPHS: Missetting angles:A:1, 2, 3, 4: $$\n')
    fout.write('Batch PhiX PhiY PhiZ $$ Batch PhiX PhiY PhiZ $$\n')

    for image in sorted(postref_result):
      phix = postref_result[image].get('phix', 0.0)
      phiy = postref_result[image].get('phiy', 0.0)
      phiz = postref_result[image].get('phiz', 0.0)

      fout.write('%d %5.2f %5.2f %5.2f\n' % \
                 (image, phix, phiy, phiz))

    fout.write('$$\n')
    fout.close()

    if self.get_integrater_sweep_name():
      pname, xname, dname = self.get_integrater_project_info()
      FileHandler.record_log_file('%s %s %s %s postrefinement' % \
                                  (self.get_integrater_sweep_name(),
                                   pname, xname, dname),
                                  postref_log)

    hklouts.sort()

    hklout = os.path.join(self.get_working_directory(),
                          os.path.split(hklouts[0])[-1])

    Debug.write('Sorting data to %s' % hklout)
    for hklin in hklouts:
      Debug.write('<= %s' % hklin)

    sortmtz = Sortmtz()
    sortmtz.set_hklout(hklout)
    for hklin in hklouts:
      sortmtz.add_hklin(hklin)

    sortmtz.sort()

    self._mosflm_hklout = hklout

    return self._mosflm_hklout