Exemple #1
0
  def __call__(self, rlist):
    ''' Analyse the strong spots. '''
    from dials.util.command_line import Command

    # Check we have the required fields
    print "Analysing strong spots"
    if not ensure_required(rlist, self.required):
      return

    # Remove I_sigma <= 0
    selection = rlist['intensity.sum.variance'] <= 0
    if selection.count(True) > 0:
      rlist.del_selected(selection)
      print ' Removing %d reflections with variance <= 0' % \
        selection.count(True)

    if 'flags' in rlist:
      # Select only strong reflections
      Command.start(" Selecting only strong reflections")
      mask = rlist.get_flags(rlist.flags.strong)
      if mask.count(True) > 0:
        threshold = 10
        rlist = rlist.select(mask)
      Command.end(" Selected %d strong reflections" % len(rlist))

    # Look at distribution of spot counts
    self.spot_count_per_image(rlist)
    self.spot_count_per_panel(rlist)
Exemple #2
0
    def run(self, args=None):
        """Run the script."""
        # Parse the command line arguments
        params, options = self.parser.parse_args(args, show_diff_phil=True)
        if len(params.input.reflections) == 0:
            self.parser.print_help()
            return
        if len(params.input.reflections) <= 1:
            sys.exit("more than 1 reflection table must be specified")
        tables = [p.data for p in params.input.reflections]

        # Get the number of rows and columns
        nrows = [t.nrows() for t in tables]
        ncols = [t.ncols() for t in tables]

        # Merge the reflection lists
        if params.method == "update":
            assert all(n == nrows[0] for n in nrows[1:])
            table = tables[0]
            for t in tables[1:]:
                table.update(t)
        elif params.method == "extend":
            assert all(n == ncols[0] for n in ncols[1:])
            table = tables[0]
            for t in tables[1:]:
                table.extend(t)
        else:
            raise RuntimeError(f"unknown method, {params.method}")

        # Write the reflections to the file
        Command.start(f"Writing {len(table)} reflections to {params.output}")
        table.as_file(params.output)
        Command.end(f"Wrote {len(table)} reflections to {params.output}")
    def __call__(self, rlist):
        """ Analyse the strong spots. """
        # Check we have the required fields
        print("Analysing strong spots")
        if not ensure_required(rlist, self.required):
            return

        # Remove I_sigma <= 0
        selection = rlist["intensity.sum.variance"] <= 0
        if selection.count(True) > 0:
            rlist.del_selected(selection)
            print(" Removing %d reflections with variance <= 0" %
                  selection.count(True))

        if "flags" in rlist:
            # Select only strong reflections
            Command.start(" Selecting only strong reflections")
            mask = rlist.get_flags(rlist.flags.strong)
            if mask.count(True) > 0:
                rlist = rlist.select(mask)
            Command.end(" Selected %d strong reflections" % len(rlist))

        # Look at distribution of spot counts
        self.spot_count_per_image(rlist)
        self.spot_count_per_panel(rlist)
    def __call__(self, rlist):
        """ Analyse the reference profiles. """
        # Check we have the required fields
        print("Analysing reference profiles")
        if not ensure_required(rlist, self.required):
            return

        # Select only integrated reflections
        Command.start(" Selecting only integated reflections")
        mask = rlist.get_flags(rlist.flags.integrated)
        if mask.count(True) == 0:
            return

        rlist = rlist.select(mask)
        Command.end(" Selected %d integrated reflections" % len(rlist))

        # Analyse distribution of reference spots
        print(" Analysing reference profile distribution vs x/y")
        self.reference_xy(rlist)
        print(" Analysing reference profile distribution vs z")
        self.reference_z(rlist)

        # Look at correlations between profiles
        def ideal_correlations(filename, rlist):
            """ Call for reference spots and all reflections. """

            print(" Analysing reflection profile correlations")
            self.ideal_reflection_corr_hist(rlist, filename)

            print(" Analysing reflection profile correlations vs x/y")
            self.ideal_reflection_corr_vs_xy(rlist, filename)

            print(" Analysing reflection profile correlations vs z")
            self.ideal_reflection_corr_vs_z(rlist, filename)

            print(" Analysing reflection profile correlations vs I/Sigma")
            self.ideal_reflection_corr_vs_ios(rlist, filename)

        # Look at correlations between profiles
        def correlations(filename, rlist):
            """ Call for reference spots and all reflections. """

            print(" Analysing reflection profile correlations")
            self.reflection_corr_hist(rlist, filename)

            print(" Analysing reflection profile correlations vs x/y")
            self.reflection_corr_vs_xy(rlist, filename)

            print(" Analysing reflection profile correlations vs z")
            self.reflection_corr_vs_z(rlist, filename)

            print(" Analysing reflection profile correlations vs I/Sigma")
            self.reflection_corr_vs_ios(rlist, filename)

        mask = rlist.get_flags(rlist.flags.reference_spot)
        correlations("reference", rlist.select(mask))
        correlations("reflection", rlist)
        ideal_correlations("reference", rlist.select(mask))
        ideal_correlations("reflection", rlist)
Exemple #5
0
  def run(self):
    '''Execute the script.'''
    from dials.util.command_line import Command
    from dials.array_family import flex
    from dials.util.options import flatten_experiments

    # Parse the command line
    params, options = self.parser.parse_args(show_diff_phil=True)

    # Check the number of experiments
    experiments = flatten_experiments(params.input.experiments)
    if len(experiments) == 0:
      self.parser.print_help()
      return

    predicted_all = flex.reflection_table()

    for i_expt, expt in enumerate(experiments):
      if params.buffer_size > 0:
        # Hack to make the predicter predict reflections outside of the range
        # of the scan
        scan = expt.scan
        image_range = scan.get_image_range()
        oscillation = scan.get_oscillation()
        scan.set_image_range((image_range[0]-params.buffer_size,
                              image_range[1]+params.buffer_size))
        scan.set_oscillation((oscillation[0]-params.buffer_size*oscillation[1],
                              oscillation[1]))

      # Populate the reflection table with predictions
      predicted = flex.reflection_table.from_predictions(
        expt,
        force_static=params.force_static,
        dmin=params.d_min)
      predicted['id'] = flex.int(len(predicted), i_expt)
      predicted_all.extend(predicted)

    # if we are not ignoring shadows, look for reflections in the masked
    # region, see https://github.com/dials/dials/issues/349

    if not params.ignore_shadows:
      from dials.algorithms.shadowing.filter import filter_shadowed_reflections

      shadowed = filter_shadowed_reflections(experiments, predicted_all,
                                             experiment_goniometer=True)
      predicted_all = predicted_all.select(~shadowed)

    try:
      predicted_all.compute_bbox(experiments)
    except Exception:
      pass

    # Save the reflections to file
    Command.start('Saving {0} reflections to {1}'.format(
        len(predicted_all), params.output))
    predicted_all.as_pickle(params.output)
    Command.end('Saved {0} reflections to {1}'.format(
        len(predicted_all), params.output))
  def compute_background(self, reflections):
    ''' Compute the backgrond. '''
    from dials.util.command_line import Command

    # Do the background subtraction
    Command.start('Calculating reflection background')
    mask = self._subtractor(reflections['shoebox'])
    reflections.del_selected(mask != True)
    Command.end('Calculated {0} background values'.format(len(reflections)))
    def compute_background(self, reflections):
        """Compute the background."""
        from dials.util.command_line import Command

        # Do the background subtraction
        Command.start("Calculating reflection background")
        mask = self._subtractor(reflections["shoebox"])
        reflections.del_selected(mask != True)
        Command.end("Calculated {0} background values".format(
            len(reflections)))
Exemple #8
0
    def run(self, args=None):
        """Execute the script."""
        # Parse the command line
        params, options = self.parser.parse_args(args, show_diff_phil=True)

        # Check the number of experiments
        experiments = flatten_experiments(params.input.experiments)
        if len(experiments) == 0:
            self.parser.print_help()
            return

        predicted_all = flex.reflection_table()

        for i_expt, expt in enumerate(experiments):
            if params.buffer_size > 0:
                # Hack to make the predicter predict reflections outside of the range
                # of the scan
                scan = expt.scan
                image_range = scan.get_image_range()
                oscillation = scan.get_oscillation()
                scan.set_image_range((
                    image_range[0] - params.buffer_size,
                    image_range[1] + params.buffer_size,
                ))
                scan.set_oscillation((
                    oscillation[0] - params.buffer_size * oscillation[1],
                    oscillation[1],
                ))

            # Populate the reflection table with predictions
            predicted = flex.reflection_table.from_predictions(
                expt, force_static=params.force_static, dmin=params.d_min)
            predicted["id"] = flex.int(len(predicted), i_expt)
            predicted_all.extend(predicted)

        # if we are not ignoring shadows, look for reflections in the masked
        # region, see https://github.com/dials/dials/issues/349

        if not params.ignore_shadows:
            shadowed = filter_shadowed_reflections(experiments,
                                                   predicted_all,
                                                   experiment_goniometer=True)
            predicted_all = predicted_all.select(~shadowed)

        try:
            predicted_all.compute_bbox(experiments)
        except Exception:
            pass

        # Save the reflections to file
        Command.start(
            f"Saving {len(predicted_all)} reflections to {params.output}")
        predicted_all.as_file(params.output)
        Command.end(
            f"Saved {len(predicted_all)} reflections to {params.output}")
Exemple #9
0
  def __call__(self, rlist):
    ''' Analyse the relfection background. '''
    from dials.util.command_line import Command

    # Check we have the required fields
    print "Analysing reflection backgrounds"
    if not ensure_required(rlist, self.required):
      return

    selection = rlist['intensity.sum.variance'] <= 0
    if selection.count(True) > 0:
      rlist.del_selected(selection)
      print ' Removing %d reflections with variance <= 0' % \
        selection.count(True)

    selection = rlist['background.mse'] < 0
    if selection.count(True) > 0:
      rlist.del_selected(selection)
      print ' Removing %d reflections with negative background model RMSD' % \
        selection.count(True)

    selection = rlist['background.mean'] <= 0
    if selection.count(True) > 0:
      rlist.del_selected(selection)
      print ' Removing %d reflections with mean background <= 0' % \
        selection.count(True)

    # Select only integrated reflections
    Command.start(" Selecting only integated reflections")
    mask = rlist.get_flags(rlist.flags.integrated)
    if mask.count(True) == 0:
      return

    rlist = rlist.select(mask)
    Command.end(" Selected %d integrated reflections" % len(rlist))

    # Look at distribution of I/Sigma
    print " Analysing distribution of background mean"
    self.mean_hist(rlist)
    print " Analysing distribution of background mean vs XY"
    self.mean_vs_xy(rlist)
    print " Analysing distribution of background mean vs z"
    self.mean_vs_z(rlist)
    print " Analysing distribution of background mean vs I/Sigma"
    self.mean_vs_ios(rlist)
    print " Analysing distribution of background CVRMSD"
    self.rmsd_hist(rlist)
    print " Analysing distribution of background CVRMSD vs XY"
    self.rmsd_vs_xy(rlist)
    print " Analysing distribution of background CVRMSD vs z"
    self.rmsd_vs_z(rlist)
    print " Analysing distribution of background CVRMSD vs I/Sigma"
    self.rmsd_vs_ios(rlist)
Exemple #10
0
    def __call__(self, rlist):
        """ Analyse the relfection background. """
        # Check we have the required fields
        print("Analysing reflection backgrounds")
        if not ensure_required(rlist, self.required):
            return

        selection = rlist["intensity.sum.variance"] <= 0
        if selection.count(True) > 0:
            rlist.del_selected(selection)
            print(" Removing %d reflections with variance <= 0" %
                  selection.count(True))

        selection = rlist["background.mse"] < 0
        if selection.count(True) > 0:
            rlist.del_selected(selection)
            print(
                " Removing %d reflections with negative background model RMSD"
                % selection.count(True))

        selection = rlist["background.mean"] <= 0
        if selection.count(True) > 0:
            rlist.del_selected(selection)
            print(" Removing %d reflections with mean background <= 0" %
                  selection.count(True))

        # Select only integrated reflections
        Command.start(" Selecting only integated reflections")
        mask = rlist.get_flags(rlist.flags.integrated)
        if mask.count(True) == 0:
            return

        rlist = rlist.select(mask)
        Command.end(" Selected %d integrated reflections" % len(rlist))

        # Look at distribution of I/Sigma
        print(" Analysing distribution of background mean")
        self.mean_hist(rlist)
        print(" Analysing distribution of background mean vs XY")
        self.mean_vs_xy(rlist)
        print(" Analysing distribution of background mean vs z")
        self.mean_vs_z(rlist)
        print(" Analysing distribution of background mean vs I/Sigma")
        self.mean_vs_ios(rlist)
        print(" Analysing distribution of background CVRMSD")
        self.rmsd_hist(rlist)
        print(" Analysing distribution of background CVRMSD vs XY")
        self.rmsd_vs_xy(rlist)
        print(" Analysing distribution of background CVRMSD vs z")
        self.rmsd_vs_z(rlist)
        print(" Analysing distribution of background CVRMSD vs I/Sigma")
        self.rmsd_vs_ios(rlist)
    def _filter_reflections(self, params, experiments, reflections):
        ''' Filter the reflections to integrate. '''
        from dials.util.command_line import Command
        from dials.algorithms import filtering
        from dials.array_family import flex

        # Set all reflections which overlap bad pixels to zero
        Command.start('Filtering reflections by detector mask')
        if experiments[0].scan == None:
            array_range = 1
        else:
            array_range = experiments[0].scan.get_array_range()
        mask = filtering.by_detector_mask(
            reflections['bbox'],
            experiments[0].imageset.get_raw_data(0)[0] >= 0, array_range)
        reflections.del_selected(not mask)
        Command.end('Filtered %d reflections by detector mask' %
                    len(reflections))

        # Filter the reflections by zeta
        min_zeta = params.integration.filter.by_zeta
        if min_zeta > 0:
            Command.start('Filtering reflections by zeta >= %f' % min_zeta)
            zeta = reflections.compute_zeta(experiments[0])
            reflections.del_selected(flex.abs(zeta) < min_zeta)
            n = len(reflections)
            Command.end('Filtered %d reflections by zeta >= %f' %
                        (n, min_zeta))
            return reflections
Exemple #12
0
  def _filter_reflections(self, params, experiments, reflections):
    ''' Filter the reflections to integrate. '''
    from dials.util.command_line import Command
    from dials.algorithms import filtering
    from dials.array_family import flex

    # Set all reflections which overlap bad pixels to zero
    Command.start('Filtering reflections by detector mask')
    if experiments[0].scan == None:
      array_range = 1
    else:
      array_range = experiments[0].scan.get_array_range()
    mask = filtering.by_detector_mask(
      reflections['bbox'],
      experiments[0].imageset.get_raw_data(0)[0] >= 0,
      array_range)
    reflections.del_selected(not mask)
    Command.end('Filtered %d reflections by detector mask' % len(reflections))

    # Filter the reflections by zeta
    min_zeta = params.integration.filter.by_zeta
    if min_zeta > 0:
      Command.start('Filtering reflections by zeta >= %f' % min_zeta)
      zeta = reflections.compute_zeta(experiments[0])
      reflections.del_selected(flex.abs(zeta) < min_zeta)
      n = len(reflections)
      Command.end('Filtered %d reflections by zeta >= %f' % (n, min_zeta))
      return reflections
    def _match_with_reference(self, predicted, reference):
        ''' Match predictions with reference spots. '''

        from dials.algorithms.spot_finding.spot_matcher import SpotMatcher
        from dials.util.command_line import Command
        Command.start("Matching reference spots with predicted reflections")
        match = SpotMatcher(max_separation=1)
        rind, pind = match(reference, predicted)
        h1 = predicted.select(pind)['miller_index']
        h2 = reference.select(rind)['miller_index']
        mask = (h1 == h2)
        predicted.set_flags(pind.select(mask), predicted.flags.reference_spot)
        Command.end("Matched %d reference spots with predicted reflections" %
                    mask.count(True))
        return predicted
Exemple #14
0
  def __call__(self):
    """Run the script."""
    import cPickle as pickle
    from dials.model.data import ReflectionList # import dependency
    from dials.util.command_line import Command

    # Read the pickle file
    Command.start('Reading reflection file.')
    with open(self.reflections_filename, 'rb') as f:
      self.reflections = pickle.load(f)

    Command.end('Read {0} spots from reflection file.'.format(
        len(self.reflections)))

    self.view()
def process_reference(reference):
    ''' Load the reference spots. '''
    from dials.util.command_line import Command
    from dials.array_family import flex
    if reference is None:
      return None
    assert("miller_index" in reference)
    Command.start('Removing reference spots with invalid coordinates')
    mask = flex.bool([x == (0, 0, 0) for x in reference['xyzcal.mm']])
    reference.del_selected(mask)
    mask = flex.bool([h == (0, 0, 0) for h in reference['miller_index']])
    reference.del_selected(mask)
    Command.end('Removed reference spots with invalid coordinates, %d remaining' %
                len(reference))
    return reference
Exemple #16
0
  def _match_with_reference(self, predicted, reference):
    ''' Match predictions with reference spots. '''

    from dials.algorithms.spot_finding.spot_matcher import SpotMatcher
    from dials.util.command_line import Command
    Command.start("Matching reference spots with predicted reflections")
    match = SpotMatcher(max_separation=1)
    rind, pind = match(reference, predicted)
    h1 = predicted.select(pind)['miller_index']
    h2 = reference.select(rind)['miller_index']
    mask = (h1 == h2)
    predicted.set_flags(pind.select(mask), predicted.flags.reference_spot)
    Command.end("Matched %d reference spots with predicted reflections" %
                mask.count(True))
    return predicted
Exemple #17
0
  def run(self):
    '''Execute the script.'''
    from dials.util.command_line import Command
    from dials.array_family import flex
    from dials.util.options import flatten_experiments

    # Parse the command line
    params, options = self.parser.parse_args(show_diff_phil=True)

    # Check the number of experiments
    experiments = flatten_experiments(params.input.experiments)
    if len(experiments) == 0:
      self.parser.print_help()
      return

    predicted_all = flex.reflection_table()

    for i_expt, expt in enumerate(experiments):
      if params.buffer_size > 0:
        # Hack to make the predicter predict reflections outside of the range
        # of the scan
        scan = expt.scan
        image_range = scan.get_image_range()
        oscillation = scan.get_oscillation()
        scan.set_image_range((image_range[0]-params.buffer_size,
                              image_range[1]+params.buffer_size))
        scan.set_oscillation((oscillation[0]-params.buffer_size*oscillation[1],
                              oscillation[1]))

      # Populate the reflection table with predictions
      predicted = flex.reflection_table.from_predictions(
        expt,
        force_static=params.force_static,
        dmin=params.d_min)
      predicted['id'] = flex.int(len(predicted), i_expt)
      predicted_all.extend(predicted)

    try:
      predicted_all.compute_bbox(experiments)
    except Exception:
      pass

    # Save the reflections to file
    Command.start('Saving {0} reflections to {1}'.format(
        len(predicted_all), params.output))
    predicted_all.as_pickle(params.output)
    Command.end('Saved {0} reflections to {1}'.format(
        len(predicted_all), params.output))
def process_reference(reference):
    """Load the reference spots."""
    from dials.util.command_line import Command
    from dials.array_family import flex

    if reference is None:
        return None
    assert "miller_index" in reference
    Command.start("Removing reference spots with invalid coordinates")
    mask = flex.bool([x == (0, 0, 0) for x in reference["xyzcal.mm"]])
    reference.del_selected(mask)
    mask = flex.bool([h == (0, 0, 0) for h in reference["miller_index"]])
    reference.del_selected(mask)
    Command.end(
        "Removed reference spots with invalid coordinates, %d remaining" %
        len(reference))
    return reference
Exemple #19
0
  def __call__(self, rlist):
    ''' Analyse the reflection centroids. '''
    from dials.util.command_line import Command

    # FIXME Do the same and a comparison for intensity.prf

    # Check we have the required fields
    print "Analysing reflection intensities"
    if not ensure_required(rlist, self.required):
      return

    selection = rlist['intensity.sum.variance'] <= 0
    if selection.count(True) > 0:
      rlist.del_selected(selection)
      print ' Removing %d reflections with variance <= 0' % \
        selection.count(True)

    selection = rlist['intensity.sum.value'] <= 0
    if selection.count(True) > 0:
      rlist.del_selected(selection)
      print ' Removing %d reflections with intensity <= 0' % \
        selection.count(True)

    # Select only integrated reflections
    Command.start(" Selecting only integated reflections")
    mask = rlist.get_flags(rlist.flags.integrated)
    if mask.count(True) == 0:
      return

    rlist = rlist.select(mask)
    Command.end(" Selected %d integrated reflections" % len(rlist))

    # Look at distribution of I/Sigma
    print " Analysing distribution of I/Sigma"
    self.i_over_s_hist(rlist)
    print " Analysing distribution of I/Sigma vs xy"
    self.i_over_s_vs_xy(rlist, "sum")
    print " Analysing distribution of I/Sigma vs xy"
    self.i_over_s_vs_xy(rlist, "prf")
    print " Analysing distribution of I/Sigma vs z"
    self.i_over_s_vs_z(rlist)
    print " Analysing number of background pixels used"
    self.num_background_hist(rlist)
    print " Analysing number of foreground pixels used"
    self.num_foreground_hist(rlist)
Exemple #20
0
    def __call__(self, rlist):
        """ Analyse the reflection centroids. """
        # FIXME Do the same and a comparison for intensity.prf

        # Check we have the required fields
        print("Analysing reflection intensities")
        if not ensure_required(rlist, self.required):
            return

        selection = rlist["intensity.sum.variance"] <= 0
        if selection.count(True) > 0:
            rlist.del_selected(selection)
            print(" Removing %d reflections with variance <= 0" %
                  selection.count(True))

        selection = rlist["intensity.sum.value"] <= 0
        if selection.count(True) > 0:
            rlist.del_selected(selection)
            print(" Removing %d reflections with intensity <= 0" %
                  selection.count(True))

        # Select only integrated reflections
        Command.start(" Selecting only integated reflections")
        mask = rlist.get_flags(rlist.flags.integrated)
        if mask.count(True) == 0:
            return

        rlist = rlist.select(mask)
        Command.end(" Selected %d integrated reflections" % len(rlist))

        # Look at distribution of I/Sigma
        print(" Analysing distribution of I/Sigma")
        self.i_over_s_hist(rlist)
        print(" Analysing distribution of I/Sigma vs xy")
        self.i_over_s_vs_xy(rlist, "sum")
        print(" Analysing distribution of I/Sigma vs xy")
        self.i_over_s_vs_xy(rlist, "prf")
        print(" Analysing distribution of I/Sigma vs z")
        self.i_over_s_vs_z(rlist)
        print(" Analysing number of background pixels used")
        self.num_background_hist(rlist)
        print(" Analysing number of foreground pixels used")
        self.num_foreground_hist(rlist)
    def run(self):
        ''' Run the script. '''
        from dials.array_family import flex
        from dials.util.command_line import Command
        from libtbx.utils import Sorry

        # Parse the command line arguments
        params, options = self.parser.parse_args(show_diff_phil=True)
        if len(params.input.reflections) == 0:
            self.parser.print_help()
            return
        if len(params.input.reflections) <= 1:
            raise Sorry('more than 1 reflection table must be specified')
        tables = [p.data for p in params.input.reflections]

        # Get the number of rows and columns
        nrows = [t.nrows() for t in tables]
        ncols = [t.ncols() for t in tables]

        # Merge the reflection lists
        if params.method == "update":
            assert (all(n == nrows[0] for n in nrows[1:]))
            table = tables[0]
            for t in tables[1:]:
                table.update(t)
        elif params.method == "extend":
            assert (all(n == ncols[0] for n in ncols[1:]))
            table = tables[0]
            for t in tables[1:]:
                table.extend(t)
        else:
            raise RuntimeError('unknown method, %s' % params.method)

        # Write the reflections to the file
        Command.start('Writing %d reflections to %s' %
                      (len(table), params.output))
        table.as_pickle(params.output)
        Command.end('Wrote %d reflections to %s' % (len(table), params.output))
  def run(self):
    ''' Run the script. '''
    from dials.array_family import flex
    from dials.util.command_line import Command
    from libtbx.utils import Sorry

    # Parse the command line arguments
    params, options = self.parser.parse_args(show_diff_phil=True)
    if len(params.input.reflections) == 0:
      self.parser.print_help()
      return
    if len(params.input.reflections) <= 1:
      raise Sorry('more than 1 reflection table must be specified')
    tables = [p.data for p in params.input.reflections]

    # Get the number of rows and columns
    nrows = [t.nrows() for t in tables]
    ncols = [t.ncols() for t in tables]

    # Merge the reflection lists
    if params.method == "update":
      assert(all(n == nrows[0] for n in nrows[1:]))
      table = tables[0]
      for t in tables[1:]:
        table.update(t)
    elif params.method == "extend":
      assert(all(n == ncols[0] for n in ncols[1:]))
      table = tables[0]
      for t in tables[1:]:
        table.extend(t)
    else:
      raise RuntimeError('unknown method, %s' % params.method)

    # Write the reflections to the file
    Command.start('Writing %d reflections to %s' % (len(table), params.output))
    table.as_pickle(params.output)
    Command.end('Wrote %d reflections to %s' % (len(table), params.output))
Exemple #23
0
    def __call__(self, rlist):
        """ Analyse the reflection centroids. """
        # Check we have the required fields
        print("Analysing reflection centroids")
        if not ensure_required(rlist, self.required):
            return

        # Remove I_sigma <= 0
        selection = rlist["intensity.sum.variance"] <= 0
        if selection.count(True) > 0:
            rlist.del_selected(selection)
            print(" Removing %d reflections with variance <= 0" %
                  selection.count(True))

        # Remove partial reflections as their observed centroids won't be accurate
        if "partiality" in rlist:
            selection = rlist["partiality"] < 0.99
            if selection.count(True) > 0 and selection.count(
                    True) < selection.size():
                rlist.del_selected(selection)
                print(" Removing %d partial reflections" %
                      selection.count(True))

        # Select only integrated reflections
        Command.start(" Selecting only summation-integated reflections")
        mask = rlist.get_flags(rlist.flags.integrated_sum)
        if mask.count(True) > 0:
            threshold = 10
            rlist = rlist.select(mask)
            Command.end(" Selected %d summation-integrated reflections" %
                        len(rlist))
        else:
            # Select only those reflections used in refinement
            threshold = 0
            mask = rlist.get_flags(rlist.flags.used_in_refinement)
            rlist = rlist.select(mask)
            Command.end(" Selected %d refined reflections" % len(rlist))

        # Look at differences in calculated/observed position
        print(" Analysing centroid differences with I/Sigma > %s" % threshold)
        self.centroid_diff_hist(rlist, threshold)
        print(" Analysing centroid differences in x/y with I/Sigma > %s" %
              threshold)
        self.centroid_diff_xy(rlist, threshold)
        self.centroid_xy_xz_zy_residuals(rlist, threshold)
        print(" Analysing centroid differences in z with I/Sigma > %s" %
              threshold)
        self.centroid_diff_z(rlist, threshold)
        print(" Analysing centroid differences vs phi with I/Sigma > %s" %
              threshold)
        self.centroid_mean_diff_vs_phi(rlist, threshold)
Exemple #24
0
  def __call__(self, rlist):
    ''' Analyse the reflection centroids. '''
    from dials.util.command_line import Command

    # Check we have the required fields
    print "Analysing reflection centroids"
    if not ensure_required(rlist, self.required):
      return

    # Remove I_sigma <= 0
    selection = rlist['intensity.sum.variance'] <= 0
    if selection.count(True) > 0:
      rlist.del_selected(selection)
      print ' Removing %d reflections with variance <= 0' % \
        selection.count(True)

    # Remove partial reflections as their observed centroids won't be accurate
    if 'partiality' in rlist:
      selection = rlist['partiality'] < 0.99
      if selection.count(True) > 0 and selection.count(True) < selection.size():
        rlist.del_selected(selection)
        print ' Removing %d partial reflections' % \
          selection.count(True)

    # Select only integrated reflections
    Command.start(" Selecting only summation-integated reflections")
    mask = rlist.get_flags(rlist.flags.integrated_sum)
    if mask.count(True) > 0:
      threshold = 10
      rlist = rlist.select(mask)
      Command.end(" Selected %d summation-integrated reflections" % len(rlist))
    else:
      # Select only those reflections used in refinement
      threshold = 0
      mask = rlist.get_flags(rlist.flags.used_in_refinement)
      rlist = rlist.select(mask)
      Command.end(" Selected %d refined reflections" % len(rlist))

    # Look at differences in calculated/observed position
    print " Analysing centroid differences with I/Sigma > %s" %threshold
    self.centroid_diff_hist(rlist, threshold)
    print " Analysing centroid differences in x/y with I/Sigma > %s" %threshold
    self.centroid_diff_xy(rlist, threshold)
    self.centroid_xy_xz_zy_residuals(rlist, threshold)
    print " Analysing centroid differences in z with I/Sigma > %s" %threshold
    self.centroid_diff_z(rlist, threshold)
    print " Analysing centroid differences vs phi with I/Sigma > %s" %threshold
    self.centroid_mean_diff_vs_phi(rlist, threshold)
    import libtbx.load_env

    usage = ("usage: %s [options] reflections1.refl reflections2.refl" %
             libtbx.env.dispatcher_name)
    parser = OptionParser(usage)

    # Parse the command line arguments
    (options, args) = parser.parse_args()

    # Ensure there are only two reflection lists
    if len(args) != 2:
        parser.print_help()
        exit(0)

    # Read the first batch of reflections
    Command.start("Reading reflections from %s" % args[0])
    refl1 = flex.reflection_table.from_file(args[0])
    mask = flex.bool(xyz == (0, 0, 0) for xyz in refl1["xyzobs.px.value"])
    refl1.del_selected(mask)
    Command.end("Read %d reflections from %s" % (len(refl1), args[0]))

    # Read the second batch of reflections
    Command.start("Reading reflections from %s" % args[1])
    refl2 = flex.reflection_table.from_file(args[1])
    mask = refl2["intensity.sum.value"] <= 0.0
    refl2.del_selected(mask)
    mask = refl2["intensity.sum.value"]**2 < refl2["intensity.sum.variance"]
    refl2.del_selected(mask)
    Command.end("Read %d reflections from %s" % (len(refl2), args[1]))

    # perform the match
Exemple #26
0
    def run(self, args=None):
        """Run the script."""
        from dials.algorithms.profile_model.factory import ProfileModelFactory
        from dials.array_family import flex
        from dials.util import Sorry, log
        from dials.util.command_line import Command
        from dials.util.options import reflections_and_experiments_from_files

        log.config()

        # Parse the command line
        params, options = self.parser.parse_args(args, show_diff_phil=True)
        reflections, experiments = reflections_and_experiments_from_files(
            params.input.reflections, params.input.experiments)
        if len(reflections) == 0 and len(experiments) == 0:
            self.parser.print_help()
            return
        if len(reflections) != 1:
            raise Sorry("exactly 1 reflection table must be specified")
        if len(experiments) == 0:
            raise Sorry("no experiments were specified")
        if ("background.mean"
                not in reflections[0]) and params.subtract_background:
            raise Sorry(
                "for subtract_background need background.mean in reflections")

        reflections, _ = self.process_reference(reflections[0], params)

        # Check pixels don't belong to neighbours
        self.filter_reference_pixels(reflections, experiments)

        # Predict the reflections
        logger.info("")
        logger.info("=" * 80)
        logger.info("")
        logger.info("Predicting reflections")
        logger.info("")
        predicted = flex.reflection_table.from_predictions_multi(
            experiments,
            dmin=params.prediction.d_min,
            dmax=params.prediction.d_max,
            margin=params.prediction.margin,
            force_static=params.prediction.force_static,
            padding=params.prediction.padding,
        )

        # Match with predicted
        matched, reflections, unmatched = predicted.match_with_reference(
            reflections)
        assert len(matched) == len(predicted)
        assert matched.count(True) <= len(reflections)
        if matched.count(True) == 0:
            raise Sorry("""
        Invalid input for reference reflections.
        Zero reference spots were matched to predictions
      """)
        elif len(unmatched) != 0:
            logger.info("")
            logger.info("*" * 80)
            logger.info(
                "Warning: %d reference spots were not matched to predictions",
                len(unmatched),
            )
            logger.info("*" * 80)
            logger.info("")

        # Create the profile model
        experiments = ProfileModelFactory.create(params, experiments,
                                                 reflections)
        for model in experiments:
            sigma_b = model.profile.sigma_b(deg=True)
            sigma_m = model.profile.sigma_m(deg=True)
            if model.profile.is_scan_varying():  # scan varying
                mean_sigma_b = sum(sigma_b) / len(sigma_b)
                mean_sigma_m = sum(sigma_m) / len(sigma_m)
                logger.info("Sigma B: %f", mean_sigma_b)
                logger.info("Sigma M: %f", mean_sigma_m)
            else:
                logger.info("Sigma B: %f", sigma_b)
                logger.info("Sigma M: %f", sigma_m)

        # Write the parameters
        Command.start(f"Writing experiments to {params.output}")
        experiments.as_file(params.output)
        Command.end(f"Wrote experiments to {params.output}")
Exemple #27
0
    # save cbf file
    if self.params.dispatch.dump_strong:
      self.save_image(cspad_img, self.params, os.path.join(self.params.output.output_dir, "hit-" + s))

      # save strong reflections.  self.find_spots() would have done this, but we only
      # want to save data if it is enough to try and index it
      if self.strong_filename_template:
        if "%s" in self.strong_filename_template:
          strong_filename = self.strong_filename_template%("hit-" + s)
        else:
          strong_filename = self.strong_filename_template
        strong_filename = os.path.join(self.params.output.output_dir, strong_filename)

        from dials.util.command_line import Command
        Command.start('Saving {0} reflections to {1}'.format(
            len(observed), os.path.basename(strong_filename)))
        observed.as_pickle(strong_filename)
        Command.end('Saved {0} observed to {1}'.format(
            len(observed), os.path.basename(strong_filename)))

    if not self.params.dispatch.index:
      self.debug_write("strong_shot_%d"%len(observed), "done")
      self.log_frame(None, None, run.run(), len(observed), timestamp)
      return

    # index and refine
    self.debug_write("index_start")
    try:
      experiments, indexed = self.index(datablock, observed)
    except Exception, e:
      import traceback; traceback.print_exc()
Exemple #28
0
    def __call__(self, sweep):
        """The main function of the spot finder. Select the pixels from
        the sweep and then group the pixels into spots. Return the data
        in the form of a reflection list.

        Params:
            sweep The sweep object

        Returns:
            The reflection list

        """
        from dials.util.command_line import Command

        # Set a command indent to 4
        Command.indent = 4
        print("\nFinding spot in {0} images...".format(len(sweep)))

        # Extract the image pixels from the sweep
        Command.start("Extracting pixels from sweep")
        coords, intensity = self._extract_pixels(sweep)
        Command.end("Extracted {0} strong pixels".format(len(coords)))

        # Label the pixels and group into spots
        Command.start("Labelling connected components")
        labels = self._label_pixels(coords, sweep)
        Command.end("Found {0} connected components".format(max(labels) + 1))

        # Filter spots that are too small
        Command.start("Filtering spots by size")
        spots = self._filter_spots(labels)
        Command.end("Filtered {0} spots by size".format(len(spots)))

        # Calculate the bounding box for each spot
        Command.start("Calculating bounding boxes")
        bbox = self._calculate_bbox(coords, spots, sweep)
        Command.end("Calculated {0} bounding boxes".format(len(bbox)))

        # Calculate the spot centroids
        Command.start("Calculating centroids")
        cpos, cvar = self._calculate_centroids(coords, intensity, spots)
        Command.end("Calculated {0} centroids".format(len(cpos)))

        # Filter the spots by centroid-maxmimum distance
        Command.start("Filtering spots by distance")
        index = self._filter_maximum_centroid(coords, intensity, spots, cpos)
        Command.end("Filtered {0} spots by distance".format(len(index)))

        # Create a reflection list and return
        return self._create_reflection_list(coords, intensity, spots, bbox,
                                            cpos, cvar, index)
sweep = load.sweep(os.path.join(path, 'sweep.json'))
crystal = load.crystal(os.path.join(path, 'crystal.json'))

extract = ReflectionExtractor(3)
refl = extract(sweep, crystal)


import copy

refl2 = copy.deepcopy(refl)


from dials.algorithms.reflection_basis import transform
from dials.util.command_line import Command

Command.start('Init transform')
trans = transform.Forward(sweep, crystal, 3, 4)
Command.end('Init transform')

Command.start('Transform')
for i in range(1):
  trans(refl)
Command.end('Transform')

Command.start('Init Transform')
spec = transform.TransformSpec(sweep, crystal, 3, 4)
Command.end('Init Transform')

Command.start('Transform')
for i in range(1):
  transform.forward_batch(spec, refl2)
Exemple #30
0
    def __call__(self, params, options):
        """Import the spot.xds file."""
        # Read the SPOT.XDS file
        Command.start("Reading SPOT.XDS")
        handle = spot_xds.reader()
        handle.read_file(self._spot_xds)
        centroid = handle.centroid
        intensity = handle.intensity
        try:
            miller_index = handle.miller_index
        except AttributeError:
            miller_index = None
        Command.end(f"Read {len(centroid)} spots from SPOT.XDS file.")

        # Create the reflection list
        Command.start("Creating reflection list")
        table = flex.reflection_table()
        table["id"] = flex.int(len(centroid), 0)
        table["panel"] = flex.size_t(len(centroid), 0)
        if miller_index:
            table["miller_index"] = flex.miller_index(miller_index)
        table["xyzobs.px.value"] = flex.vec3_double(centroid)
        table["intensity.sum.value"] = flex.double(intensity)
        Command.end("Created reflection list")

        # Remove invalid reflections
        Command.start("Removing invalid reflections")
        if miller_index and params.remove_invalid:
            flags = flex.bool([h != (0, 0, 0) for h in table["miller_index"]])
            table = table.select(flags)
        Command.end(f"Removed invalid reflections, {len(table)} remaining")

        # Fill empty standard columns
        if params.add_standard_columns:
            Command.start("Adding standard columns")
            rt = flex.reflection_table.empty_standard(len(table))
            rt.update(table)
            table = rt
            # set variances to unity
            table["xyzobs.mm.variance"] = flex.vec3_double(len(table), (1, 1, 1))
            table["xyzobs.px.variance"] = flex.vec3_double(len(table), (1, 1, 1))
            Command.end("Standard columns added")

        # Output the table to pickle file
        if params.output.filename is None:
            params.output.filename = "spot_xds.refl"
        Command.start(f"Saving reflection table to {params.output.filename}")
        table.as_file(params.output.filename)
        Command.end(f"Saved reflection table to {params.output.filename}")
Exemple #31
0
    def __call__(self, params, options):
        ''' Import the spot.xds file. '''
        from iotbx.xds import spot_xds
        from dials.util.command_line import Command
        from dials.array_family import flex

        # Read the SPOT.XDS file
        Command.start('Reading SPOT.XDS')
        handle = spot_xds.reader()
        handle.read_file(self._spot_xds)
        centroid = handle.centroid
        intensity = handle.intensity
        try:
            miller_index = handle.miller_index
        except AttributeError:
            miller_index = None
        Command.end('Read {0} spots from SPOT.XDS file.'.format(len(centroid)))

        # Create the reflection list
        Command.start('Creating reflection list')
        table = flex.reflection_table()
        table['id'] = flex.int(len(centroid), 0)
        table['panel'] = flex.size_t(len(centroid), 0)
        if miller_index:
            table['miller_index'] = flex.miller_index(miller_index)
        table['xyzobs.px.value'] = flex.vec3_double(centroid)
        table['intensity.sum.value'] = flex.double(intensity)
        Command.end('Created reflection list')

        # Remove invalid reflections
        Command.start('Removing invalid reflections')
        if miller_index and params.remove_invalid:
            flags = flex.bool([h != (0, 0, 0) for h in table['miller_index']])
            table = table.select(flags)
        Command.end('Removed invalid reflections, %d remaining' % len(table))

        # Fill empty standard columns
        if params.add_standard_columns:
            Command.start('Adding standard columns')
            rt = flex.reflection_table.empty_standard(len(table))
            rt.update(table)
            table = rt
            # set variances to unity
            table['xyzobs.mm.variance'] = flex.vec3_double(
                len(table), (1, 1, 1))
            table['xyzobs.px.variance'] = flex.vec3_double(
                len(table), (1, 1, 1))
            Command.end('Standard columns added')

        # Output the table to pickle file
        if params.output.filename is None:
            params.output.filename = 'spot_xds.pickle'
        Command.start('Saving reflection table to %s' % params.output.filename)
        table.as_pickle(params.output.filename)
        Command.end('Saved reflection table to %s' % params.output.filename)
  def __call__(self, observed, predicted):
    '''Calculate the divegence/mosaicity parameters.

    First match the observed reflections to the predicted reflections
    by finding the nearest neighbour based on the observed centroid
    and predicted bragg maximum. Then calculate the standard deviation
    of the beam divergence followed by the standard deviation of the
    mosaicity.

    The updated list of reflections can be accessed via the
    self.reflections member function.

    Params:
        observed The observed list of reflections
        predicted The predicted list of reflections

    Returns:
        sigma_d, sigma_m

    '''
    from math import pi
    from dials.util.command_line import Command

    # Setup the output
    Command.indent = 4
    print '\nCalculating e.s.d of beam divergence and mosaicity...'

    # Map observed to predicted reflections
    Command.start('Matching observed and predicted reflections.')
    self._data = self._match_observed_w_predicted(observed, predicted)
    Command.end('Matched {0} observed reflections.'.format(len(self._data)))

    # Calculate the standard deviation of the beam divergence
    Command.start('Calculating e.s.d of the beam divergence')
    sigma_d = self._calculate_esd_beam_divergence(self._data)
    Command.end('Calculated e.s.d of the beam divergence = {0:.3f} deg' \
        .format(sigma_d * 180 / pi))

    # Calculate the standard deviation of the reflecting range (mosaicity)
    Command.start('Calculating the mosaicity')
    sigma_m = self._calculate_esd_reflecting_range(self._data)
    Command.end('Calculated mosaicity = {0:.3f} deg'\
        .format(sigma_m * 180 / pi))

    # Return the parameters
    return sigma_d, sigma_m
Exemple #33
0
    def __call__(self, reflections):
        """Process the reflections.

        Params:
            reflections The reflections to process

        Returns:
            The list of integrated reflections

        """
        from dlstbx.algorithms.integration.fit_image import ImageSpaceProfileFitting
        from dlstbx.algorithms.integration.fit_image import Spec
        from dials.algorithms.integration.integrator import job_id
        from dials.array_family import flex
        from dials.util.command_line import Command

        # Get the flags
        flags = flex.reflection_table.flags

        # Create the algorithm
        algorithm = ImageSpaceProfileFitting(self._grid_size)

        # Add the specs
        for experiment, model in zip(self._experiments, self._profile_model):
            algorithm.add(
                Spec(
                    experiment.beam,
                    experiment.detector,
                    experiment.goniometer,
                    experiment.scan,
                    model.delta_b(deg=False),
                    model.delta_m(deg=False),
                )
            )

        # Perform the integration
        num = reflections.get_flags(flags.dont_integrate).count(False)
        Command.start("Integrating %d reflections with profile fitting" % num)
        profiles = algorithm.execute(reflections)

        # Print the number integrated
        num = reflections.get_flags(flags.integrated_prf).count(True)
        Command.end("Integrated %d reflections with profile fitting" % num)

        # Output the reference profiles
        if self._debug:
            import cPickle as pickle

            filename = "debug_%d.pickle" % job_id()
            print("Saving debugging information to %s" % filename)
            reference = [profiles.data(i) for i in range(len(profiles))]
            rprofiles = []
            for r in reflections:
                rprofiles.append(
                    profiles.get(
                        r["id"], r["panel"], r["s1"], r["xyzcal.mm"][2], r["bbox"]
                    )
                )
            output = {
                "reflections": reflections,
                "experiments": self._experiments,
                "profile_model": self._profile_model,
                "reference": reference,
                "profiles": rprofiles,
            }
            with open(filename, "wb") as outfile:
                pickle.dump(output, outfile, protocol=pickle.HIGHEST_PROTOCOL)

        # Return the reflections
        return reflections
Exemple #34
0
    import libtbx.load_env

    usage = "usage: %s [options] reflections1.pickle reflections2.pickle" \
      % libtbx.env.dispatcher_name
    parser = OptionParser(usage)

    # Parse the command line arguments
    (options, args) = parser.parse_args()

    # Ensure there are only two reflection lists
    if len(args) != 2:
        parser.print_help()
        exit(0)

    # Read the first batch of reflections
    Command.start('Reading reflections from %s' % args[0])
    refl1 = flex.reflection_table.from_pickle(args[0])
    mask = flex.bool(xyz == (0, 0, 0) for xyz in refl1['xyzobs.px.value'])
    refl1.del_selected(mask)
    Command.end('Read %d reflections from %s' % (len(refl1), args[0]))

    # Read the second batch of reflections
    Command.start('Reading reflections from %s' % args[1])
    refl2 = flex.reflection_table.from_pickle(args[1])
    mask = refl2['intensity.sum.value'] <= 0.0
    refl2.del_selected(mask)
    mask = refl2['intensity.sum.value']**2 < refl2['intensity.sum.variance']
    refl2.del_selected(mask)
    Command.end('Read %d reflections from %s' % (len(refl2), args[1]))

    # perform the match
Exemple #35
0
  def __call__(self, rlist):
    ''' Analyse the reference profiles. '''
    from dials.util.command_line import Command

    # Check we have the required fields
    print "Analysing reference profiles"
    if not ensure_required(rlist, self.required):
      return

    # Select only integrated reflections
    Command.start(" Selecting only integated reflections")
    mask = rlist.get_flags(rlist.flags.integrated)
    if mask.count(True) == 0:
      return

    rlist = rlist.select(mask)
    Command.end(" Selected %d integrated reflections" % len(rlist))

    # Analyse distribution of reference spots
    print " Analysing reference profile distribution vs x/y"
    self.reference_xy(rlist)
    print " Analysing reference profile distribution vs z"
    self.reference_z(rlist)

    # Look at correlations between profiles
    def ideal_correlations(filename, rlist):
      ''' Call for reference spots and all reflections. '''

      print " Analysing reflection profile correlations"
      self.ideal_reflection_corr_hist(rlist, filename)

      print " Analysing reflection profile correlations vs x/y"
      self.ideal_reflection_corr_vs_xy(rlist, filename)

      print " Analysing reflection profile correlations vs z"
      self.ideal_reflection_corr_vs_z(rlist, filename)

      print " Analysing reflection profile correlations vs I/Sigma"
      self.ideal_reflection_corr_vs_ios(rlist, filename)

    # Look at correlations between profiles
    def correlations(filename, rlist):
      ''' Call for reference spots and all reflections. '''

      print " Analysing reflection profile correlations"
      self.reflection_corr_hist(rlist, filename)

      print " Analysing reflection profile correlations vs x/y"
      self.reflection_corr_vs_xy(rlist, filename)

      print " Analysing reflection profile correlations vs z"
      self.reflection_corr_vs_z(rlist, filename)

      print " Analysing reflection profile correlations vs I/Sigma"
      self.reflection_corr_vs_ios(rlist, filename)

    mask = rlist.get_flags(rlist.flags.reference_spot)
    correlations("reference",  rlist.select(mask))
    correlations("reflection", rlist)
    ideal_correlations("reference", rlist.select(mask))
    ideal_correlations("reflection", rlist)
sweep = load.sweep(os.path.join(path, "sweep.json"))
crystal = load.crystal(os.path.join(path, "crystal.json"))

extract = ReflectionExtractor(3)
refl = extract(sweep, crystal)


import copy

refl2 = copy.deepcopy(refl)


from dials.algorithms.reflection_basis import transform
from dials.util.command_line import Command

Command.start("Init transform")
trans = transform.Forward(sweep, crystal, 3, 4)
Command.end("Init transform")

Command.start("Transform")
for i in range(1):
    trans(refl)
Command.end("Transform")

Command.start("Init Transform")
spec = transform.TransformSpec(sweep, crystal, 3, 4)
Command.end("Init Transform")

Command.start("Transform")
for i in range(1):
    transform.forward_batch(spec, refl2)
  import libtbx.load_env

  usage = "usage: %s [options] reflections1.pickle reflections2.pickle" \
    % libtbx.env.dispatcher_name
  parser = OptionParser(usage)

  # Parse the command line arguments
  (options, args) = parser.parse_args()

  # Ensure there are only two reflection lists
  if len(args) != 2:
    parser.print_help()
    exit(0)

  # Read the first batch of reflections
  Command.start('Reading reflections from %s' % args[0])
  refl1 = flex.reflection_table.from_pickle(args[0])
  mask = flex.bool(xyz == (0, 0, 0) for xyz in refl1['xyzobs.px.value'])
  refl1.del_selected(mask)
  Command.end('Read %d reflections from %s' % (len(refl1), args[0]))

  # Read the second batch of reflections
  Command.start('Reading reflections from %s' % args[1])
  refl2 = flex.reflection_table.from_pickle(args[1])
  mask = refl2['intensity.sum.value'] <= 0.0
  refl2.del_selected(mask)
  mask = refl2['intensity.sum.value']**2 < refl2['intensity.sum.variance']
  refl2.del_selected(mask)
  Command.end('Read %d reflections from %s' % (len(refl2), args[1]))

  # perform the match
Exemple #38
0
  def __call__(self, params, options):
    ''' Import the spot.xds file. '''
    from iotbx.xds import spot_xds
    from dials.util.command_line import Command
    from dials.array_family import flex

    # Read the SPOT.XDS file
    Command.start('Reading SPOT.XDS')
    handle = spot_xds.reader()
    handle.read_file(self._spot_xds)
    centroid = handle.centroid
    intensity = handle.intensity
    try:
      miller_index = handle.miller_index
    except AttributeError:
      miller_index = None
    Command.end('Read {0} spots from SPOT.XDS file.'.format(len(centroid)))

    # Create the reflection list
    Command.start('Creating reflection list')
    table = flex.reflection_table()
    table['id'] = flex.int(len(centroid), 0)
    table['panel'] = flex.size_t(len(centroid), 0)
    if miller_index:
      table['miller_index'] = flex.miller_index(miller_index)
    table['xyzobs.px.value'] = flex.vec3_double(centroid)
    table['intensity.sum.value'] = flex.double(intensity)
    Command.end('Created reflection list')

    # Remove invalid reflections
    Command.start('Removing invalid reflections')
    if miller_index and params.remove_invalid:
      flags = flex.bool([h != (0, 0, 0) for h in table['miller_index']])
      table = table.select(flags)
    Command.end('Removed invalid reflections, %d remaining' % len(table))

    # Fill empty standard columns
    if params.add_standard_columns:
      Command.start('Adding standard columns')
      rt = flex.reflection_table.empty_standard(len(table))
      rt.update(table)
      table = rt
      # set variances to unity
      table['xyzobs.mm.variance'] = flex.vec3_double(len(table), (1,1,1))
      table['xyzobs.px.variance'] = flex.vec3_double(len(table), (1,1,1))
      Command.end('Standard columns added')

    # Output the table to pickle file
    if params.output.filename is None:
      params.output.filename = 'spot_xds.pickle'
    Command.start('Saving reflection table to %s' % params.output.filename)
    table.as_pickle(params.output.filename)
    Command.end('Saved reflection table to %s' % params.output.filename)
Exemple #39
0
    def generate_predictions(self, N):
        """ Generate some reflections. """
        from dials.algorithms.profile_model.gaussian_rs import MaskCalculator3D
        from dials.array_family import flex
        from dials.util.command_line import Command
        from dials.algorithms import filtering
        from dials.algorithms.shoebox import MaskCode
        from dials.algorithms.profile_model.gaussian_rs import Model as ProfileModel
        import random

        # Set the profile model
        self.experiment.profile = ProfileModel(None, self.n_sigma, self.sigma_b, self.sigma_m)

        # Generate a list of reflections
        refl = flex.reflection_table.from_predictions(self.experiment)
        refl["id"] = flex.int(len(refl), 0)

        # Filter by zeta
        zeta = 0.05
        Command.start("Filtering by zeta >= %f" % zeta)
        mask = filtering.by_zeta(self.experiment.goniometer, self.experiment.beam, refl["s1"], zeta)
        refl.del_selected(mask != True)
        Command.end("Filtered %d reflections by zeta >= %f" % (len(refl), zeta))

        # Compute the bounding box
        refl.compute_bbox([self.experiment])
        index = []
        image_size = self.experiment.detector[0].get_image_size()
        array_range = self.experiment.scan.get_array_range()
        bbox = refl["bbox"]
        for i in range(len(refl)):
            x0, x1, y0, y1, z0, z1 = bbox[i]
            if (
                x0 < 0
                or x1 > image_size[0]
                or y0 < 0
                or y1 > image_size[1]
                or z0 < array_range[0]
                or z1 > array_range[1]
            ):
                index.append(i)
        refl.del_selected(flex.size_t(index))

        # Sample if specified
        index = random.sample(range(len(refl)), N)
        refl = refl.select(flex.size_t(index))

        # Compute the bounding box
        # Create a load of shoeboxes
        Command.start("Creating shoeboxes for %d reflections" % len(refl))
        refl["shoebox"] = flex.shoebox(refl["panel"], refl["bbox"])
        refl["shoebox"].allocate_with_value(MaskCode.Valid)
        Command.end("Created shoeboxes for %d reflections" % len(refl))

        # Get the function object to mask the foreground
        Command.start("Masking Foreground for %d reflections" % len(refl))
        mask_foreground = MaskCalculator3D(
            self.experiment.beam,
            self.experiment.detector,
            self.experiment.goniometer,
            self.experiment.scan,
            self.n_sigma * self.sigma_b,
            self.n_sigma * self.sigma_m,
        )

        # Mask the foreground
        mask_foreground(refl["shoebox"], refl["s1"], refl["xyzcal.px"].parts()[2], refl["panel"])
        Command.end("Masked foreground for %d reflections" % len(refl))

        # Return the reflections
        return refl
Exemple #40
0
                cspad_img, self.params,
                os.path.join(self.params.output.output_dir, "hit-" + s))

            # save strong reflections.  self.find_spots() would have done this, but we only
            # want to save data if it is enough to try and index it
            if self.strong_filename_template:
                if "%s" in self.strong_filename_template:
                    strong_filename = self.strong_filename_template % ("hit-" +
                                                                       s)
                else:
                    strong_filename = self.strong_filename_template
                strong_filename = os.path.join(self.params.output.output_dir,
                                               strong_filename)

                from dials.util.command_line import Command
                Command.start('Saving {0} reflections to {1}'.format(
                    len(observed), os.path.basename(strong_filename)))
                observed.as_pickle(strong_filename)
                Command.end('Saved {0} observed to {1}'.format(
                    len(observed), os.path.basename(strong_filename)))

        if not self.params.dispatch.index:
            self.debug_write("strong_shot_%d" % len(observed), "done")
            self.log_frame(None, None, run.run(), len(observed), timestamp)
            return

        # index and refine
        self.debug_write("index_start")
        try:
            experiments, indexed = self.index(datablock, observed)
        except Exception, e:
            import traceback
Exemple #41
0
    def __call__(self, params, options):
        ''' Import the integrate.hkl file. '''

        from iotbx.xds import integrate_hkl
        from dials.array_family import flex
        from dials.util.command_line import Command
        from cctbx import sgtbx

        # Get the unit cell to calculate the resolution
        uc = self._experiment.crystal.get_unit_cell()

        # Read the INTEGRATE.HKL file
        Command.start('Reading INTEGRATE.HKL')
        handle = integrate_hkl.reader()
        handle.read_file(self._integrate_hkl)
        hkl = flex.miller_index(handle.hkl)
        xyzcal = flex.vec3_double(handle.xyzcal)
        xyzobs = flex.vec3_double(handle.xyzobs)
        iobs = flex.double(handle.iobs)
        sigma = flex.double(handle.sigma)
        rlp = flex.double(handle.rlp)
        peak = flex.double(handle.peak) * 0.01
        if len(handle.iseg):
            panel = flex.size_t(handle.iseg) - 1
        else:
            panel = flex.size_t(len(hkl), 0)
        Command.end('Read %d reflections from INTEGRATE.HKL file.' % len(hkl))

        if len(self._experiment.detector) > 1:
            for p_id, p in enumerate(self._experiment.detector):
                sel = (panel == p_id)
                offset = p.get_raw_image_offset()
                xyzcal.set_selected(
                    sel,
                    xyzcal.select(sel) - (offset[0], offset[1], 0))
                xyzobs.set_selected(
                    sel,
                    xyzobs.select(sel) - (offset[0], offset[1], 0))

        # Derive the reindex matrix
        rdx = self.derive_reindex_matrix(handle)
        print('Reindex matrix:\n%d %d %d\n%d %d %d\n%d %d %d' % (rdx.elems))

        # Reindex the reflections
        Command.start('Reindexing reflections')
        cb_op = sgtbx.change_of_basis_op(sgtbx.rt_mx(sgtbx.rot_mx(rdx.elems)))
        hkl = cb_op.apply(hkl)
        Command.end('Reindexed %d reflections' % len(hkl))

        # Create the reflection list
        Command.start('Creating reflection table')
        table = flex.reflection_table()
        table['id'] = flex.int(len(hkl), 0)
        table['panel'] = panel
        table['miller_index'] = hkl
        table['xyzcal.px'] = xyzcal
        table['xyzobs.px.value'] = xyzobs
        table['intensity.cor.value'] = iobs
        table['intensity.cor.variance'] = sigma**2
        table['intensity.prf.value'] = iobs * peak / rlp
        table['intensity.prf.variance'] = (sigma * peak / rlp)**2
        table['lp'] = 1.0 / rlp
        table['d'] = flex.double(uc.d(h) for h in hkl)
        Command.end('Created table with {0} reflections'.format(len(table)))

        # Output the table to pickle file
        if params.output.filename is None:
            params.output.filename = 'integrate_hkl.pickle'
        Command.start('Saving reflection table to %s' % params.output.filename)
        table.as_pickle(params.output.filename)
        Command.end('Saved reflection table to %s' % params.output.filename)
Exemple #42
0
  def run(self):
    ''' Run the script. '''
    from dials.algorithms.profile_model.factory import ProfileModelFactory
    from dials.util.command_line import Command
    from dials.array_family import flex
    from dials.util.options import flatten_reflections, flatten_experiments
    from dxtbx.model.experiment.experiment_list import ExperimentListDumper
    from libtbx.utils import Sorry
    from dials.util import log

    log.config()

    # Parse the command line
    params, options = self.parser.parse_args(show_diff_phil=True)
    reflections = flatten_reflections(params.input.reflections)
    experiments = flatten_experiments(params.input.experiments)
    if len(reflections) == 0 and len(experiments) == 0:
      self.parser.print_help()
      return
    if len(reflections) != 1:
      raise Sorry('exactly 1 reflection table must be specified')
    if len(experiments) == 0:
      raise Sorry('no experiments were specified')
    if (not 'background.mean' in reflections[0]) and params.subtract_background:
      raise Sorry('for subtract_background need background.mean in reflections')

    reflections, _ = self.process_reference(reflections[0], params)

    # Predict the reflections
    logger.info("")
    logger.info("=" * 80)
    logger.info("")
    logger.info("Predicting reflections")
    logger.info("")
    predicted = flex.reflection_table.from_predictions_multi(
      experiments,
      dmin=params.prediction.d_min,
      dmax=params.prediction.d_max,
      margin=params.prediction.margin,
      force_static=params.prediction.force_static)

    # Match with predicted
    matched, reflections, unmatched = predicted.match_with_reference(reflections)
    assert(len(matched) == len(predicted))
    assert(matched.count(True) <= len(reflections))
    if matched.count(True) == 0:
      raise Sorry('''
        Invalid input for reference reflections.
        Zero reference spots were matched to predictions
      ''')
    elif len(unmatched) != 0:
      logger.info('')
      logger.info('*' * 80)
      logger.info('Warning: %d reference spots were not matched to predictions' % (
        len(unmatched)))
      logger.info('*' * 80)
      logger.info('')

    # Create the profile model
    experiments = ProfileModelFactory.create(params, experiments, reflections)
    for model in experiments:
      sigma_b = model.profile.sigma_b(deg=True)
      sigma_m = model.profile.sigma_m(deg=True)
      if type(sigma_b) == type(1.0):
        logger.info('Sigma B: %f' % sigma_b)
        logger.info('Sigma M: %f' % sigma_m)
      else: # scan varying
        mean_sigma_b = sum(sigma_b) / len(sigma_b)
        mean_sigma_m = sum(sigma_m) / len(sigma_m)
        logger.info('Sigma B: %f' % mean_sigma_b)
        logger.info('Sigma M: %f' % mean_sigma_m)

    # Wrtie the parameters
    Command.start("Writing experiments to %s" % params.output)
    dump = ExperimentListDumper(experiments)
    with open(params.output, "w") as outfile:
      outfile.write(dump.as_json())
    Command.end("Wrote experiments to %s" % params.output)
Exemple #43
0
    def __call__(self, params, options):
        """Import the integrate.hkl file."""
        # Get the unit cell to calculate the resolution
        uc = self._experiment.crystal.get_unit_cell()

        # Read the INTEGRATE.HKL file
        Command.start("Reading INTEGRATE.HKL")
        handle = integrate_hkl.reader()
        handle.read_file(self._integrate_hkl)
        hkl = flex.miller_index(handle.hkl)
        xyzcal = flex.vec3_double(handle.xyzcal)
        xyzobs = flex.vec3_double(handle.xyzobs)
        iobs = flex.double(handle.iobs)
        sigma = flex.double(handle.sigma)
        rlp = flex.double(handle.rlp)
        peak = flex.double(handle.peak) * 0.01
        if len(handle.iseg):
            panel = flex.size_t(handle.iseg) - 1
        else:
            panel = flex.size_t(len(hkl), 0)
        Command.end(f"Read {len(hkl)} reflections from INTEGRATE.HKL file.")

        if len(self._experiment.detector) > 1:
            for p_id, p in enumerate(self._experiment.detector):
                sel = panel == p_id
                offset = p.get_raw_image_offset()
                xyzcal.set_selected(sel, xyzcal.select(sel) - (offset[0], offset[1], 0))
                xyzobs.set_selected(sel, xyzobs.select(sel) - (offset[0], offset[1], 0))

        # Derive the reindex matrix
        rdx = self.derive_reindex_matrix(handle)
        print("Reindex matrix:\n%d %d %d\n%d %d %d\n%d %d %d" % (rdx.elems))

        # Reindex the reflections
        Command.start("Reindexing reflections")
        cb_op = sgtbx.change_of_basis_op(sgtbx.rt_mx(sgtbx.rot_mx(rdx.elems)))
        hkl = cb_op.apply(hkl)
        Command.end(f"Reindexed {len(hkl)} reflections")

        # Create the reflection list
        Command.start("Creating reflection table")
        table = flex.reflection_table()
        table["id"] = flex.int(len(hkl), 0)
        table["panel"] = panel
        table["miller_index"] = hkl
        table["xyzcal.px"] = xyzcal
        table["xyzobs.px.value"] = xyzobs
        table["intensity.cor.value"] = iobs
        table["intensity.cor.variance"] = flex.pow2(sigma)
        table["intensity.prf.value"] = iobs * peak / rlp
        table["intensity.prf.variance"] = flex.pow2(sigma * peak / rlp)
        table["lp"] = 1.0 / rlp
        table["d"] = flex.double(uc.d(h) for h in hkl)
        Command.end(f"Created table with {len(table)} reflections")

        # Output the table to pickle file
        if params.output.filename is None:
            params.output.filename = "integrate_hkl.refl"
        Command.start(f"Saving reflection table to {params.output.filename}")
        table.as_file(params.output.filename)
        Command.end(f"Saved reflection table to {params.output.filename}")
Exemple #44
0
  def __call__(self, params, options):
    ''' Import the integrate.hkl file. '''

    from iotbx.xds import integrate_hkl
    from dials.array_family import flex
    from dials.util.command_line import Command
    from cctbx import sgtbx

    # Get the unit cell to calculate the resolution
    uc = self._experiment.crystal.get_unit_cell()

    # Read the INTEGRATE.HKL file
    Command.start('Reading INTEGRATE.HKL')
    handle = integrate_hkl.reader()
    handle.read_file(self._integrate_hkl)
    hkl    = flex.miller_index(handle.hkl)
    xyzcal = flex.vec3_double(handle.xyzcal)
    xyzobs = flex.vec3_double(handle.xyzobs)
    iobs   = flex.double(handle.iobs)
    sigma  = flex.double(handle.sigma)
    rlp = flex.double(handle.rlp)
    peak = flex.double(handle.peak) * 0.01
    if len(handle.iseg):
      panel = flex.size_t(handle.iseg) - 1
    else:
      panel = flex.size_t(len(hkl), 0)
    Command.end('Read %d reflections from INTEGRATE.HKL file.' % len(hkl))

    if len(self._experiment.detector) > 1:
      for p_id, p in enumerate(self._experiment.detector):
        sel = (panel == p_id)
        offset = p.get_raw_image_offset()
        xyzcal.set_selected(sel, xyzcal.select(sel) - (offset[0], offset[1], 0))
        xyzobs.set_selected(sel, xyzobs.select(sel) - (offset[0], offset[1], 0))

    # Derive the reindex matrix
    rdx = self.derive_reindex_matrix(handle)
    print 'Reindex matrix:\n%d %d %d\n%d %d %d\n%d %d %d' % (rdx.elems)

    # Reindex the reflections
    Command.start('Reindexing reflections')
    cb_op = sgtbx.change_of_basis_op(sgtbx.rt_mx(sgtbx.rot_mx(rdx.elems)))
    hkl = cb_op.apply(hkl)
    Command.end('Reindexed %d reflections' % len(hkl))

    # Create the reflection list
    Command.start('Creating reflection table')
    table = flex.reflection_table()
    table['id'] = flex.int(len(hkl), 0)
    table['panel'] = panel
    table['miller_index'] = hkl
    table['xyzcal.px'] = xyzcal
    table['xyzobs.px.value'] = xyzobs
    table['intensity.cor.value'] = iobs
    table['intensity.cor.variance'] = sigma**2
    table['intensity.prf.value'] = iobs * peak / rlp
    table['intensity.prf.variance'] = (sigma * peak / rlp)**2
    table['lp'] = 1.0 / rlp
    table['d'] = flex.double(uc.d(h) for h in hkl)
    Command.end('Created table with {0} reflections'.format(len(table)))

    # Output the table to pickle file
    if params.output.filename is None:
      params.output.filename = 'integrate_hkl.pickle'
    Command.start('Saving reflection table to %s' % params.output.filename)
    table.as_pickle(params.output.filename)
    Command.end('Saved reflection table to %s' % params.output.filename)
Exemple #45
0
    def generate_predictions(self, N):
        """Generate some reflections."""
        from dials.algorithms import filtering
        from dials.algorithms.profile_model.gaussian_rs import MaskCalculator3D
        from dials.algorithms.profile_model.gaussian_rs import Model as ProfileModel
        from dials.algorithms.shoebox import MaskCode
        from dials.util.command_line import Command

        # Set the profile model
        self.experiment.profile = ProfileModel(None, self.n_sigma,
                                               self.sigma_b, self.sigma_m)

        # Generate a list of reflections
        refl = flex.reflection_table.from_predictions(self.experiment)
        refl["id"] = flex.int(len(refl), 0)

        # Filter by zeta
        zeta = 0.05
        Command.start(f"Filtering by zeta >= {zeta:f}")
        mask = filtering.by_zeta(self.experiment.goniometer,
                                 self.experiment.beam, refl["s1"], zeta)
        refl.del_selected(~mask)
        Command.end(f"Filtered {len(refl)} reflections by zeta >= {zeta:f}")

        # Compute the bounding box
        refl.compute_bbox([self.experiment])
        index = []
        image_size = self.experiment.detector[0].get_image_size()
        array_range = self.experiment.scan.get_array_range()
        bbox = refl["bbox"]
        for i in range(len(refl)):
            x0, x1, y0, y1, z0, z1 = bbox[i]
            if (x0 < 0 or x1 > image_size[0] or y0 < 0 or y1 > image_size[1]
                    or z0 < array_range[0] or z1 > array_range[1]):
                index.append(i)
        refl.del_selected(flex.size_t(index))

        # Sample if specified
        index = random.sample(range(len(refl)), N)
        refl = refl.select(flex.size_t(index))

        # Compute the bounding box
        # Create a load of shoeboxes
        Command.start(f"Creating shoeboxes for {len(refl)} reflections")
        refl["shoebox"] = flex.shoebox(refl["panel"], refl["bbox"])
        refl["shoebox"].allocate_with_value(MaskCode.Valid)
        Command.end(f"Created shoeboxes for {len(refl)} reflections")

        # Get the function object to mask the foreground
        Command.start(f"Masking Foreground for {len(refl)} reflections")
        mask_foreground = MaskCalculator3D(
            self.experiment.beam,
            self.experiment.detector,
            self.experiment.goniometer,
            self.experiment.scan,
            self.n_sigma * self.sigma_b,
            self.n_sigma * self.sigma_m,
        )

        # Mask the foreground
        mask_foreground(refl["shoebox"], refl["s1"],
                        refl["xyzcal.px"].parts()[2], refl["panel"])
        Command.end(f"Masked foreground for {len(refl)} reflections")

        # Return the reflections
        return refl
Exemple #46
0
  def __call__(self, sweep):
    '''The main function of the spot finder. Select the pixels from
    the sweep and then group the pixels into spots. Return the data
    in the form of a reflection list.

    Params:
        sweep The sweep object

    Returns:
        The reflection list

    '''
    from dials.util.command_line import Command

    # Set a command indent to 4
    Command.indent = 4
    print '\nFinding spot in {0} images...'.format(len(sweep))

    # Extract the image pixels from the sweep
    Command.start('Extracting pixels from sweep')
    coords, intensity = self._extract_pixels(sweep)
    Command.end('Extracted {0} strong pixels'.format(len(coords)))

    # Label the pixels and group into spots
    Command.start('Labelling connected components')
    labels = self._label_pixels(coords, sweep)
    Command.end('Found {0} connected components'.format(max(labels) + 1))

    # Filter spots that are too small
    Command.start('Filtering spots by size')
    spots = self._filter_spots(labels)
    Command.end('Filtered {0} spots by size'.format(len(spots)))

    # Calculate the bounding box for each spot
    Command.start('Calculating bounding boxes')
    bbox = self._calculate_bbox(coords, spots, sweep)
    Command.end('Calculated {0} bounding boxes'.format(len(bbox)))

    # Calculate the spot centroids
    Command.start('Calculating centroids')
    cpos, cvar = self._calculate_centroids(coords, intensity, spots)
    Command.end('Calculated {0} centroids'.format(len(cpos)))

    # Filter the spots by centroid-maxmimum distance
    Command.start('Filtering spots by distance')
    index = self._filter_maximum_centroid(coords, intensity, spots, cpos)
    Command.end('Filtered {0} spots by distance'.format(len(index)))

    # Create a reflection list and return
    return self._create_reflection_list(
        coords, intensity, spots, bbox, cpos, cvar, index)