コード例 #1
0
def run(args):
    phil = iotbx.phil.process_command_line(args=args,
                                           master_string=master_phil).show()
    work_params = phil.work.extract()
    from xfel.merging.phil_validation import application, samosa
    application(work_params)
    samosa(work_params)
    if ("--help" in args):
        libtbx.phil.parse(master_phil.show())
        return

    if ((work_params.d_min is None) or (work_params.data is None)):
        command_name = os.environ["LIBTBX_DISPATCHER_NAME"]
        raise Usage(command_name + " "
                    "d_min=4.0 "
                    "data=~/scratch/r0220/006/strong/ "
                    "model=3bz1_3bz2_core.pdb")
    if ((work_params.rescale_with_average_cell)
            and (not work_params.set_average_unit_cell)):
        raise Usage(
            "If rescale_with_average_cell=True, you must also specify " +
            "set_average_unit_cell=True.")
    if work_params.raw_data.sdfac_auto and work_params.raw_data.sdfac_refine:
        raise Usage("Cannot specify both sdfac_auto and sdfac_refine")

    # Read Nat's reference model from an MTZ file.  XXX The observation
    # type is given as F, not I--should they be squared?  Check with Nat!
    log = open("%s.log" % work_params.output.prefix, "w")
    out = multi_out()
    out.register("log", log, atexit_send_to=None)
    out.register("stdout", sys.stdout)
    print >> out, "I model"
    if work_params.model is not None:
        from xfel.merging.general_fcalc import run
        i_model = run(work_params)
        work_params.target_unit_cell = i_model.unit_cell()
        work_params.target_space_group = i_model.space_group_info()
        i_model.show_summary()
    else:
        i_model = None

    print >> out, "Target unit cell and space group:"
    print >> out, "  ", work_params.target_unit_cell
    print >> out, "  ", work_params.target_space_group

    miller_set, i_model = consistent_set_and_model(work_params, i_model)

    frame_files = get_observations(work_params)
    scaler = scaling_manager(miller_set=miller_set,
                             i_model=i_model,
                             params=work_params,
                             log=out)
    scaler.scale_all(frame_files)
    if scaler.n_accepted == 0:
        return None
    scaler.show_unit_cell_histograms()
    if (work_params.rescale_with_average_cell):
        average_cell_abc = scaler.uc_values.get_average_cell_dimensions()
        average_cell = uctbx.unit_cell(
            list(average_cell_abc) +
            list(work_params.target_unit_cell.parameters()[3:]))
        work_params.target_unit_cell = average_cell
        print >> out, ""
        print >> out, "#" * 80
        print >> out, "RESCALING WITH NEW TARGET CELL"
        print >> out, "  average cell: %g %g %g %g %g %g" % \
          work_params.target_unit_cell.parameters()
        print >> out, ""
        scaler.reset()
        scaler.scale_all(frame_files)
        scaler.show_unit_cell_histograms()
    if False:  #(work_params.output.show_plots) :
        try:
            plot_overall_completeness(completeness)
        except Exception as e:
            print "ERROR: can't show plots"
            print "  %s" % str(e)
    print >> out, "\n"

    # Sum the observations of I and I/sig(I) for each reflection.
    sum_I = flex.double(miller_set.size(), 0.)
    sum_I_SIGI = flex.double(miller_set.size(), 0.)
    for i in xrange(miller_set.size()):
        index = miller_set.indices()[i]
        if index in scaler.ISIGI:
            for t in scaler.ISIGI[index]:
                sum_I[i] += t[0]
                sum_I_SIGI[i] += t[1]

    miller_set_avg = miller_set.customized_copy(
        unit_cell=work_params.target_unit_cell)
    table1 = show_overall_observations(obs=miller_set_avg,
                                       redundancy=scaler.completeness,
                                       summed_wt_I=scaler.summed_wt_I,
                                       summed_weight=scaler.summed_weight,
                                       ISIGI=scaler.ISIGI,
                                       n_bins=work_params.output.n_bins,
                                       title="Statistics for all reflections",
                                       out=out,
                                       work_params=work_params)
    print >> out, ""
    n_refl, corr = ((scaler.completeness > 0).count(True), 0)
    print >> out, "\n"
    table2 = show_overall_observations(
        obs=miller_set_avg,
        redundancy=scaler.summed_N,
        summed_wt_I=scaler.summed_wt_I,
        summed_weight=scaler.summed_weight,
        ISIGI=scaler.ISIGI,
        n_bins=work_params.output.n_bins,
        title="Statistics for reflections where I > 0",
        out=out,
        work_params=work_params)
    #from libtbx import easy_pickle
    #easy_pickle.dump(file_name="stats.pickle", obj=stats)
    #stats.report(plot=work_params.plot)
    #miller_counts = miller_set_p1.array(data=stats.counts.as_double()).select(
    #  stats.counts != 0)
    #miller_counts.as_mtz_dataset(column_root_label="NOBS").mtz_object().write(
    #  file_name="nobs.mtz")
    if work_params.data_subsubsets.subsubset is not None and work_params.data_subsubsets.subsubset_total is not None:
        easy_pickle.dump(
            "scaler_%d.pickle" % work_params.data_subsubsets.subsubset, scaler)
    print >> out, ""
    mtz_file, miller_array = scaler.finalize_and_save_data()
    #table_pickle_file = "%s_graphs.pkl" % work_params.output.prefix
    #easy_pickle.dump(table_pickle_file, [table1, table2])
    loggraph_file = os.path.abspath("%s_graphs.log" %
                                    work_params.output.prefix)
    f = open(loggraph_file, "w")
    f.write(table1.format_loggraph())
    f.write("\n")
    f.write(table2.format_loggraph())
    f.close()
    result = scaling_result(miller_array=miller_array,
                            plots=scaler.get_plot_statistics(),
                            mtz_file=mtz_file,
                            loggraph_file=loggraph_file,
                            obs_table=table1,
                            all_obs_table=table2,
                            n_reflections=n_refl,
                            overall_correlation=corr)
    easy_pickle.dump("%s.pkl" % work_params.output.prefix, result)
    return result
コード例 #2
0
ファイル: cxi_xmerge.py プロジェクト: cctbx/cctbx-playground
def run(args):
  phil = iotbx.phil.process_command_line(args=args, master_string=master_phil).show()
  work_params = phil.work.extract()
  from xfel.merging.phil_validation import application
  application(work_params)
  if ("--help" in args) :
    libtbx.phil.parse(master_phil.show())
    return

  if ((work_params.d_min is None) or
      (work_params.data is None) or
      ( (work_params.model is None) and work_params.scaling.algorithm != "mark1") ) :
    raise Usage("cxi.merge "
                "d_min=4.0 "
                "data=~/scratch/r0220/006/strong/ "
                "model=3bz1_3bz2_core.pdb")
  if ((work_params.rescale_with_average_cell) and
      (not work_params.set_average_unit_cell)) :
    raise Usage("If rescale_with_average_cell=True, you must also specify "+
      "set_average_unit_cell=True.")
  if work_params.raw_data.sdfac_auto and work_params.raw_data.sdfac_refine:
    raise Usage("Cannot specify both sdfac_auto and sdfac_refine")

  log = open("%s_%s.log" % (work_params.output.prefix,work_params.scaling.algorithm), "w")
  out = multi_out()
  out.register("log", log, atexit_send_to=None)
  out.register("stdout", sys.stdout)

  # Verify that the externally supplied isomorphous reference, if
  # present, defines a suitable column of intensities, and exit with
  # error if it does not.  Then warn if it is necessary to generate
  # Bijvoet mates.  Failure to catch these issues here would lead to
  # possibly obscure problems in cxi/cxi_cc.py later on.
  try:
    data_SR = mtz.object(work_params.scaling.mtz_file)
  except RuntimeError:
    pass
  else:
    array_SR = None
    obs_labels = []
    for array in data_SR.as_miller_arrays():
      this_label = array.info().label_string().lower()
      if array.observation_type() is not None:
        obs_labels.append(this_label.split(',')[0])
      if this_label.find('fobs')>=0:
        array_SR = array.as_intensity_array()
        break
      if this_label.find('imean')>=0:
        array_SR = array.as_intensity_array()
        break
      if this_label.find(work_params.scaling.mtz_column_F)==0:
        array_SR = array.as_intensity_array()
        break

    if array_SR is None:
      known_labels = ['fobs', 'imean', work_params.scaling.mtz_column_F]
      raise Usage(work_params.scaling.mtz_file +
                  " does not contain any observations labelled [" +
                  ", ".join(known_labels) +
                  "].  Please set scaling.mtz_column_F to one of [" +
                  ",".join(obs_labels) + "].")
    elif not work_params.merge_anomalous and not array_SR.anomalous_flag():
      print >> out, "Warning: Preserving anomalous contributors, but %s " \
        "has anomalous contributors merged.  Generating identical Bijvoet " \
        "mates." % work_params.scaling.mtz_file

  # Read Nat's reference model from an MTZ file.  XXX The observation
  # type is given as F, not I--should they be squared?  Check with Nat!
  print >> out, "I model"
  if work_params.model is not None:
    from xfel.merging.general_fcalc import run
    i_model = run(work_params)
    work_params.target_unit_cell = i_model.unit_cell()
    work_params.target_space_group = i_model.space_group_info()
    i_model.show_summary()
  else:
    i_model = None

  print >> out, "Target unit cell and space group:"
  print >> out, "  ", work_params.target_unit_cell
  print >> out, "  ", work_params.target_space_group

  miller_set = symmetry(
      unit_cell=work_params.target_unit_cell,
      space_group_info=work_params.target_space_group
    ).build_miller_set(
      anomalous_flag=not work_params.merge_anomalous,
      d_max=work_params.d_max,
      d_min=work_params.d_min / math.pow(
        1 + work_params.unit_cell_length_tolerance, 1 / 3))
  miller_set = miller_set.change_basis(
    work_params.model_reindex_op).map_to_asu()

  if i_model is not None:
    matches = miller.match_indices(i_model.indices(), miller_set.indices())
    assert not matches.have_singles()
    miller_set = miller_set.select(matches.permutation())

# ---- Augment this code with any special procedures for x scaling
  scaler = xscaling_manager(
    miller_set=miller_set,
    i_model=i_model,
    params=work_params,
    log=out)
  scaler.scale_all()
  if scaler.n_accepted == 0:
    return None
# --- End of x scaling
  scaler.uc_values = unit_cell_distribution()
  for icell in xrange(len(scaler.frames["unit_cell"])):
    if scaler.params.model is None:
      scaler.uc_values.add_cell(
      unit_cell=scaler.frames["unit_cell"][icell])
    else:
      scaler.uc_values.add_cell(
      unit_cell=scaler.frames["unit_cell"][icell],
      rejected=(scaler.frames["cc"][icell] < scaler.params.min_corr))

  scaler.show_unit_cell_histograms()
  if (work_params.rescale_with_average_cell) :
    average_cell_abc = scaler.uc_values.get_average_cell_dimensions()
    average_cell = uctbx.unit_cell(list(average_cell_abc) +
      list(work_params.target_unit_cell.parameters()[3:]))
    work_params.target_unit_cell = average_cell
    print >> out, ""
    print >> out, "#" * 80
    print >> out, "RESCALING WITH NEW TARGET CELL"
    print >> out, "  average cell: %g %g %g %g %g %g" % \
      work_params.target_unit_cell.parameters()
    print >> out, ""
    scaler.reset()
    scaler = xscaling_manager(
      miller_set=miller_set,
      i_model=i_model,
      params=work_params,
      log=out)
    scaler.scale_all()
    scaler.uc_values = unit_cell_distribution()
    for icell in xrange(len(scaler.frames["unit_cell"])):
      if scaler.params.model is None:
        scaler.uc_values.add_cell(
        unit_cell=scaler.frames["unit_cell"][icell])
      else:
        scaler.uc_values.add_cell(
        unit_cell=scaler.frames["unit_cell"][icell],
        rejected=(scaler.frames["cc"][icell] < scaler.params.min_corr))
    scaler.show_unit_cell_histograms()
  if False : #(work_params.output.show_plots) :
    try :
      plot_overall_completeness(completeness)
    except Exception, e :
      print "ERROR: can't show plots"
      print "  %s" % str(e)
コード例 #3
0
          a_values=self.unit_cell_statistics.all_uc_a_values,
          b_values=self.unit_cell_statistics.all_uc_b_values,
          c_values=self.unit_cell_statistics.all_uc_c_values,
          n_slots=n_slots,
          title=\
            "Unit cell length distribution (all frames with compatible indexing): %s" % self.prefix)
        plt.show()
        fig = plt.figure(figsize=(9, 12))
        self.plot_statistics_histograms(figure=fig, n_slots=n_slots)
        plt.show()

    def plot_statistics_histograms(self, figure, n_slots=20):
        ax3 = figure.add_axes([0.1, 0.7, 0.8, 0.25])
        ax3.hist(self.frame_d_min, n_slots, color=[0.0, 0.5, 1.0])
        ax3.set_xlabel("Integrated resolution limit")
        ax3.set_title("Resolution by frame (%s)" % self.prefix)


if (__name__ == "__main__"):
    show_plots = False
    if ("--plots" in sys.argv):
        sys.argv.remove("--plots")
        show_plots = True
    result = run(args=sys.argv[1:])
    if result is None:
        sys.exit(1)
    if (show_plots):
        result.plots.show_all_pyplot()
        from wxtbx.command_line import loggraph
        loggraph.run([result.loggraph_file])
コード例 #4
0
ファイル: cxi_xmerge.py プロジェクト: cctbx/cctbx-playground
  if work_params.merging.reverse_lookup is not None:
    for key in scaler.reverse_lookup:
      if reindexing_ops.get(scaler.reverse_lookup[key], None) is None:
        reindexing_ops[scaler.reverse_lookup[key]]=0
      reindexing_ops[scaler.reverse_lookup[key]]+=1

  from xfel.cxi.cxi_cc import run_cc
  for key in reindexing_ops.keys():
    run_cc(work_params,reindexing_op=key,output=out)

  return result

if (__name__ == "__main__"):
  show_plots = False
  if ("--plots" in sys.argv) :
    sys.argv.remove("--plots")
    show_plots = True
  result = run(args=sys.argv[1:])
  if result is None:
    sys.exit(1)
  if (show_plots) :
    try :
      result.plots.show_all_pyplot()
      from wxtbx.command_line import loggraph
      loggraph.run([result.loggraph_file])
    except Exception, e :
      print "Can't display plots"
      print "You should be able to view them by running this command:"
      print "  wxtbx.loggraph %s" % result.loggraph_file
      raise e
コード例 #5
0
def run(args):
    phil = iotbx.phil.process_command_line(args=args,
                                           master_string=master_phil).show()
    work_params = phil.work.extract()
    from xfel.merging.phil_validation import application
    application(work_params)
    if ("--help" in args):
        libtbx.phil.parse(master_phil.show())
        return

    if ((work_params.d_min is None) or (work_params.data is None)
            or ((work_params.model is None)
                and work_params.scaling.algorithm != "mark1")):
        raise Usage("cxi.merge "
                    "d_min=4.0 "
                    "data=~/scratch/r0220/006/strong/ "
                    "model=3bz1_3bz2_core.pdb")
    if ((work_params.rescale_with_average_cell)
            and (not work_params.set_average_unit_cell)):
        raise Usage(
            "If rescale_with_average_cell=True, you must also specify " +
            "set_average_unit_cell=True.")
    if work_params.raw_data.sdfac_auto and work_params.raw_data.sdfac_refine:
        raise Usage("Cannot specify both sdfac_auto and sdfac_refine")

    log = open(
        "%s_%s.log" %
        (work_params.output.prefix, work_params.scaling.algorithm), "w")
    out = multi_out()
    out.register("log", log, atexit_send_to=None)
    out.register("stdout", sys.stdout)

    # Verify that the externally supplied isomorphous reference, if
    # present, defines a suitable column of intensities, and exit with
    # error if it does not.  Then warn if it is necessary to generate
    # Bijvoet mates.  Failure to catch these issues here would lead to
    # possibly obscure problems in cxi/cxi_cc.py later on.
    try:
        data_SR = mtz.object(work_params.scaling.mtz_file)
    except RuntimeError:
        pass
    else:
        array_SR = None
        obs_labels = []
        for array in data_SR.as_miller_arrays():
            this_label = array.info().label_string().lower()
            if array.observation_type() is not None:
                obs_labels.append(this_label.split(',')[0])
            if this_label.find('fobs') >= 0:
                array_SR = array.as_intensity_array()
                break
            if this_label.find('imean') >= 0:
                array_SR = array.as_intensity_array()
                break
            if this_label.find(work_params.scaling.mtz_column_F) == 0:
                array_SR = array.as_intensity_array()
                break

        if array_SR is None:
            known_labels = ['fobs', 'imean', work_params.scaling.mtz_column_F]
            raise Usage(work_params.scaling.mtz_file +
                        " does not contain any observations labelled [" +
                        ", ".join(known_labels) +
                        "].  Please set scaling.mtz_column_F to one of [" +
                        ",".join(obs_labels) + "].")
        elif not work_params.merge_anomalous and not array_SR.anomalous_flag():
            print >> out, "Warning: Preserving anomalous contributors, but %s " \
              "has anomalous contributors merged.  Generating identical Bijvoet " \
              "mates." % work_params.scaling.mtz_file

    # Read Nat's reference model from an MTZ file.  XXX The observation
    # type is given as F, not I--should they be squared?  Check with Nat!
    print >> out, "I model"
    if work_params.model is not None:
        from xfel.merging.general_fcalc import run
        i_model = run(work_params)
        work_params.target_unit_cell = i_model.unit_cell()
        work_params.target_space_group = i_model.space_group_info()
        i_model.show_summary()
    else:
        i_model = None

    print >> out, "Target unit cell and space group:"
    print >> out, "  ", work_params.target_unit_cell
    print >> out, "  ", work_params.target_space_group

    miller_set, i_model = consistent_set_and_model(work_params, i_model)

    # ---- Augment this code with any special procedures for x scaling
    scaler = xscaling_manager(miller_set=miller_set,
                              i_model=i_model,
                              params=work_params,
                              log=out)
    scaler.scale_all()
    if scaler.n_accepted == 0:
        return None


# --- End of x scaling
    scaler.uc_values = unit_cell_distribution()
    for icell in xrange(len(scaler.frames["unit_cell"])):
        if scaler.params.model is None:
            scaler.uc_values.add_cell(
                unit_cell=scaler.frames["unit_cell"][icell])
        else:
            scaler.uc_values.add_cell(
                unit_cell=scaler.frames["unit_cell"][icell],
                rejected=(scaler.frames["cc"][icell] < scaler.params.min_corr))

    scaler.show_unit_cell_histograms()
    if (work_params.rescale_with_average_cell):
        average_cell_abc = scaler.uc_values.get_average_cell_dimensions()
        average_cell = uctbx.unit_cell(
            list(average_cell_abc) +
            list(work_params.target_unit_cell.parameters()[3:]))
        work_params.target_unit_cell = average_cell
        print >> out, ""
        print >> out, "#" * 80
        print >> out, "RESCALING WITH NEW TARGET CELL"
        print >> out, "  average cell: %g %g %g %g %g %g" % \
          work_params.target_unit_cell.parameters()
        print >> out, ""
        scaler.reset()
        scaler = xscaling_manager(miller_set=miller_set,
                                  i_model=i_model,
                                  params=work_params,
                                  log=out)
        scaler.scale_all()
        scaler.uc_values = unit_cell_distribution()
        for icell in xrange(len(scaler.frames["unit_cell"])):
            if scaler.params.model is None:
                scaler.uc_values.add_cell(
                    unit_cell=scaler.frames["unit_cell"][icell])
            else:
                scaler.uc_values.add_cell(
                    unit_cell=scaler.frames["unit_cell"][icell],
                    rejected=(scaler.frames["cc"][icell] <
                              scaler.params.min_corr))
        scaler.show_unit_cell_histograms()
    if False:  #(work_params.output.show_plots) :
        try:
            plot_overall_completeness(completeness)
        except Exception, e:
            print "ERROR: can't show plots"
            print "  %s" % str(e)
コード例 #6
0
ファイル: join.py プロジェクト: hainm/cctbx_project
def run(args):
    phil = iotbx.phil.process_command_line(args=args,
                                           master_string=master_phil).show()
    work_params = phil.work.extract()
    from xfel.merging.phil_validation import application, samosa
    application(work_params)
    samosa(work_params)
    if ("--help" in args):
        libtbx.phil.parse(master_phil.show())
        return

    if ((work_params.d_min is None) or (work_params.data is None)):
        command_name = os.environ["LIBTBX_DISPATCHER_NAME"]
        raise Usage(command_name + " "
                    "d_min=4.0 "
                    "data=~/scratch/r0220/006/strong/ "
                    "model=3bz1_3bz2_core.pdb")
    if ((work_params.rescale_with_average_cell)
            and (not work_params.set_average_unit_cell)):
        raise Usage(
            "If rescale_with_average_cell=True, you must also specify " +
            "set_average_unit_cell=True.")
    if work_params.raw_data.sdfac_auto and work_params.raw_data.sdfac_refine:
        raise Usage("Cannot specify both sdfac_auto and sdfac_refine")

    # Read Nat's reference model from an MTZ file.  XXX The observation
    # type is given as F, not I--should they be squared?  Check with Nat!
    log = open("%s.log" % work_params.output.prefix, "w")
    out = multi_out()
    out.register("log", log, atexit_send_to=None)
    out.register("stdout", sys.stdout)
    print >> out, "I model"
    if work_params.model is not None:
        from xfel.merging.general_fcalc import run
        i_model = run(work_params)
        work_params.target_unit_cell = i_model.unit_cell()
        work_params.target_space_group = i_model.space_group_info()
        i_model.show_summary()
    else:
        i_model = None

    print >> out, "Target unit cell and space group:"
    print >> out, "  ", work_params.target_unit_cell
    print >> out, "  ", work_params.target_space_group

    miller_set, i_model = consistent_set_and_model(work_params, i_model)

    frame_files = get_observations(work_params)
    scaler = scaling_manager(miller_set=miller_set,
                             i_model=i_model,
                             params=work_params,
                             log=out)
    scaler.scale_all(frame_files)
    if scaler.n_accepted == 0:
        return None
    scaler.show_unit_cell_histograms()
    if (work_params.rescale_with_average_cell):
        average_cell_abc = scaler.uc_values.get_average_cell_dimensions()
        average_cell = uctbx.unit_cell(
            list(average_cell_abc) +
            list(work_params.target_unit_cell.parameters()[3:]))
        work_params.target_unit_cell = average_cell
        print >> out, ""
        print >> out, "#" * 80
        print >> out, "RESCALING WITH NEW TARGET CELL"
        print >> out, "  average cell: %g %g %g %g %g %g" % \
          work_params.target_unit_cell.parameters()
        print >> out, ""
        scaler.reset()
        scaler.scale_all(frame_files)
        scaler.show_unit_cell_histograms()
    if False:  #(work_params.output.show_plots) :
        try:
            plot_overall_completeness(completeness)
        except Exception, e:
            print "ERROR: can't show plots"
            print "  %s" % str(e)
コード例 #7
0
def run(args):
    phil = iotbx.phil.process_command_line(args=args,
                                           master_string=master_phil).show()
    work_params = phil.work.extract()
    from xfel.merging.phil_validation import application
    application(work_params)
    if ("--help" in args):
        libtbx.phil.parse(master_phil.show())
        return

    if ((work_params.d_min is None) or (work_params.data is None)
            or ((work_params.model is None)
                and work_params.scaling.algorithm != "mark1")):
        raise Usage("cxi.merge "
                    "d_min=4.0 "
                    "data=~/scratch/r0220/006/strong/ "
                    "model=3bz1_3bz2_core.pdb")
    if ((work_params.rescale_with_average_cell)
            and (not work_params.set_average_unit_cell)):
        raise Usage(
            "If rescale_with_average_cell=True, you must also specify " +
            "set_average_unit_cell=True.")
    if work_params.raw_data.sdfac_auto and work_params.raw_data.sdfac_refine:
        raise Usage("Cannot specify both sdfac_auto and sdfac_refine")
    if not work_params.include_negatives_fix_27May2018:
        work_params.include_negatives = False  # use old behavior

    log = open(
        "%s_%s.log" %
        (work_params.output.prefix, work_params.scaling.algorithm), "w")
    out = multi_out()
    out.register("log", log, atexit_send_to=None)
    out.register("stdout", sys.stdout)

    # Verify that the externally supplied isomorphous reference, if
    # present, defines a suitable column of intensities, and exit with
    # error if it does not.  Then warn if it is necessary to generate
    # Bijvoet mates.  Failure to catch these issues here would lead to
    # possibly obscure problems in cxi/cxi_cc.py later on.
    try:
        data_SR = mtz.object(work_params.scaling.mtz_file)
    except RuntimeError:
        pass
    else:
        array_SR = None
        obs_labels = []
        for array in data_SR.as_miller_arrays():
            this_label = array.info().label_string().lower()
            if array.observation_type() is not None:
                obs_labels.append(this_label.split(',')[0])
            if this_label.find('fobs') >= 0:
                array_SR = array.as_intensity_array()
                break
            if this_label.find('imean') >= 0:
                array_SR = array.as_intensity_array()
                break
            if this_label.find(work_params.scaling.mtz_column_F) == 0:
                array_SR = array.as_intensity_array()
                break

        if array_SR is None:
            known_labels = ['fobs', 'imean', work_params.scaling.mtz_column_F]
            raise Usage(work_params.scaling.mtz_file +
                        " does not contain any observations labelled [" +
                        ", ".join(known_labels) +
                        "].  Please set scaling.mtz_column_F to one of [" +
                        ",".join(obs_labels) + "].")
        elif not work_params.merge_anomalous and not array_SR.anomalous_flag():
            print("Warning: Preserving anomalous contributors, but %s " \
              "has anomalous contributors merged.  Generating identical Bijvoet " \
              "mates." % work_params.scaling.mtz_file, file=out)

    # Read Nat's reference model from an MTZ file.  XXX The observation
    # type is given as F, not I--should they be squared?  Check with Nat!
    print("I model", file=out)
    if work_params.model is not None:
        from xfel.merging.general_fcalc import run
        i_model = run(work_params)
        work_params.target_unit_cell = i_model.unit_cell()
        work_params.target_space_group = i_model.space_group_info()
        i_model.show_summary()
    else:
        i_model = None

    print("Target unit cell and space group:", file=out)
    print("  ", work_params.target_unit_cell, file=out)
    print("  ", work_params.target_space_group, file=out)

    miller_set, i_model = consistent_set_and_model(work_params, i_model)

    # ---- Augment this code with any special procedures for x scaling
    scaler = xscaling_manager(miller_set=miller_set,
                              i_model=i_model,
                              params=work_params,
                              log=out)
    scaler.scale_all()
    if scaler.n_accepted == 0:
        return None


# --- End of x scaling
    scaler.uc_values = unit_cell_distribution()
    for icell in range(len(scaler.frames["unit_cell"])):
        if scaler.params.model is None:
            scaler.uc_values.add_cell(
                unit_cell=scaler.frames["unit_cell"][icell])
        else:
            scaler.uc_values.add_cell(
                unit_cell=scaler.frames["unit_cell"][icell],
                rejected=(scaler.frames["cc"][icell] < scaler.params.min_corr))

    scaler.show_unit_cell_histograms()
    if (work_params.rescale_with_average_cell):
        average_cell_abc = scaler.uc_values.get_average_cell_dimensions()
        average_cell = uctbx.unit_cell(
            list(average_cell_abc) +
            list(work_params.target_unit_cell.parameters()[3:]))
        work_params.target_unit_cell = average_cell
        print("", file=out)
        print("#" * 80, file=out)
        print("RESCALING WITH NEW TARGET CELL", file=out)
        print("  average cell: %g %g %g %g %g %g" % \
          work_params.target_unit_cell.parameters(), file=out)
        print("", file=out)
        scaler.reset()
        scaler = xscaling_manager(miller_set=miller_set,
                                  i_model=i_model,
                                  params=work_params,
                                  log=out)
        scaler.scale_all()
        scaler.uc_values = unit_cell_distribution()
        for icell in range(len(scaler.frames["unit_cell"])):
            if scaler.params.model is None:
                scaler.uc_values.add_cell(
                    unit_cell=scaler.frames["unit_cell"][icell])
            else:
                scaler.uc_values.add_cell(
                    unit_cell=scaler.frames["unit_cell"][icell],
                    rejected=(scaler.frames["cc"][icell] <
                              scaler.params.min_corr))
        scaler.show_unit_cell_histograms()
    if False:  #(work_params.output.show_plots) :
        try:
            plot_overall_completeness(completeness)
        except Exception as e:
            print("ERROR: can't show plots")
            print("  %s" % str(e))
    print("\n", file=out)

    reserve_prefix = work_params.output.prefix
    for data_subset in [1, 2, 0]:
        work_params.data_subset = data_subset
        work_params.output.prefix = "%s_s%1d_%s" % (
            reserve_prefix, data_subset, work_params.scaling.algorithm)

        if work_params.data_subset == 0:
            scaler.frames["data_subset"] = flex.bool(
                scaler.frames["frame_id"].size(), True)
        elif work_params.data_subset == 1:
            scaler.frames["data_subset"] = scaler.frames["odd_numbered"]
        elif work_params.data_subset == 2:
            scaler.frames["data_subset"] = scaler.frames[
                "odd_numbered"] == False

    # --------- New code ------------------
    #sanity check
        for mod, obs in zip(miller_set.indices(),
                            scaler.millers["merged_asu_hkl"]):
            if mod != obs:
                raise Exception(
                    "miller index lists inconsistent--check d_min are equal for merge and xmerge scripts"
                )
            assert mod == obs
        """Sum the observations of I and I/sig(I) for each reflection.
    sum_I = flex.double(i_model.size(), 0.)
    sum_I_SIGI = flex.double(i_model.size(), 0.)
    scaler.completeness = flex.int(i_model.size(), 0)
    scaler.summed_N = flex.int(i_model.size(), 0)
    scaler.summed_wt_I = flex.double(i_model.size(), 0.)
    scaler.summed_weight = flex.double(i_model.size(), 0.)
    scaler.n_rejected = flex.double(scaler.frames["frame_id"].size(), 0.)
    scaler.n_obs = flex.double(scaler.frames["frame_id"].size(), 0.)
    scaler.d_min_values = flex.double(scaler.frames["frame_id"].size(), 0.)
    scaler.ISIGI = {}"""

        from xfel import scaling_results, get_scaling_results, get_isigi_dict
        results = scaling_results(scaler._observations, scaler._frames,
                                  scaler.millers["merged_asu_hkl"],
                                  scaler.frames["data_subset"],
                                  work_params.include_negatives)
        results.__getattribute__(work_params.scaling.algorithm)(
            scaler.params.min_corr, scaler.params.target_unit_cell)

        sum_I, sum_I_SIGI, \
        scaler.completeness, scaler.summed_N, \
        scaler.summed_wt_I, scaler.summed_weight, scaler.n_rejected, scaler.n_obs, \
        scaler.d_min_values, hkl_ids, i_sigi_list = get_scaling_results(results)

        scaler.ISIGI = get_isigi_dict(results)

        if work_params.merging.refine_G_Imodel:
            from xfel.cxi.merging.refine import find_scale

            my_find_scale = find_scale(scaler, work_params)

            sum_I, sum_I_SIGI, \
              scaler.completeness, scaler.summed_N, \
              scaler.summed_wt_I, scaler.summed_weight, scaler.n_rejected, \
              scaler.n_obs, scaler.d_min_values, hkl_ids, i_sigi_list \
              = my_find_scale.get_scaling_results(results, scaler)
            scaler.ISIGI = get_isigi_dict(results)

        scaler.wavelength = scaler.frames["wavelength"]
        scaler.corr_values = scaler.frames["cc"]

        scaler.rejected_fractions = flex.double(
            scaler.frames["frame_id"].size(), 0.)
        for irej in range(len(scaler.rejected_fractions)):
            if scaler.n_obs[irej] > 0:
                scaler.rejected_fractions = scaler.n_rejected[
                    irej] / scaler.n_obs[irej]
    # ---------- End of new code ----------------

        if work_params.raw_data.sdfac_refine or work_params.raw_data.errors_from_sample_residuals:
            if work_params.raw_data.sdfac_refine:
                if work_params.raw_data.error_models.sdfac_refine.minimizer == 'simplex':
                    from xfel.merging.algorithms.error_model.sdfac_refine import sdfac_refine as error_modeler
                elif work_params.raw_data.error_models.sdfac_refine.minimizer == 'lbfgs':
                    from xfel.merging.algorithms.error_model.sdfac_refine_lbfgs import sdfac_refine_refltable_lbfgs as error_modeler
                elif self.params.raw_data.error_models.sdfac_refine.minimizer == 'LevMar':
                    from xfel.merging.algorithms.error_model.sdfac_refine_levmar import sdfac_refine_refltable_levmar as error_modeler

            if work_params.raw_data.errors_from_sample_residuals:
                from xfel.merging.algorithms.error_model.errors_from_residuals import errors_from_residuals as error_modeler

            error_modeler(scaler).adjust_errors()

        if work_params.raw_data.reduced_chi_squared_correction:
            from xfel.merging.algorithms.error_model.reduced_chi_squared import reduced_chi_squared
            reduced_chi_squared(scaler).compute()

        miller_set_avg = miller_set.customized_copy(
            unit_cell=work_params.target_unit_cell)

        table1 = show_overall_observations(
            obs=miller_set_avg,
            redundancy=scaler.completeness,
            redundancy_to_edge=None,
            summed_wt_I=scaler.summed_wt_I,
            summed_weight=scaler.summed_weight,
            ISIGI=scaler.ISIGI,
            n_bins=work_params.output.n_bins,
            title="Statistics for all reflections",
            out=out,
            work_params=work_params)
        if table1 is None:
            raise Exception("table could not be constructed")
        print("", file=out)
        if work_params.scaling.algorithm == 'mark0':
            n_refl, corr = scaler.get_overall_correlation(sum_I)
        else:
            n_refl, corr = ((scaler.completeness > 0).count(True), 0)
        print("\n", file=out)
        table2 = show_overall_observations(
            obs=miller_set_avg,
            redundancy=scaler.summed_N,
            redundancy_to_edge=None,
            summed_wt_I=scaler.summed_wt_I,
            summed_weight=scaler.summed_weight,
            ISIGI=scaler.ISIGI,
            n_bins=work_params.output.n_bins,
            title="Statistics for reflections where I > 0",
            out=out,
            work_params=work_params)
        if table2 is None:
            raise Exception("table could not be constructed")

        print("", file=out)
        mtz_file, miller_array = scaler.finalize_and_save_data()

        loggraph_file = os.path.abspath("%s_graphs.log" %
                                        work_params.output.prefix)
        f = open(loggraph_file, "w")
        f.write(table1.format_loggraph())
        f.write("\n")
        f.write(table2.format_loggraph())
        f.close()
        result = scaling_result(miller_array=miller_array,
                                plots=scaler.get_plot_statistics(),
                                mtz_file=mtz_file,
                                loggraph_file=loggraph_file,
                                obs_table=table1,
                                all_obs_table=table2,
                                n_reflections=n_refl,
                                overall_correlation=corr)
        easy_pickle.dump("%s.pkl" % work_params.output.prefix, result)
    work_params.output.prefix = reserve_prefix

    # Output table with number of images contribution reflections per
    # resolution bin.
    from libtbx import table_utils

    miller_set_avg.setup_binner(d_max=100000,
                                d_min=work_params.d_min,
                                n_bins=work_params.output.n_bins)
    table_data = [["Bin", "Resolution Range", "# images", "%accept"]]
    if work_params.model is None:
        appropriate_min_corr = -1.1  # lowest possible c.c.
    else:
        appropriate_min_corr = work_params.min_corr
    n_frames = (scaler.frames['cc'] > appropriate_min_corr).count(True)
    iselect = 1
    while iselect < work_params.output.n_bins:
        col_count1 = results.count_frames(
            appropriate_min_corr,
            miller_set_avg.binner().selection(iselect))
        print("colcount1", col_count1)
        if col_count1 > 0: break
        iselect += 1
    if col_count1 == 0: raise Exception("no reflections in any bins")
    for i_bin in miller_set_avg.binner().range_used():
        col_count = '%8d' % results.count_frames(
            appropriate_min_corr,
            miller_set_avg.binner().selection(i_bin))
        col_legend = '%-13s' % miller_set_avg.binner().bin_legend(
            i_bin=i_bin,
            show_bin_number=False,
            show_bin_range=False,
            show_d_range=True,
            show_counts=False)
        xpercent = results.count_frames(
            appropriate_min_corr,
            miller_set_avg.binner().selection(i_bin)) / float(n_frames)
        percent = '%5.2f' % (100. * xpercent)
        table_data.append(['%3d' % i_bin, col_legend, col_count, percent])

    table_data.append([""] * len(table_data[0]))
    table_data.append(["All", "", '%8d' % n_frames])
    print(file=out)
    print(table_utils.format(table_data,
                             has_header=1,
                             justify='center',
                             delim=' '),
          file=out)

    reindexing_ops = {
        "h,k,l": 0
    }  # get a list of all reindexing ops for this dataset
    if work_params.merging.reverse_lookup is not None:
        for key in scaler.reverse_lookup:
            if reindexing_ops.get(scaler.reverse_lookup[key], None) is None:
                reindexing_ops[scaler.reverse_lookup[key]] = 0
            reindexing_ops[scaler.reverse_lookup[key]] += 1

    from xfel.cxi.cxi_cc import run_cc
    for key in reindexing_ops.keys():
        run_cc(work_params, reindexing_op=key, output=out)

    if isinstance(scaler.ISIGI, dict):
        from xfel.merging import isigi_dict_to_reflection_table
        refls = isigi_dict_to_reflection_table(scaler.miller_set.indices(),
                                               scaler.ISIGI)
    else:
        refls = scaler.ISIGI
    easy_pickle.dump("%s.refl" % work_params.output.prefix, refls)

    return result
コード例 #8
0
ファイル: join.py プロジェクト: cctbx/cctbx-playground
def run(args):
  phil = iotbx.phil.process_command_line(args=args, master_string=master_phil).show()
  work_params = phil.work.extract()
  from xfel.merging.phil_validation import application,samosa
  application(work_params)
  samosa(work_params)
  if ("--help" in args) :
    libtbx.phil.parse(master_phil.show())
    return

  if ((work_params.d_min is None) or
      (work_params.data is None) ) :
    command_name = os.environ["LIBTBX_DISPATCHER_NAME"]
    raise Usage(command_name + " "
                "d_min=4.0 "
                "data=~/scratch/r0220/006/strong/ "
                "model=3bz1_3bz2_core.pdb")
  if ((work_params.rescale_with_average_cell) and
      (not work_params.set_average_unit_cell)) :
    raise Usage("If rescale_with_average_cell=True, you must also specify "+
      "set_average_unit_cell=True.")
  if work_params.raw_data.sdfac_auto and work_params.raw_data.sdfac_refine:
    raise Usage("Cannot specify both sdfac_auto and sdfac_refine")

  # Read Nat's reference model from an MTZ file.  XXX The observation
  # type is given as F, not I--should they be squared?  Check with Nat!
  log = open("%s.log" % work_params.output.prefix, "w")
  out = multi_out()
  out.register("log", log, atexit_send_to=None)
  out.register("stdout", sys.stdout)
  print >> out, "I model"
  if work_params.model is not None:
    from xfel.merging.general_fcalc import run
    i_model = run(work_params)
    work_params.target_unit_cell = i_model.unit_cell()
    work_params.target_space_group = i_model.space_group_info()
    i_model.show_summary()
  else:
    i_model = None

  print >> out, "Target unit cell and space group:"
  print >> out, "  ", work_params.target_unit_cell
  print >> out, "  ", work_params.target_space_group

  # Adjust the minimum d-spacing of the generated Miller set to assure
  # that the desired high-resolution limit is included even if the
  # observed unit cell differs slightly from the target.  If a
  # reference model is present, ensure that Miller indices are ordered
  # identically.
  miller_set = symmetry(
      unit_cell=work_params.target_unit_cell,
      space_group_info=work_params.target_space_group
    ).build_miller_set(
      anomalous_flag=not work_params.merge_anomalous,
      d_max=work_params.d_max,
      d_min=work_params.d_min / math.pow(
        1 + work_params.unit_cell_length_tolerance, 1 / 3))
  miller_set = miller_set.change_basis(
    work_params.model_reindex_op).map_to_asu()

  if i_model is not None:
    matches = miller.match_indices(i_model.indices(), miller_set.indices())
    assert not matches.have_singles()
    miller_set = miller_set.select(matches.permutation())

  frame_files = get_observations(work_params)
  scaler = scaling_manager(
    miller_set=miller_set,
    i_model=i_model,
    params=work_params,
    log=out)
  scaler.scale_all(frame_files)
  if scaler.n_accepted == 0:
    return None
  scaler.show_unit_cell_histograms()
  if (work_params.rescale_with_average_cell) :
    average_cell_abc = scaler.uc_values.get_average_cell_dimensions()
    average_cell = uctbx.unit_cell(list(average_cell_abc) +
      list(work_params.target_unit_cell.parameters()[3:]))
    work_params.target_unit_cell = average_cell
    print >> out, ""
    print >> out, "#" * 80
    print >> out, "RESCALING WITH NEW TARGET CELL"
    print >> out, "  average cell: %g %g %g %g %g %g" % \
      work_params.target_unit_cell.parameters()
    print >> out, ""
    scaler.reset()
    scaler.scale_all(frame_files)
    scaler.show_unit_cell_histograms()
  if False : #(work_params.output.show_plots) :
    try :
      plot_overall_completeness(completeness)
    except Exception, e :
      print "ERROR: can't show plots"
      print "  %s" % str(e)