Ejemplo n.º 1
0
def run(show_plots,args):
  from xfel.command_line.cxi_merge import master_phil
  phil = iotbx.phil.process_command_line(args=args, master_string=master_phil).show()
  work_params = phil.work.extract()
  from xfel.merging.phil_validation import application,samosa
  application(work_params)
  samosa(work_params)
  if ("--help" in args) :
    libtbx.phil.parse(master_phil.show())
    return

  datadir = "."
  written_files = []
  if work_params.levmar.compute_cc_half:

    for half_data_flag in [1,2,0]:
      case = execute_case(datadir, work_params, plot=show_plots, half_data_flag=half_data_flag)
      assert len(case.Fit_I)==len(case.ordered_intensities.indices())==len(case.reference_millers.indices())
      model_subset = case.reference_millers[0:len(case.Fit_I)]
      fitted_miller_array = miller.array (miller_set = model_subset,
                                        data = case.Fit_I, sigmas = case.Fit_I_stddev)
      fitted_miller_array.set_observation_type_xray_intensity()
      output_result = fitted_miller_array.select(case.I_visited==1)
      outfile = "%s_s%1d_levmar.mtz"%(work_params.output.prefix,half_data_flag)
      output_result.show_summary(prefix="%s: "%outfile)
      mtz_out = output_result.as_mtz_dataset(column_root_label="Iobs",title=outfile,wavelength=None)
      mtz_obj = mtz_out.mtz_object()
      mtz_obj.write(outfile)
      written_files.append(outfile)
      print "OK s%1d"%half_data_flag
      #raw_input("OK?")

    """Guest code to retrieve the modified orientations after rotational fitting is done"""
    if "Rxy" in work_params.levmar.parameter_flags:
      all_A = [e.crystal.get_A() for e in case.experiments.get_experiments()]
      all_files = case.experiments.get_files()
      all_x = case.Fit["Ax"]
      all_y = case.Fit["Ay"]

      from scitbx import matrix
      x_axis = matrix.col((1.,0.,0.))
      y_axis = matrix.col((0.,1.,0.))
      out = open("aaaaa","w")
      for x in xrange(len(all_A)):
        Rx = x_axis.axis_and_angle_as_r3_rotation_matrix(angle=all_x[x], deg=True)
        Ry = y_axis.axis_and_angle_as_r3_rotation_matrix(angle=all_y[x], deg=True)
        modified_A = Rx * Ry * all_A[x]
        filename = all_files[x]
        print >>out, filename, " ".join([str(a) for a in modified_A.elems])



    work_params.scaling.algorithm="levmar"
    from xfel.cxi.cxi_cc import run_cc
    run_cc(work_params,work_params.model_reindex_op,sys.stdout)

  else:
    execute_case(datadir, work_params, plot=show_plots)
Ejemplo n.º 2
0
def run(show_plots,args):
  from xfel.command_line.cxi_merge import master_phil
  phil = iotbx.phil.process_command_line(args=args, master_string=master_phil).show()
  work_params = phil.work.extract()
  from xfel.merging.phil_validation import application,samosa
  application(work_params)
  samosa(work_params)
  if ("--help" in args) :
    libtbx.phil.parse(master_phil.show())
    return

  datadir = "."
  written_files = []
  if work_params.levmar.compute_cc_half:

    for half_data_flag in [1,2,0]:
      case = execute_case(datadir, work_params, plot=show_plots, half_data_flag=half_data_flag)
      assert len(case.Fit_I)==len(case.ordered_intensities.indices())==len(case.reference_millers.indices())
      model_subset = case.reference_millers[0:len(case.Fit_I)]
      fitted_miller_array = miller.array (miller_set = model_subset,
                                        data = case.Fit_I, sigmas = case.Fit_I_stddev)
      fitted_miller_array.set_observation_type_xray_intensity()
      output_result = fitted_miller_array.select(case.I_visited==1)
      outfile = "%s_s%1d_levmar.mtz"%(work_params.output.prefix,half_data_flag)
      output_result.show_summary(prefix="%s: "%outfile)
      mtz_out = output_result.as_mtz_dataset(column_root_label="Iobs",title=outfile,wavelength=None)
      mtz_obj = mtz_out.mtz_object()
      mtz_obj.write(outfile)
      written_files.append(outfile)
      print "OK s%1d"%half_data_flag
      #raw_input("OK?")

    """Guest code to retrieve the modified orientations after rotational fitting is done"""
    if "Rxy" in work_params.levmar.parameter_flags:
      all_A = [e.crystal.get_A() for e in case.experiments.get_experiments()]
      all_files = case.experiments.get_files()
      all_x = case.Fit["Ax"]
      all_y = case.Fit["Ay"]

      from scitbx import matrix
      x_axis = matrix.col((1.,0.,0.))
      y_axis = matrix.col((0.,1.,0.))
      out = open("aaaaa","w")
      for x in xrange(len(all_A)):
        Rx = x_axis.axis_and_angle_as_r3_rotation_matrix(angle=all_x[x], deg=True)
        Ry = y_axis.axis_and_angle_as_r3_rotation_matrix(angle=all_y[x], deg=True)
        modified_A = Rx * Ry * all_A[x]
        filename = all_files[x]
        print >>out, filename, " ".join([str(a) for a in modified_A.elems])



    work_params.scaling.algorithm="levmar"
    from xfel.cxi.cxi_cc import run_cc
    run_cc(work_params,work_params.model_reindex_op,sys.stdout)
Ejemplo n.º 3
0
def run(args):
    phil = iotbx.phil.process_command_line(args=args,
                                           master_string=master_phil).show()
    work_params = phil.work.extract()
    from xfel.merging.phil_validation import application, samosa
    application(work_params)
    samosa(work_params)
    if ("--help" in args):
        libtbx.phil.parse(master_phil.show())
        return

    if ((work_params.d_min is None) or (work_params.data is None)):
        command_name = os.environ["LIBTBX_DISPATCHER_NAME"]
        raise Usage(command_name + " "
                    "d_min=4.0 "
                    "data=~/scratch/r0220/006/strong/ "
                    "model=3bz1_3bz2_core.pdb")
    if ((work_params.rescale_with_average_cell)
            and (not work_params.set_average_unit_cell)):
        raise Usage(
            "If rescale_with_average_cell=True, you must also specify " +
            "set_average_unit_cell=True.")
    if work_params.raw_data.sdfac_auto and work_params.raw_data.sdfac_refine:
        raise Usage("Cannot specify both sdfac_auto and sdfac_refine")

    # Read Nat's reference model from an MTZ file.  XXX The observation
    # type is given as F, not I--should they be squared?  Check with Nat!
    log = open("%s.log" % work_params.output.prefix, "w")
    out = multi_out()
    out.register("log", log, atexit_send_to=None)
    out.register("stdout", sys.stdout)
    print >> out, "I model"
    if work_params.model is not None:
        from xfel.merging.general_fcalc import run
        i_model = run(work_params)
        work_params.target_unit_cell = i_model.unit_cell()
        work_params.target_space_group = i_model.space_group_info()
        i_model.show_summary()
    else:
        i_model = None

    print >> out, "Target unit cell and space group:"
    print >> out, "  ", work_params.target_unit_cell
    print >> out, "  ", work_params.target_space_group

    miller_set, i_model = consistent_set_and_model(work_params, i_model)

    frame_files = get_observations(work_params)
    scaler = scaling_manager(miller_set=miller_set,
                             i_model=i_model,
                             params=work_params,
                             log=out)
    scaler.scale_all(frame_files)
    if scaler.n_accepted == 0:
        return None
    scaler.show_unit_cell_histograms()
    if (work_params.rescale_with_average_cell):
        average_cell_abc = scaler.uc_values.get_average_cell_dimensions()
        average_cell = uctbx.unit_cell(
            list(average_cell_abc) +
            list(work_params.target_unit_cell.parameters()[3:]))
        work_params.target_unit_cell = average_cell
        print >> out, ""
        print >> out, "#" * 80
        print >> out, "RESCALING WITH NEW TARGET CELL"
        print >> out, "  average cell: %g %g %g %g %g %g" % \
          work_params.target_unit_cell.parameters()
        print >> out, ""
        scaler.reset()
        scaler.scale_all(frame_files)
        scaler.show_unit_cell_histograms()
    if False:  #(work_params.output.show_plots) :
        try:
            plot_overall_completeness(completeness)
        except Exception as e:
            print "ERROR: can't show plots"
            print "  %s" % str(e)
    print >> out, "\n"

    # Sum the observations of I and I/sig(I) for each reflection.
    sum_I = flex.double(miller_set.size(), 0.)
    sum_I_SIGI = flex.double(miller_set.size(), 0.)
    for i in xrange(miller_set.size()):
        index = miller_set.indices()[i]
        if index in scaler.ISIGI:
            for t in scaler.ISIGI[index]:
                sum_I[i] += t[0]
                sum_I_SIGI[i] += t[1]

    miller_set_avg = miller_set.customized_copy(
        unit_cell=work_params.target_unit_cell)
    table1 = show_overall_observations(obs=miller_set_avg,
                                       redundancy=scaler.completeness,
                                       summed_wt_I=scaler.summed_wt_I,
                                       summed_weight=scaler.summed_weight,
                                       ISIGI=scaler.ISIGI,
                                       n_bins=work_params.output.n_bins,
                                       title="Statistics for all reflections",
                                       out=out,
                                       work_params=work_params)
    print >> out, ""
    n_refl, corr = ((scaler.completeness > 0).count(True), 0)
    print >> out, "\n"
    table2 = show_overall_observations(
        obs=miller_set_avg,
        redundancy=scaler.summed_N,
        summed_wt_I=scaler.summed_wt_I,
        summed_weight=scaler.summed_weight,
        ISIGI=scaler.ISIGI,
        n_bins=work_params.output.n_bins,
        title="Statistics for reflections where I > 0",
        out=out,
        work_params=work_params)
    #from libtbx import easy_pickle
    #easy_pickle.dump(file_name="stats.pickle", obj=stats)
    #stats.report(plot=work_params.plot)
    #miller_counts = miller_set_p1.array(data=stats.counts.as_double()).select(
    #  stats.counts != 0)
    #miller_counts.as_mtz_dataset(column_root_label="NOBS").mtz_object().write(
    #  file_name="nobs.mtz")
    if work_params.data_subsubsets.subsubset is not None and work_params.data_subsubsets.subsubset_total is not None:
        easy_pickle.dump(
            "scaler_%d.pickle" % work_params.data_subsubsets.subsubset, scaler)
    print >> out, ""
    mtz_file, miller_array = scaler.finalize_and_save_data()
    #table_pickle_file = "%s_graphs.pkl" % work_params.output.prefix
    #easy_pickle.dump(table_pickle_file, [table1, table2])
    loggraph_file = os.path.abspath("%s_graphs.log" %
                                    work_params.output.prefix)
    f = open(loggraph_file, "w")
    f.write(table1.format_loggraph())
    f.write("\n")
    f.write(table2.format_loggraph())
    f.close()
    result = scaling_result(miller_array=miller_array,
                            plots=scaler.get_plot_statistics(),
                            mtz_file=mtz_file,
                            loggraph_file=loggraph_file,
                            obs_table=table1,
                            all_obs_table=table2,
                            n_reflections=n_refl,
                            overall_correlation=corr)
    easy_pickle.dump("%s.pkl" % work_params.output.prefix, result)
    return result
Ejemplo n.º 4
0
def find_merge_common_images(args):
    phil = iotbx.phil.process_command_line(args = args,
                                           master_string = master_phil).show()
    work_params = phil.work.extract()
    if ("--help" in args) :
        libtbx.phil.parse(master_phil.show())
        return

    if ((work_params.d_min is None) or
        (work_params.data is None) or
        ((work_params.model is None) and
         work_params.scaling.algorithm != "mark1")) :
        raise Usage("cxi.merge "
                    "d_min=4.0 "
                    "data=~/scratch/r0220/006/strong/ "
                    "model=3bz1_3bz2_core.pdb")
    if ((work_params.rescale_with_average_cell) and
        (not work_params.set_average_unit_cell)) :
        raise Usage(
            "If rescale_with_average_cell=True, you must also specify "+
            "set_average_unit_cell=True.")

    # Read Nat's reference model from an MTZ file.  XXX The observation
    # type is given as F, not I--should they be squared?  Check with Nat!
    log = open("%s_%s_scale.log" % (work_params.output.prefix,
                                    work_params.scaling.algorithm), "w")
    out = multi_out()
    out.register("log", log, atexit_send_to=None)
    out.register("stdout", sys.stdout)

    print >> out, "Target unit cell and space group:"
    print >> out, "  ", work_params.target_unit_cell
    print >> out, "  ", work_params.target_space_group

    uc = work_params.target_unit_cell

    miller_set = symmetry(
        unit_cell=work_params.target_unit_cell,
        space_group_info=work_params.target_space_group
        ).build_miller_set(
        anomalous_flag=not work_params.merge_anomalous,
        d_min=work_params.d_min)
    print 'Miller set size: %d' % len(miller_set.indices())
    from xfel.cxi.merging.general_fcalc import random_structure
    i_model = random_structure(work_params)

    # ---- Augment this code with any special procedures for x scaling
    scaler = xscaling_manager(
        miller_set=miller_set,
        i_model=i_model,
        params=work_params,
        log=out)
    scaler.read_all()
    print "finished reading"
    sg = miller_set.space_group()
    pg = sg.build_derived_laue_group()
    miller_set.show_summary()

    hkl_asu = scaler.observations["hkl_id"]
    imageno = scaler.observations["frame_id"]
    intensi = scaler.observations["i"]
    sigma_i = scaler.observations["sigi"]
    
    lookup = scaler.millers["merged_asu_hkl"]    

    # construct table of start / end indices for frames: now using Python
    # range indexing

    starts = [0]
    ends = []
    
    for x in xrange(1, len(scaler.observations["hkl_id"])):
        if imageno[x] != imageno[x - 1]:
            ends.append(x)
            starts.append(x)
            
    ends.append(len(scaler.observations["hkl_id"]))

    keep_start = []
    keep_end = []

    def nint(a):
        return int(round(a))

    from collections import defaultdict
    i_scale = 0.1
    i_hist = defaultdict(int)

    for j, se in enumerate(zip(starts, ends)):
        s, e = se

        for i in intensi[s:e]:
            i_hist[nint(i_scale * i)] += 1
        
        isig = sum(i / s for i, s in zip(intensi[s:e], sigma_i[s:e])) / (e - s)
        dmin = 100.0
        for x in xrange(s, e):
            d = uc.d(lookup[hkl_asu[x]])
            if d < dmin:
                dmin = d
        if isig > 6.0 and dmin < 3.2:
            keep_start.append(s)
            keep_end.append(e)

    fout = open('i_hist.dat', 'w')
    for i in i_hist:
        fout.write('%.2f %d\n' % (i / i_scale, i_hist[i]))
    fout.close()

    starts = keep_start
    ends = keep_end

    print 'Keeping %d frames' % len(starts)

    frames = []

    odd = 0
    even = 0

    for s, e in zip(starts, ends):

        for x in range(s, e):
            hkl = lookup[hkl_asu[x]]
            
            if (hkl[0] + hkl[1] + hkl[2]) % 2 == 1:
                odd += 1
            else:
                even += 1
        
        indices = [tuple(lookup[hkl_asu[x]]) for x in range(s, e)]
        intensities = intensi[s:e]
        sigmas = sigma_i[s:e]

        frames.append(Frame(uc, indices, intensities, sigmas))

    # pre-scale the data - first determine average ln(k), B; then apply

    kbs = [f.kb() for f in frames]

    mn_k = sum([kb[0] for kb in kbs]) / len(kbs)
    mn_B = sum([kb[1] for kb in kbs]) / len(kbs)

    n_lt_500 = 0
    n_gt_500 = 0

    for j, f in enumerate(frames):
        s_i = f.scale_to_kb(mn_k, mn_B)
        fout = open('frame-s-i-%05d.dat' % j, 'w')
        for s, i, si in s_i:
            fout.write('%f %f %f\n' % (s, i, si))
            if i < 500:
                n_lt_500 += 1
            else:
                n_gt_500 += 1
        fout.close()

    from collections import defaultdict

    hist = defaultdict(int)

    fout = open('kb.dat', 'w')

    for j, f in enumerate(frames):
        kb = f.kb()
        fout.write('%4d %6.3f %6.3f\n' % (j, kb[0], kb[1]))
        hist[int(round(kb[1]))] += 1

    fout.close()

    for b in sorted(hist):
        print b, hist[b]


    print odd, even
    print n_lt_500, n_gt_500
    

    return
Ejemplo n.º 5
0
def run(args):
  phil = iotbx.phil.process_command_line(args=args, master_string=master_phil).show()
  work_params = phil.work.extract()
  from xfel.merging.phil_validation import application
  application(work_params)
  if ("--help" in args) :
    libtbx.phil.parse(master_phil.show())
    return

  if ((work_params.d_min is None) or
      (work_params.data is None) or
      ( (work_params.model is None) and work_params.scaling.algorithm != "mark1") ) :
    raise Usage("cxi.merge "
                "d_min=4.0 "
                "data=~/scratch/r0220/006/strong/ "
                "model=3bz1_3bz2_core.pdb")
  if ((work_params.rescale_with_average_cell) and
      (not work_params.set_average_unit_cell)) :
    raise Usage("If rescale_with_average_cell=True, you must also specify "+
      "set_average_unit_cell=True.")
  if work_params.raw_data.sdfac_auto and work_params.raw_data.sdfac_refine:
    raise Usage("Cannot specify both sdfac_auto and sdfac_refine")

  log = open("%s_%s.log" % (work_params.output.prefix,work_params.scaling.algorithm), "w")
  out = multi_out()
  out.register("log", log, atexit_send_to=None)
  out.register("stdout", sys.stdout)

  # Verify that the externally supplied isomorphous reference, if
  # present, defines a suitable column of intensities, and exit with
  # error if it does not.  Then warn if it is necessary to generate
  # Bijvoet mates.  Failure to catch these issues here would lead to
  # possibly obscure problems in cxi/cxi_cc.py later on.
  try:
    data_SR = mtz.object(work_params.scaling.mtz_file)
  except RuntimeError:
    pass
  else:
    array_SR = None
    obs_labels = []
    for array in data_SR.as_miller_arrays():
      this_label = array.info().label_string().lower()
      if array.observation_type() is not None:
        obs_labels.append(this_label.split(',')[0])
      if this_label.find('fobs')>=0:
        array_SR = array.as_intensity_array()
        break
      if this_label.find('imean')>=0:
        array_SR = array.as_intensity_array()
        break
      if this_label.find(work_params.scaling.mtz_column_F)==0:
        array_SR = array.as_intensity_array()
        break

    if array_SR is None:
      known_labels = ['fobs', 'imean', work_params.scaling.mtz_column_F]
      raise Usage(work_params.scaling.mtz_file +
                  " does not contain any observations labelled [" +
                  ", ".join(known_labels) +
                  "].  Please set scaling.mtz_column_F to one of [" +
                  ",".join(obs_labels) + "].")
    elif not work_params.merge_anomalous and not array_SR.anomalous_flag():
      print >> out, "Warning: Preserving anomalous contributors, but %s " \
        "has anomalous contributors merged.  Generating identical Bijvoet " \
        "mates." % work_params.scaling.mtz_file

  # Read Nat's reference model from an MTZ file.  XXX The observation
  # type is given as F, not I--should they be squared?  Check with Nat!
  print >> out, "I model"
  if work_params.model is not None:
    from xfel.merging.general_fcalc import run
    i_model = run(work_params)
    work_params.target_unit_cell = i_model.unit_cell()
    work_params.target_space_group = i_model.space_group_info()
    i_model.show_summary()
  else:
    i_model = None

  print >> out, "Target unit cell and space group:"
  print >> out, "  ", work_params.target_unit_cell
  print >> out, "  ", work_params.target_space_group

  miller_set = symmetry(
      unit_cell=work_params.target_unit_cell,
      space_group_info=work_params.target_space_group
    ).build_miller_set(
      anomalous_flag=not work_params.merge_anomalous,
      d_max=work_params.d_max,
      d_min=work_params.d_min / math.pow(
        1 + work_params.unit_cell_length_tolerance, 1 / 3))
  miller_set = miller_set.change_basis(
    work_params.model_reindex_op).map_to_asu()

  if i_model is not None:
    matches = miller.match_indices(i_model.indices(), miller_set.indices())
    assert not matches.have_singles()
    miller_set = miller_set.select(matches.permutation())

# ---- Augment this code with any special procedures for x scaling
  scaler = xscaling_manager(
    miller_set=miller_set,
    i_model=i_model,
    params=work_params,
    log=out)
  scaler.scale_all()
  if scaler.n_accepted == 0:
    return None
# --- End of x scaling
  scaler.uc_values = unit_cell_distribution()
  for icell in xrange(len(scaler.frames["unit_cell"])):
    if scaler.params.model is None:
      scaler.uc_values.add_cell(
      unit_cell=scaler.frames["unit_cell"][icell])
    else:
      scaler.uc_values.add_cell(
      unit_cell=scaler.frames["unit_cell"][icell],
      rejected=(scaler.frames["cc"][icell] < scaler.params.min_corr))

  scaler.show_unit_cell_histograms()
  if (work_params.rescale_with_average_cell) :
    average_cell_abc = scaler.uc_values.get_average_cell_dimensions()
    average_cell = uctbx.unit_cell(list(average_cell_abc) +
      list(work_params.target_unit_cell.parameters()[3:]))
    work_params.target_unit_cell = average_cell
    print >> out, ""
    print >> out, "#" * 80
    print >> out, "RESCALING WITH NEW TARGET CELL"
    print >> out, "  average cell: %g %g %g %g %g %g" % \
      work_params.target_unit_cell.parameters()
    print >> out, ""
    scaler.reset()
    scaler = xscaling_manager(
      miller_set=miller_set,
      i_model=i_model,
      params=work_params,
      log=out)
    scaler.scale_all()
    scaler.uc_values = unit_cell_distribution()
    for icell in xrange(len(scaler.frames["unit_cell"])):
      if scaler.params.model is None:
        scaler.uc_values.add_cell(
        unit_cell=scaler.frames["unit_cell"][icell])
      else:
        scaler.uc_values.add_cell(
        unit_cell=scaler.frames["unit_cell"][icell],
        rejected=(scaler.frames["cc"][icell] < scaler.params.min_corr))
    scaler.show_unit_cell_histograms()
  if False : #(work_params.output.show_plots) :
    try :
      plot_overall_completeness(completeness)
    except Exception, e :
      print "ERROR: can't show plots"
      print "  %s" % str(e)
Ejemplo n.º 6
0
def run(args):
    phil = iotbx.phil.process_command_line(args=args,
                                           master_string=master_phil).show()
    work_params = phil.work.extract()
    from xfel.merging.phil_validation import application
    application(work_params)
    if ("--help" in args):
        libtbx.phil.parse(master_phil.show())
        return

    if ((work_params.d_min is None) or (work_params.data is None)
            or ((work_params.model is None)
                and work_params.scaling.algorithm != "mark1")):
        raise Usage("cxi.merge "
                    "d_min=4.0 "
                    "data=~/scratch/r0220/006/strong/ "
                    "model=3bz1_3bz2_core.pdb")
    if ((work_params.rescale_with_average_cell)
            and (not work_params.set_average_unit_cell)):
        raise Usage(
            "If rescale_with_average_cell=True, you must also specify " +
            "set_average_unit_cell=True.")
    if work_params.raw_data.sdfac_auto and work_params.raw_data.sdfac_refine:
        raise Usage("Cannot specify both sdfac_auto and sdfac_refine")

    log = open(
        "%s_%s.log" %
        (work_params.output.prefix, work_params.scaling.algorithm), "w")
    out = multi_out()
    out.register("log", log, atexit_send_to=None)
    out.register("stdout", sys.stdout)

    # Verify that the externally supplied isomorphous reference, if
    # present, defines a suitable column of intensities, and exit with
    # error if it does not.  Then warn if it is necessary to generate
    # Bijvoet mates.  Failure to catch these issues here would lead to
    # possibly obscure problems in cxi/cxi_cc.py later on.
    try:
        data_SR = mtz.object(work_params.scaling.mtz_file)
    except RuntimeError:
        pass
    else:
        array_SR = None
        obs_labels = []
        for array in data_SR.as_miller_arrays():
            this_label = array.info().label_string().lower()
            if array.observation_type() is not None:
                obs_labels.append(this_label.split(',')[0])
            if this_label.find('fobs') >= 0:
                array_SR = array.as_intensity_array()
                break
            if this_label.find('imean') >= 0:
                array_SR = array.as_intensity_array()
                break
            if this_label.find(work_params.scaling.mtz_column_F) == 0:
                array_SR = array.as_intensity_array()
                break

        if array_SR is None:
            known_labels = ['fobs', 'imean', work_params.scaling.mtz_column_F]
            raise Usage(work_params.scaling.mtz_file +
                        " does not contain any observations labelled [" +
                        ", ".join(known_labels) +
                        "].  Please set scaling.mtz_column_F to one of [" +
                        ",".join(obs_labels) + "].")
        elif not work_params.merge_anomalous and not array_SR.anomalous_flag():
            print >> out, "Warning: Preserving anomalous contributors, but %s " \
              "has anomalous contributors merged.  Generating identical Bijvoet " \
              "mates." % work_params.scaling.mtz_file

    # Read Nat's reference model from an MTZ file.  XXX The observation
    # type is given as F, not I--should they be squared?  Check with Nat!
    print >> out, "I model"
    if work_params.model is not None:
        from xfel.merging.general_fcalc import run
        i_model = run(work_params)
        work_params.target_unit_cell = i_model.unit_cell()
        work_params.target_space_group = i_model.space_group_info()
        i_model.show_summary()
    else:
        i_model = None

    print >> out, "Target unit cell and space group:"
    print >> out, "  ", work_params.target_unit_cell
    print >> out, "  ", work_params.target_space_group

    miller_set, i_model = consistent_set_and_model(work_params, i_model)

    # ---- Augment this code with any special procedures for x scaling
    scaler = xscaling_manager(miller_set=miller_set,
                              i_model=i_model,
                              params=work_params,
                              log=out)
    scaler.scale_all()
    if scaler.n_accepted == 0:
        return None


# --- End of x scaling
    scaler.uc_values = unit_cell_distribution()
    for icell in xrange(len(scaler.frames["unit_cell"])):
        if scaler.params.model is None:
            scaler.uc_values.add_cell(
                unit_cell=scaler.frames["unit_cell"][icell])
        else:
            scaler.uc_values.add_cell(
                unit_cell=scaler.frames["unit_cell"][icell],
                rejected=(scaler.frames["cc"][icell] < scaler.params.min_corr))

    scaler.show_unit_cell_histograms()
    if (work_params.rescale_with_average_cell):
        average_cell_abc = scaler.uc_values.get_average_cell_dimensions()
        average_cell = uctbx.unit_cell(
            list(average_cell_abc) +
            list(work_params.target_unit_cell.parameters()[3:]))
        work_params.target_unit_cell = average_cell
        print >> out, ""
        print >> out, "#" * 80
        print >> out, "RESCALING WITH NEW TARGET CELL"
        print >> out, "  average cell: %g %g %g %g %g %g" % \
          work_params.target_unit_cell.parameters()
        print >> out, ""
        scaler.reset()
        scaler = xscaling_manager(miller_set=miller_set,
                                  i_model=i_model,
                                  params=work_params,
                                  log=out)
        scaler.scale_all()
        scaler.uc_values = unit_cell_distribution()
        for icell in xrange(len(scaler.frames["unit_cell"])):
            if scaler.params.model is None:
                scaler.uc_values.add_cell(
                    unit_cell=scaler.frames["unit_cell"][icell])
            else:
                scaler.uc_values.add_cell(
                    unit_cell=scaler.frames["unit_cell"][icell],
                    rejected=(scaler.frames["cc"][icell] <
                              scaler.params.min_corr))
        scaler.show_unit_cell_histograms()
    if False:  #(work_params.output.show_plots) :
        try:
            plot_overall_completeness(completeness)
        except Exception, e:
            print "ERROR: can't show plots"
            print "  %s" % str(e)
Ejemplo n.º 7
0
def run(args):
  phil = iotbx.phil.process_command_line(
    args = args, master_string = master_phil)
  work_params = phil.work.extract()
  if ("--help" in args) :
    libtbx.phil.parse(master_phil.show())
    return

  if ((work_params.d_min is None) or
      (work_params.data is None) or
      ((work_params.model is None) and
       work_params.scaling.algorithm != "mark1")):
    raise Usage("cxi.merge "
                "d_min=4.0 "
                "data=~/scratch/r0220/006/strong/ "
                "model=3bz1_3bz2_core.pdb")
  
  if ((work_params.rescale_with_average_cell) and
      (not work_params.set_average_unit_cell)) :
    raise Usage("If rescale_with_average_cell=True, you must also specify "+
      "set_average_unit_cell=True.")
  
  miller_set = symmetry(
      unit_cell = work_params.target_unit_cell,
      space_group_info = work_params.target_space_group
    ).build_miller_set(
      anomalous_flag = not work_params.merge_anomalous,
      d_min = work_params.d_min)
  from xfel.cxi.merging.general_fcalc import random_structure
  i_model = random_structure(work_params)

# ---- Augment this code with any special procedures for x scaling
  scaler = xscaling_manager(
    miller_set = miller_set,
    i_model = i_model,
    params = work_params)
  
  scaler.read_all()
  sg = miller_set.space_group()
  pg = sg.build_derived_laue_group()
  rational_ops = []
  for symop in pg:
    rational_ops.append((matrix.sqr(symop.r().transpose().as_rational()),
                         symop.r().as_hkl()))

  # miller_set.show_summary()
    
  uc = work_params.target_unit_cell
    
  hkl_asu = scaler.observations["hkl_id"]
  imageno = scaler.observations["frame_id"]
  intensi = scaler.observations["i"]
  sigma_i = scaler.observations["sigi"]
  lookup = scaler.millers["merged_asu_hkl"]
  origH = scaler.observations["H"]
  origK = scaler.observations["K"]
  origL = scaler.observations["L"]

  from cctbx.miller import map_to_asu
  sgtype = miller_set.space_group_info().type()
  aflag = miller_set.anomalous_flag()
  from cctbx.array_family import flex

  # FIXME in here perform the mapping to ASU for both the original and other
  # index as an array-wise manipulation to make things a bunch faster...
  # however this also uses a big chunk of RAM... FIXME also in here use
  # cb_op.apply(indices) to get the indices reindexed...

  original_indices = flex.miller_index()
  for x in xrange(len(scaler.observations["hkl_id"])):
    original_indices.append(lookup[hkl_asu[x]])

  from cctbx.sgtbx import change_of_basis_op

  I23 = change_of_basis_op('k, -h, l')

  other_indices = I23.apply(original_indices)

  map_to_asu(sgtype, aflag, original_indices)
  map_to_asu(sgtype, aflag, other_indices)

  # FIXME would be useful in here to have a less expensive way of finding the
  # symmetry operation which gave the map to the ASU - perhaps best way is to
  # make a new C++ map_to_asu which records this.
  
  # FIXME in here recover the original frame structure of the data to
  # logical frame objetcs - N.B. the frame will need to be augmented to test
  # alternative indexings

  # construct table of start / end indices for frames: now using Python
  # range indexing

  starts = [0]
  ends = []
    
  for x in xrange(1, len(scaler.observations["hkl_id"])):
    if imageno[x] != imageno[x - 1]:
      ends.append(x)
      starts.append(x)
            
  ends.append(len(scaler.observations["hkl_id"]))
  
  keep_start = []
  keep_end = []
  
  for j, se in enumerate(zip(starts, ends)):
    print 'processing frame %d: %d to %d' % (j, se[0], se[1])
    s, e = se
    isig = sum(i / s for i, s in zip(intensi[s:e], sigma_i[s:e])) / (e - s)
    dmin = 100.0
    for x in xrange(s, e):
      d = uc.d(lookup[hkl_asu[x]])
      if d < dmin:
        dmin = d
    if isig > 6.0 and dmin < 3.2:
      keep_start.append(s)
      keep_end.append(e)

  starts = keep_start
  ends = keep_end

  print 'Keeping %d frames' % len(starts)

  # then start running the comparison code

  frames = []

  for s, e in zip(starts, ends):
    # FIXME need this from remap to ASU
    misym = [0 for x in range(s, e)]
    indices = [original_indices[x] for x in range(s, e)]
    other = [other_indices[x] for x in range(s, e)]
    intensities = intensi[s:e]
    sigmas = sigma_i[s:e]

    frames.append(Frame(uc, indices, other, intensities, sigmas))

  reference = FrameFromReferenceMTZ()

  fout = open('cc_reference.log', 'w')

  for j, f in enumerate(frames):
    _cc = reference.cc(f)
    _oo = reference.cc_other(f)
    print '%d %d %d %d %f %d %f' % (j, starts[j], ends[j], _cc[0], _cc[1],
                                    _oo[0], _oo[1])

    fout.write('%d %d %d %d %f %d %f\n' % (j, starts[j], ends[j],
                                           _cc[0], _cc[1], _oo[0], _oo[1]))

  fout.close()

  return
Ejemplo n.º 8
0
def run(args):
    phil = iotbx.phil.process_command_line(args=args,
                                           master_string=master_phil).show()
    work_params = phil.work.extract()
    from xfel.merging.phil_validation import application, samosa
    application(work_params)
    samosa(work_params)
    if ("--help" in args):
        libtbx.phil.parse(master_phil.show())
        return

    if ((work_params.d_min is None) or (work_params.data is None)):
        command_name = os.environ["LIBTBX_DISPATCHER_NAME"]
        raise Usage(command_name + " "
                    "d_min=4.0 "
                    "data=~/scratch/r0220/006/strong/ "
                    "model=3bz1_3bz2_core.pdb")
    if ((work_params.rescale_with_average_cell)
            and (not work_params.set_average_unit_cell)):
        raise Usage(
            "If rescale_with_average_cell=True, you must also specify " +
            "set_average_unit_cell=True.")
    if work_params.raw_data.sdfac_auto and work_params.raw_data.sdfac_refine:
        raise Usage("Cannot specify both sdfac_auto and sdfac_refine")

    # Read Nat's reference model from an MTZ file.  XXX The observation
    # type is given as F, not I--should they be squared?  Check with Nat!
    log = open("%s.log" % work_params.output.prefix, "w")
    out = multi_out()
    out.register("log", log, atexit_send_to=None)
    out.register("stdout", sys.stdout)
    print >> out, "I model"
    if work_params.model is not None:
        from xfel.merging.general_fcalc import run
        i_model = run(work_params)
        work_params.target_unit_cell = i_model.unit_cell()
        work_params.target_space_group = i_model.space_group_info()
        i_model.show_summary()
    else:
        i_model = None

    print >> out, "Target unit cell and space group:"
    print >> out, "  ", work_params.target_unit_cell
    print >> out, "  ", work_params.target_space_group

    miller_set, i_model = consistent_set_and_model(work_params, i_model)

    frame_files = get_observations(work_params)
    scaler = scaling_manager(miller_set=miller_set,
                             i_model=i_model,
                             params=work_params,
                             log=out)
    scaler.scale_all(frame_files)
    if scaler.n_accepted == 0:
        return None
    scaler.show_unit_cell_histograms()
    if (work_params.rescale_with_average_cell):
        average_cell_abc = scaler.uc_values.get_average_cell_dimensions()
        average_cell = uctbx.unit_cell(
            list(average_cell_abc) +
            list(work_params.target_unit_cell.parameters()[3:]))
        work_params.target_unit_cell = average_cell
        print >> out, ""
        print >> out, "#" * 80
        print >> out, "RESCALING WITH NEW TARGET CELL"
        print >> out, "  average cell: %g %g %g %g %g %g" % \
          work_params.target_unit_cell.parameters()
        print >> out, ""
        scaler.reset()
        scaler.scale_all(frame_files)
        scaler.show_unit_cell_histograms()
    if False:  #(work_params.output.show_plots) :
        try:
            plot_overall_completeness(completeness)
        except Exception, e:
            print "ERROR: can't show plots"
            print "  %s" % str(e)
Ejemplo n.º 9
0
def run(args):
    phil = iotbx.phil.process_command_line(args=args,
                                           master_string=master_phil).show()
    work_params = phil.work.extract()
    from xfel.merging.phil_validation import application
    application(work_params)
    if ("--help" in args):
        libtbx.phil.parse(master_phil.show())
        return

    if ((work_params.d_min is None) or (work_params.data is None)
            or ((work_params.model is None)
                and work_params.scaling.algorithm != "mark1")):
        raise Usage("cxi.merge "
                    "d_min=4.0 "
                    "data=~/scratch/r0220/006/strong/ "
                    "model=3bz1_3bz2_core.pdb")
    if ((work_params.rescale_with_average_cell)
            and (not work_params.set_average_unit_cell)):
        raise Usage(
            "If rescale_with_average_cell=True, you must also specify " +
            "set_average_unit_cell=True.")
    if work_params.raw_data.sdfac_auto and work_params.raw_data.sdfac_refine:
        raise Usage("Cannot specify both sdfac_auto and sdfac_refine")
    if not work_params.include_negatives_fix_27May2018:
        work_params.include_negatives = False  # use old behavior

    log = open(
        "%s_%s.log" %
        (work_params.output.prefix, work_params.scaling.algorithm), "w")
    out = multi_out()
    out.register("log", log, atexit_send_to=None)
    out.register("stdout", sys.stdout)

    # Verify that the externally supplied isomorphous reference, if
    # present, defines a suitable column of intensities, and exit with
    # error if it does not.  Then warn if it is necessary to generate
    # Bijvoet mates.  Failure to catch these issues here would lead to
    # possibly obscure problems in cxi/cxi_cc.py later on.
    try:
        data_SR = mtz.object(work_params.scaling.mtz_file)
    except RuntimeError:
        pass
    else:
        array_SR = None
        obs_labels = []
        for array in data_SR.as_miller_arrays():
            this_label = array.info().label_string().lower()
            if array.observation_type() is not None:
                obs_labels.append(this_label.split(',')[0])
            if this_label.find('fobs') >= 0:
                array_SR = array.as_intensity_array()
                break
            if this_label.find('imean') >= 0:
                array_SR = array.as_intensity_array()
                break
            if this_label.find(work_params.scaling.mtz_column_F) == 0:
                array_SR = array.as_intensity_array()
                break

        if array_SR is None:
            known_labels = ['fobs', 'imean', work_params.scaling.mtz_column_F]
            raise Usage(work_params.scaling.mtz_file +
                        " does not contain any observations labelled [" +
                        ", ".join(known_labels) +
                        "].  Please set scaling.mtz_column_F to one of [" +
                        ",".join(obs_labels) + "].")
        elif not work_params.merge_anomalous and not array_SR.anomalous_flag():
            print("Warning: Preserving anomalous contributors, but %s " \
              "has anomalous contributors merged.  Generating identical Bijvoet " \
              "mates." % work_params.scaling.mtz_file, file=out)

    # Read Nat's reference model from an MTZ file.  XXX The observation
    # type is given as F, not I--should they be squared?  Check with Nat!
    print("I model", file=out)
    if work_params.model is not None:
        from xfel.merging.general_fcalc import run
        i_model = run(work_params)
        work_params.target_unit_cell = i_model.unit_cell()
        work_params.target_space_group = i_model.space_group_info()
        i_model.show_summary()
    else:
        i_model = None

    print("Target unit cell and space group:", file=out)
    print("  ", work_params.target_unit_cell, file=out)
    print("  ", work_params.target_space_group, file=out)

    miller_set, i_model = consistent_set_and_model(work_params, i_model)

    # ---- Augment this code with any special procedures for x scaling
    scaler = xscaling_manager(miller_set=miller_set,
                              i_model=i_model,
                              params=work_params,
                              log=out)
    scaler.scale_all()
    if scaler.n_accepted == 0:
        return None


# --- End of x scaling
    scaler.uc_values = unit_cell_distribution()
    for icell in range(len(scaler.frames["unit_cell"])):
        if scaler.params.model is None:
            scaler.uc_values.add_cell(
                unit_cell=scaler.frames["unit_cell"][icell])
        else:
            scaler.uc_values.add_cell(
                unit_cell=scaler.frames["unit_cell"][icell],
                rejected=(scaler.frames["cc"][icell] < scaler.params.min_corr))

    scaler.show_unit_cell_histograms()
    if (work_params.rescale_with_average_cell):
        average_cell_abc = scaler.uc_values.get_average_cell_dimensions()
        average_cell = uctbx.unit_cell(
            list(average_cell_abc) +
            list(work_params.target_unit_cell.parameters()[3:]))
        work_params.target_unit_cell = average_cell
        print("", file=out)
        print("#" * 80, file=out)
        print("RESCALING WITH NEW TARGET CELL", file=out)
        print("  average cell: %g %g %g %g %g %g" % \
          work_params.target_unit_cell.parameters(), file=out)
        print("", file=out)
        scaler.reset()
        scaler = xscaling_manager(miller_set=miller_set,
                                  i_model=i_model,
                                  params=work_params,
                                  log=out)
        scaler.scale_all()
        scaler.uc_values = unit_cell_distribution()
        for icell in range(len(scaler.frames["unit_cell"])):
            if scaler.params.model is None:
                scaler.uc_values.add_cell(
                    unit_cell=scaler.frames["unit_cell"][icell])
            else:
                scaler.uc_values.add_cell(
                    unit_cell=scaler.frames["unit_cell"][icell],
                    rejected=(scaler.frames["cc"][icell] <
                              scaler.params.min_corr))
        scaler.show_unit_cell_histograms()
    if False:  #(work_params.output.show_plots) :
        try:
            plot_overall_completeness(completeness)
        except Exception as e:
            print("ERROR: can't show plots")
            print("  %s" % str(e))
    print("\n", file=out)

    reserve_prefix = work_params.output.prefix
    for data_subset in [1, 2, 0]:
        work_params.data_subset = data_subset
        work_params.output.prefix = "%s_s%1d_%s" % (
            reserve_prefix, data_subset, work_params.scaling.algorithm)

        if work_params.data_subset == 0:
            scaler.frames["data_subset"] = flex.bool(
                scaler.frames["frame_id"].size(), True)
        elif work_params.data_subset == 1:
            scaler.frames["data_subset"] = scaler.frames["odd_numbered"]
        elif work_params.data_subset == 2:
            scaler.frames["data_subset"] = scaler.frames[
                "odd_numbered"] == False

    # --------- New code ------------------
    #sanity check
        for mod, obs in zip(miller_set.indices(),
                            scaler.millers["merged_asu_hkl"]):
            if mod != obs:
                raise Exception(
                    "miller index lists inconsistent--check d_min are equal for merge and xmerge scripts"
                )
            assert mod == obs
        """Sum the observations of I and I/sig(I) for each reflection.
    sum_I = flex.double(i_model.size(), 0.)
    sum_I_SIGI = flex.double(i_model.size(), 0.)
    scaler.completeness = flex.int(i_model.size(), 0)
    scaler.summed_N = flex.int(i_model.size(), 0)
    scaler.summed_wt_I = flex.double(i_model.size(), 0.)
    scaler.summed_weight = flex.double(i_model.size(), 0.)
    scaler.n_rejected = flex.double(scaler.frames["frame_id"].size(), 0.)
    scaler.n_obs = flex.double(scaler.frames["frame_id"].size(), 0.)
    scaler.d_min_values = flex.double(scaler.frames["frame_id"].size(), 0.)
    scaler.ISIGI = {}"""

        from xfel import scaling_results, get_scaling_results, get_isigi_dict
        results = scaling_results(scaler._observations, scaler._frames,
                                  scaler.millers["merged_asu_hkl"],
                                  scaler.frames["data_subset"],
                                  work_params.include_negatives)
        results.__getattribute__(work_params.scaling.algorithm)(
            scaler.params.min_corr, scaler.params.target_unit_cell)

        sum_I, sum_I_SIGI, \
        scaler.completeness, scaler.summed_N, \
        scaler.summed_wt_I, scaler.summed_weight, scaler.n_rejected, scaler.n_obs, \
        scaler.d_min_values, hkl_ids, i_sigi_list = get_scaling_results(results)

        scaler.ISIGI = get_isigi_dict(results)

        if work_params.merging.refine_G_Imodel:
            from xfel.cxi.merging.refine import find_scale

            my_find_scale = find_scale(scaler, work_params)

            sum_I, sum_I_SIGI, \
              scaler.completeness, scaler.summed_N, \
              scaler.summed_wt_I, scaler.summed_weight, scaler.n_rejected, \
              scaler.n_obs, scaler.d_min_values, hkl_ids, i_sigi_list \
              = my_find_scale.get_scaling_results(results, scaler)
            scaler.ISIGI = get_isigi_dict(results)

        scaler.wavelength = scaler.frames["wavelength"]
        scaler.corr_values = scaler.frames["cc"]

        scaler.rejected_fractions = flex.double(
            scaler.frames["frame_id"].size(), 0.)
        for irej in range(len(scaler.rejected_fractions)):
            if scaler.n_obs[irej] > 0:
                scaler.rejected_fractions = scaler.n_rejected[
                    irej] / scaler.n_obs[irej]
    # ---------- End of new code ----------------

        if work_params.raw_data.sdfac_refine or work_params.raw_data.errors_from_sample_residuals:
            if work_params.raw_data.sdfac_refine:
                if work_params.raw_data.error_models.sdfac_refine.minimizer == 'simplex':
                    from xfel.merging.algorithms.error_model.sdfac_refine import sdfac_refine as error_modeler
                elif work_params.raw_data.error_models.sdfac_refine.minimizer == 'lbfgs':
                    from xfel.merging.algorithms.error_model.sdfac_refine_lbfgs import sdfac_refine_refltable_lbfgs as error_modeler
                elif self.params.raw_data.error_models.sdfac_refine.minimizer == 'LevMar':
                    from xfel.merging.algorithms.error_model.sdfac_refine_levmar import sdfac_refine_refltable_levmar as error_modeler

            if work_params.raw_data.errors_from_sample_residuals:
                from xfel.merging.algorithms.error_model.errors_from_residuals import errors_from_residuals as error_modeler

            error_modeler(scaler).adjust_errors()

        if work_params.raw_data.reduced_chi_squared_correction:
            from xfel.merging.algorithms.error_model.reduced_chi_squared import reduced_chi_squared
            reduced_chi_squared(scaler).compute()

        miller_set_avg = miller_set.customized_copy(
            unit_cell=work_params.target_unit_cell)

        table1 = show_overall_observations(
            obs=miller_set_avg,
            redundancy=scaler.completeness,
            redundancy_to_edge=None,
            summed_wt_I=scaler.summed_wt_I,
            summed_weight=scaler.summed_weight,
            ISIGI=scaler.ISIGI,
            n_bins=work_params.output.n_bins,
            title="Statistics for all reflections",
            out=out,
            work_params=work_params)
        if table1 is None:
            raise Exception("table could not be constructed")
        print("", file=out)
        if work_params.scaling.algorithm == 'mark0':
            n_refl, corr = scaler.get_overall_correlation(sum_I)
        else:
            n_refl, corr = ((scaler.completeness > 0).count(True), 0)
        print("\n", file=out)
        table2 = show_overall_observations(
            obs=miller_set_avg,
            redundancy=scaler.summed_N,
            redundancy_to_edge=None,
            summed_wt_I=scaler.summed_wt_I,
            summed_weight=scaler.summed_weight,
            ISIGI=scaler.ISIGI,
            n_bins=work_params.output.n_bins,
            title="Statistics for reflections where I > 0",
            out=out,
            work_params=work_params)
        if table2 is None:
            raise Exception("table could not be constructed")

        print("", file=out)
        mtz_file, miller_array = scaler.finalize_and_save_data()

        loggraph_file = os.path.abspath("%s_graphs.log" %
                                        work_params.output.prefix)
        f = open(loggraph_file, "w")
        f.write(table1.format_loggraph())
        f.write("\n")
        f.write(table2.format_loggraph())
        f.close()
        result = scaling_result(miller_array=miller_array,
                                plots=scaler.get_plot_statistics(),
                                mtz_file=mtz_file,
                                loggraph_file=loggraph_file,
                                obs_table=table1,
                                all_obs_table=table2,
                                n_reflections=n_refl,
                                overall_correlation=corr)
        easy_pickle.dump("%s.pkl" % work_params.output.prefix, result)
    work_params.output.prefix = reserve_prefix

    # Output table with number of images contribution reflections per
    # resolution bin.
    from libtbx import table_utils

    miller_set_avg.setup_binner(d_max=100000,
                                d_min=work_params.d_min,
                                n_bins=work_params.output.n_bins)
    table_data = [["Bin", "Resolution Range", "# images", "%accept"]]
    if work_params.model is None:
        appropriate_min_corr = -1.1  # lowest possible c.c.
    else:
        appropriate_min_corr = work_params.min_corr
    n_frames = (scaler.frames['cc'] > appropriate_min_corr).count(True)
    iselect = 1
    while iselect < work_params.output.n_bins:
        col_count1 = results.count_frames(
            appropriate_min_corr,
            miller_set_avg.binner().selection(iselect))
        print("colcount1", col_count1)
        if col_count1 > 0: break
        iselect += 1
    if col_count1 == 0: raise Exception("no reflections in any bins")
    for i_bin in miller_set_avg.binner().range_used():
        col_count = '%8d' % results.count_frames(
            appropriate_min_corr,
            miller_set_avg.binner().selection(i_bin))
        col_legend = '%-13s' % miller_set_avg.binner().bin_legend(
            i_bin=i_bin,
            show_bin_number=False,
            show_bin_range=False,
            show_d_range=True,
            show_counts=False)
        xpercent = results.count_frames(
            appropriate_min_corr,
            miller_set_avg.binner().selection(i_bin)) / float(n_frames)
        percent = '%5.2f' % (100. * xpercent)
        table_data.append(['%3d' % i_bin, col_legend, col_count, percent])

    table_data.append([""] * len(table_data[0]))
    table_data.append(["All", "", '%8d' % n_frames])
    print(file=out)
    print(table_utils.format(table_data,
                             has_header=1,
                             justify='center',
                             delim=' '),
          file=out)

    reindexing_ops = {
        "h,k,l": 0
    }  # get a list of all reindexing ops for this dataset
    if work_params.merging.reverse_lookup is not None:
        for key in scaler.reverse_lookup:
            if reindexing_ops.get(scaler.reverse_lookup[key], None) is None:
                reindexing_ops[scaler.reverse_lookup[key]] = 0
            reindexing_ops[scaler.reverse_lookup[key]] += 1

    from xfel.cxi.cxi_cc import run_cc
    for key in reindexing_ops.keys():
        run_cc(work_params, reindexing_op=key, output=out)

    if isinstance(scaler.ISIGI, dict):
        from xfel.merging import isigi_dict_to_reflection_table
        refls = isigi_dict_to_reflection_table(scaler.miller_set.indices(),
                                               scaler.ISIGI)
    else:
        refls = scaler.ISIGI
    easy_pickle.dump("%s.refl" % work_params.output.prefix, refls)

    return result
Ejemplo n.º 10
0
def find_merge_common_images(args):
    phil = iotbx.phil.process_command_line(args = args,
                                           master_string = master_phil).show()
    work_params = phil.work.extract()
    if ("--help" in args) :
        libtbx.phil.parse(master_phil.show())
        return

    if ((work_params.d_min is None) or
        (work_params.data is None) or
        ((work_params.model is None) and
         work_params.scaling.algorithm != "mark1")) :
        raise Usage("cxi.merge "
                    "d_min=4.0 "
                    "data=~/scratch/r0220/006/strong/ "
                    "model=3bz1_3bz2_core.pdb")
    if ((work_params.rescale_with_average_cell) and
        (not work_params.set_average_unit_cell)) :
        raise Usage(
            "If rescale_with_average_cell=True, you must also specify "+
            "set_average_unit_cell=True.")

    # Read Nat's reference model from an MTZ file.  XXX The observation
    # type is given as F, not I--should they be squared?  Check with Nat!
    log = open("%s_%s_scale.log" % (work_params.output.prefix,
                                    work_params.scaling.algorithm), "w")
    out = multi_out()
    out.register("log", log, atexit_send_to=None)
    out.register("stdout", sys.stdout)

    print >> out, "Target unit cell and space group:"
    print >> out, "  ", work_params.target_unit_cell
    print >> out, "  ", work_params.target_space_group

    uc = work_params.target_unit_cell

    miller_set = symmetry(
        unit_cell=work_params.target_unit_cell,
        space_group_info=work_params.target_space_group
        ).build_miller_set(
        anomalous_flag=not work_params.merge_anomalous,
        d_min=work_params.d_min)
    from xfel.cxi.merging.general_fcalc import random_structure
    i_model = random_structure(work_params)

    # ---- Augment this code with any special procedures for x scaling
    scaler = xscaling_manager(
        miller_set=miller_set,
        i_model=i_model,
        params=work_params,
        log=out)
    scaler.read_all()
    print "finished reading"
    sg = miller_set.space_group()
    pg = sg.build_derived_laue_group()
    miller_set.show_summary()

    hkl_asu = scaler.observations["hkl_id"]
    imageno = scaler.observations["frame_id"]
    intensi = scaler.observations["i"]
    sigma_i = scaler.observations["sigi"]
    
    lookup = scaler.millers["merged_asu_hkl"]    

    # construct table of start / end indices for frames: now using Python
    # range indexing

    starts = [0]
    ends = []
    
    for x in xrange(1, len(scaler.observations["hkl_id"])):
        if imageno[x] != imageno[x - 1]:
            ends.append(x)
            starts.append(x)
            
    ends.append(len(scaler.observations["hkl_id"]))

    keep_start = []
    keep_end = []

    for j, se in enumerate(zip(starts, ends)):
        s, e = se
        isig = sum(i / s for i, s in zip(intensi[s:e], sigma_i[s:e])) / (e - s)
        dmin = 100.0
        for x in xrange(s, e):
            d = uc.d(lookup[hkl_asu[x]])
            if d < dmin:
                dmin = d
        if isig > 6.0 and dmin < 3.2:
            keep_start.append(s)
            keep_end.append(e)

    starts = keep_start
    ends = keep_end

    print 'Keeping %d frames' % len(starts)

    frames = []

    for s, e in zip(starts, ends):
        indices = [tuple(lookup[hkl_asu[x]]) for x in range(s, e)]
        intensities = intensi[s:e]
        sigmas = sigma_i[s:e]

        frames.append(Frame(uc, indices, intensities, sigmas))

    cycle = 0

    total_nref = sum([len(f.get_indices()) for f in frames])

    # pre-scale the data - first determine average ln(k), B; then apply

    kbs = [f.kb() for f in frames]

    mn_k = sum([kb[0] for kb in kbs]) / len(kbs)
    mn_B = sum([kb[1] for kb in kbs]) / len(kbs)

    for f in frames:
        f.scale_to_kb(mn_k, mn_B)
    
    while True:

        print 'Analysing %d frames' % len(frames)
        print 'Cycle %d' % cycle
        cycle += 1

        print 'Power spectrum'
        fn = frame_numbers(frames)
        for j in sorted(fn):
            print '%4d %4d' % (j, fn[j])
            
        nref_cycle = sum([len(f.get_indices()) for f in frames])
        assert(nref_cycle == total_nref)

        common_reflections = numpy.zeros((len(frames), len(frames)),
                                         dtype = numpy.short)

        obs = { } 

        from cctbx.sgtbx import rt_mx, change_of_basis_op
        oh = change_of_basis_op(rt_mx('h,l,k'))

        for j, f in enumerate(frames):
            indices = set(f.get_indices())
            for i in indices:
                _i = tuple(i)
                if not _i in obs:
                    obs[_i] = []
                obs[_i].append(j)

        # work through unique observations ignoring those which include no
        # hand information
 
        for hkl in obs:
            if hkl == oh.apply(hkl):
                continue
            obs[hkl].sort()
            for j, f1 in enumerate(obs[hkl][:-1]):
                for f2 in obs[hkl][j + 1:]:
                    common_reflections[(f1, f2)] += 1

        cmn_rfl_list = []

        for f1 in range(len(frames)):
            for f2 in range(f1 + 1, len(frames)):
                if common_reflections[(f1, f2)] > 20:
                    cmn_rfl_list.append((common_reflections[(f1, f2)], f1, f2))

        cmn_rfl_list.sort()
        cmn_rfl_list.reverse()
    
        joins = []
        used = []
    
        for n, f1, f2 in cmn_rfl_list:

            if f1 in used or f2 in used:
                continue
            
            _cc = frames[f1].cc(frames[f2])

            # really only need to worry about f2 which will get merged...
            # merging multiple files together should be OK provided they are
            # correctly sorted (though the order should not matter anyhow?)
            # anyhow they are sorted anyway... ah as f2 > f1 then just sorting
            # the list by f2 will make sure the data cascase correctly.

            # p-value very small for cc > 0.75 for > 20 observations - necessary
            # as will be correlated due to Wilson curves

            if _cc[0] > 20 and _cc[1] > 0.75:
                print '%4d %.3f' % _cc, f1, f2
                joins.append((f2, f1))
                # used.append(f1)
                used.append(f2)

        if not joins:
            print 'No pairs found'
            break

        joins.sort()
        joins.reverse()
        
        for j2, j1 in joins:
            rmerge = frames[j1].merge(frames[j2])
            if rmerge:
                print 'R: %4d %4d %6.3f' % (j1, j2, rmerge)
            else:
                print 'R: %4d %4d ------' % (j1, j2)
                
        continue

    frames.sort()

    print 'Biggest few: #frames; #unique refl'
    j = -1
    while frames[j].get_frames() > 1:
        print frames[j].get_frames(), frames[j].get_unique_indices()
        j -= 1

    return
Ejemplo n.º 11
0
def run(args):
  phil = iotbx.phil.process_command_line(
    args = args, master_string = master_phil)
  work_params = phil.work.extract()
  if ("--help" in args) :
    libtbx.phil.parse(master_phil.show())
    return

  if ((work_params.d_min is None) or
      (work_params.data is None) or
      ((work_params.model is None) and
       work_params.scaling.algorithm != "mark1")):
    raise Usage("cxi.merge "
                "d_min=4.0 "
                "data=~/scratch/r0220/006/strong/ "
                "model=3bz1_3bz2_core.pdb")
  
  if ((work_params.rescale_with_average_cell) and
      (not work_params.set_average_unit_cell)) :
    raise Usage("If rescale_with_average_cell=True, you must also specify "+
      "set_average_unit_cell=True.")
  
  miller_set = symmetry(
      unit_cell = work_params.target_unit_cell,
      space_group_info = work_params.target_space_group
    ).build_miller_set(
      anomalous_flag = not work_params.merge_anomalous,
      d_min = work_params.d_min)
  from xfel.cxi.merging.general_fcalc import random_structure
  i_model = random_structure(work_params)

# ---- Augment this code with any special procedures for x scaling
  scaler = xscaling_manager(
    miller_set = miller_set,
    i_model = i_model,
    params = work_params)
  
  scaler.read_all()
  sg = miller_set.space_group()
  pg = sg.build_derived_laue_group()
  rational_ops = []
  for symop in pg:
    rational_ops.append((matrix.sqr(symop.r().transpose().as_rational()),
                         symop.r().as_hkl()))

  # miller_set.show_summary()
    
  uc = work_params.target_unit_cell
    
  hkl_asu = scaler.observations["hkl_id"]
  imageno = scaler.observations["frame_id"]
  intensi = scaler.observations["i"]
  sigma_i = scaler.observations["sigi"]
  lookup = scaler.millers["merged_asu_hkl"]
  origH = scaler.observations["H"]
  origK = scaler.observations["K"]
  origL = scaler.observations["L"]

  from cctbx.miller import map_to_asu
  sgtype = miller_set.space_group_info().type()
  aflag = miller_set.anomalous_flag()
  from cctbx.array_family import flex

  # FIXME in here perform the mapping to ASU for both the original and other
  # index as an array-wise manipulation to make things a bunch faster...
  # however this also uses a big chunk of RAM... FIXME also in here use
  # cb_op.apply(indices) to get the indices reindexed...

  original_indices = flex.miller_index()
  for x in xrange(len(scaler.observations["hkl_id"])):
    original_indices.append(lookup[hkl_asu[x]])

  from cctbx.sgtbx import change_of_basis_op

  I23 = change_of_basis_op('k, -h, l')

  other_indices = I23.apply(original_indices)

  map_to_asu(sgtype, aflag, original_indices)
  map_to_asu(sgtype, aflag, other_indices)

  # FIXME would be useful in here to have a less expensive way of finding the
  # symmetry operation which gave the map to the ASU - perhaps best way is to
  # make a new C++ map_to_asu which records this.
  
  # FIXME in here recover the original frame structure of the data to
  # logical frame objetcs - N.B. the frame will need to be augmented to test
  # alternative indexings

  # construct table of start / end indices for frames: now using Python
  # range indexing

  starts = [0]
  ends = []
    
  for x in xrange(1, len(scaler.observations["hkl_id"])):
    if imageno[x] != imageno[x - 1]:
      ends.append(x)
      starts.append(x)
            
  ends.append(len(scaler.observations["hkl_id"]))
  
  keep_start = []
  keep_end = []
  
  for j, se in enumerate(zip(starts, ends)):
    print 'processing frame %d: %d to %d' % (j, se[0], se[1])
    s, e = se
    isig = sum(i / s for i, s in zip(intensi[s:e], sigma_i[s:e])) / (e - s)
    dmin = 100.0
    for x in xrange(s, e):
      d = uc.d(lookup[hkl_asu[x]])
      if d < dmin:
        dmin = d
    if isig > 6.0 and dmin < 3.2:
      keep_start.append(s)
      keep_end.append(e)

  starts = keep_start
  ends = keep_end

  print 'Keeping %d frames' % len(starts)

  # then start running the comparison code

  frames = []

  for s, e in zip(starts, ends):
    # FIXME need this from remap to ASU
    misym = [0 for x in range(s, e)]
    indices = [original_indices[x] for x in range(s, e)]
    other = [other_indices[x] for x in range(s, e)]
    intensities = intensi[s:e]
    sigmas = sigma_i[s:e]

    frames.append(Frame(uc, indices, other, intensities, sigmas))

  cycle = 0

  total_nref = sum([len(f.get_indices()) for f in frames])

  # pre-scale the data - first determine average ln(k), B; then apply

  kbs = [f.kb() for f in frames]

  mn_k = sum([kb[0] for kb in kbs]) / len(kbs)
  mn_B = sum([kb[1] for kb in kbs]) / len(kbs)

  for f in frames:
    f.scale_to_kb(mn_k, mn_B)
    
  while True:

    print 'Analysing %d frames' % len(frames)
    print 'Cycle %d' % cycle
    cycle += 1

    print 'Power spectrum'
    fn = frame_numbers(frames)
    for j in sorted(fn):
      print '%4d %4d' % (j, fn[j])
            
    nref_cycle = sum([len(f.get_indices()) for f in frames])
    assert(nref_cycle == total_nref)

    # first work on the original indices

    import numpy

    common_reflections = numpy.zeros((len(frames), len(frames)),
                                     dtype = numpy.short)
    
    obs = { } 

    # for other hand add -j

    for j, f in enumerate(frames):
      indices = set(f.get_indices())
      for i in indices:
        _i = tuple(i)
        if not _i in obs:
          obs[_i] = []
        obs[_i].append(j)

    for hkl in obs:
      obs[hkl].sort()
      for j, f1 in enumerate(obs[hkl][:-1]):
        for f2 in obs[hkl][j + 1:]:
          if f1 * f2 > 0:
            common_reflections[(abs(f1), abs(f2))] += 1

    cmn_rfl_list = []

    for f1 in range(len(frames)):
      for f2 in range(f1 + 1, len(frames)):
        if common_reflections[(f1, f2)] > 10:
          cmn_rfl_list.append((common_reflections[(f1, f2)], f1, f2))

    cmn_rfl_list.sort()
    cmn_rfl_list.reverse()
    
    joins = []
    used = []
    
    for n, f1, f2 in cmn_rfl_list:
      
      if f1 in used or f2 in used:
        continue
            
      _cc = frames[f1].cc(frames[f2])

      # really only need to worry about f2 which will get merged...
      # merging multiple files together should be OK provided they are
      # correctly sorted (though the order should not matter anyhow?)
      # anyhow they are sorted anyway... ah as f2 > f1 then just sorting
      # the list by f2 will make sure the data cascase correctly.

      # p-value small (3% ish) for cc > 0.6 for > 10 observations -
      # necessary as will be correlated due to Wilson curves though
      # with B factor < 10 this is less of an issue

      if _cc[0] > 10 and _cc[1] > 0.6:
        print '%4d %.3f' % _cc, f1, f2
        joins.append((f2, f1))
        used.append(f2)

    if not joins:
      print 'No pairs found'
      break

    joins.sort()
    joins.reverse()
        
    for j2, j1 in joins:
      rmerge = frames[j1].merge(frames[j2])
      if rmerge:
        print 'R: %4d %4d %6.3f' % (j1, j2, rmerge)
      else:
        print 'R: %4d %4d ------' % (j1, j2)

    all_joins = [j for j in joins]

    # then do the same for the alternative indices

    other_reflections = numpy.zeros((len(frames), len(frames)),
                                    dtype = numpy.short)

    obs = { } 

    # for other hand add -j

    for j, f in enumerate(frames):
      indices = set(f.get_indices())
      for i in indices:
        _i = tuple(i)
        if not _i in obs:
          obs[_i] = []
        obs[_i].append(j)

      indices = set(f.get_other())
      for i in indices:
        _i = tuple(i)
        if not _i in obs:
          obs[_i] = []
        obs[_i].append(-j)

    for hkl in obs:
      obs[hkl].sort()
      for j, f1 in enumerate(obs[hkl][:-1]):
        for f2 in obs[hkl][j + 1:]:
          if f1 * f2 < 0:
            other_reflections[(abs(f1), abs(f2))] += 1

    oth_rfl_list = []

    for f1 in range(len(frames)):
      for f2 in range(f1 + 1, len(frames)):
        if other_reflections[(f1, f2)] > 10:
          oth_rfl_list.append((other_reflections[(f1, f2)], f1, f2))
    
    joins = []

    oth_rfl_list.sort()
    oth_rfl_list.reverse()
        
    for n, f1, f2 in oth_rfl_list:
      
      if f1 in used or f2 in used:
        continue
            
      _cc = frames[f1].cc_other(frames[f2])

      # really only need to worry about f2 which will get merged...
      # merging multiple files together should be OK provided they are
      # correctly sorted (though the order should not matter anyhow?)
      # anyhow they are sorted anyway... ah as f2 > f1 then just sorting
      # the list by f2 will make sure the data cascase correctly.

      # p-value small (3% ish) for cc > 0.6 for > 10 observations -
      # necessary as will be correlated due to Wilson curves though
      # with B factor < 10 this is less of an issue

      if _cc[0] > 10 and _cc[1] > 0.6:
        print '%4d %.3f' % _cc, f1, f2
        joins.append((f2, f1))
        used.append(f2)

    all_joins += joins

    if not all_joins:
      break
      
    joins.sort()
    joins.reverse()
        
    for j2, j1 in joins:
      frames[j2].reindex()
      rmerge = frames[j1].merge(frames[j2])
      if rmerge:
        print 'R: %4d %4d %6.3f' % (j1, j2, rmerge)
      else:
        print 'R: %4d %4d ------' % (j1, j2)
        
    continue

  frames.sort()

  print 'Biggest few: #frames; #unique refl'
  j = -1
  while frames[j].get_frames() > 1:
    print frames[j].get_frames(), frames[j].get_unique_indices()
    frames[j].output_as_scalepack(sg, 'scalepack-%d.sca' % j)
    j -= 1

  return
Ejemplo n.º 12
0
def find_merge_common_images(args):
    phil = iotbx.phil.process_command_line(args = args,
                                           master_string = master_phil).show()
    work_params = phil.work.extract()
    if ("--help" in args) :
        libtbx.phil.parse(master_phil.show())
        return

    if ((work_params.d_min is None) or
        (work_params.data is None) or
        ((work_params.model is None) and
         work_params.scaling.algorithm != "mark1")) :
        raise Usage("cxi.merge "
                    "d_min=4.0 "
                    "data=~/scratch/r0220/006/strong/ "
                    "model=3bz1_3bz2_core.pdb")
    if ((work_params.rescale_with_average_cell) and
        (not work_params.set_average_unit_cell)) :
        raise Usage(
            "If rescale_with_average_cell=True, you must also specify "+
            "set_average_unit_cell=True.")

    # Read Nat's reference model from an MTZ file.  XXX The observation
    # type is given as F, not I--should they be squared?  Check with Nat!
    log = open("%s_%s_scale.log" % (work_params.output.prefix,
                                    work_params.scaling.algorithm), "w")
    out = multi_out()
    out.register("log", log, atexit_send_to=None)
    out.register("stdout", sys.stdout)

    print >> out, "Target unit cell and space group:"
    print >> out, "  ", work_params.target_unit_cell
    print >> out, "  ", work_params.target_space_group

    uc = work_params.target_unit_cell

    miller_set = symmetry(
        unit_cell=work_params.target_unit_cell,
        space_group_info=work_params.target_space_group
        ).build_miller_set(
        anomalous_flag=not work_params.merge_anomalous,
        d_min=work_params.d_min)
    from xfel.cxi.merging.general_fcalc import random_structure
    i_model = random_structure(work_params)

    # ---- Augment this code with any special procedures for x scaling
    scaler = xscaling_manager(
        miller_set=miller_set,
        i_model=i_model,
        params=work_params,
        log=out)
    scaler.read_all()
    print "finished reading"
    sg = miller_set.space_group()
    pg = sg.build_derived_laue_group()
    miller_set.show_summary()

    hkl_asu = scaler.observations["hkl_id"]
    imageno = scaler.observations["frame_id"]
    intensi = scaler.observations["i"]
    sigma_i = scaler.observations["sigi"]
    
    lookup = scaler.millers["merged_asu_hkl"]    

    # construct table of start / end indices for frames: now using Python
    # range indexing

    starts = [0]
    ends = []
    
    for x in xrange(1, len(scaler.observations["hkl_id"])):
        if imageno[x] != imageno[x - 1]:
            ends.append(x)
            starts.append(x)
            
    ends.append(len(scaler.observations["hkl_id"]))

    keep_start = []
    keep_end = []

    for j, se in enumerate(zip(starts, ends)):
        s, e = se
        isig = sum(i / s for i, s in zip(intensi[s:e], sigma_i[s:e])) / (e - s)
        dmin = 100.0
        for x in xrange(s, e):
            d = uc.d(lookup[hkl_asu[x]])
            if d < dmin:
                dmin = d
        if isig > 6.0 and dmin < 3.2:
            keep_start.append(s)
            keep_end.append(e)

    starts = keep_start
    ends = keep_end

    print 'Keeping %d frames' % len(starts)

    frames = []

    for s, e in zip(starts, ends):
        indices = [tuple(lookup[hkl_asu[x]]) for x in range(s, e)]
        intensities = intensi[s:e]
        sigmas = sigma_i[s:e]

        frames.append(Frame(uc, indices, intensities, sigmas))

    from collections import defaultdict

    hist = defaultdict(int)

    fout = open('common_oh.dat', 'w')

    for j, f in enumerate(frames):
        hp = f.hand_pairs()
        fout.write('%4d %d\n' % (j, hp))
        hist[int(hp)] += 1

    fout.close()

    for b in sorted(hist):
        print b, hist[b]


    

    return
Ejemplo n.º 13
0
def run(args):
  phil = iotbx.phil.process_command_line(args=args, master_string=master_phil).show()
  work_params = phil.work.extract()
  from xfel.merging.phil_validation import application,samosa
  application(work_params)
  samosa(work_params)
  if ("--help" in args) :
    libtbx.phil.parse(master_phil.show())
    return

  if ((work_params.d_min is None) or
      (work_params.data is None) ) :
    command_name = os.environ["LIBTBX_DISPATCHER_NAME"]
    raise Usage(command_name + " "
                "d_min=4.0 "
                "data=~/scratch/r0220/006/strong/ "
                "model=3bz1_3bz2_core.pdb")
  if ((work_params.rescale_with_average_cell) and
      (not work_params.set_average_unit_cell)) :
    raise Usage("If rescale_with_average_cell=True, you must also specify "+
      "set_average_unit_cell=True.")
  if work_params.raw_data.sdfac_auto and work_params.raw_data.sdfac_refine:
    raise Usage("Cannot specify both sdfac_auto and sdfac_refine")

  # Read Nat's reference model from an MTZ file.  XXX The observation
  # type is given as F, not I--should they be squared?  Check with Nat!
  log = open("%s.log" % work_params.output.prefix, "w")
  out = multi_out()
  out.register("log", log, atexit_send_to=None)
  out.register("stdout", sys.stdout)
  print >> out, "I model"
  if work_params.model is not None:
    from xfel.merging.general_fcalc import run
    i_model = run(work_params)
    work_params.target_unit_cell = i_model.unit_cell()
    work_params.target_space_group = i_model.space_group_info()
    i_model.show_summary()
  else:
    i_model = None

  print >> out, "Target unit cell and space group:"
  print >> out, "  ", work_params.target_unit_cell
  print >> out, "  ", work_params.target_space_group

  # Adjust the minimum d-spacing of the generated Miller set to assure
  # that the desired high-resolution limit is included even if the
  # observed unit cell differs slightly from the target.  If a
  # reference model is present, ensure that Miller indices are ordered
  # identically.
  miller_set = symmetry(
      unit_cell=work_params.target_unit_cell,
      space_group_info=work_params.target_space_group
    ).build_miller_set(
      anomalous_flag=not work_params.merge_anomalous,
      d_max=work_params.d_max,
      d_min=work_params.d_min / math.pow(
        1 + work_params.unit_cell_length_tolerance, 1 / 3))
  miller_set = miller_set.change_basis(
    work_params.model_reindex_op).map_to_asu()

  if i_model is not None:
    matches = miller.match_indices(i_model.indices(), miller_set.indices())
    assert not matches.have_singles()
    miller_set = miller_set.select(matches.permutation())

  frame_files = get_observations(work_params)
  scaler = scaling_manager(
    miller_set=miller_set,
    i_model=i_model,
    params=work_params,
    log=out)
  scaler.scale_all(frame_files)
  if scaler.n_accepted == 0:
    return None
  scaler.show_unit_cell_histograms()
  if (work_params.rescale_with_average_cell) :
    average_cell_abc = scaler.uc_values.get_average_cell_dimensions()
    average_cell = uctbx.unit_cell(list(average_cell_abc) +
      list(work_params.target_unit_cell.parameters()[3:]))
    work_params.target_unit_cell = average_cell
    print >> out, ""
    print >> out, "#" * 80
    print >> out, "RESCALING WITH NEW TARGET CELL"
    print >> out, "  average cell: %g %g %g %g %g %g" % \
      work_params.target_unit_cell.parameters()
    print >> out, ""
    scaler.reset()
    scaler.scale_all(frame_files)
    scaler.show_unit_cell_histograms()
  if False : #(work_params.output.show_plots) :
    try :
      plot_overall_completeness(completeness)
    except Exception, e :
      print "ERROR: can't show plots"
      print "  %s" % str(e)
Ejemplo n.º 14
0
def find_merge_common_images(args):
    phil = iotbx.phil.process_command_line(args = args,
                                           master_string = master_phil).show()
    work_params = phil.work.extract()
    if ("--help" in args) :
        libtbx.phil.parse(master_phil.show())
        return

    if ((work_params.d_min is None) or
        (work_params.data is None) or
        ((work_params.model is None) and
         work_params.scaling.algorithm != "mark1")) :
        raise Usage("cxi.merge "
                    "d_min=4.0 "
                    "data=~/scratch/r0220/006/strong/ "
                    "model=3bz1_3bz2_core.pdb")
    if ((work_params.rescale_with_average_cell) and
        (not work_params.set_average_unit_cell)) :
        raise Usage(
            "If rescale_with_average_cell=True, you must also specify "+
            "set_average_unit_cell=True.")

    # Read Nat's reference model from an MTZ file.  XXX The observation
    # type is given as F, not I--should they be squared?  Check with Nat!
    log = open("%s_%s_scale.log" % (work_params.output.prefix,
                                    work_params.scaling.algorithm), "w")
    out = multi_out()
    out.register("log", log, atexit_send_to=None)
    out.register("stdout", sys.stdout)

    print >> out, "Target unit cell and space group:"
    print >> out, "  ", work_params.target_unit_cell
    print >> out, "  ", work_params.target_space_group

    uc = work_params.target_unit_cell

    miller_set = symmetry(
        unit_cell=work_params.target_unit_cell,
        space_group_info=work_params.target_space_group
        ).build_miller_set(
        anomalous_flag=not work_params.merge_anomalous,
        d_min=work_params.d_min)
    from xfel.cxi.merging.general_fcalc import random_structure
    i_model = random_structure(work_params)

    # ---- Augment this code with any special procedures for x scaling
    scaler = xscaling_manager(
        miller_set=miller_set,
        i_model=i_model,
        params=work_params,
        log=out)
    scaler.read_all()
    print "finished reading"
    sg = miller_set.space_group()
    pg = sg.build_derived_laue_group()
    miller_set.show_summary()

    hkl_asu = scaler.observations["hkl_id"]
    imageno = scaler.observations["frame_id"]
    intensi = scaler.observations["i"]
    sigma_i = scaler.observations["sigi"]
    
    lookup = scaler.millers["merged_asu_hkl"]    

    # construct table of start / end indices for frames: now using Python
    # range indexing

    starts = [0]
    ends = []
    
    for x in xrange(1, len(scaler.observations["hkl_id"])):
        if imageno[x] != imageno[x - 1]:
            ends.append(x)
            starts.append(x)
            
    ends.append(len(scaler.observations["hkl_id"]))

    keep_start = []
    keep_end = []

    for j, se in enumerate(zip(starts, ends)):
        s, e = se
        isig = sum(i / s for i, s in zip(intensi[s:e], sigma_i[s:e])) / (e - s)
        dmin = 100.0
        for x in xrange(s, e):
            d = uc.d(lookup[hkl_asu[x]])
            if d < dmin:
                dmin = d
        if isig > 6.0 and dmin < 3.2:
            keep_start.append(s)
            keep_end.append(e)

    starts = keep_start
    ends = keep_end

    print 'Keeping %d frames' % len(starts)

    frames = []

    for s, e in zip(starts, ends):
        indices = [tuple(lookup[hkl_asu[x]]) for x in range(s, e)]
        intensities = intensi[s:e]
        sigmas = sigma_i[s:e]

        frames.append(Frame(indices, intensities, sigmas))

    while True:

        print 'Analysing %d frames' % len(frames)

        common_reflections = numpy.zeros((len(frames), len(frames)),
                                         dtype = numpy.short)

        obs = { } 

        for j, f in enumerate(frames):
            indices = set(f.get_indices())
            for i in indices:
                _i = tuple(i)
                if not _i in obs:
                    obs[_i] = []
                obs[_i].append(j)

        for hkl in obs:
            obs[hkl].sort()
            for j, f1 in enumerate(obs[hkl][:-1]):
                for f2 in obs[hkl][j + 1:]:
                    common_reflections[(f1, f2)] += 1

        cmn_rfl_list = []

        for f1 in range(len(frames)):
            for f2 in range(f1, len(frames)):
                cmn_rfl_list.append((common_reflections[(f1, f2)], f1, f2))

        cmn_rfl_list.sort()
        cmn_rfl_list.reverse()
    
        joins = []
    
        for n, f1, f2 in cmn_rfl_list:
            _cc = frames[f1].cc(frames[f2])

            if _cc[0] > 20 and _cc[1] > 0.75:
                # print '===> %d %d' % (n, frames[f1].common(frames[f2]))
                print '%4d %.3f' % _cc, f1, f2
                joins.append((f1, f2))

        if not joins:
            print 'No pairs found'
            break

        joins.sort()
        joins.reverse()
        
        for j1, j2 in joins:
            frames[j1].merge(frames[j2])
            frames.remove(frames[j2])
            
        continue

    

    return