Esempio n. 1
0
def exercise_01():
    """
  Sanity check - don't crash when mean intensity for a bin is zero.
  """
    xrs = random_structure.xray_structure(unit_cell=(50, 50, 50, 90, 90, 90),
                                          space_group_symbol="P1",
                                          n_scatterers=1200,
                                          elements="random")
    fc = abs(xrs.structure_factors(d_min=1.5).f_calc())
    fc = fc.set_observation_type_xray_amplitude()
    cs = fc.complete_set(d_min=1.4)
    ls = cs.lone_set(other=fc)
    f_zero = ls.array(data=flex.double(ls.size(), 0))
    f_zero.set_observation_type_xray_amplitude()
    fc = fc.concatenate(other=f_zero)
    sigf = flex.double(fc.size(), 0.1) + (fc.data() * 0.03)
    fc = fc.customized_copy(sigmas=sigf)
    try:
        fc_fc = french_wilson.french_wilson_scale(miller_array=fc,
                                                  log=null_out())
    except Sorry:
        pass
    else:
        raise Exception_expected
    ic = fc.f_as_f_sq().set_observation_type_xray_intensity()
    fc_fc = french_wilson.french_wilson_scale(miller_array=ic, log=null_out())
def exercise_01 () :
  """
  Sanity check - don't crash when mean intensity for a bin is zero.
  """
  xrs = random_structure.xray_structure(
    unit_cell=(50,50,50,90,90,90),
    space_group_symbol="P1",
    n_scatterers=1200,
    elements="random")
  fc = abs(xrs.structure_factors(d_min=1.5).f_calc())
  fc = fc.set_observation_type_xray_amplitude()
  cs = fc.complete_set(d_min=1.4)
  ls = cs.lone_set(other=fc)
  f_zero = ls.array(data=flex.double(ls.size(), 0))
  f_zero.set_observation_type_xray_amplitude()
  fc = fc.concatenate(other=f_zero)
  sigf = flex.double(fc.size(), 0.1) + (fc.data() * 0.03)
  fc = fc.customized_copy(sigmas=sigf)
  try :
    fc_fc = french_wilson.french_wilson_scale(miller_array=fc, log=null_out())
  except Sorry :
    pass
  else :
    raise Exception_expected
  ic = fc.f_as_f_sq().set_observation_type_xray_intensity()
  fc_fc = french_wilson.french_wilson_scale(miller_array=ic, log=null_out())
Esempio n. 3
0
    def data_as_f_obs(self):
        """
    Convert input data array to amplitudes, adjusting the data type and
    applying additional filters if necessary.

    :param f_obs: selected input data
    :returns: :py:class:`cctbx.miller.array` of real numbers with observation
      type set to amplitudes
    """
        if (self.raw_data_truncated is None): return None
        f_obs = self.raw_data_truncated.deep_copy()
        # Convert to non-anomalous if requested
        if (self.force_non_anomalous):
            f_obs = f_obs.average_bijvoet_mates()
        #
        d_min = f_obs.d_min()
        if (d_min < 0.25):
            raise Sorry("Resolution of data is too high: %-6.4f A" % d_min)
        if (f_obs.is_complex_array()): f_obs = abs(f_obs)
        if (f_obs.is_xray_intensity_array()):
            if (self.parameters.french_wilson_scale):
                try:
                    sigI_cutoff = self.parameters.sigma_iobs_rejection_criterion
                    f_obs = french_wilson.french_wilson_scale(
                        miller_array=f_obs,
                        params=self.parameters.french_wilson,
                        sigma_iobs_rejection_criterion=sigI_cutoff,
                        log=self.log)
                except Exception as e:
                    print(str(e), file=self.log)
                    print("Using alternative Iobs->Fobs conversion.",
                          file=self.log)
                    f_obs = f_obs.f_sq_as_f()
        #
        f_obs.set_observation_type_xray_amplitude()
        if (self.parameters.force_anomalous_flag_to_be_equal_to is not None):
            if (not self.parameters.force_anomalous_flag_to_be_equal_to):
                if (f_obs.anomalous_flag()):
                    merged = f_obs.as_non_anomalous_array().merge_equivalents()
                    f_obs = merged.array().set_observation_type(f_obs)
            elif (not f_obs.anomalous_flag()):
                observation_type = f_obs.observation_type()
                f_obs = f_obs.generate_bijvoet_mates()
                f_obs.set_observation_type(observation_type)
        else:
            f_obs = f_obs.convert_to_non_anomalous_if_ratio_pairs_lone_less_than(
                threshold=self.parameters.
                convert_to_non_anomalous_if_ratio_pairs_lone_less_than_threshold
            )
        f_obs.set_info(self.raw_data.info())
        return f_obs
Esempio n. 4
0
def run(args=None, params=None, out=sys.stdout):
    assert [args, params].count(None) == 1
    if args is not None:
        if (len(args) == 0) or ("--help" in args):
            raise Usage("""
  phenix.cc_star model.pdb data.mtz unmerged_data=data.hkl [n_bins=X] [options]
  phenix.cc_star model_refine_001.mtz unmerged_data=data.hkl [...]

Implementation of the method for assessing data and model quality described in:
  Karplus PA & Diederichs K (2012) Science 336:1030-3.

Full parameters:
  %s
  """ % master_phil.as_str(prefix=" ", attributes_level=1))
        import iotbx.phil
        cmdline = iotbx.phil.process_command_line_with_files(
            args=args,
            master_phil=master_phil,
            pdb_file_def="model",
            reflection_file_def="data")
        params = cmdline.work.extract()
    import mmtbx.command_line
    import mmtbx.validation.experimental
    from iotbx import merging_statistics
    from iotbx import file_reader
    if (params.data is None):
        raise Sorry("Please specify a data file (usually MTZ format).")
    if (params.unmerged_data is None):
        raise Sorry("Please specify unmerged_data file")
    hkl_in = file_reader.any_file(params.data, force_type="hkl")
    hkl_in.check_file_type("hkl")
    f_model = f_obs = r_free_flags = None
    f_models = []
    data_arrays = []
    f_model_labels = []
    if (params.f_model_labels is None):
        for array in hkl_in.file_server.miller_arrays:
            labels = array.info().label_string()
            if (array.is_complex_array()):
                if (labels.startswith("F-model")
                        or labels.startswith("FMODEL")):
                    f_models.append(array)
                    f_model_labels.append(labels)
        if (len(f_models) > 1):
            raise Sorry(
                ("Multiple F(model) arrays found:\n%s\nPlease specify the " +
                 "'labels' parameter.") % "\n".join(f_model_labels))
        elif (len(f_models) == 1):
            f_model = f_models[0]
            if (f_model.anomalous_flag()):
                info = f_model.info()
                f_model = f_model.average_bijvoet_mates().set_info(info)
            print("F(model):", file=out)
            f_model.show_summary(f=out, prefix="  ")
        else:
            data_array = hkl_in.file_server.get_xray_data(
                file_name=params.data,
                labels=params.f_obs_labels,
                ignore_all_zeros=True,
                parameter_scope="")
            if (data_array.is_xray_intensity_array()):
                from cctbx import french_wilson
                f_obs = french_wilson.french_wilson_scale(
                    miller_array=data_array, out=out)
            else:
                f_obs = data_array
    else:
        for array in hkl_in.file_server.miller_arrays:
            array_labels = array.info().label_string()
            if (array_labels == params.f_model_labels):
                if (array.is_complex_array()):
                    f_model = array
                    break
                else:
                    raise Sorry(
                        "The data in %s are not of the required type." %
                        array_labels)
    if (f_model is not None):
        assert (f_obs is None)
        for array in hkl_in.file_server.miller_arrays:
            labels = array.info().label_string()
            if (labels == params.f_obs_labels):
                f_obs = array
                break
        else:
            try:
                f_obs = hkl_in.file_server.get_amplitudes(
                    file_name=params.f_obs_labels,
                    labels=None,
                    convert_to_amplitudes_if_necessary=False,
                    parameter_name="f_obs_labels",
                    parameter_scope="",
                    strict=True)
            except Sorry:
                raise Sorry(
                    "You must supply a file containing both F-obs and F-model "
                    + "if you want to use a pre-calculated F-model array.")
    assert (f_obs.is_xray_amplitude_array())
    if (f_obs.anomalous_flag()):
        info = f_obs.info()
        f_obs = f_obs.average_bijvoet_mates().set_info(info)
    print("F(obs):", file=out)
    f_obs.show_summary(f=out, prefix="  ")
    print("", file=out)
    r_free_flags, test_flag_value = hkl_in.file_server.get_r_free_flags(
        file_name=params.data,
        label=params.r_free_flags.label,
        test_flag_value=params.r_free_flags.test_flag_value,
        disable_suitability_test=False,
        parameter_scope="")
    info = r_free_flags.info()
    r_free_flags = r_free_flags.customized_copy(
        data=r_free_flags.data() == test_flag_value).set_info(info)
    if (r_free_flags.anomalous_flag()):
        r_free_flags = r_free_flags.average_bijvoet_mates().set_info(info)
    print("R-free flags:", file=out)
    r_free_flags.show_summary(f=out, prefix="  ")
    print("", file=out)
    unmerged_i_obs = mmtbx.command_line.load_and_validate_unmerged_data(
        f_obs=f_obs,
        file_name=params.unmerged_data,
        data_labels=params.unmerged_labels,
        log=out)
    print("Unmerged intensities:", file=out)
    unmerged_i_obs.show_summary(f=out, prefix="  ")
    print("", file=out)
    if (f_model is None):
        assert (f_obs is not None)
        if (params.model is None):
            raise Sorry(
                "A PDB file is required if F(model) is not pre-calculated.")
        make_sub_header("Calculating F(model)", out=out)
        pdb_in = file_reader.any_file(params.model, force_type="pdb")
        pdb_in.check_file_type("pdb")
        pdb_symm = pdb_in.file_object.crystal_symmetry()
        if (pdb_symm is None):
            pdb_symm = f_obs
        else:
            if (f_obs.crystal_symmetry() is None):
                f_obs = f_obs.customized_copy(crystal_symmetry=pdb_symm)
            elif (not pdb_symm.is_similar_symmetry(f_obs)):
                mmtbx.command_line.show_symmetry_error(file1="PDB file",
                                                       file2="data file",
                                                       symm1=pdb_symm,
                                                       symm2=f_obs)
        xray_structure = pdb_in.file_object.xray_structure_simple(
            crystal_symmetry=pdb_symm)
        from mmtbx.utils import fmodel_simple
        # XXX this gets done anyway later, but they need to be consistent before
        # creating the fmodel manager
        if (f_obs.anomalous_flag()):
            f_obs = f_obs.average_bijvoet_mates()
        f_obs = f_obs.eliminate_sys_absent()
        f_obs, r_free_flags = f_obs.map_to_asu().common_sets(
            other=r_free_flags.map_to_asu())
        fmodel = fmodel_simple(f_obs=f_obs,
                               r_free_flags=r_free_flags,
                               xray_structures=[xray_structure],
                               skip_twin_detection=True,
                               scattering_table="n_gaussian")
        fmodel.show(log=out)
        f_model = fmodel.f_model()
        f_obs = fmodel.f_obs()
        r_free_flags = fmodel.r_free_flags()
    else:
        if (f_model.anomalous_flag()):
            f_model = f_model.average_bijvoet_mates()

    stats = mmtbx.validation.experimental.merging_and_model_statistics(
        f_model=f_model,
        f_obs=f_obs,
        r_free_flags=r_free_flags,
        unmerged_i_obs=unmerged_i_obs,
        n_bins=params.n_bins,
        sigma_filtering=params.sigma_filtering)
    stats.show_cc_star(out=out)
    if (params.loggraph):
        stats.show_loggraph(out=out)
    print("", file=out)
    print("Reference:", file=out)
    print("  Karplus PA & Diederichs K (2012) Science 336:1030-3.", file=out)
    print("", file=out)
    return stats
Esempio n. 5
0
def run(args, out=sys.stdout):
  from cctbx import french_wilson
  from iotbx import file_reader
  hkl_file = None
  sources = []
  interpreter = master_phil.command_line_argument_interpreter()
  for arg in args :
    if os.path.isfile(arg):
      input_file = file_reader.any_file(arg)
      if (input_file.file_type == "hkl"):
        hkl_file = input_file
        sources.append(interpreter.process(arg="file_name=\"%s\"" % arg))
      elif (input_file.file_type == "phil"):
        sources.append(input_file.file_object)
    else :
      arg_phil = interpreter.process(arg=arg)
      sources.append(arg_phil)
  work_phil = master_phil.fetch(sources=sources)
  work_params = work_phil.extract()
  if (work_params.french_wilson.file_name is None):
    if (hkl_file is None):
      raise Usage("phenix.french_wilson data.mtz [params.eff] [options ...]")
    else :
      work_params.french_wilson.file_name = hkl_file.file_name
  elif (hkl_file is None):
    hkl_file = file_reader.any_file(work_params.french_wilson.file_name)
  params = work_params.french_wilson
  xray_data_server = hkl_file.file_server
  crystal_symmetry = xray_data_server.miller_arrays[0].crystal_symmetry()
  if (crystal_symmetry is None):
    raise Sorry("No crystal symmetry found.  This program requires an input "+
      "format with complete symmetry information.")
  unit_cell = xray_data_server.miller_arrays[0].unit_cell()
  if (unit_cell is None):
    raise Sorry("No unit cell found.  This program requires an input "+
      "format with complete unit cell information.")
  i_obs = None
  i_obs = xray_data_server.get_xray_data(
    file_name = params.file_name,
    labels = params.intensity_labels,
    ignore_all_zeros = True,
    parameter_scope = 'french_wilson',
    parameter_name = 'intensity_labels')
  import cStringIO
  xray_data_server.err = cStringIO.StringIO()
  try :
    r_free_flags, test_flag_value = xray_data_server.get_r_free_flags(
      file_name = params.file_name,
      label = params.r_free_flags.label,
      test_flag_value = None,
      disable_suitability_test = False,
      parameter_scope = "french_wilson.r_free_flags")
  except Sorry as e :
    r_free_flags = None
  if (i_obs is None):
    raise Sorry("Couldn't find intensities!")
  wavelength = params.wavelength
  if (wavelength is None):
    info = i_obs.info()
    if (info is not None):
      wavelength = info.wavelength
      if (wavelength is not None):
        print("Using wavelength=%g from input file" % wavelength, file=out)
  sigma_iobs_rejection_criterion = work_params.french_wilson.\
    sigma_iobs_rejection_criterion
  if (not i_obs.is_unique_set_under_symmetry()):
    print("Merging symmetry-equivalent reflections", file=out)
    i_obs = i_obs.merge_equivalents().array()
  f_obs = french_wilson.french_wilson_scale(miller_array=i_obs,
    params=params,
    sigma_iobs_rejection_criterion=sigma_iobs_rejection_criterion,
    log=out)
  if f_obs is None:
    raise Sorry("Not enough data to accurately apply the French-Wilson method."+\
                " Exiting.")
  if params.output_file == None:
    output_file = "french_wilson.mtz"
  else:
    output_file = params.output_file
  mtz_dataset = i_obs.as_mtz_dataset(
    column_root_label = "I",
    wavelength = wavelength)
  mtz_dataset.add_miller_array(
    miller_array      = f_obs,
    column_root_label = "F")
  if (r_free_flags is not None) and (params.keep_r_free_flags):
    mtz_dataset.add_miller_array(
      miller_array      = r_free_flags,
      column_root_label = "R-free-flags")
  mtz_object = mtz_dataset.mtz_object()
  mtz_object.write(file_name = output_file)
  print("Wrote %s" % output_file, file=out)
  return output_file
Esempio n. 6
0
     raise Sorry("Couldn't find intensities!")
 wavelength = params.wavelength
 if (wavelength is None):
     info = i_obs.info()
     if (info is not None):
         wavelength = info.wavelength
         if (wavelength is not None):
             print >> out, "Using wavelength=%g from input file" % wavelength
 sigma_iobs_rejection_criterion = work_params.french_wilson.\
   sigma_iobs_rejection_criterion
 if (not i_obs.is_unique_set_under_symmetry()):
     print >> out, "Merging symmetry-equivalent reflections"
     i_obs = i_obs.merge_equivalents().array()
 f_obs = french_wilson.french_wilson_scale(
     miller_array=i_obs,
     params=params,
     sigma_iobs_rejection_criterion=sigma_iobs_rejection_criterion,
     log=out)
 if f_obs is None:
     raise Sorry("Not enough data to accurately apply the French-Wilson method."+\
                 " Exiting.")
 if params.output_file == None:
     output_file = "french_wilson.mtz"
 else:
     output_file = params.output_file
 mtz_dataset = i_obs.as_mtz_dataset(column_root_label="I",
                                    wavelength=wavelength)
 mtz_dataset.add_miller_array(miller_array=f_obs, column_root_label="F")
 if (r_free_flags is not None) and (params.keep_r_free_flags):
     mtz_dataset.add_miller_array(miller_array=r_free_flags,
                                  column_root_label="R-free-flags")
  def __init__ (self,
      obs,
      r_free_flags,
      test_flag_value,
      phases=None,
      d_min=None,
      d_max=None,
      r_free_flags_params=None,
      merge_anomalous=False,
      log=sys.stdout,
      verbose=True) :
    assert (log is not None) and (obs is not None)
    if (r_free_flags_params is None) :
      from cctbx.r_free_utils import generate_r_free_params_str
      r_free_flags_params = libtbx.phil.parse(
        generate_r_free_params_str).extract()
    obs_info = obs.info()
    r_free_flags_info = phases_info = None
    sg = obs.space_group_info()
    obs = obs.map_to_asu().merge_equivalents().array()
    obs = obs.eliminate_sys_absent(log=log)
    obs = obs.resolution_filter(d_min=d_min, d_max=d_max)
    if (obs.is_xray_intensity_array()) :
      from cctbx import french_wilson
      if (verbose) :
        fw_out = log
      else :
        fw_out = null_out()
      obs = french_wilson.french_wilson_scale(
        miller_array=obs,
        params=None,
        log=fw_out)
    assert (obs is not None)
    merged_obs = obs.average_bijvoet_mates()
    if (merged_obs.completeness() < 0.9) :
      print >> log, """
  WARNING: data are incomplete (%.1f%% of possible reflections measured to
  %.2fA).  This may cause problems if you plan to use the maps for building
  and/or ligand fitting!
    """ % (100*merged_obs.completeness(), merged_obs.d_min())
    # XXX this is kind of a hack (the reconstructed arrays break some of my
    # assumptions about labels)
    if (merge_anomalous) :
      obs = obs.average_bijvoet_mates()
    if (r_free_flags is not None) :
      r_free_flags_info = r_free_flags.info()
      format = "cns"
      if (test_flag_value == 0) :
        format = "ccp4"
      elif (test_flag_value == -1) :
        format = "shelx"
      if (r_free_flags.anomalous_flag()) :
        r_free_flags = r_free_flags.average_bijvoet_mates()
      is_compatible_symmetry = False
      obs_pg = obs.space_group().build_derived_point_group()
      flags_pg = r_free_flags.space_group().build_derived_point_group()
      if (obs_pg.type().number() == flags_pg.type().number()) :
        is_compatible_symmetry = True
      else :
        pass # TODO unit cell comparison?
      if (is_compatible_symmetry) :
        r_free_flags = r_free_flags.map_to_asu().merge_equivalents().array()
        r_free_flags = r_free_flags.eliminate_sys_absent(log=log)
        if (format == "cns") :
          r_free_flags = r_free_flags.customized_copy(
            crystal_symmetry=obs.crystal_symmetry(),
            data=(r_free_flags.data() == test_flag_value))
          test_flag_value = True
        obs_tmp = obs.deep_copy()
        if (obs.anomalous_flag()) :
          obs_tmp = obs.average_bijvoet_mates()
        r_free_flags = r_free_flags.common_set(other=obs_tmp)
        n_r_free = r_free_flags.indices().size()
        n_obs = obs_tmp.indices().size()
        if ((test_flag_value is None) or
            (r_free_flags.data().all_eq(r_free_flags.data()[0]))) :
          print >> log, """
  WARNING: uniform R-free flags detected; a new test set will be generated,
  but this will bias the refinement statistics.
"""
          r_free_flags = None
        elif (n_r_free != n_obs) :
          missing_set = obs_tmp.lone_set(other=r_free_flags)
          n_missing = missing_set.indices().size()
          if (n_missing > 0) :
            print >> log, """
  WARNING: R-free flags are incomplete relative to experimental
  data (%d vs. %d reflections).  The flags will be extended to
  complete the set, but we recommend supplying flags that are already
  generated to the maximum expected resolution.
""" % (n_r_free, n_obs)
            if (n_missing < 20) : # FIXME
              if (format == "cns") :
                missing_flags = missing_set.array(data=flex.bool(n_missing,
                  False))
              else :
                missing_flags = missing_set.array(data=flex.int(n_missing, 1))
            else :
              missing_flags = missing_set.generate_r_free_flags(
                fraction=(r_free_flags.data().count(test_flag_value)/n_r_free),
                max_free=None,
                use_lattice_symmetry=True,
                format=format)
            r_free_flags = r_free_flags.concatenate(other=missing_flags)
        if (r_free_flags is not None) :
          assert (r_free_flags.indices().size() == obs_tmp.indices().size())
      else :
        print >> log, """
    NOTE: incompatible symmetry between the data and the R-free flags:
         Data  : %s  %s
         Flags : %s  %s
       A new test set will be generated.
""" % (str(obs.space_group_info()),
          " ".join([ "%g" % x for x in obs.unit_cell().parameters() ]),
          str(r_free_flags.space_group_info()),
          " ".join(["%g" % x for x in r_free_flags.unit_cell().parameters()]))
    else :
      print >> log, """
 WARNING: R-free flags not supplied.  This may bias the refinement if the
     structures are very nearly isomorphous!
"""
    self._generate_new = False
    if (r_free_flags is None) :
      r_free_flags = obs.generate_r_free_flags(
        fraction=r_free_flags_params.fraction,
        max_free=r_free_flags_params.max_free,
        use_lattice_symmetry=r_free_flags_params.use_lattice_symmetry,
        use_dataman_shells=r_free_flags_params.use_dataman_shells,
        n_shells=r_free_flags_params.n_shells,
        format="ccp4")
      test_flag_value = 0
      self._generate_new = True
    if (r_free_flags.anomalous_flag()) :
      r_free_flags = r_free_flags.average_bijvoet_mates()
    if (phases is not None) :
      phases_info = phases.info()
      phases = phases.map_to_asu().resolution_filter(d_min=d_min, d_max=d_max)
    assert (obs.is_xray_amplitude_array())
    self.f_obs = obs.set_info(obs_info)
    self.r_free_flags = r_free_flags.set_info(r_free_flags_info)
    self.test_flag_value = test_flag_value
    self.phases = None
    if (phases is not None) :
      self.phases = phases.set_info(phases_info)
Esempio n. 8
0
  def __init__(self,
      obs,
      r_free_flags,
      test_flag_value,
      phases=None,
      d_min=None,
      d_max=None,
      r_free_flags_params=None,
      merge_anomalous=False,
      log=sys.stdout,
      verbose=True):
    assert (log is not None) and (obs is not None)
    if (r_free_flags_params is None):
      from cctbx.r_free_utils import generate_r_free_params_str
      r_free_flags_params = libtbx.phil.parse(
        generate_r_free_params_str).extract()
    obs_info = obs.info()
    r_free_flags_info = phases_info = None
    sg = obs.space_group_info()
    obs = obs.map_to_asu().merge_equivalents().array()
    obs = obs.eliminate_sys_absent(log=log)
    obs = obs.resolution_filter(d_min=d_min, d_max=d_max)
    if (obs.is_xray_intensity_array()):
      from cctbx import french_wilson
      if (verbose):
        fw_out = log
      else :
        fw_out = null_out()
      obs = french_wilson.french_wilson_scale(
        miller_array=obs,
        params=None,
        log=fw_out)
    assert (obs is not None)
    merged_obs = obs.average_bijvoet_mates()
    if (merged_obs.completeness() < 0.9):
      print >> log, """
  WARNING: data are incomplete (%.1f%% of possible reflections measured to
  %.2fA).  This may cause problems if you plan to use the maps for building
  and/or ligand fitting!
    """ % (100*merged_obs.completeness(), merged_obs.d_min())
    # XXX this is kind of a hack (the reconstructed arrays break some of my
    # assumptions about labels)
    if (merge_anomalous):
      obs = obs.average_bijvoet_mates()
    if (r_free_flags is not None):
      r_free_flags_info = r_free_flags.info()
      format = "cns"
      if (test_flag_value == 0):
        format = "ccp4"
      elif (test_flag_value == -1):
        format = "shelx"
      if (r_free_flags.anomalous_flag()):
        r_free_flags = r_free_flags.average_bijvoet_mates()
      is_compatible_symmetry = False
      obs_pg = obs.space_group().build_derived_point_group()
      flags_pg = r_free_flags.space_group().build_derived_point_group()
      if (obs_pg.type().number() == flags_pg.type().number()):
        is_compatible_symmetry = True
      else :
        pass # TODO unit cell comparison?
      if (is_compatible_symmetry):
        r_free_flags = r_free_flags.map_to_asu().merge_equivalents().array()
        r_free_flags = r_free_flags.eliminate_sys_absent(log=log)
        if (format == "cns"):
          r_free_flags = r_free_flags.customized_copy(
            crystal_symmetry=obs.crystal_symmetry(),
            data=(r_free_flags.data() == test_flag_value))
          test_flag_value = True
        obs_tmp = obs.deep_copy()
        if (obs.anomalous_flag()):
          obs_tmp = obs.average_bijvoet_mates()
        r_free_flags = r_free_flags.common_set(other=obs_tmp)
        n_r_free = r_free_flags.indices().size()
        n_obs = obs_tmp.indices().size()
        if ((test_flag_value is None) or
            (r_free_flags.data().all_eq(r_free_flags.data()[0]))):
          print >> log, """
  WARNING: uniform R-free flags detected; a new test set will be generated,
  but this will bias the refinement statistics.
"""
          r_free_flags = None
        elif (n_r_free != n_obs):
          missing_set = obs_tmp.lone_set(other=r_free_flags)
          n_missing = missing_set.indices().size()
          if (n_missing > 0):
            print >> log, """
  WARNING: R-free flags are incomplete relative to experimental
  data (%d vs. %d reflections).  The flags will be extended to
  complete the set, but we recommend supplying flags that are already
  generated to the maximum expected resolution.
""" % (n_r_free, n_obs)
            if (n_missing < 20) : # FIXME
              if (format == "cns"):
                missing_flags = missing_set.array(data=flex.bool(n_missing,
                  False))
              else :
                missing_flags = missing_set.array(data=flex.int(n_missing, 1))
            else :
              missing_flags = missing_set.generate_r_free_flags(
                fraction=(r_free_flags.data().count(test_flag_value)/n_r_free),
                max_free=None,
                use_lattice_symmetry=True,
                format=format)
            r_free_flags = r_free_flags.concatenate(other=missing_flags)
        if (r_free_flags is not None):
          assert (r_free_flags.indices().size() == obs_tmp.indices().size())
      else :
        print >> log, """
    NOTE: incompatible symmetry between the data and the R-free flags:
         Data  : %s  %s
         Flags : %s  %s
       A new test set will be generated.
""" % (str(obs.space_group_info()),
          " ".join([ "%g" % x for x in obs.unit_cell().parameters() ]),
          str(r_free_flags.space_group_info()),
          " ".join(["%g" % x for x in r_free_flags.unit_cell().parameters()]))
    else :
      print >> log, """
 WARNING: R-free flags not supplied.  This may bias the refinement if the
     structures are very nearly isomorphous!
"""
    self._generate_new = False
    if (r_free_flags is None):
      r_free_flags = obs.generate_r_free_flags(
        fraction=r_free_flags_params.fraction,
        max_free=r_free_flags_params.max_free,
        use_lattice_symmetry=r_free_flags_params.use_lattice_symmetry,
        use_dataman_shells=r_free_flags_params.use_dataman_shells,
        n_shells=r_free_flags_params.n_shells,
        format="ccp4")
      test_flag_value = 0
      self._generate_new = True
    if (r_free_flags.anomalous_flag()):
      r_free_flags = r_free_flags.average_bijvoet_mates()
    if (phases is not None):
      phases_info = phases.info()
      phases = phases.map_to_asu().resolution_filter(d_min=d_min, d_max=d_max)
    assert (obs.is_xray_amplitude_array())
    self.f_obs = obs.set_info(obs_info)
    self.r_free_flags = r_free_flags.set_info(r_free_flags_info)
    self.test_flag_value = test_flag_value
    self.phases = None
    if (phases is not None):
      self.phases = phases.set_info(phases_info)
def run (args, out=sys.stdout) :
  from iotbx import file_reader
  import iotbx.phil
  if (len(args) == 0) :
    raise Usage("""\
iotbx.simple_map_coefficients data_phases.mtz [options]

Full parameters:
%s""" % iotbx.phil.parse(master_phil).as_str(attributes_level=1, prefix=" "))
  cmdline = iotbx.phil.process_command_line_with_files(
    args=args,
    master_phil_string=master_phil,
    reflection_file_def="file_name")
  params = cmdline.work.extract()
  if (params.file_name is None) :
    raise Sorry("No reflection file specified.")
  hkl_in = file_reader.any_file(params.file_name).check_file_type("hkl")
  hkl_server = hkl_in.file_server
  data = hkl_server.get_xray_data(
    file_name=params.file_name,
    labels=params.data_labels,
    ignore_all_zeros=False,
    parameter_name="data_labels",
    parameter_scope="",
    prefer_anomalous=True,
    prefer_amplitudes=True)
  data_labels = data.info().label_string()
  if (data.is_xray_intensity_array()) :
    from cctbx.french_wilson import french_wilson_scale
    data = french_wilson_scale(data, log=out)
  phases = hkl_server.get_phases_deg(
    file_name=params.file_name,
    labels=params.phase_labels,
    convert_to_phases_if_necessary=True,
    original_phase_units=None,
    parameter_scope="",
    parameter_name="phase_labels",
    minimum_score=2)
  assert (not phases.anomalous_flag())
  deg = True # FIXME
  weights = None
  if (params.use_weights in [Auto, True]) :
    # FIXME centralize this in iotbx.reflection_file_utils
    for array in hkl_server.miller_arrays :
      if (array.is_real_array()) :
        label_string = array.info().label_string()
        if ((label_string == params.weight_labels) or
            ((params.weight_labels is None) and ("FOM" in label_string))) :
          weights = array
          break
  amplitudes = data
  if (params.map_type == "anom") :
    if (not data.anomalous_flag()) :
      raise Sorry("Anomalous map requested, but selected data are merged.")
    amplitudes = data.anomalous_differences()
    print >> out, "Using anomalous differences in %s" % data_labels
  else :
    print >> out, "Using amplitudes in %s" % data_labels
    if (data.anomalous_flag()) :
      amplitudes = data.average_bijvoet_mates()
  if (params.use_weights is Auto) and (weights is not None) :
    if (params.map_type != "anom") :
      params.use_weights = True
  elif (params.use_weights == True) and (weights is None) :
    raise Sorry("No weights (FOM, etc.) found in input file.")
  if (params.use_weights == True) :
    assert (not weights.anomalous_flag())
    print >> out, "Applying weights in %s" % weights.info().label_string()
    amplitudes, weights = amplitudes.common_sets(other=weights)
    amplitudes = amplitudes.customized_copy(
      data=amplitudes.data()*weights.data())
  amplitudes = amplitudes.customized_copy(sigmas=None)
  print >> out, "Applying phases in %s" % phases.info().label_string()
  amplitudes, phases = amplitudes.common_sets(phases)
  coeffs = amplitudes.phase_transfer(phases,
    deg=deg).set_observation_type(None) # FIXME
  if (params.map_type == "anom") : # apply 90-degree phase shift
    coeffs = coeffs.customized_copy(data=coeffs.data()/(2j))
  assert (coeffs.is_complex_array())
  column_root_label = "F"
  decorator = None
  if (params.map_type == "anom") :
    column_root_label = "ANOM"
  elif (params.use_weights == True) :
    column_root_label = "FWT"
    decorator = iotbx.mtz.ccp4_label_decorator()
  import iotbx.mtz
  mtz_dataset = coeffs.as_mtz_dataset(
    column_root_label=column_root_label,
    label_decorator=decorator)
  if (params.output_file is None) :
    params.output_file = "map_coeffs.mtz"
  mtz_dataset.mtz_object().write(params.output_file)
  print >> out, "Wrote %s" % params.output_file
  return os.path.abspath(params.output_file)
Esempio n. 10
0
def run (args=None, params=None, out=sys.stdout) :
  assert [args, params].count(None) == 1
  if args is not None:
    if (len(args) == 0) or ("--help" in args) :
      raise Usage("""
  phenix.cc_star model.pdb data.mtz unmerged_data=data.hkl [n_bins=X] [options]
  phenix.cc_star model_refine_001.mtz unmerged_data=data.hkl [...]

Implementation of the method for assessing data and model quality described in:
  Karplus PA & Diederichs K (2012) Science 336:1030-3.

Full parameters:
  %s
  """ % master_phil.as_str(prefix=" ", attributes_level=1))
    import iotbx.phil
    cmdline = iotbx.phil.process_command_line_with_files(
      args=args,
      master_phil=master_phil,
      pdb_file_def="model",
      reflection_file_def="data")
    params = cmdline.work.extract()
  import mmtbx.command_line
  import mmtbx.validation.experimental
  from iotbx import merging_statistics
  from iotbx import file_reader
  if (params.data is None) :
    raise Sorry("Please specify a data file (usually MTZ format).")
  if (params.unmerged_data is None) :
    raise Sorry("Please specify unmerged_data file")
  hkl_in = file_reader.any_file(params.data, force_type="hkl")
  hkl_in.check_file_type("hkl")
  f_model = f_obs = r_free_flags = None
  f_models = []
  data_arrays = []
  f_model_labels = []
  if (params.f_model_labels is None) :
    for array in hkl_in.file_server.miller_arrays :
      labels = array.info().label_string()
      if (array.is_complex_array()) :
        if (labels.startswith("F-model") or labels.startswith("FMODEL")) :
          f_models.append(array)
          f_model_labels.append(labels)
    if (len(f_models) > 1) :
      raise Sorry(("Multiple F(model) arrays found:\n%s\nPlease specify the "+
        "'labels' parameter.") % "\n".join(f_model_labels))
    elif (len(f_models) == 1) :
      f_model = f_models[0]
      if (f_model.anomalous_flag()) :
        info = f_model.info()
        f_model = f_model.average_bijvoet_mates().set_info(info)
      print >> out, "F(model):"
      f_model.show_summary(f=out, prefix="  ")
    else :
      data_array = hkl_in.file_server.get_xray_data(
        file_name=params.data,
        labels=params.f_obs_labels,
        ignore_all_zeros=True,
        parameter_scope="")
      if (data_array.is_xray_intensity_array()) :
        from cctbx import french_wilson
        f_obs = french_wilson.french_wilson_scale(
          miller_array=data_array,
          out=out)
      else :
        f_obs = data_array
  else :
    for array in hkl_in.file_server.miller_arrays :
      array_labels = array.info().label_string()
      if (array_labels == params.f_model_labels) :
        if (array.is_complex_array()) :
          f_model = array
          break
        else :
          raise Sorry("The data in %s are not of the required type." %
            array_labels)
  if (f_model is not None) :
    assert (f_obs is None)
    for array in hkl_in.file_server.miller_arrays :
      labels = array.info().label_string()
      if (labels == params.f_obs_labels) :
        f_obs = array
        break
    else :
      try :
        f_obs = hkl_in.file_server.get_amplitudes(
          file_name=params.f_obs_labels,
          labels=None,
          convert_to_amplitudes_if_necessary=False,
          parameter_name="f_obs_labels",
          parameter_scope="",
          strict=True)
      except Sorry :
        raise Sorry("You must supply a file containing both F-obs and F-model "+
          "if you want to use a pre-calculated F-model array.")
  assert (f_obs.is_xray_amplitude_array())
  if (f_obs.anomalous_flag()) :
    info = f_obs.info()
    f_obs = f_obs.average_bijvoet_mates().set_info(info)
  print >> out, "F(obs):"
  f_obs.show_summary(f=out, prefix="  ")
  print >> out, ""
  r_free_flags, test_flag_value = hkl_in.file_server.get_r_free_flags(
    file_name=params.data,
    label=params.r_free_flags.label,
    test_flag_value=params.r_free_flags.test_flag_value,
    disable_suitability_test=False,
    parameter_scope="")
  info = r_free_flags.info()
  r_free_flags = r_free_flags.customized_copy(
    data=r_free_flags.data()==test_flag_value).set_info(info)
  if (r_free_flags.anomalous_flag()) :
    r_free_flags = r_free_flags.average_bijvoet_mates().set_info(info)
  print >> out, "R-free flags:"
  r_free_flags.show_summary(f=out, prefix="  ")
  print >> out, ""
  unmerged_i_obs = mmtbx.command_line.load_and_validate_unmerged_data(
    f_obs=f_obs,
    file_name=params.unmerged_data,
    data_labels=params.unmerged_labels,
    log=out)
  print >> out, "Unmerged intensities:"
  unmerged_i_obs.show_summary(f=out, prefix="  ")
  print >> out, ""
  if (f_model is None) :
    assert (f_obs is not None)
    if (params.model is None) :
      raise Sorry("A PDB file is required if F(model) is not pre-calculated.")
    make_sub_header("Calculating F(model)", out=out)
    pdb_in = file_reader.any_file(params.model, force_type="pdb")
    pdb_in.check_file_type("pdb")
    pdb_symm = pdb_in.file_object.crystal_symmetry()
    if (pdb_symm is None) :
      pdb_symm = f_obs
    else :
      if (f_obs.crystal_symmetry() is None) :
        f_obs = f_obs.customized_copy(crystal_symmetry=pdb_symm)
      elif (not pdb_symm.is_similar_symmetry(f_obs)) :
        mmtbx.command_line.show_symmetry_error(
          file1="PDB file",
          file2="data file",
          symm1=pdb_symm,
          symm2=f_obs)
    xray_structure = pdb_in.file_object.xray_structure_simple(
      crystal_symmetry=pdb_symm)
    from mmtbx.utils import fmodel_simple
    # XXX this gets done anyway later, but they need to be consistent before
    # creating the fmodel manager
    if (f_obs.anomalous_flag()) :
      f_obs = f_obs.average_bijvoet_mates()
    f_obs = f_obs.eliminate_sys_absent()
    f_obs, r_free_flags = f_obs.map_to_asu().common_sets(
      other=r_free_flags.map_to_asu())
    fmodel = fmodel_simple(
      f_obs=f_obs,
      r_free_flags=r_free_flags,
      xray_structures=[xray_structure],
      skip_twin_detection=True,
      scattering_table="n_gaussian")
    fmodel.show(log=out)
    f_model = fmodel.f_model()
    r_free_flags = f_model.customized_copy(data=fmodel.arrays.free_sel)
  else :
    if (f_model.anomalous_flag()) :
      f_model = f_model.average_bijvoet_mates()
    f_model, r_free_flags = f_model.common_sets(other=r_free_flags)
  stats = mmtbx.validation.experimental.merging_and_model_statistics(
    f_model=f_model,
    f_obs=f_obs,
    r_free_flags=r_free_flags,
    unmerged_i_obs=unmerged_i_obs,
    n_bins=params.n_bins,
    sigma_filtering=params.sigma_filtering)
  stats.show_cc_star(out=out)
  if (params.loggraph) :
    stats.show_loggraph(out=out)
  print >> out, ""
  print >> out, "Reference:"
  print >> out, "  Karplus PA & Diederichs K (2012) Science 336:1030-3."
  print >> out, ""
  return stats
Esempio n. 11
0
 if (i_obs is None) :
   raise Sorry("Couldn't find intensities!")
 wavelength = params.wavelength
 if (wavelength is None) :
   info = i_obs.info()
   if (info is not None) :
     wavelength = info.wavelength
     if (wavelength is not None) :
       print >> out, "Using wavelength=%g from input file" % wavelength
 sigma_iobs_rejection_criterion = work_params.french_wilson.\
   sigma_iobs_rejection_criterion
 if (not i_obs.is_unique_set_under_symmetry()) :
   print >> out, "Merging symmetry-equivalent reflections"
   i_obs = i_obs.merge_equivalents().array()
 f_obs = french_wilson.french_wilson_scale(miller_array=i_obs,
   params=params,
   sigma_iobs_rejection_criterion=sigma_iobs_rejection_criterion,
   log=out)
 if f_obs is None:
   raise Sorry("Not enough data to accurately apply the French-Wilson method."+\
               " Exiting.")
 if params.output_file == None:
   output_file = "french_wilson.mtz"
 else:
   output_file = params.output_file
 mtz_dataset = i_obs.as_mtz_dataset(
   column_root_label = "I",
   wavelength = wavelength)
 mtz_dataset.add_miller_array(
   miller_array      = f_obs,
   column_root_label = "F")
 if (r_free_flags is not None) and (params.keep_r_free_flags):
Esempio n. 12
0
def run(args, out=sys.stdout):
    from iotbx import file_reader
    import iotbx.phil
    if (len(args) == 0):
        raise Usage("""\
iotbx.simple_map_coefficients data_phases.mtz [options]

Full parameters:
%s""" % iotbx.phil.parse(master_phil).as_str(attributes_level=1, prefix=" "))
    cmdline = iotbx.phil.process_command_line_with_files(
        args=args,
        master_phil_string=master_phil,
        reflection_file_def="file_name")
    params = cmdline.work.extract()
    if (params.file_name is None):
        raise Sorry("No reflection file specified.")
    hkl_in = file_reader.any_file(params.file_name).check_file_type("hkl")
    hkl_server = hkl_in.file_server
    data = hkl_server.get_xray_data(file_name=params.file_name,
                                    labels=params.data_labels,
                                    ignore_all_zeros=False,
                                    parameter_name="data_labels",
                                    parameter_scope="",
                                    prefer_anomalous=True,
                                    prefer_amplitudes=True)
    data_labels = data.info().label_string()
    if (data.is_xray_intensity_array()):
        from cctbx.french_wilson import french_wilson_scale
        data = french_wilson_scale(data, log=out)
    phases = hkl_server.get_phases_deg(file_name=params.file_name,
                                       labels=params.phase_labels,
                                       convert_to_phases_if_necessary=True,
                                       original_phase_units=None,
                                       parameter_scope="",
                                       parameter_name="phase_labels",
                                       minimum_score=2)
    assert (not phases.anomalous_flag())
    deg = True  # FIXME
    weights = None
    if (params.use_weights in [Auto, True]):
        # FIXME centralize this in iotbx.reflection_file_utils
        for array in hkl_server.miller_arrays:
            if (array.is_real_array()):
                label_string = array.info().label_string()
                if ((label_string == params.weight_labels)
                        or ((params.weight_labels is None) and
                            ("FOM" in label_string))):
                    weights = array
                    break
    amplitudes = data
    if (params.map_type == "anom"):
        if (not data.anomalous_flag()):
            raise Sorry(
                "Anomalous map requested, but selected data are merged.")
        amplitudes = data.anomalous_differences()
        print >> out, "Using anomalous differences in %s" % data_labels
    else:
        print >> out, "Using amplitudes in %s" % data_labels
        if (data.anomalous_flag()):
            amplitudes = data.average_bijvoet_mates()
    if (params.use_weights is Auto) and (weights is not None):
        if (params.map_type != "anom"):
            params.use_weights = True
    elif (params.use_weights == True) and (weights is None):
        raise Sorry("No weights (FOM, etc.) found in input file.")
    if (params.use_weights == True):
        assert (not weights.anomalous_flag())
        print >> out, "Applying weights in %s" % weights.info().label_string()
        amplitudes, weights = amplitudes.common_sets(other=weights)
        amplitudes = amplitudes.customized_copy(data=amplitudes.data() *
                                                weights.data())
    amplitudes = amplitudes.customized_copy(sigmas=None)
    print >> out, "Applying phases in %s" % phases.info().label_string()
    amplitudes, phases = amplitudes.common_sets(phases)
    coeffs = amplitudes.phase_transfer(phases, deg=deg).set_observation_type(
        None)  # FIXME
    if (params.map_type == "anom"):  # apply 90-degree phase shift
        coeffs = coeffs.customized_copy(data=coeffs.data() / (2j))
    assert (coeffs.is_complex_array())
    column_root_label = "F"
    decorator = None
    if (params.map_type == "anom"):
        column_root_label = "ANOM"
    elif (params.use_weights == True):
        column_root_label = "FWT"
        decorator = iotbx.mtz.ccp4_label_decorator()
    import iotbx.mtz
    mtz_dataset = coeffs.as_mtz_dataset(column_root_label=column_root_label,
                                        label_decorator=decorator)
    if (params.output_file is None):
        params.output_file = "map_coeffs.mtz"
    mtz_dataset.mtz_object().write(params.output_file)
    print >> out, "Wrote %s" % params.output_file
    return os.path.abspath(params.output_file)