예제 #1
0
def run(processed_args, params):
    if (params.scattering_table
            not in ["n_gaussian", "wk1995", "it1992", "neutron"]):
        raise Sorry("Incorrect scattering_table.")
    crystal_symmetry = None
    crystal_symmetries = []
    for f in [str(params.pdb_file_name), str(params.reflection_file_name)]:
        cs = crystal_symmetry_from_any.extract_from(f)
        if (cs is not None): crystal_symmetries.append(cs)
    if (len(crystal_symmetries) == 1): crystal_symmetry = crystal_symmetries[0]
    elif (len(crystal_symmetries) == 0):
        raise Sorry("No crystal symmetry found.")
    else:
        if (not crystal_symmetries[0].is_similar_symmetry(
                crystal_symmetries[1])):
            raise Sorry("Crystal symmetry mismatch between different files.")
        crystal_symmetry = crystal_symmetries[0]
    f_obs = None
    r_free_flags = None
    if (params.reflection_file_name is not None):
        reflection_file = reflection_file_reader.any_reflection_file(
            file_name=params.reflection_file_name, ensure_read_access=True)
        rfs = reflection_file_server(crystal_symmetry=crystal_symmetry,
                                     reflection_files=[reflection_file])
        parameters = utils.data_and_flags_master_params().extract()
        if (params.data_labels is not None):
            parameters.labels = [processed_args.data_labels]
        determine_data_and_flags_result = utils.determine_data_and_flags(
            reflection_file_server=rfs,
            parameters=parameters,
            keep_going=True,
            log=StringIO())
        f_obs = determine_data_and_flags_result.f_obs
        r_free_flags = determine_data_and_flags_result.r_free_flags
        if (r_free_flags is None):
            r_free_flags = f_obs.array(
                data=flex.bool(f_obs.data().size(), False))
            test_flag_value = None
    xray_structure = iotbx.pdb.input(
        file_name=params.pdb_file_name).xray_structure_simple()
    xray_structure_da = None
    if (params.external_da_pdb_file_name is not None):
        xray_structure_da = iotbx.pdb.input(
            file_name=params.external_da_pdb_file_name).xray_structure_simple(
            )
    if (f_obs is not None):
        f_obs = f_obs.resolution_filter(d_min=params.high_resolution,
                                        d_max=params.low_resolution)
        r_free_flags = r_free_flags.resolution_filter(
            d_min=params.high_resolution, d_max=params.low_resolution)
    #
    assert params.mode in ["build", "build_and_refine"]
    grow_density(f_obs=f_obs,
                 r_free_flags=r_free_flags,
                 xray_structure=xray_structure,
                 xray_structure_da=xray_structure_da,
                 params=params)
예제 #2
0
def run(processed_args, params):
  if(params.scattering_table not in ["n_gaussian","wk1995",
     "it1992","neutron"]):
    raise Sorry("Incorrect scattering_table.")
  crystal_symmetry = None
  crystal_symmetries = []
  for f in [str(params.pdb_file_name), str(params.reflection_file_name)]:
    cs = crystal_symmetry_from_any.extract_from(f)
    if(cs is not None): crystal_symmetries.append(cs)
  if(len(crystal_symmetries) == 1): crystal_symmetry = crystal_symmetries[0]
  elif(len(crystal_symmetries) == 0):
    raise Sorry("No crystal symmetry found.")
  else:
    if(not crystal_symmetries[0].is_similar_symmetry(crystal_symmetries[1])):
      raise Sorry("Crystal symmetry mismatch between different files.")
    crystal_symmetry = crystal_symmetries[0]
  f_obs = None
  r_free_flags = None
  if(params.reflection_file_name is not None):
    reflection_file = reflection_file_reader.any_reflection_file(
      file_name = params.reflection_file_name, ensure_read_access = True)
    rfs = reflection_file_server(
      crystal_symmetry = crystal_symmetry,
      reflection_files = [reflection_file])
    parameters = utils.data_and_flags_master_params().extract()
    if(params.data_labels is not None):
      parameters.labels = [processed_args.data_labels]
    determine_data_and_flags_result = utils.determine_data_and_flags(
      reflection_file_server  = rfs,
      parameters              = parameters,
      keep_going              = True,
      log                     = StringIO())
    f_obs = determine_data_and_flags_result.f_obs
    r_free_flags = determine_data_and_flags_result.r_free_flags
    if(r_free_flags is None):
      r_free_flags=f_obs.array(data=flex.bool(f_obs.data().size(), False))
      test_flag_value=None
  xray_structure = iotbx.pdb.input(file_name =
    params.pdb_file_name).xray_structure_simple()
  xray_structure_da = None
  if(params.external_da_pdb_file_name is not None):
    xray_structure_da = iotbx.pdb.input(file_name =
      params.external_da_pdb_file_name).xray_structure_simple()
  if(f_obs is not None):
    f_obs = f_obs.resolution_filter(d_min = params.high_resolution,
      d_max = params.low_resolution)
    r_free_flags = r_free_flags.resolution_filter(d_min = params.high_resolution,
      d_max = params.low_resolution)
  #
  assert params.mode in ["build", "build_and_refine"]
  grow_density(f_obs             = f_obs,
               r_free_flags      = r_free_flags,
               xray_structure    = xray_structure,
               xray_structure_da = xray_structure_da,
               params            = params)
예제 #3
0
    def _get_f_obs_r_free(self):
        """ Get f_obs and r_free_flags from pdb and mtz via self.inputs

        Returns
        -------
        f_obs: cctbx.miller.array
            observed data

        r_free_flags: cctbx.miller.array
            r free flags
        """

        rfs = reflection_file_utils.reflection_file_server(
            crystal_symmetry=self.inputs.crystal_symmetry,
            force_symmetry=True,
            reflection_files=self.inputs.reflection_files,
            err=StringIO(),
        )

        # Parameter object descrinign the may to process data
        # From mmtbx.utils
        data_flags_params = data_and_flags_master_params().extract()
        data_flags_params.labels = self.params.exhaustive.options.column_type

        determined_data_and_flags = mmtbx.utils.determine_data_and_flags(
            reflection_file_server=rfs,
            parameters=data_flags_params,
            keep_going=True,
            log=StringIO(),
        )

        # Extract the need parts of determined_data_and_flags
        f_obs = determined_data_and_flags.f_obs
        r_free_flags = determined_data_and_flags.r_free_flags

        return f_obs, r_free_flags
예제 #4
0
def run(args,
        command_name             = "mmtbx.model_vs_data",
        show_geometry_statistics = True,
        model_size_max_atoms     = 80000,
        data_size_max_reflections= 1000000,
        unit_cell_max_dimension  = 800.,
        return_fmodel_and_pdb    = False,
        out                      = None,
        log                      = sys.stdout):
  import mmtbx.f_model_info
  if(len(args)==0) or (args == ["--help"]) :
    print >> log, msg
    defaults(log=log, silent=False)
    return
  parsed = defaults(log=log, silent=True)
  #
  mvd_obj = mvd()
  #
  processed_args = utils.process_command_line_args(args = args,
    log = log, master_params = parsed)
  params = processed_args.params.extract()
  #
  reflection_files = processed_args.reflection_files
  if(len(reflection_files) == 0):
    raise Sorry("No reflection file found.")
  crystal_symmetry = processed_args.crystal_symmetry
  if(crystal_symmetry is None):
    raise Sorry("No crystal symmetry found.")
  if(len(processed_args.pdb_file_names) == 0):
    raise Sorry("No PDB file found.")
  pdb_file_names = processed_args.pdb_file_names
  #
  rfs = reflection_file_server(
    crystal_symmetry = crystal_symmetry,
    reflection_files = reflection_files)
  parameters = utils.data_and_flags_master_params().extract()
  if(params.f_obs_label is not None):
    parameters.labels = params.f_obs_label
  if(params.r_free_flags_label is not None):
    parameters.r_free_flags.label = params.r_free_flags_label
  if (params.high_resolution is not None) :
    parameters.high_resolution = params.high_resolution
  determine_data_and_flags_result = utils.determine_data_and_flags(
    reflection_file_server  = rfs,
    parameters              = parameters,
    data_parameter_scope    = "refinement.input.xray_data",
    flags_parameter_scope   = "refinement.input.xray_data.r_free_flags",
    data_description        = "X-ray data",
    keep_going              = True,
    log                     = StringIO())
  f_obs = determine_data_and_flags_result.f_obs
  number_of_reflections = f_obs.indices().size()
  if(params.ignore_giant_models_and_datasets and
     number_of_reflections > data_size_max_reflections):
    raise Sorry("Too many reflections: %d"%number_of_reflections)
  #
  max_unit_cell_dimension = max(f_obs.unit_cell().parameters()[:3])
  if(params.ignore_giant_models_and_datasets and
     max_unit_cell_dimension > unit_cell_max_dimension):
    raise Sorry("Too large unit cell (max dimension): %s"%
      str(max_unit_cell_dimension))
  #
  r_free_flags = determine_data_and_flags_result.r_free_flags
  test_flag_value = determine_data_and_flags_result.test_flag_value
  if(r_free_flags is None):
    r_free_flags=f_obs.array(data=flex.bool(f_obs.data().size(), False))
    test_flag_value=None
  #
  mmtbx_pdb_file = mmtbx.utils.pdb_file(
    pdb_file_names        = pdb_file_names,
    cif_objects           = processed_args.cif_objects,
    crystal_symmetry      = crystal_symmetry,
    use_neutron_distances = (params.scattering_table=="neutron"),
    ignore_unknown_nonbonded_energy_types = not show_geometry_statistics,
    log                   = log)
  mmtbx_pdb_file.set_ppf(stop_if_duplicate_labels = False)
  processed_pdb_file = mmtbx_pdb_file.processed_pdb_file
  pdb_raw_records = mmtbx_pdb_file.pdb_raw_records
  pdb_inp = mmtbx_pdb_file.pdb_inp
  #
  # just to avoid going any further with bad PDB file....
  pdb_inp.xray_structures_simple()
  #
  acp = processed_pdb_file.all_chain_proxies
  atom_selections = group_args(
    all           = acp.selection(string = "all"),
    macromolecule = acp.selection(string = "protein or dna or rna"),
    solvent       = acp.selection(string = "water"), # XXX single_atom_residue
    ligand        = acp.selection(string = "not (protein or dna or rna or water)"),
    backbone      = acp.selection(string = "backbone"),
    sidechain     = acp.selection(string = "sidechain"))
  #
  scattering_table = params.scattering_table
  exptl_method = pdb_inp.get_experiment_type()
  if (exptl_method is not None) and ("NEUTRON" in exptl_method) :
    scattering_table = "neutron"
  xsfppf = mmtbx.utils.xray_structures_from_processed_pdb_file(
    processed_pdb_file = processed_pdb_file,
    scattering_table   = scattering_table,
    d_min              = f_obs.d_min())
  xray_structures = xsfppf.xray_structures
  if(0): #XXX normalize occupancies if all models have occ=1 so the total=1
    n_models = len(xray_structures)
    for xrs in xray_structures:
      occ = xrs.scatterers().extract_occupancies()
      occ = occ/n_models
      xrs.set_occupancies(occ)
  model_selections = xsfppf.model_selections
  mvd_obj.collect(crystal = group_args(
    uc       = f_obs.unit_cell(),
    sg       = f_obs.crystal_symmetry().space_group_info().symbol_and_number(),
    n_sym_op = f_obs.crystal_symmetry().space_group_info().type().group().order_z(),
    uc_vol   = f_obs.unit_cell().volume()))
  #
  hierarchy = pdb_inp.construct_hierarchy()
  pdb_atoms = hierarchy.atoms()
  pdb_atoms.reset_i_seq()
  #
  # Extract TLS
  pdb_tls = None
  pdb_inp_tls = pdb_inp.extract_tls_params(hierarchy)
  pdb_tls = group_args(pdb_inp_tls           = pdb_inp_tls,
                       tls_selections        = [],
                       tls_selection_strings = [])
  # XXX no TLS + multiple models
  if(pdb_inp_tls.tls_present and pdb_inp_tls.error_string is None and
     len(xray_structures)==1):
    pdb_tls = mmtbx.tls.tools.extract_tls_from_pdb(
      pdb_inp_tls       = pdb_inp_tls,
      all_chain_proxies = mmtbx_pdb_file.processed_pdb_file.all_chain_proxies,
      xray_structure    = xsfppf.xray_structure_all)
    if(len(pdb_tls.tls_selections)==len(pdb_inp_tls.tls_params) and
       len(pdb_inp_tls.tls_params) > 0):
      xray_structures = [utils.extract_tls_and_u_total_from_pdb(
        f_obs          = f_obs,
        r_free_flags   = r_free_flags,
        xray_structure = xray_structures[0], # XXX no TLS + multiple models
        tls_selections = pdb_tls.tls_selections,
        tls_groups     = pdb_inp_tls.tls_params)]
  ###########################
  geometry_statistics = show_geometry(
    xray_structures          = xray_structures,
    processed_pdb_file       = processed_pdb_file,
    scattering_table         = scattering_table,
    hierarchy                = hierarchy,
    model_selections         = model_selections,
    show_geometry_statistics = show_geometry_statistics,
    mvd_obj                  = mvd_obj,
    atom_selections          = atom_selections)
  ###########################
  mp = mmtbx.masks.mask_master_params.extract()
  f_obs_labels = f_obs.info().label_string()
  f_obs = f_obs.sort(reverse=True, by_value="packed_indices")
  r_free_flags = r_free_flags.sort(reverse=True, by_value="packed_indices")
  fmodel = utils.fmodel_simple(
    xray_structures     = xray_structures,
    scattering_table    = scattering_table,
    mask_params         = mp,
    f_obs               = f_obs,
    r_free_flags        = r_free_flags,
    skip_twin_detection = params.skip_twin_detection)
  n_outl = f_obs.data().size() - fmodel.f_obs().data().size()
  mvd_obj.collect(model_vs_data = show_model_vs_data(fmodel))
  # Extract information from PDB file header and output (if any)
  pub_r_work       = None
  pub_r_free       = None
  pub_high         = None
  pub_low          = None
  pub_sigma        = None
  pub_program_name = None
  pub_solv_cont    = None
  pub_matthews     = None
  published_results = pdb_inp.get_r_rfree_sigma(file_name=pdb_file_names[0])
  if(published_results is not None):
    pub_r_work = published_results.r_work
    pub_r_free = published_results.r_free
    pub_high   = published_results.high
    pub_low    = published_results.low
    pub_sigma  = published_results.sigma
  pub_program_name = pdb_inp.get_program_name()
  pub_solv_cont    = pdb_inp.get_solvent_content()
  pub_matthews     = pdb_inp.get_matthews_coeff()
  mvd_obj.collect(pdb_header = group_args(
    program_name    = pub_program_name,
    year            = pdb_inp.extract_header_year(),
    r_work          = pub_r_work,
    r_free          = pub_r_free,
    high_resolution = pub_high,
    low_resolution  = pub_low,
    sigma_cutoff    = pub_sigma,
    matthews_coeff  = pub_matthews,
    solvent_cont    = pub_solv_cont,
    tls             = pdb_tls,
    exptl_method    = exptl_method))
  #
  # Recompute R-factors using published cutoffs
  fmodel_cut = fmodel
  tmp_sel = flex.bool(fmodel.f_obs().data().size(), True)
  if(pub_sigma is not None and fmodel.f_obs().sigmas() is not None):
    tmp_sel &= fmodel.f_obs().data() > fmodel.f_obs().sigmas()*pub_sigma
  if(pub_high is not None and abs(pub_high-fmodel.f_obs().d_min()) > 0.03):
    tmp_sel &= fmodel.f_obs().d_spacings().data() > pub_high
  if(pub_low is not None and abs(pub_low-fmodel.f_obs().d_max_min()[0]) > 0.03):
    tmp_sel &= fmodel.f_obs().d_spacings().data() < pub_low
  if(tmp_sel.count(True) != tmp_sel.size() and tmp_sel.count(True) > 0):
    fmodel_cut = utils.fmodel_simple(
      xray_structures     = xray_structures,
      scattering_table    = scattering_table,
      f_obs               = fmodel.f_obs().select(tmp_sel),
      r_free_flags        = fmodel.r_free_flags().select(tmp_sel),
      skip_twin_detection = params.skip_twin_detection)
  mvd_obj.collect(misc = group_args(
    r_work_cutoff = fmodel_cut.r_work(),
    r_free_cutoff = fmodel_cut.r_free(),
    n_refl_cutoff = fmodel_cut.f_obs().data().size()))
  mvd_obj.collect(data =
    show_data(fmodel          = fmodel,
              n_outl          = n_outl,
              test_flag_value = test_flag_value,
              f_obs_labels    = f_obs_labels,
              fmodel_cut      = fmodel_cut))
  # map statistics
  if(len(xray_structures)==1): # XXX no multi-model support yet
    mvd_obj.collect(maps = maps(fmodel = fmodel, mvd_obj = mvd_obj))
  # CC* and friends
  cc_star_stats = None
  if (params.unmerged_data is not None) :
    import mmtbx.validation.experimental
    import mmtbx.command_line
    f_obs = fmodel.f_obs().average_bijvoet_mates()
    unmerged_i_obs = mmtbx.command_line.load_and_validate_unmerged_data(
      f_obs=f_obs,
      file_name=params.unmerged_data,
      data_labels=params.unmerged_labels,
      log=null_out())
    cc_star_stats = mmtbx.validation.experimental.merging_and_model_statistics(
      f_model=fmodel.f_model().average_bijvoet_mates(),
      f_obs=f_obs,
      r_free_flags=fmodel.r_free_flags().average_bijvoet_mates(),
      unmerged_i_obs=unmerged_i_obs,
      n_bins=params.n_bins)
  mvd_obj.show(log=out)
  if (cc_star_stats is not None) :
    cc_star_stats.show_model_vs_data(out=out, prefix="  ")
  if return_fmodel_and_pdb :
    mvd_obj.pdb_file = processed_pdb_file
    mvd_obj.fmodel = fmodel
  if(len(params.map) > 0):
    for map_name_string in params.map:
      map_type_obj = mmtbx.map_names(map_name_string = map_name_string)
      map_params = mmtbx.maps.map_and_map_coeff_master_params().fetch(
        mmtbx.maps.cast_map_coeff_params(map_type_obj)).extract()
      maps_obj = mmtbx.maps.compute_map_coefficients(fmodel = fmodel_cut, params =
        map_params.map_coefficients)
      fn = os.path.basename(processed_args.reflection_file_names[0])
      if(fn.count(".")):
        prefix = fn[:fn.index(".")]
      else: prefix= fn
      file_name = prefix+"_%s_map_coeffs.mtz"%map_type_obj.format()
      maps_obj.write_mtz_file(file_name = file_name)
  # statistics in bins
  if(not fmodel.twin):
    print >> log, "Statistics in resolution bins:"
    mmtbx.f_model_info.r_work_and_completeness_in_resolution_bins(
      fmodel = fmodel, out = log, prefix="  ")
  # report map cc
  if(params.comprehensive and not fmodel_cut.twin and
     fmodel_cut.xray_structure is not None):
    rsc_params = real_space_correlation.master_params().extract()
    rsc_params.scattering_table = scattering_table
    real_space_correlation.simple(
       fmodel        = fmodel_cut,
       pdb_hierarchy = hierarchy,
       params        = rsc_params,
       log           = log,
       show_results  = True)
  #
  if(params.dump_result_object_as_pickle):
    output_prefixes = []
    for op in processed_args.pdb_file_names+processed_args.reflection_file_names:
      op = os.path.basename(op)
      try: op = op[:op.index(".")]
      except Exception: pass
      if(not op in output_prefixes): output_prefixes.append(op)
    output_prefix = "_".join(output_prefixes)
    easy_pickle.dump("%s.pickle"%output_prefix, mvd_obj)
  return mvd_obj
예제 #5
0
def run(args, command_name="phenix.twin_map_utils"):
    log = sys.stdout
    params = None
    if (len(args) == 0 or "--help" in args or "--h" in args or "-h" in args):
        print_help(command_name=command_name)
    else:
        log = multi_out()
        if (not "--quiet" in args):
            log.register(label="stdout", file_object=sys.stdout)
        string_buffer = StringIO()
        string_buffer_plots = StringIO()
        log.register(label="log_buffer", file_object=string_buffer)

        phil_objects = []
        argument_interpreter = master_params.command_line_argument_interpreter(
            home_scope="map_coefs")

        for arg in args:
            command_line_params = None
            arg_is_processed = False
            # is it a file?
            if (os.path.isfile(arg)):  ## is this a file name?
                # check if it is a phil file
                try:
                    command_line_params = iotbx.phil.parse(file_name=arg)
                    if command_line_params is not None:
                        phil_objects.append(command_line_params)
                        arg_is_processed = True
                except KeyboardInterrupt:
                    raise
                except Exception:
                    pass
            else:
                try:
                    command_line_params = argument_interpreter.process(arg=arg)
                    if command_line_params is not None:
                        phil_objects.append(command_line_params)
                        arg_is_processed = True
                except KeyboardInterrupt:
                    raise
                except Exception:
                    pass

            if not arg_is_processed:
                print("##----------------------------------------------##",
                      file=log)
                print("## Unknown file or keyword:", arg, file=log)
                print("##----------------------------------------------##",
                      file=log)
                print(file=log)
                raise Sorry("Unknown file or keyword: %s" % arg)

        effective_params = master_params.fetch(sources=phil_objects)
        params = effective_params.extract()
        """
    new_params =  master_params.format(python_object=params)
    new_params.show(out=log)
    """
        # now get the unit cell from the pdb file

        hkl_xs = crystal_symmetry_from_any.extract_from(
            file_name=params.twin_utils.input.xray_data.file_name)
        pdb_xs = crystal_symmetry_from_any.extract_from(
            file_name=params.twin_utils.input.model.file_name)

        phil_xs = None
        if ([
                params.twin_utils.input.unit_cell,
                params.twin_utils.input.space_group
        ]).count(None) < 2:
            phil_xs = crystal.symmetry(
                unit_cell=params.twin_utils.input.unit_cell,
                space_group_info=params.twin_utils.input.space_group)

        combined_xs = crystal.select_crystal_symmetry(None, phil_xs, [pdb_xs],
                                                      [hkl_xs])

        # inject the unit cell and symmetry in the phil scope please
        params.twin_utils.input.unit_cell = combined_xs.unit_cell()
        params.twin_utils.input.space_group = \
          sgtbx.space_group_info( group = combined_xs.space_group() )

        new_params = master_params.format(python_object=params)
        new_params.show(out=log)

        if params.twin_utils.input.unit_cell is None:
            raise Sorry("unit cell not specified")
        if params.twin_utils.input.space_group is None:
            raise Sorry("space group not specified")
        if params.twin_utils.input.xray_data.file_name is None:
            raise Sorry("Xray data not specified")
        if params.twin_utils.input.model.file_name is None:
            raise Sorry("pdb file with  model not specified")

        #-----------------------------------------------------------
        #
        # step 1: read in the reflection file
        #
        phil_xs = crystal.symmetry(
            unit_cell=params.twin_utils.input.unit_cell,
            space_group_info=params.twin_utils.input.space_group)

        xray_data_server = reflection_file_utils.reflection_file_server(
            crystal_symmetry=phil_xs, force_symmetry=True, reflection_files=[])

        miller_array = None
        free_flags = None

        tmp_params = utils.data_and_flags_master_params().extract()
        # insert proper values please
        tmp_params.file_name = params.twin_utils.input.xray_data.file_name
        tmp_params.labels = params.twin_utils.input.xray_data.obs_labels
        tmp_params.r_free_flags.file_name = params.twin_utils.input.xray_data.file_name
        tmp_params.r_free_flags.label = params.twin_utils.input.xray_data.free_flag

        tmp_object = utils.determine_data_and_flags(
            reflection_file_server=xray_data_server,
            parameters=tmp_params,
            log=log)

        miller_array = tmp_object.extract_data()
        if miller_array.is_xray_intensity_array():
            miller_array = miller_array.f_sq_as_f()
        assert miller_array.is_xray_amplitude_array()

        free_flags = tmp_object.extract_flags(data=miller_array)
        print(file=log)
        print("Attempting to extract Free R flags", file=log)

        free_flags = free_flags.customized_copy(data=flex.bool(
            free_flags.data() == 1))
        if free_flags is None:
            free_flags = miller_array.generate_r_free_flags(
                use_lattice_symmetry=True)

        assert miller_array.observation_type() is not None

        print(file=log)
        print("Summary info of observed data", file=log)
        print("=============================", file=log)
        miller_array.show_summary(f=log)
        print(file=log)

        if miller_array.indices().size() == 0:
            raise Sorry("No data available")

        #----------------------------------------------------------------
        # Step 2: get an xray structure from the PDB file
        #
        model = pdb.input(file_name=params.twin_utils.input.model.file_name
                          ).xray_structure_simple(crystal_symmetry=phil_xs)
        print("Atomic model summary", file=log)
        print("====================", file=log)
        model.show_summary(f=log)
        print(file=log)

        #----------------------------------------------------------------
        # step 3: get the twin laws for this xs
        twin_laws = twin_analyses.twin_laws(
          miller_array,
          lattice_symmetry_max_delta=\
             params.twin_utils.parameters.twinning.max_delta,
          out=log)

        print(file=log)
        print("Preliminary data analyses", file=log)
        print("==========================", file=log)
        twin_laws.show(out=log)

        #---------
        # step 3:
        # make twin model managers for all twin laws
        print(file=log)
        print(
            "Overall and bulk solvent scale paranmeters and twin fraction estimation",
            file=log)
        print(
            "=======================================================================",
            file=log)
        twin_models = []
        operator_count = 0

        if params.twin_utils.parameters.twinning.twin_law is not None:
            tmp_law = sgtbx.rt_mx(
                params.twin_utils.parameters.twinning.twin_law)
            tmp_law = twin_analyses.twin_law(tmp_law, None, None, None, None,
                                             None)
            twin_laws.operators = [tmp_law]
        for twin_law in twin_laws.operators:
            operator_count += 1
            operator_hkl = sgtbx.change_of_basis_op(twin_law.operator).as_hkl()
            twin_model = twin_f_model.twin_model_manager(
                f_obs=miller_array,
                r_free_flags=free_flags,
                xray_structure=model,
                twin_law=twin_law.operator,
                detwin_mode=params.twin_utils.parameters.twinning.detwin_mode,
                out=log)

            print("--- bulk solvent scaling ---", file=log)
            twin_model.update_all_scales()
            twin_model.r_values()
            twin_model.target()
            twin_model.show_k_sol_b_sol_b_cart_target()
            twin_model.show_essential()

            wfofc = twin_model.map_coefficients(map_type="mFo-DFc")
            wtfofc = twin_model.map_coefficients(map_type="2mFo-DFc")
            grad = twin_model.map_coefficients(map_type="gradient")

            mtz_dataset = wtfofc.as_mtz_dataset(column_root_label="FWT")
            mtz_dataset = mtz_dataset.add_miller_array(
                miller_array=wfofc, column_root_label="DFWT")
            mtz_dataset = mtz_dataset.add_miller_array(
                miller_array=grad, column_root_label="GRAD")
            name = params.twin_utils.output.map_coeffs_root + "_" + str(
                operator_count) + ".mtz"
            print(file=log)
            print("Writing %s for twin law %s" % (name, operator_hkl),
                  file=log)
            print(file=log)
            mtz_dataset.mtz_object().write(file_name=name)

            if params.twin_utils.output.obs_and_calc is not None:
                # i want also a Fobs and Fmodel combined dataset please
                mtz_dataset = miller_array.as_mtz_dataset(
                    column_root_label="FOBS")
                mtz_dataset = mtz_dataset.add_miller_array(
                    miller_array=twin_model.f_model(),
                    column_root_label="FMODEL")
                name = params.twin_utils.output.obs_and_calc
                mtz_dataset.mtz_object().write(file_name=name)

        if len(twin_laws.operators) == 0:
            print(file=log)
            print("No twin laws were found", file=log)
            print("Performing maximum likelihood based bulk solvent scaling",
                  file=log)
            f_model_object = f_model.manager(f_obs=miller_array,
                                             r_free_flags=free_flags,
                                             xray_structure=model)
            f_model_object.update_all_scales(log=log)
            tfofc = f_model_object.map_coefficients(map_type="2mFobs-DFmodel")
            fofc = f_model_object.map_coefficients(map_type="mFobs-DFmodel")
            mtz_dataset = tfofc.as_mtz_dataset(column_root_label="FWT")
            mtz_dataset = mtz_dataset.add_miller_array(
                miller_array=fofc, column_root_label="DELFWT")
            name = params.twin_utils.output.map_coeffs_root + "_ML.mtz"
            mtz_dataset.mtz_object().write(file_name=name)

            if params.twin_utils.output.obs_and_calc is not None:
                # i want also a Fobs and Fmodel combined dataset please
                mtz_dataset = miller_array.as_mtz_dataset(
                    column_root_label="FOBS")
                mtz_dataset = mtz_dataset.add_miller_array(
                    miller_array=f_model_object.f_model(),
                    column_root_label="FMODEL")
                name = params.twin_utils.output.obs_and_calc
                mtz_dataset.mtz_object().write(file_name=name)

        print(file=log)
        print(file=log)
        print("All done \n", file=log)
        logfile = open(params.twin_utils.output.logfile, 'w')
        print(string_buffer.getvalue(), file=logfile)
        print(file=log)
예제 #6
0
def run(args, command_name="phenix.fobs_minus_fobs_map", log=None):
    if (len(args) == 0): args = ["--help"]
    examples = """Examples:

phenix.fobs_minus_fobs_map f_obs_1_file=data1.mtz f_obs_2_file=data2.sca \
f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 model.pdb

phenix.fobs_minus_fobs_map f_obs_1_file=data.mtz f_obs_2_file=data.mtz \
f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 phase_source=model.pdb \
high_res=2.0 sigma_cutoff=2 scattering_table=neutron"""

    command_line = (iotbx_option_parser(
        usage="%s [options]" % command_name,
        description=examples).option("--silent",
                                     action="store_true",
                                     help="Suppress output to the screen.").
                    enable_symmetry_comprehensive()).process(args=args)
    #
    if (log is None):
        log = sys.stdout
    if (not command_line.options.silent):
        utils.print_header("phenix.fobs_minus_fobs_map", out=log)
        print("Command line arguments: ", file=log)
        print(args, file=log)
        print(file=log)
    #
    processed_args = utils.process_command_line_args(
        args=command_line.args,
        cmd_cs=command_line.symmetry,
        master_params=fo_minus_fo_master_params(),
        absolute_angle_tolerance=5,
        absolute_length_tolerance=1,
        log=log,
        suppress_symmetry_related_errors=True)
    working_phil = processed_args.params
    if (not command_line.options.silent):
        print("*** Parameters:", file=log)
        working_phil.show(out=log)
        print(file=log)
    params = working_phil.extract()
    consensus_symmetry = None
    if (params.ignore_non_isomorphous_unit_cells):
        if (None in [
                params.f_obs_1_file_name, params.f_obs_2_file_name,
                params.phase_source
        ]):
            raise Sorry(
                "The file parameters (f_obs_1_file_name, f_obs_2_file_name, " +
                "phase_source) must be specified explicitly when " +
                "ignore_non_isomorphous_unit_cells=True.")
        symm_manager = iotbx.symmetry.manager()
        pdb_in = iotbx.file_reader.any_file(params.phase_source,
                                            force_type="pdb")
        symm_manager.process_pdb_file(pdb_in)
        hkl_in_1 = iotbx.file_reader.any_file(params.f_obs_1_file_name,
                                              force_type="hkl")
        sg_err_1, uc_err_1 = symm_manager.process_reflections_file(hkl_in_1)
        hkl_in_2 = iotbx.file_reader.any_file(params.f_obs_2_file_name,
                                              force_type="hkl")
        sg_err_2, uc_err_2 = symm_manager.process_reflections_file(hkl_in_2)
        out = StringIO()
        symm_manager.show(out=out)
        if (sg_err_1) or (sg_err_2):
            raise Sorry((
                "Incompatible space groups in input files:\n%s\nAll files " +
                "must have the same point group (and ideally the same space group). "
                +
                "Please note that any symmetry information in the PDB file will be "
                + "used first.") % out.getvalue())
        elif (uc_err_1) or (uc_err_2):
            libtbx.call_back(
                message="warn",
                data=
                ("Crystal symmetry mismatch:\n%s\nCalculations will continue "
                 +
                 "using the symmetry in the PDB file (or if not available, the "
                 +
                 "first reflection file), but the maps should be treated with "
                 + "extreme suspicion.") % out.getvalue())
        crystal_symmetry = symm_manager.as_symmetry_object()
    else:
        processed_args = utils.process_command_line_args(
            args=command_line.args,
            cmd_cs=command_line.symmetry,
            master_params=fo_minus_fo_master_params(),
            suppress_symmetry_related_errors=False,
            absolute_angle_tolerance=5,
            absolute_length_tolerance=1,
            log=StringIO())
        crystal_symmetry = processed_args.crystal_symmetry
    #
    pdb_file_names = processed_args.pdb_file_names
    if (len(processed_args.pdb_file_names) == 0):
        if (params.phase_source is not None):
            pdb_file_names = [params.phase_source]
        else:
            raise Sorry("No PDB file found.")
    # Extaract Fobs1, Fobs2
    f_obss = []
    if (len(processed_args.reflection_files) == 2):
        for reflection_file in processed_args.reflection_files:
            reflection_file_server = reflection_file_utils.reflection_file_server(
                crystal_symmetry=crystal_symmetry,
                force_symmetry=True,
                reflection_files=[reflection_file],
                err=null_out())
            # XXX UGLY !!!
            try:
                parameters = utils.data_and_flags_master_params().extract()
                if (params.f_obs_1_label is not None):
                    parameters.labels = [params.f_obs_1_label]
                determine_data_and_flags_result = utils.determine_data_and_flags(
                    reflection_file_server=reflection_file_server,
                    keep_going=True,
                    parameters=parameters,
                    log=null_out())
            except:  # intentional
                parameters = utils.data_and_flags_master_params().extract()
                if (params.f_obs_2_label is not None):
                    parameters.labels = [params.f_obs_2_label]
                determine_data_and_flags_result = utils.determine_data_and_flags(
                    reflection_file_server=reflection_file_server,
                    keep_going=True,
                    parameters=parameters,
                    log=null_out())
            f_obss.append(determine_data_and_flags_result.f_obs)
    else:
        if ([params.f_obs_1_file_name,
             params.f_obs_2_file_name].count(None) == 2):
            raise Sorry("No reflection data file found.")
        for file_name, label in zip(
            [params.f_obs_1_file_name, params.f_obs_2_file_name],
            [params.f_obs_1_label, params.f_obs_2_label]):
            reflection_file = reflection_file_reader.any_reflection_file(
                file_name=file_name, ensure_read_access=False)
            reflection_file_server = reflection_file_utils.reflection_file_server(
                crystal_symmetry=crystal_symmetry,
                force_symmetry=True,
                reflection_files=[reflection_file],
                err=null_out())
            parameters = utils.data_and_flags_master_params().extract()
            if (label is not None):
                parameters.labels = [label]
            determine_data_and_flags_result = utils.determine_data_and_flags(
                reflection_file_server=reflection_file_server,
                parameters=parameters,
                keep_going=True,
                log=null_out())
            f_obss.append(determine_data_and_flags_result.f_obs)
    if (len(f_obss) != 2):
        raise Sorry(" ".join(errors))
    if (not command_line.options.silent):
        for ifobs, fobs in enumerate(f_obss):
            print("*** Summary for data set %d:" % ifobs, file=log)
            fobs.show_comprehensive_summary(f=log)
            print(file=log)
    pdb_combined = combine_unique_pdb_files(file_names=pdb_file_names)
    pdb_combined.report_non_unique(out=log)
    if (len(pdb_combined.unique_file_names) == 0):
        raise Sorry("No coordinate file given.")
    #
    raw_recs = flex.std_string()
    for rec in pdb_combined.raw_records:
        if (rec.upper().count("CRYST1") == 0):
            raw_recs.append(rec)
    raw_recs.append(
        iotbx.pdb.format_cryst1_record(crystal_symmetry=crystal_symmetry))
    #
    pdb_in = iotbx.pdb.input(source_info=None, lines=raw_recs)
    model = mmtbx.model.manager(model_input=pdb_in)
    d_min = min(f_obss[0].d_min(), f_obss[1].d_min())
    model.setup_scattering_dictionaries(
        scattering_table=params.scattering_table, d_min=d_min)
    xray_structure = model.get_xray_structure()
    hierarchy = model.get_hierarchy()
    #
    omit_sel = flex.bool(hierarchy.atoms_size(), False)
    if (params.advanced.omit_selection is not None):
        print("Will omit selection from phasing model:", file=log)
        print("  " + params.advanced.omit_selection, file=log)
        omit_sel = hierarchy.atom_selection_cache().selection(
            params.advanced.omit_selection)
        print("%d atoms selected for removal" % omit_sel.count(True), file=log)
    del hierarchy
    xray_structure = xray_structure.select(~omit_sel)
    if (not command_line.options.silent):
        print("*** Model summary:", file=log)
        xray_structure.show_summary(f=log)
        print(file=log)
    info0 = f_obss[0].info()
    info1 = f_obss[1].info()
    f_obss[0] = f_obss[0].resolution_filter(
        d_min=params.high_resolution,
        d_max=params.low_resolution).set_info(info0)
    f_obss[1] = f_obss[1].resolution_filter(
        d_min=params.high_resolution,
        d_max=params.low_resolution).set_info(info1)
    if (params.sigma_cutoff is not None):
        for i in [0, 1]:
            if (f_obss[i].sigmas() is not None):
                sel = f_obss[i].data(
                ) > f_obss[i].sigmas() * params.sigma_cutoff
                f_obss[i] = f_obss[i].select(sel).set_info(info0)
    for k, f_obs in enumerate(f_obss):
        if (f_obs.indices().size() == 0):
            raise Sorry(
                "No data left in array %d (labels=%s) after filtering!" %
                (k + 1, f_obs.info().label_string()))
    output_file_name = params.output_file
    if (output_file_name is None) and (params.file_name_prefix is not None):
        output_file_name = "%s_%s.mtz" % (params.file_name_prefix,
                                          params.job_id)
    output_files = compute_fo_minus_fo_map(
        data_arrays=f_obss,
        xray_structure=xray_structure,
        log=log,
        silent=command_line.options.silent,
        output_file=output_file_name,
        peak_search=params.find_peaks_holes,
        map_cutoff=params.map_cutoff,
        peak_search_params=params.peak_search,
        multiscale=params.advanced.multiscale,
        anomalous=params.advanced.anomalous).file_names
    return output_files
예제 #7
0
def run(args, command_name = "phenix.fobs_minus_fobs_map", log=None):
  if(len(args) == 0): args = ["--help"]
  examples = """Examples:

phenix.fobs_minus_fobs_map f_obs_1_file=data1.mtz f_obs_2_file=data2.sca \
f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 model.pdb

phenix.fobs_minus_fobs_map f_obs_1_file=data.mtz f_obs_2_file=data.mtz \
f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 phase_source=model.pdb \
high_res=2.0 sigma_cutoff=2 scattering_table=neutron"""

  command_line = (iotbx_option_parser(
    usage="%s [options]" % command_name,
    description=examples)
    .option("--silent",
      action="store_true",
      help="Suppress output to the screen.")
    .enable_symmetry_comprehensive()
    ).process(args=args)
  #
  if (log is None) :
    log = sys.stdout
  if(not command_line.options.silent):
    utils.print_header("phenix.fobs_minus_fobs_map", out = log)
    print >> log, "Command line arguments: "
    print >> log, args
    print >> log
  #
  processed_args = utils.process_command_line_args(
    args=command_line.args,
    cmd_cs=command_line.symmetry,
    master_params=fo_minus_fo_master_params(),
    absolute_angle_tolerance=5,
    absolute_length_tolerance=1,
    log=log,
    suppress_symmetry_related_errors=True)
  working_phil = processed_args.params
  if(not command_line.options.silent):
    print >> log, "*** Parameters:"
    working_phil.show(out = log)
    print >> log
  params = working_phil.extract()
  consensus_symmetry = None
  if (params.ignore_non_isomorphous_unit_cells) :
    if (None in [params.f_obs_1_file_name, params.f_obs_2_file_name,
        params.phase_source]):
      raise Sorry("The file parameters (f_obs_1_file_name, f_obs_2_file_name, "+
        "phase_source) must be specified explicitly when "+
        "ignore_non_isomorphous_unit_cells=True.")
    symm_manager = iotbx.symmetry.manager()
    pdb_in = iotbx.file_reader.any_file(params.phase_source, force_type="pdb")
    symm_manager.process_pdb_file(pdb_in)
    hkl_in_1 = iotbx.file_reader.any_file(params.f_obs_1_file_name,
      force_type="hkl")
    sg_err_1, uc_err_1 = symm_manager.process_reflections_file(hkl_in_1)
    hkl_in_2 = iotbx.file_reader.any_file(params.f_obs_2_file_name,
      force_type="hkl")
    sg_err_2, uc_err_2 = symm_manager.process_reflections_file(hkl_in_2)
    out = StringIO()
    symm_manager.show(out=out)
    if (sg_err_1) or (sg_err_2) :
      raise Sorry(("Incompatible space groups in input files:\n%s\nAll files "+
        "must have the same point group (and ideally the same space group). "+
        "Please note that any symmetry information in the PDB file will be "+
        "used first.") % out.getvalue())
    elif (uc_err_1) or (uc_err_2) :
      libtbx.call_back(message="warn",
        data=("Crystal symmetry mismatch:\n%s\nCalculations will continue "+
          "using the symmetry in the PDB file (or if not available, the "+
          "first reflection file), but the maps should be treated with "+
          "extreme suspicion.") % out.getvalue())
    crystal_symmetry = symm_manager.as_symmetry_object()
  else :
    processed_args = utils.process_command_line_args(
      args=command_line.args,
      cmd_cs=command_line.symmetry,
      master_params=fo_minus_fo_master_params(),
      absolute_angle_tolerance=5,
      absolute_length_tolerance=1,
      log=StringIO())
    crystal_symmetry = processed_args.crystal_symmetry
  #
  pdb_file_names = processed_args.pdb_file_names
  if(len(processed_args.pdb_file_names) == 0):
    if(params.phase_source is not None):
      pdb_file_names = [params.phase_source]
    else:
      raise Sorry("No PDB file found.")
  # Extaract Fobs1, Fobs2
  f_obss = []
  if(len(processed_args.reflection_files)==2):
    for reflection_file in processed_args.reflection_files:
      reflection_file_server = reflection_file_utils.reflection_file_server(
        crystal_symmetry = crystal_symmetry,
        force_symmetry   = True,
        reflection_files = [reflection_file],
        err              = null_out())
      # XXX UGLY !!!
      try:
        parameters = utils.data_and_flags_master_params().extract()
        if(params.f_obs_1_label is not None):
          parameters.labels = [params.f_obs_1_label]
        determine_data_and_flags_result = utils.determine_data_and_flags(
          reflection_file_server  = reflection_file_server,
          keep_going              = True,
          parameters              = parameters,
          log                     = null_out())
      except: # intentional
        parameters = utils.data_and_flags_master_params().extract()
        if(params.f_obs_2_label is not None):
          parameters.labels = [params.f_obs_2_label]
        determine_data_and_flags_result = utils.determine_data_and_flags(
          reflection_file_server  = reflection_file_server,
          keep_going              = True,
          parameters              = parameters,
          log                     = null_out())
      f_obss.append(determine_data_and_flags_result.f_obs)
  else:
    if([params.f_obs_1_file_name,params.f_obs_2_file_name].count(None)==2):
      raise Sorry("No reflection data file found.")
    for file_name, label in zip([params.f_obs_1_file_name,params.f_obs_2_file_name],
                                [params.f_obs_1_label,params.f_obs_2_label]):
      reflection_file = reflection_file_reader.any_reflection_file(
        file_name = file_name, ensure_read_access = False)
      reflection_file_server = reflection_file_utils.reflection_file_server(
        crystal_symmetry = crystal_symmetry,
        force_symmetry   = True,
        reflection_files = [reflection_file],
        err              = null_out())
      parameters = utils.data_and_flags_master_params().extract()
      if(label is not None):
        parameters.labels = [label]
      determine_data_and_flags_result = utils.determine_data_and_flags(
          reflection_file_server  = reflection_file_server,
          parameters              = parameters,
          keep_going              = True,
          log                     = null_out())
      f_obss.append(determine_data_and_flags_result.f_obs)
  if(len(f_obss)!=2):
    raise Sorry(" ".join(errors))
  if(not command_line.options.silent):
    for ifobs, fobs in enumerate(f_obss):
      print >> log, "*** Summary for data set %d:"%ifobs
      fobs.show_comprehensive_summary(f = log)
      print >> log
  pdb_combined = combine_unique_pdb_files(file_names = pdb_file_names)
  pdb_combined.report_non_unique(out = log)
  if(len(pdb_combined.unique_file_names) == 0):
    raise Sorry("No coordinate file given.")
  #
  raw_recs = flex.std_string()
  for rec in pdb_combined.raw_records:
    if(rec.upper().count("CRYST1")==0):
      raw_recs.append(rec)
  raw_recs.append(iotbx.pdb.format_cryst1_record(
    crystal_symmetry = crystal_symmetry))
  #
  pdb_in = iotbx.pdb.input(source_info = None, lines = raw_recs)
  hierarchy = pdb_in.construct_hierarchy()
  omit_sel = flex.bool(hierarchy.atoms_size(), False)
  if (params.advanced.omit_selection is not None) :
    print >> log, "Will omit selection from phasing model:"
    print >> log, "  " + params.advanced.omit_selection
    omit_sel = hierarchy.atom_selection_cache().selection(
      params.advanced.omit_selection)
    print >> log, "%d atoms selected for removal" % omit_sel.count(True)
  xray_structure = pdb_in.xray_structure_simple()
  xray_structure = xray_structure.select(~omit_sel)
  if(not command_line.options.silent):
    print >> log, "*** Model summary:"
    xray_structure.show_summary(f = log)
    print >> log
  info0 = f_obss[0].info()
  info1 = f_obss[1].info()
  f_obss[0] = f_obss[0].resolution_filter(d_min = params.high_resolution,
    d_max = params.low_resolution).set_info(info0)
  f_obss[1] = f_obss[1].resolution_filter(d_min = params.high_resolution,
    d_max = params.low_resolution).set_info(info1)
  if(params.sigma_cutoff is not None):
    for i in [0,1]:
      if(f_obss[i].sigmas() is not None):
        sel = f_obss[i].data() > f_obss[i].sigmas()*params.sigma_cutoff
        f_obss[i] = f_obss[i].select(sel).set_info(info0)
  for k, f_obs in enumerate(f_obss) :
    if (f_obs.indices().size() == 0) :
      raise Sorry("No data left in array %d (labels=%s) after filtering!" % (k+1,
        f_obs.info().label_string()))
  output_file_name = params.output_file
  if (output_file_name is None) and (params.file_name_prefix is not None) :
    output_file_name = "%s_%s.mtz" % (params.file_name_prefix, params.job_id)
  output_files = compute_fo_minus_fo_map(
    data_arrays = f_obss,
    xray_structure = xray_structure,
    log = log,
    silent = command_line.options.silent,
    output_file = output_file_name,
    peak_search=params.find_peaks_holes,
    map_cutoff=params.map_cutoff,
    peak_search_params=params.peak_search,
    multiscale=params.advanced.multiscale,
    anomalous=params.advanced.anomalous).file_names
  return output_files
예제 #8
0
def run(args, out=None, log=sys.stdout):
    if (len(args) == 0) or (args == ["--help"]):
        print(msg, file=log)
        defaults(log=log, silent=False)
        return
    parsed = defaults(log=log, silent=True)
    #
    processed_args = utils.process_command_line_args(args=args,
                                                     log=log,
                                                     master_params=parsed)
    params = processed_args.params.extract()
    #
    reflection_files = processed_args.reflection_files
    if (len(reflection_files) == 0):
        raise Sorry("No reflection file found.")
    crystal_symmetry = processed_args.crystal_symmetry
    if (crystal_symmetry is None):
        raise Sorry("No crystal symmetry found.")
    if (len(processed_args.pdb_file_names) == 0):
        raise Sorry("No PDB file found.")
    pdb_file_names = processed_args.pdb_file_names
    #
    rfs = reflection_file_server(crystal_symmetry=crystal_symmetry,
                                 reflection_files=reflection_files)
    parameters = utils.data_and_flags_master_params().extract()
    if (params.f_obs_label is not None):
        parameters.labels = params.f_obs_label
    if (params.r_free_flags_label is not None):
        parameters.r_free_flags.label = params.r_free_flags_label
    if (params.high_resolution is not None):
        parameters.high_resolution = params.high_resolution
    determine_data_and_flags_result = utils.determine_data_and_flags(
        reflection_file_server=rfs,
        parameters=parameters,
        data_parameter_scope="refinement.input.xray_data",
        flags_parameter_scope="refinement.input.xray_data.r_free_flags",
        data_description="X-ray data",
        keep_going=True,
        log=StringIO())
    f_obs = determine_data_and_flags_result.f_obs
    # Data
    show_header(l="Data:", log=log)
    f_obs.show_comprehensive_summary(prefix="  ", f=log)
    # R-free-flags
    show_header(l="R-free-flags:", log=log)
    r_free_flags = determine_data_and_flags_result.r_free_flags
    test_flag_value = determine_data_and_flags_result.test_flag_value
    if (r_free_flags is None):
        r_free_flags = f_obs.array(data=flex.bool(f_obs.data().size(), False))
        test_flag_value = None
        print("  not available", file=log)
    else:
        print("  flag value:", test_flag_value, file=log)
    # Model
    pdb_combined = iotbx.pdb.combine_unique_pdb_files(
        file_names=processed_args.pdb_file_names)
    pdb_combined.report_non_unique(out=log)
    if (len(pdb_combined.unique_file_names) == 0):
        raise Sorry("No coordinate file given.")
    raw_records = pdb_combined.raw_records
    try:
        pdb_inp = iotbx.pdb.input(source_info=None,
                                  lines=flex.std_string(raw_records))
    except ValueError as e:
        raise Sorry("Model format (PDB or mmCIF) error:\n%s" % str(e))
    model = mmtbx.model.manager(model_input=pdb_inp,
                                crystal_symmetry=crystal_symmetry,
                                log=StringIO())
    #
    scattering_table = params.scattering_table
    exptl_method = pdb_inp.get_experiment_type()
    if (exptl_method is not None) and ("NEUTRON" in exptl_method):
        scattering_table = "neutron"
    model.setup_scattering_dictionaries(scattering_table=scattering_table,
                                        d_min=f_obs.d_min())
    #
    # Model vs data
    #
    show_header(l="Model vs Data:", log=log)
    fmodel = mmtbx.f_model.manager(xray_structure=model.get_xray_structure(),
                                   f_obs=f_obs,
                                   r_free_flags=r_free_flags,
                                   twin_law=params.twin_law)
    fmodel.update_all_scales(update_f_part1=True)
    fmodel.show(log=log, show_header=False, show_approx=False)
    print("  r_work: %6.4f" % fmodel.r_work(), file=log)
    if (test_flag_value is not None):
        print("  r_free: %6.4f" % fmodel.r_free(), file=log)
    else:
        print("  r_free: None", file=log)
    print(file=log)
    n_outl = f_obs.data().size() - fmodel.f_obs().data().size()
    print("  Number of F-obs outliers:", n_outl, file=log)
    #
    # Extract information from PDB file header and output (if any)
    #
    pub_r_work = None
    pub_r_free = None
    pub_high = None
    pub_low = None
    pub_sigma = None
    pub_program_name = None
    pub_solv_cont = None
    pub_matthews = None
    published_results = pdb_inp.get_r_rfree_sigma(file_name=pdb_file_names[0])
    if (published_results is not None):
        pub_r_work = published_results.r_work
        pub_r_free = published_results.r_free
        pub_high = published_results.high
        pub_low = published_results.low
        pub_sigma = published_results.sigma
    pub_program_name = pdb_inp.get_program_name()
    pub_solv_cont = pdb_inp.get_solvent_content()
    pub_matthews = pdb_inp.get_matthews_coeff()
    #
    show_header(l="Information extracted from PDB file header:", log=log)
    print("  program_name    : %-s" % format_value("%s", pub_program_name),
          file=log)
    print("  year            : %-s" %
          format_value("%s", pdb_inp.extract_header_year()),
          file=log)
    print("  r_work          : %-s" % format_value("%s", pub_r_work), file=log)
    print("  r_free          : %-s" % format_value("%s", pub_r_free), file=log)
    print("  high_resolution : %-s" % format_value("%s", pub_high), file=log)
    print("  low_resolution  : %-s" % format_value("%s", pub_low), file=log)
    print("  sigma_cutoff    : %-s" % format_value("%s", pub_sigma), file=log)
    print("  matthews_coeff  : %-s" % format_value("%s", pub_matthews),
          file=log)
    print("  solvent_cont    : %-s" % format_value("%s", pub_solv_cont),
          file=log)
    if (exptl_method is not None):
        print("  exptl_method    : %-s" % format_value("%s", exptl_method),
              file=log)
    #
    # Recompute R-factors using published cutoffs
    fmodel_cut = fmodel
    tmp_sel = flex.bool(fmodel.f_obs().data().size(), True)
    if (pub_sigma is not None and fmodel.f_obs().sigmas() is not None):
        tmp_sel &= fmodel.f_obs().data() > fmodel.f_obs().sigmas() * pub_sigma
    if (pub_high is not None
            and abs(pub_high - fmodel.f_obs().d_min()) > 0.03):
        tmp_sel &= fmodel.f_obs().d_spacings().data() > pub_high
    if (pub_low is not None
            and abs(pub_low - fmodel.f_obs().d_max_min()[0]) > 0.03):
        tmp_sel &= fmodel.f_obs().d_spacings().data() < pub_low
    if (tmp_sel.count(True) != tmp_sel.size() and tmp_sel.count(True) > 0):
        show_header(l="After applying resolution and sigma cutoffs:", log=log)
        fmodel = mmtbx.f_model.manager(
            xray_structure=model.get_xray_structure(),
            f_obs=fmodel.f_obs().select(tmp_sel),
            r_free_flags=fmodel.r_free_flags().select(tmp_sel),
            twin_law=params.twin_law)
        fmodel.update_all_scales(update_f_part1=True)
        fmodel.show(log=log, show_header=False, show_approx=False)
        print("  r_work: %6.4f" % fmodel.r_work(), file=log)
        if (test_flag_value is not None):
            print("  r_free: %6.4f" % fmodel.r_free(), file=log)
        else:
            print("  r_free: None", file=log)
        print(file=log)
        n_outl = f_obs.data().size() - fmodel.f_obs().data().size()
        print("  Number of F-obs outliers:", n_outl, file=log)
예제 #9
0
def run(args, out=None, log=sys.stdout):
    if (len(args) == 0) or (args == ["--help"]):
        print >> log, msg
        defaults(log=log, silent=False)
        return
    parsed = defaults(log=log, silent=True)
    #
    processed_args = utils.process_command_line_args(args=args,
                                                     log=log,
                                                     master_params=parsed)
    params = processed_args.params.extract()
    #
    reflection_files = processed_args.reflection_files
    if (len(reflection_files) == 0):
        raise Sorry("No reflection file found.")
    crystal_symmetry = processed_args.crystal_symmetry
    if (crystal_symmetry is None):
        raise Sorry("No crystal symmetry found.")
    if (len(processed_args.pdb_file_names) == 0):
        raise Sorry("No PDB file found.")
    pdb_file_names = processed_args.pdb_file_names
    #
    rfs = reflection_file_server(crystal_symmetry=crystal_symmetry,
                                 reflection_files=reflection_files)
    parameters = utils.data_and_flags_master_params().extract()
    if (params.f_obs_label is not None):
        parameters.labels = params.f_obs_label
    if (params.r_free_flags_label is not None):
        parameters.r_free_flags.label = params.r_free_flags_label
    if (params.high_resolution is not None):
        parameters.high_resolution = params.high_resolution
    determine_data_and_flags_result = utils.determine_data_and_flags(
        reflection_file_server=rfs,
        parameters=parameters,
        data_parameter_scope="refinement.input.xray_data",
        flags_parameter_scope="refinement.input.xray_data.r_free_flags",
        data_description="X-ray data",
        keep_going=True,
        log=StringIO())
    f_obs = determine_data_and_flags_result.f_obs
    # Data
    show_header(l="Data:", log=log)
    f_obs.show_comprehensive_summary(prefix="  ", f=log)
    # R-free-flags
    show_header(l="R-free-flags:", log=log)
    r_free_flags = determine_data_and_flags_result.r_free_flags
    test_flag_value = determine_data_and_flags_result.test_flag_value
    if (r_free_flags is None):
        r_free_flags = f_obs.array(data=flex.bool(f_obs.data().size(), False))
        test_flag_value = None
        print >> log, "  not available"
    else:
        print >> log, "  flag value:", test_flag_value
    # Model
    pdb_combined = iotbx.pdb.combine_unique_pdb_files(
        file_names=processed_args.pdb_file_names)
    pdb_combined.report_non_unique(out=log)
    if (len(pdb_combined.unique_file_names) == 0):
        raise Sorry("No coordinate file given.")
    raw_records = pdb_combined.raw_records
    try:
        pdb_inp = iotbx.pdb.input(source_info=None,
                                  lines=flex.std_string(raw_records))
    except ValueError, e:
        raise Sorry("Model format (PDB or mmCIF) error:\n%s" % str(e))
예제 #10
0
def run(args,
        command_name="mmtbx.model_vs_data",
        show_geometry_statistics=True,
        model_size_max_atoms=80000,
        data_size_max_reflections=1000000,
        unit_cell_max_dimension=800.,
        return_fmodel_and_pdb=False,
        out=None,
        log=sys.stdout):
    import mmtbx.f_model.f_model_info
    if (len(args) == 0) or (args == ["--help"]):
        print >> log, msg
        defaults(log=log, silent=False)
        return
    parsed = defaults(log=log, silent=True)
    #
    mvd_obj = mvd()
    #
    processed_args = utils.process_command_line_args(args=args,
                                                     log=log,
                                                     master_params=parsed)
    params = processed_args.params.extract()
    #
    reflection_files = processed_args.reflection_files
    if (len(reflection_files) == 0):
        raise Sorry("No reflection file found.")
    crystal_symmetry = processed_args.crystal_symmetry
    if (crystal_symmetry is None):
        raise Sorry("No crystal symmetry found.")
    if (len(processed_args.pdb_file_names) == 0):
        raise Sorry("No PDB file found.")
    pdb_file_names = processed_args.pdb_file_names
    #
    rfs = reflection_file_server(crystal_symmetry=crystal_symmetry,
                                 reflection_files=reflection_files)
    parameters = utils.data_and_flags_master_params().extract()
    if (params.f_obs_label is not None):
        parameters.labels = params.f_obs_label
    if (params.r_free_flags_label is not None):
        parameters.r_free_flags.label = params.r_free_flags_label
    if (params.high_resolution is not None):
        parameters.high_resolution = params.high_resolution
    determine_data_and_flags_result = utils.determine_data_and_flags(
        reflection_file_server=rfs,
        parameters=parameters,
        data_parameter_scope="refinement.input.xray_data",
        flags_parameter_scope="refinement.input.xray_data.r_free_flags",
        data_description="X-ray data",
        keep_going=True,
        log=StringIO())
    f_obs = determine_data_and_flags_result.f_obs
    number_of_reflections = f_obs.indices().size()
    if (params.ignore_giant_models_and_datasets
            and number_of_reflections > data_size_max_reflections):
        raise Sorry("Too many reflections: %d" % number_of_reflections)
    #
    max_unit_cell_dimension = max(f_obs.unit_cell().parameters()[:3])
    if (params.ignore_giant_models_and_datasets
            and max_unit_cell_dimension > unit_cell_max_dimension):
        raise Sorry("Too large unit cell (max dimension): %s" %
                    str(max_unit_cell_dimension))
    #
    r_free_flags = determine_data_and_flags_result.r_free_flags
    test_flag_value = determine_data_and_flags_result.test_flag_value
    if (r_free_flags is None):
        r_free_flags = f_obs.array(data=flex.bool(f_obs.data().size(), False))
        test_flag_value = None
    #
    mmtbx_pdb_file = mmtbx.utils.pdb_file(
        pdb_file_names=pdb_file_names,
        cif_objects=processed_args.cif_objects,
        crystal_symmetry=crystal_symmetry,
        use_neutron_distances=(params.scattering_table == "neutron"),
        ignore_unknown_nonbonded_energy_types=not show_geometry_statistics,
        log=log)
    mmtbx_pdb_file.set_ppf(stop_if_duplicate_labels=False)
    processed_pdb_file = mmtbx_pdb_file.processed_pdb_file
    pdb_raw_records = mmtbx_pdb_file.pdb_raw_records
    pdb_inp = mmtbx_pdb_file.pdb_inp
    #
    # just to avoid going any further with bad PDB file....
    pdb_inp.xray_structures_simple()
    #
    acp = processed_pdb_file.all_chain_proxies
    atom_selections = group_args(
        all=acp.selection(string="all"),
        macromolecule=acp.selection(string="protein or dna or rna"),
        solvent=acp.selection(string="water"),  # XXX single_atom_residue
        ligand=acp.selection(string="not (protein or dna or rna or water)"),
        backbone=acp.selection(string="backbone"),
        sidechain=acp.selection(string="sidechain"))
    #
    scattering_table = params.scattering_table
    exptl_method = pdb_inp.get_experiment_type()
    if (exptl_method is not None) and ("NEUTRON" in exptl_method):
        scattering_table = "neutron"
    xsfppf = mmtbx.utils.xray_structures_from_processed_pdb_file(
        processed_pdb_file=processed_pdb_file,
        scattering_table=scattering_table,
        d_min=f_obs.d_min())
    xray_structures = xsfppf.xray_structures
    if (0):  #XXX normalize occupancies if all models have occ=1 so the total=1
        n_models = len(xray_structures)
        for xrs in xray_structures:
            occ = xrs.scatterers().extract_occupancies()
            occ = occ / n_models
            xrs.set_occupancies(occ)
    model_selections = xsfppf.model_selections
    mvd_obj.collect(crystal=group_args(
        uc=f_obs.unit_cell(),
        sg=f_obs.crystal_symmetry().space_group_info().symbol_and_number(),
        n_sym_op=f_obs.crystal_symmetry().space_group_info().type().group(
        ).order_z(),
        uc_vol=f_obs.unit_cell().volume()))
    #
    hierarchy = pdb_inp.construct_hierarchy()
    pdb_atoms = hierarchy.atoms()
    pdb_atoms.reset_i_seq()
    #
    # Extract TLS
    pdb_tls = None
    pdb_inp_tls = pdb_inp.extract_tls_params(hierarchy)
    pdb_tls = group_args(pdb_inp_tls=pdb_inp_tls,
                         tls_selections=[],
                         tls_selection_strings=[])
    # XXX no TLS + multiple models
    if (pdb_inp_tls.tls_present and pdb_inp_tls.error_string is None
            and len(xray_structures) == 1):
        pdb_tls = mmtbx.tls.tools.extract_tls_from_pdb(
            pdb_inp_tls=pdb_inp_tls,
            all_chain_proxies=mmtbx_pdb_file.processed_pdb_file.
            all_chain_proxies,
            xray_structure=xsfppf.xray_structure_all)
        if (len(pdb_tls.tls_selections) == len(pdb_inp_tls.tls_params)
                and len(pdb_inp_tls.tls_params) > 0):
            xray_structures = [
                utils.extract_tls_and_u_total_from_pdb(
                    f_obs=f_obs,
                    r_free_flags=r_free_flags,
                    xray_structure=xray_structures[
                        0],  # XXX no TLS + multiple models
                    tls_selections=pdb_tls.tls_selections,
                    tls_groups=pdb_inp_tls.tls_params)
            ]
    ###########################
    geometry_statistics = show_geometry(
        xray_structures=xray_structures,
        processed_pdb_file=processed_pdb_file,
        scattering_table=scattering_table,
        hierarchy=hierarchy,
        model_selections=model_selections,
        show_geometry_statistics=show_geometry_statistics,
        mvd_obj=mvd_obj,
        atom_selections=atom_selections)
    ###########################
    mp = mmtbx.masks.mask_master_params.extract()
    f_obs_labels = f_obs.info().label_string()
    f_obs = f_obs.sort(reverse=True, by_value="packed_indices")
    r_free_flags = r_free_flags.sort(reverse=True, by_value="packed_indices")
    fmodel = utils.fmodel_simple(
        xray_structures=xray_structures,
        scattering_table=scattering_table,
        mask_params=mp,
        f_obs=f_obs,
        r_free_flags=r_free_flags,
        skip_twin_detection=params.skip_twin_detection)
    n_outl = f_obs.data().size() - fmodel.f_obs().data().size()
    mvd_obj.collect(model_vs_data=show_model_vs_data(fmodel))
    # Extract information from PDB file header and output (if any)
    pub_r_work = None
    pub_r_free = None
    pub_high = None
    pub_low = None
    pub_sigma = None
    pub_program_name = None
    pub_solv_cont = None
    pub_matthews = None
    published_results = pdb_inp.get_r_rfree_sigma(file_name=pdb_file_names[0])
    if (published_results is not None):
        pub_r_work = published_results.r_work
        pub_r_free = published_results.r_free
        pub_high = published_results.high
        pub_low = published_results.low
        pub_sigma = published_results.sigma
    pub_program_name = pdb_inp.get_program_name()
    pub_solv_cont = pdb_inp.get_solvent_content()
    pub_matthews = pdb_inp.get_matthews_coeff()
    mvd_obj.collect(pdb_header=group_args(program_name=pub_program_name,
                                          year=pdb_inp.extract_header_year(),
                                          r_work=pub_r_work,
                                          r_free=pub_r_free,
                                          high_resolution=pub_high,
                                          low_resolution=pub_low,
                                          sigma_cutoff=pub_sigma,
                                          matthews_coeff=pub_matthews,
                                          solvent_cont=pub_solv_cont,
                                          tls=pdb_tls,
                                          exptl_method=exptl_method))
    #
    # Recompute R-factors using published cutoffs
    fmodel_cut = fmodel
    tmp_sel = flex.bool(fmodel.f_obs().data().size(), True)
    if (pub_sigma is not None and fmodel.f_obs().sigmas() is not None):
        tmp_sel &= fmodel.f_obs().data() > fmodel.f_obs().sigmas() * pub_sigma
    if (pub_high is not None
            and abs(pub_high - fmodel.f_obs().d_min()) > 0.03):
        tmp_sel &= fmodel.f_obs().d_spacings().data() > pub_high
    if (pub_low is not None
            and abs(pub_low - fmodel.f_obs().d_max_min()[0]) > 0.03):
        tmp_sel &= fmodel.f_obs().d_spacings().data() < pub_low
    if (tmp_sel.count(True) != tmp_sel.size() and tmp_sel.count(True) > 0):
        fmodel_cut = utils.fmodel_simple(
            xray_structures=xray_structures,
            scattering_table=scattering_table,
            f_obs=fmodel.f_obs().select(tmp_sel),
            r_free_flags=fmodel.r_free_flags().select(tmp_sel),
            skip_twin_detection=params.skip_twin_detection)
    mvd_obj.collect(
        misc=group_args(r_work_cutoff=fmodel_cut.r_work(),
                        r_free_cutoff=fmodel_cut.r_free(),
                        n_refl_cutoff=fmodel_cut.f_obs().data().size()))
    mvd_obj.collect(data=show_data(fmodel=fmodel,
                                   n_outl=n_outl,
                                   test_flag_value=test_flag_value,
                                   f_obs_labels=f_obs_labels,
                                   fmodel_cut=fmodel_cut))
    # CC* and friends
    cc_star_stats = None
    if (params.unmerged_data is not None):
        import mmtbx.validation.experimental
        import mmtbx.command_line
        f_obs = fmodel.f_obs().average_bijvoet_mates()
        unmerged_i_obs = mmtbx.command_line.load_and_validate_unmerged_data(
            f_obs=f_obs,
            file_name=params.unmerged_data,
            data_labels=params.unmerged_labels,
            log=null_out())
        cc_star_stats = mmtbx.validation.experimental.merging_and_model_statistics(
            f_model=fmodel.f_model().average_bijvoet_mates(),
            f_obs=f_obs,
            r_free_flags=fmodel.r_free_flags().average_bijvoet_mates(),
            unmerged_i_obs=unmerged_i_obs,
            n_bins=params.n_bins)
    mvd_obj.show(log=out)
    if (cc_star_stats is not None):
        cc_star_stats.show_model_vs_data(out=out, prefix="  ")
    if return_fmodel_and_pdb:
        mvd_obj.pdb_file = processed_pdb_file
        mvd_obj.fmodel = fmodel
    if (len(params.map) > 0):
        for map_name_string in params.map:
            map_type_obj = mmtbx.map_names(map_name_string=map_name_string)
            map_params = mmtbx.maps.map_and_map_coeff_master_params().fetch(
                mmtbx.maps.cast_map_coeff_params(map_type_obj)).extract()
            maps_obj = mmtbx.maps.compute_map_coefficients(
                fmodel=fmodel_cut, params=map_params.map_coefficients)
            fn = os.path.basename(processed_args.reflection_file_names[0])
            if (fn.count(".")):
                prefix = fn[:fn.index(".")]
            else:
                prefix = fn
            file_name = prefix + "_%s_map_coeffs.mtz" % map_type_obj.format()
            maps_obj.write_mtz_file(file_name=file_name)
    # statistics in bins
    if (not fmodel.twin):
        print >> log, "Statistics in resolution bins:"
        mmtbx.f_model.f_model_info.r_work_and_completeness_in_resolution_bins(
            fmodel=fmodel, out=log, prefix="  ")
    # report map cc
    if (params.comprehensive and not fmodel_cut.twin
            and fmodel_cut.xray_structure is not None):
        rsc_params = real_space_correlation.master_params().extract()
        rsc_params.scattering_table = scattering_table
        real_space_correlation.simple(fmodel=fmodel_cut,
                                      pdb_hierarchy=hierarchy,
                                      params=rsc_params,
                                      log=log,
                                      show_results=True)
    #
    if (params.dump_result_object_as_pickle):
        output_prefixes = []
        for op in processed_args.pdb_file_names + processed_args.reflection_file_names:
            op = os.path.basename(op)
            try:
                op = op[:op.index(".")]
            except Exception:
                pass
            if (not op in output_prefixes): output_prefixes.append(op)
        output_prefix = "_".join(output_prefixes)
        easy_pickle.dump("%s.pickle" % output_prefix, mvd_obj)
    return mvd_obj
예제 #11
0
def run(args, command_name="phenix.twin_map_utils"):
  log=sys.stdout
  params=None
  if (len(args) == 0 or "--help" in args or "--h" in args or "-h" in args):
    print_help(command_name=command_name)
  else:
    log = multi_out()
    if (not "--quiet" in args):
      log.register(label="stdout", file_object=sys.stdout)
    string_buffer = StringIO()
    string_buffer_plots = StringIO()
    log.register(label="log_buffer", file_object=string_buffer)

    phil_objects = []
    argument_interpreter = master_params.command_line_argument_interpreter(
      home_scope="map_coefs")

    for arg in args:
      command_line_params = None
      arg_is_processed = False
      # is it a file?
      if (os.path.isfile(arg)): ## is this a file name?
        # check if it is a phil file
        try:
          command_line_params = iotbx.phil.parse(file_name=arg)
          if command_line_params is not None:
            phil_objects.append(command_line_params)
            arg_is_processed = True
        except KeyboardInterrupt: raise
        except Exception : pass
      else:
        try:
          command_line_params = argument_interpreter.process(arg=arg)
          if command_line_params is not None:
            phil_objects.append(command_line_params)
            arg_is_processed = True
        except KeyboardInterrupt: raise
        except Exception : pass

      if not arg_is_processed:
        print >> log, "##----------------------------------------------##"
        print >> log, "## Unknown file or keyword:", arg
        print >> log, "##----------------------------------------------##"
        print >> log
        raise Sorry("Unknown file or keyword: %s" % arg)

    effective_params = master_params.fetch(sources=phil_objects)
    params = effective_params.extract()
    """
    new_params =  master_params.format(python_object=params)
    new_params.show(out=log)
    """
    # now get the unit cell from the pdb file

    hkl_xs = crystal_symmetry_from_any.extract_from(
      file_name=params.twin_utils.input.xray_data.file_name)
    pdb_xs = crystal_symmetry_from_any.extract_from(
      file_name=params.twin_utils.input.model.file_name)

    phil_xs=None
    if ([params.twin_utils.input.unit_cell,
         params.twin_utils.input.space_group]).count(None)<2:
      phil_xs = crystal.symmetry(
        unit_cell=params.twin_utils.input.unit_cell,
        space_group_info=params.twin_utils.input.space_group  )


    combined_xs = crystal.select_crystal_symmetry(
      None,phil_xs, [pdb_xs],[hkl_xs])

    # inject the unit cell and symmetry in the phil scope please
    params.twin_utils.input.unit_cell = combined_xs.unit_cell()
    params.twin_utils.input.space_group = \
      sgtbx.space_group_info( group = combined_xs.space_group() )

    new_params =  master_params.format(python_object=params)
    new_params.show(out=log)

    if params.twin_utils.input.unit_cell is None:
      raise Sorry("unit cell not specified")
    if params.twin_utils.input.space_group is None:
      raise Sorry("space group not specified")
    if params.twin_utils.input.xray_data.file_name is None:
      raise Sorry("Xray data not specified")
    if params.twin_utils.input.model.file_name is None:
      raise Sorry("pdb file with  model not specified")

    #-----------------------------------------------------------
    #
    # step 1: read in the reflection file
    #
    phil_xs = crystal.symmetry(
      unit_cell=params.twin_utils.input.unit_cell,
      space_group_info=params.twin_utils.input.space_group  )

    xray_data_server =  reflection_file_utils.reflection_file_server(
      crystal_symmetry = phil_xs,
      force_symmetry = True,
      reflection_files=[])

    miller_array = None
    free_flags = None

    tmp_params = utils.data_and_flags_master_params().extract()
    # insert proper values please
    tmp_params.file_name = params.twin_utils.input.xray_data.file_name
    tmp_params.labels = params.twin_utils.input.xray_data.obs_labels
    tmp_params.r_free_flags.file_name=params.twin_utils.input.xray_data.file_name
    tmp_params.r_free_flags.label=params.twin_utils.input.xray_data.free_flag

    tmp_object = utils.determine_data_and_flags( reflection_file_server = xray_data_server,
                                                 parameters = tmp_params, log=log )

    miller_array = tmp_object.extract_data()
    if miller_array.is_xray_intensity_array():
      miller_array = miller_array.f_sq_as_f()
    assert miller_array.is_xray_amplitude_array()

    free_flags = tmp_object.extract_flags(data = miller_array)
    print >> log
    print >> log, "Attempting to extract Free R flags"

    free_flags = free_flags.customized_copy( data = flex.bool( free_flags.data()==1 ) )
    if free_flags is None:
      free_flags = miller_array.generate_r_free_flags(use_lattice_symmetry=True)

    assert miller_array.observation_type() is not None

    print >> log
    print >> log, "Summary info of observed data"
    print >> log, "============================="
    miller_array.show_summary(f=log)
    print >> log

    if miller_array.indices().size() == 0:
      raise Sorry("No data available")

    #----------------------------------------------------------------
    # Step 2: get an xray structure from the PDB file
    #
    model = pdb.input(file_name=params.twin_utils.input.model.file_name).xray_structure_simple(
      crystal_symmetry=phil_xs)
    print >> log, "Atomic model summary"
    print >> log, "===================="
    model.show_summary(f=log)
    print >> log


    #----------------------------------------------------------------
    # step 3: get the twin laws for this xs
    twin_laws = twin_analyses.twin_laws(
      miller_array,
      lattice_symmetry_max_delta=\
         params.twin_utils.parameters.twinning.max_delta,
      out=log)

    print >> log
    print >> log, "Preliminary data analyses"
    print >> log, "=========================="
    twin_laws.show(out=log)


    #---------
    # step 3:
    # make twin model managers for all twin laws
    print >> log
    print >> log, "Overall and bulk solvent scale paranmeters and twin fraction estimation"
    print >> log, "======================================================================="
    twin_models = []
    operator_count = 0

    if params.twin_utils.parameters.twinning.twin_law is not None:
      tmp_law = sgtbx.rt_mx( params.twin_utils.parameters.twinning.twin_law )
      tmp_law = twin_analyses.twin_law(tmp_law,None,None,None,None,None)
      twin_laws.operators = [ tmp_law ]
    for twin_law in twin_laws.operators:
      operator_count += 1
      operator_hkl = sgtbx.change_of_basis_op( twin_law.operator ).as_hkl()
      twin_model = twin_f_model.twin_model_manager(
        f_obs=miller_array,
        r_free_flags = free_flags,
        xray_structure=model,
        twin_law = twin_law.operator,
        detwin_mode = params.twin_utils.parameters.twinning.detwin_mode,
        out=log)


      print >> log, "--- bulk solvent scaling ---"
      twin_model.update_solvent_and_scale(update_f_part1=False)
      twin_model.r_values()
      twin_model.target()
      twin_model.show_k_sol_b_sol_b_cart_target()
      twin_model.show_essential()

      wfofc  = twin_model.map_coefficients(map_type="mFo-DFc"  )
      wtfofc = twin_model.map_coefficients(map_type="2mFo-DFc" )
      grad   = twin_model.map_coefficients(map_type="gradient"       )

      mtz_dataset = wtfofc.as_mtz_dataset(
        column_root_label="FWT")
      mtz_dataset = mtz_dataset.add_miller_array(
        miller_array = wfofc,
        column_root_label = "DFWT"
        )
      mtz_dataset = mtz_dataset.add_miller_array(
        miller_array = grad,
        column_root_label = "GRAD"
        )
      name = params.twin_utils.output.map_coeffs_root+"_"+str(operator_count)+".mtz"
      print >> log
      print >> log, "Writing %s for twin law %s"%(name,operator_hkl)
      print >> log
      mtz_dataset.mtz_object().write(
        file_name=name)

      if params.twin_utils.output.obs_and_calc is not None:
        # i want also a Fobs and Fmodel combined dataset please
        mtz_dataset = miller_array.as_mtz_dataset(
          column_root_label="FOBS")
        mtz_dataset = mtz_dataset.add_miller_array(
          miller_array = twin_model.f_model(),
          column_root_label="FMODEL")
        name = params.twin_utils.output.obs_and_calc
        mtz_dataset.mtz_object().write(
          file_name=name)

    if len(twin_laws.operators)==0:
      print >> log
      print >> log, "No twin laws were found"
      print >> log, "Performing maximum likelihood based bulk solvent scaling"
      f_model_object = f_model.manager(
        f_obs = miller_array,
        r_free_flags = free_flags,
        xray_structure = model )
      f_model_object.update_solvent_and_scale(out=log)
      tfofc =  f_model_object.map_coefficients(map_type="2mFobs-DFmodel")
      fofc = f_model_object.map_coefficients(map_type="mFobs-DFmodel")
      mtz_dataset = tfofc.as_mtz_dataset(
        column_root_label="FWT")
      mtz_dataset = mtz_dataset.add_miller_array(
        miller_array = fofc,
        column_root_label = "DELFWT"
      )
      name = params.twin_utils.output.map_coeffs_root+"_ML.mtz"
      mtz_dataset.mtz_object().write(
        file_name=name)

      if params.twin_utils.output.obs_and_calc is not None:
        # i want also a Fobs and Fmodel combined dataset please
        mtz_dataset = miller_array.as_mtz_dataset(
          column_root_label="FOBS")
        mtz_dataset = mtz_dataset.add_miller_array(
          miller_array = f_model_object.f_model(),
          column_root_label="FMODEL")
        name = params.twin_utils.output.obs_and_calc
        mtz_dataset.mtz_object().write(
          file_name=name)

    print >> log
    print >> log
    print >> log, "All done \n"
    logfile = open(params.twin_utils.output.logfile,'w')
    print >> logfile,  string_buffer.getvalue()
    print >> log
예제 #12
0
  def run(self, args, command_name, out=sys.stdout):
    command_line = (iotbx_option_parser(
      usage="%s [options]" % command_name,
      description='Example: %s data.mtz data.mtz ref_model.pdb'%command_name)
      .option(None, "--show_defaults",
        action="store_true",
        help="Show list of parameters.")
      ).process(args=args)

    cif_file = None
    processed_args = utils.process_command_line_args(
                       args          = args,
                       log           = sys.stdout,
                       master_params = master_phil)
    params = processed_args.params
    if(params is None): params = master_phil
    self.params = params.extract().ensemble_probability
    pdb_file_names = processed_args.pdb_file_names
    if len(pdb_file_names) != 1 :
      raise Sorry("Only one PDB structure may be used")
    pdb_file = file_reader.any_file(pdb_file_names[0])
    self.log = multi_out()
    self.log.register(label="stdout", file_object=sys.stdout)
    self.log.register(
      label="log_buffer",
      file_object=StringIO(),
      atexit_send_to=None)
    sys.stderr = self.log
    log_file = open(pdb_file_names[0].split('/')[-1].replace('.pdb','') + '_pensemble.log', "w")

    self.log.replace_stringio(
        old_label="log_buffer",
        new_label="log",
        new_file_object=log_file)
    utils.print_header(command_name, out = self.log)
    params.show(out = self.log)
    #
    f_obs = None
    r_free_flags = None
    reflection_files = processed_args.reflection_files

    if self.params.fobs_vs_fcalc_post_nll:
      if len(reflection_files) == 0:
        raise Sorry("Fobs from input MTZ required for fobs_vs_fcalc_post_nll")

    if len(reflection_files) > 0:
      crystal_symmetry = processed_args.crystal_symmetry
      print >> self.log, 'Reflection file : ', processed_args.reflection_file_names[0]
      utils.print_header("Model and data statistics", out = self.log)
      rfs = reflection_file_server(
        crystal_symmetry = crystal_symmetry,
        reflection_files = processed_args.reflection_files,
        log              = self.log)

      parameters = utils.data_and_flags_master_params().extract()
      determine_data_and_flags_result = utils.determine_data_and_flags(
        reflection_file_server  = rfs,
        parameters              = parameters,
        data_parameter_scope    = "refinement.input.xray_data",
        flags_parameter_scope   = "refinement.input.xray_data.r_free_flags",
        data_description        = "X-ray data",
        keep_going              = True,
        log                     = self.log)
      f_obs = determine_data_and_flags_result.f_obs
      number_of_reflections = f_obs.indices().size()
      r_free_flags = determine_data_and_flags_result.r_free_flags
      test_flag_value = determine_data_and_flags_result.test_flag_value
      if(r_free_flags is None):
        r_free_flags=f_obs.array(data=flex.bool(f_obs.data().size(), False))

    # process PDB
    pdb_file.assert_file_type("pdb")
    #
    pdb_in = hierarchy.input(file_name=pdb_file.file_name)
    ens_pdb_hierarchy = pdb_in.construct_hierarchy()
    ens_pdb_hierarchy.atoms().reset_i_seq()
    ens_pdb_xrs_s = pdb_in.input.xray_structures_simple()
    number_structures = len(ens_pdb_xrs_s)
    print >> self.log, 'Number of structure in ensemble : ', number_structures

    # Calculate sigmas from input map only
    if self.params.assign_sigma_from_map and self.params.ensemble_sigma_map_input is not None:
      # process MTZ
      input_file = file_reader.any_file(self.params.ensemble_sigma_map_input)
      if input_file.file_type == "hkl" :
        if input_file.file_object.file_type() != "ccp4_mtz" :
           raise Sorry("Only MTZ format accepted for map input")
        else:
          mtz_file = input_file
      else:
        raise Sorry("Only MTZ format accepted for map input")
      miller_arrays = mtz_file.file_server.miller_arrays
      map_coeffs_1 = miller_arrays[0]
      #
      xrs_list = []
      for n, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
        # get sigma levels from ensemble fc for each structure
        xrs = get_map_sigma(ens_pdb_hierarchy = ens_pdb_hierarchy,
                          ens_pdb_xrs       = ens_pdb_xrs,
                          map_coeffs_1      = map_coeffs_1,
                          residue_detail    = self.params.residue_detail,
                          ignore_hd         = self.params.ignore_hd,
                          log               = self.log)
        xrs_list.append(xrs)
      # write ensemble pdb file, occupancies as sigma level
      filename = pdb_file_names[0].split('/')[-1].replace('.pdb','') + '_vs_' + self.params.ensemble_sigma_map_input.replace('.mtz','') + '_pensemble.pdb'
      write_ensemble_pdb(filename = filename,
                         xrs_list = xrs_list,
                         ens_pdb_hierarchy = ens_pdb_hierarchy
                         )

    # Do full analysis vs Fobs
    else:
      model_map_coeffs = []
      fmodel = None
      # Get <fcalc>
      for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
        ens_pdb_xrs.set_occupancies(1.0)
        if model == 0:
          # If mtz not supplied get fobs from xray structure...
          # Use input Fobs for scoring against nll
          if self.params.fobs_vs_fcalc_post_nll:
            dummy_fobs = f_obs
          else:
            if f_obs == None:
              if self.params.fcalc_high_resolution == None:
                raise Sorry("Please supply high resolution limit or input mtz file.")
              dummy_dmin = self.params.fcalc_high_resolution
              dummy_dmax = self.params.fcalc_low_resolution
            else:
              print >> self.log, 'Supplied mtz used to determine high and low resolution cuttoffs'
              dummy_dmax, dummy_dmin = f_obs.d_max_min()
            #
            dummy_fobs = abs(ens_pdb_xrs.structure_factors(d_min = dummy_dmin).f_calc())
            dummy_fobs.set_observation_type_xray_amplitude()
            # If mtz supplied, free flags are over written to prevent array size error
            r_free_flags = dummy_fobs.array(data=flex.bool(dummy_fobs.data().size(),False))
          #
          fmodel = utils.fmodel_simple(
                     scattering_table         = "wk1995",
                     xray_structures          = [ens_pdb_xrs],
                     f_obs                    = dummy_fobs,
                     target_name              = 'ls',
                     bulk_solvent_and_scaling = False,
                     r_free_flags             = r_free_flags
                     )
          f_calc_ave = fmodel.f_calc().array(data = fmodel.f_calc().data()*0).deep_copy()
          # XXX Important to ensure scale is identical for each model and <model>
          fmodel.set_scale_switch = 1.0
          f_calc_ave_total = fmodel.f_calc().data().deep_copy()
        else:
          fmodel.update_xray_structure(xray_structure  = ens_pdb_xrs,
                                       update_f_calc   = True,
                                       update_f_mask   = False)
          f_calc_ave_total += fmodel.f_calc().data().deep_copy()
        print >> self.log, 'Model :', model+1
        print >> self.log, "\nStructure vs real Fobs (no bulk solvent or scaling)"
        print >> self.log, 'Rwork          : %5.4f '%fmodel.r_work()
        print >> self.log, 'Rfree          : %5.4f '%fmodel.r_free()
        print >> self.log, 'K1             : %5.4f '%fmodel.scale_k1()
        fcalc_edm        = fmodel.electron_density_map()
        fcalc_map_coeffs = fcalc_edm.map_coefficients(map_type = 'Fc')
        fcalc_mtz_dataset = fcalc_map_coeffs.as_mtz_dataset(column_root_label ='Fc')
        if self.params.output_model_and_model_ave_mtz:
          fcalc_mtz_dataset.mtz_object().write(file_name = str(model+1)+"_Fc.mtz")
        model_map_coeffs.append(fcalc_map_coeffs.deep_copy())

      fmodel.update(f_calc = f_calc_ave.array(f_calc_ave_total / number_structures))
      print >> self.log, "\nEnsemble vs real Fobs (no bulk solvent or scaling)"
      print >> self.log, 'Rwork          : %5.4f '%fmodel.r_work()
      print >> self.log, 'Rfree          : %5.4f '%fmodel.r_free()
      print >> self.log, 'K1             : %5.4f '%fmodel.scale_k1()

      # Get <Fcalc> map
      fcalc_ave_edm        = fmodel.electron_density_map()
      fcalc_ave_map_coeffs = fcalc_ave_edm.map_coefficients(map_type = 'Fc').deep_copy()
      fcalc_ave_mtz_dataset = fcalc_ave_map_coeffs.as_mtz_dataset(column_root_label ='Fc')
      if self.params.output_model_and_model_ave_mtz:
        fcalc_ave_mtz_dataset.mtz_object().write(file_name = "aveFc.mtz")
      fcalc_ave_map_coeffs = fcalc_ave_map_coeffs.fft_map()
      fcalc_ave_map_coeffs.apply_volume_scaling()
      fcalc_ave_map_data   = fcalc_ave_map_coeffs.real_map_unpadded()
      fcalc_ave_map_stats  = maptbx.statistics(fcalc_ave_map_data)

      print >> self.log, "<Fcalc> Map Stats :"
      fcalc_ave_map_stats.show_summary(f = self.log)
      offset = fcalc_ave_map_stats.min()
      model_neg_ll = []

      number_previous_scatters = 0

      # Run through structure list again and get probability
      xrs_list = []
      for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
        if self.params.verbose:
          print >> self.log, '\n\nModel                   : ', model+1
        # Get model atom sigmas vs Fcalc
        fcalc_map = model_map_coeffs[model].fft_map()
        fcalc_map.apply_volume_scaling()
        fcalc_map_data  = fcalc_map.real_map_unpadded()
        fcalc_map_stats  = maptbx.statistics(fcalc_map_data)
        if self.params.verbose:
          print >> self.log, "Fcalc map stats         :"
        fcalc_map_stats.show_summary(f = self.log)

        xrs = get_map_sigma(ens_pdb_hierarchy = ens_pdb_hierarchy,
                            ens_pdb_xrs       = ens_pdb_xrs,
                            fft_map_1         = fcalc_map,
                            model_i           = model,
                            residue_detail    = self.params.residue_detail,
                            ignore_hd         = self.params.ignore_hd,
                            number_previous_scatters = number_previous_scatters,
                            log               = self.log)
        fcalc_sigmas = xrs.scatterers().extract_occupancies()
        del fcalc_map
        # Get model atom sigmas vs <Fcalc>
        xrs = get_map_sigma(ens_pdb_hierarchy = ens_pdb_hierarchy,
                            ens_pdb_xrs       = ens_pdb_xrs,
                            fft_map_1         = fcalc_ave_map_coeffs,
                            model_i           = model,
                            residue_detail    = self.params.residue_detail,
                            ignore_hd         = self.params.ignore_hd,
                            number_previous_scatters = number_previous_scatters,
                            log               = self.log)

        ### For testing other residue averaging options
        #print xrs.residue_selections

        fcalc_ave_sigmas = xrs.scatterers().extract_occupancies()
        # Probability of model given <model>
        prob = fcalc_ave_sigmas / fcalc_sigmas
        # XXX debug option
        if False:
          for n,p in enumerate(prob):
            print >> self.log, ' {0:5d} {1:5.3f}'.format(n,p)
        # Set probabilty between 0 and 1
        # XXX Make Histogram / more stats
        prob_lss_zero = flex.bool(prob <= 0)
        prob_grt_one = flex.bool(prob > 1)
        prob.set_selected(prob_lss_zero, 0.001)
        prob.set_selected(prob_grt_one, 1.0)
        xrs.set_occupancies(prob)
        xrs_list.append(xrs)
        sum_neg_ll = sum(-flex.log(prob))
        model_neg_ll.append((sum_neg_ll, model))
        if self.params.verbose:
          print >> self.log, 'Model probability stats :'
          print >> self.log, prob.min_max_mean().show()
          print >> self.log, '  Count < 0.0 : ', prob_lss_zero.count(True)
          print >> self.log, '  Count > 1.0 : ', prob_grt_one.count(True)

        # For averaging by residue
        number_previous_scatters += ens_pdb_xrs.sites_cart().size()

      # write ensemble pdb file, occupancies as sigma level
      write_ensemble_pdb(filename = pdb_file_names[0].split('/')[-1].replace('.pdb','') + '_pensemble.pdb',
                       xrs_list = xrs_list,
                       ens_pdb_hierarchy = ens_pdb_hierarchy
                       )

      # XXX Test ordering models by nll
      # XXX Test removing nth percentile atoms
      if self.params.sort_ensemble_by_nll_score or self.params.fobs_vs_fcalc_post_nll:
        for percentile in [1.0,0.975,0.95,0.9,0.8,0.6,0.2]:
          model_neg_ll = sorted(model_neg_ll)
          f_calc_ave_total_reordered = None
          print_list = []
          for i_neg_ll in model_neg_ll:
            xrs = xrs_list[i_neg_ll[1]]
            nll_occ = xrs.scatterers().extract_occupancies()

            # Set q=0 nth percentile atoms
            sorted_nll_occ = sorted(nll_occ, reverse=True)
            number_atoms = len(sorted_nll_occ)
            percentile_prob_cutoff = sorted_nll_occ[int(number_atoms * percentile)-1]
            cutoff_selections = flex.bool(nll_occ < percentile_prob_cutoff)
            cutoff_nll_occ = flex.double(nll_occ.size(), 1.0).set_selected(cutoff_selections, 0.0)
            #XXX Debug
            if False:
              print '\nDebug'
              for x in xrange(len(cutoff_selections)):
                print cutoff_selections[x], nll_occ[x], cutoff_nll_occ[x]
              print percentile
              print percentile_prob_cutoff
              print cutoff_selections.count(True)
              print cutoff_selections.size()
              print cutoff_nll_occ.count(0.0)
              print 'Count q = 1           : ', cutoff_nll_occ.count(1.0)
              print 'Count scatterers size : ', cutoff_nll_occ.size()

            xrs.set_occupancies(cutoff_nll_occ)
            fmodel.update_xray_structure(xray_structure  = xrs,
                                         update_f_calc   = True,
                                         update_f_mask   = True)

            if f_calc_ave_total_reordered == None:
              f_calc_ave_total_reordered = fmodel.f_calc().data().deep_copy()
              f_mask_ave_total_reordered = fmodel.f_masks()[0].data().deep_copy()
              cntr = 1
            else:
              f_calc_ave_total_reordered += fmodel.f_calc().data().deep_copy()
              f_mask_ave_total_reordered += fmodel.f_masks()[0].data().deep_copy()
              cntr+=1
            fmodel.update(f_calc = f_calc_ave.array(f_calc_ave_total_reordered / cntr).deep_copy(),
                          f_mask = f_calc_ave.array(f_mask_ave_total_reordered / cntr).deep_copy()
                          )

            # Update solvent and scale
            # XXX Will need to apply_back_trace on latest version
            fmodel.set_scale_switch = 0
            fmodel.update_all_scales()

            # Reset occ for outout
            xrs.set_occupancies(nll_occ)
            # k1 updated vs Fobs
            if self.params.fobs_vs_fcalc_post_nll:
              print_list.append([cntr, i_neg_ll[0], i_neg_ll[1], fmodel.r_work(), fmodel.r_free()])

          # Order models by nll and print summary
          print >> self.log, '\nModels ranked by nll <Fcalc> R-factors recalculated'
          print >> self.log, 'Percentile cutoff : {0:5.3f}'.format(percentile)
          xrs_list_sorted_nll = []
          print >> self.log, '      |      NLL     <Rw>     <Rf>    Ens Model'
          for info in print_list:
            print >> self.log, ' {0:4d} | {1:8.1f} {2:8.4f} {3:8.4f} {4:12d}'.format(
              info[0],
              info[1],
              info[3],
              info[4],
              info[2]+1,
              )
            xrs_list_sorted_nll.append(xrs_list[info[2]])

        # Output nll ordered ensemble

        write_ensemble_pdb(filename = 'nll_ordered_' + pdb_file_names[0].split('/')[-1].replace('.pdb','') + '_pensemble.pdb',
                       xrs_list = xrs_list_sorted_nll,
                       ens_pdb_hierarchy = ens_pdb_hierarchy
                       )
예제 #13
0
phase_source = '/mnt/data4/XFEL/LR23/DED_tests/dat/Dark.pdb'
f_obs_1_file_name = 'a/ligh_old_truncate.mtz'
f_obs_2_file_name = 'a/dark_old_truncate.mtz'


def multiscale(self, other, reflections_per_bin=None):
    if (reflections_per_bin is None):
        reflections_per_bin = other.indices().size()
    assert self.indices().all_eq(other.indices())
    assert self.is_similar_symmetry(other)
    self.setup_binner(reflections_per_bin=reflections_per_bin)
    other.use_binning_of(self)
    scale = flex.double(self.indices().size(), -1)
    for i_bin in self.binner().range_used():
        sel = self.binner().selection(i_bin)
        f1 = self.select(sel)
        f2 = other.select(sel)
        scale_ = 1.0
        den = flex.sum(flex.abs(f2.data()) * flex.abs(f2.data()))
        if (den != 0):
            scale_ = flex.sum(flex.abs(f1.data()) * flex.abs(f2.data())) / den
        scale.set_selected(sel, scale_)
    assert (scale > 0).count(True) == scale.size()
    return other.array(data=other.data() * scale)


parameters = utils.data_and_flags_master_params().extract()

print(parameters.labels)
예제 #14
0
    inputs = mmtbx.utils.process_command_line_args(args=args)

    rfs = reflection_file_utils.reflection_file_server(
        crystal_symmetry=inputs.crystal_symmetry,
        force_symmetry=True,
        reflection_files=inputs.reflection_files,
        err=StringIO(),
    )

    pdb_inp = iotbx.pdb.input(file_name=inputs.pdb_file_names[0])
    ph = pdb_inp.construct_hierarchy()

    xrs = ph.extract_xray_structure(crystal_symmetry=inputs.crystal_symmetry)
    xrs.show_summary()

    data_flags_params = data_and_flags_master_params().extract()
    data_flags_params.labels = params.exhaustive.options.column_type

    determined_data_and_flags = mmtbx.utils.determine_data_and_flags(
        reflection_file_server=rfs,
        parameters=data_flags_params,
        keep_going=True,
        log=StringIO(),
    )

    f_obs = determined_data_and_flags.f_obs
    r_free_flags = determined_data_and_flags.r_free_flags

    sites_frac = xrs.sites_frac()

    f_obs.show_summary()