def exercise_01(): """ Sanity check - don't crash when mean intensity for a bin is zero. """ xrs = random_structure.xray_structure(unit_cell=(50, 50, 50, 90, 90, 90), space_group_symbol="P1", n_scatterers=1200, elements="random") fc = abs(xrs.structure_factors(d_min=1.5).f_calc()) fc = fc.set_observation_type_xray_amplitude() cs = fc.complete_set(d_min=1.4) ls = cs.lone_set(other=fc) f_zero = ls.array(data=flex.double(ls.size(), 0)) f_zero.set_observation_type_xray_amplitude() fc = fc.concatenate(other=f_zero) sigf = flex.double(fc.size(), 0.1) + (fc.data() * 0.03) fc = fc.customized_copy(sigmas=sigf) try: fc_fc = french_wilson.french_wilson_scale(miller_array=fc, log=null_out()) except Sorry: pass else: raise Exception_expected ic = fc.f_as_f_sq().set_observation_type_xray_intensity() fc_fc = french_wilson.french_wilson_scale(miller_array=ic, log=null_out())
def exercise_01 () : """ Sanity check - don't crash when mean intensity for a bin is zero. """ xrs = random_structure.xray_structure( unit_cell=(50,50,50,90,90,90), space_group_symbol="P1", n_scatterers=1200, elements="random") fc = abs(xrs.structure_factors(d_min=1.5).f_calc()) fc = fc.set_observation_type_xray_amplitude() cs = fc.complete_set(d_min=1.4) ls = cs.lone_set(other=fc) f_zero = ls.array(data=flex.double(ls.size(), 0)) f_zero.set_observation_type_xray_amplitude() fc = fc.concatenate(other=f_zero) sigf = flex.double(fc.size(), 0.1) + (fc.data() * 0.03) fc = fc.customized_copy(sigmas=sigf) try : fc_fc = french_wilson.french_wilson_scale(miller_array=fc, log=null_out()) except Sorry : pass else : raise Exception_expected ic = fc.f_as_f_sq().set_observation_type_xray_intensity() fc_fc = french_wilson.french_wilson_scale(miller_array=ic, log=null_out())
def data_as_f_obs(self): """ Convert input data array to amplitudes, adjusting the data type and applying additional filters if necessary. :param f_obs: selected input data :returns: :py:class:`cctbx.miller.array` of real numbers with observation type set to amplitudes """ if (self.raw_data_truncated is None): return None f_obs = self.raw_data_truncated.deep_copy() # Convert to non-anomalous if requested if (self.force_non_anomalous): f_obs = f_obs.average_bijvoet_mates() # d_min = f_obs.d_min() if (d_min < 0.25): raise Sorry("Resolution of data is too high: %-6.4f A" % d_min) if (f_obs.is_complex_array()): f_obs = abs(f_obs) if (f_obs.is_xray_intensity_array()): if (self.parameters.french_wilson_scale): try: sigI_cutoff = self.parameters.sigma_iobs_rejection_criterion f_obs = french_wilson.french_wilson_scale( miller_array=f_obs, params=self.parameters.french_wilson, sigma_iobs_rejection_criterion=sigI_cutoff, log=self.log) except Exception as e: print(str(e), file=self.log) print("Using alternative Iobs->Fobs conversion.", file=self.log) f_obs = f_obs.f_sq_as_f() # f_obs.set_observation_type_xray_amplitude() if (self.parameters.force_anomalous_flag_to_be_equal_to is not None): if (not self.parameters.force_anomalous_flag_to_be_equal_to): if (f_obs.anomalous_flag()): merged = f_obs.as_non_anomalous_array().merge_equivalents() f_obs = merged.array().set_observation_type(f_obs) elif (not f_obs.anomalous_flag()): observation_type = f_obs.observation_type() f_obs = f_obs.generate_bijvoet_mates() f_obs.set_observation_type(observation_type) else: f_obs = f_obs.convert_to_non_anomalous_if_ratio_pairs_lone_less_than( threshold=self.parameters. convert_to_non_anomalous_if_ratio_pairs_lone_less_than_threshold ) f_obs.set_info(self.raw_data.info()) return f_obs
def run(args=None, params=None, out=sys.stdout): assert [args, params].count(None) == 1 if args is not None: if (len(args) == 0) or ("--help" in args): raise Usage(""" phenix.cc_star model.pdb data.mtz unmerged_data=data.hkl [n_bins=X] [options] phenix.cc_star model_refine_001.mtz unmerged_data=data.hkl [...] Implementation of the method for assessing data and model quality described in: Karplus PA & Diederichs K (2012) Science 336:1030-3. Full parameters: %s """ % master_phil.as_str(prefix=" ", attributes_level=1)) import iotbx.phil cmdline = iotbx.phil.process_command_line_with_files( args=args, master_phil=master_phil, pdb_file_def="model", reflection_file_def="data") params = cmdline.work.extract() import mmtbx.command_line import mmtbx.validation.experimental from iotbx import merging_statistics from iotbx import file_reader if (params.data is None): raise Sorry("Please specify a data file (usually MTZ format).") if (params.unmerged_data is None): raise Sorry("Please specify unmerged_data file") hkl_in = file_reader.any_file(params.data, force_type="hkl") hkl_in.check_file_type("hkl") f_model = f_obs = r_free_flags = None f_models = [] data_arrays = [] f_model_labels = [] if (params.f_model_labels is None): for array in hkl_in.file_server.miller_arrays: labels = array.info().label_string() if (array.is_complex_array()): if (labels.startswith("F-model") or labels.startswith("FMODEL")): f_models.append(array) f_model_labels.append(labels) if (len(f_models) > 1): raise Sorry( ("Multiple F(model) arrays found:\n%s\nPlease specify the " + "'labels' parameter.") % "\n".join(f_model_labels)) elif (len(f_models) == 1): f_model = f_models[0] if (f_model.anomalous_flag()): info = f_model.info() f_model = f_model.average_bijvoet_mates().set_info(info) print("F(model):", file=out) f_model.show_summary(f=out, prefix=" ") else: data_array = hkl_in.file_server.get_xray_data( file_name=params.data, labels=params.f_obs_labels, ignore_all_zeros=True, parameter_scope="") if (data_array.is_xray_intensity_array()): from cctbx import french_wilson f_obs = french_wilson.french_wilson_scale( miller_array=data_array, out=out) else: f_obs = data_array else: for array in hkl_in.file_server.miller_arrays: array_labels = array.info().label_string() if (array_labels == params.f_model_labels): if (array.is_complex_array()): f_model = array break else: raise Sorry( "The data in %s are not of the required type." % array_labels) if (f_model is not None): assert (f_obs is None) for array in hkl_in.file_server.miller_arrays: labels = array.info().label_string() if (labels == params.f_obs_labels): f_obs = array break else: try: f_obs = hkl_in.file_server.get_amplitudes( file_name=params.f_obs_labels, labels=None, convert_to_amplitudes_if_necessary=False, parameter_name="f_obs_labels", parameter_scope="", strict=True) except Sorry: raise Sorry( "You must supply a file containing both F-obs and F-model " + "if you want to use a pre-calculated F-model array.") assert (f_obs.is_xray_amplitude_array()) if (f_obs.anomalous_flag()): info = f_obs.info() f_obs = f_obs.average_bijvoet_mates().set_info(info) print("F(obs):", file=out) f_obs.show_summary(f=out, prefix=" ") print("", file=out) r_free_flags, test_flag_value = hkl_in.file_server.get_r_free_flags( file_name=params.data, label=params.r_free_flags.label, test_flag_value=params.r_free_flags.test_flag_value, disable_suitability_test=False, parameter_scope="") info = r_free_flags.info() r_free_flags = r_free_flags.customized_copy( data=r_free_flags.data() == test_flag_value).set_info(info) if (r_free_flags.anomalous_flag()): r_free_flags = r_free_flags.average_bijvoet_mates().set_info(info) print("R-free flags:", file=out) r_free_flags.show_summary(f=out, prefix=" ") print("", file=out) unmerged_i_obs = mmtbx.command_line.load_and_validate_unmerged_data( f_obs=f_obs, file_name=params.unmerged_data, data_labels=params.unmerged_labels, log=out) print("Unmerged intensities:", file=out) unmerged_i_obs.show_summary(f=out, prefix=" ") print("", file=out) if (f_model is None): assert (f_obs is not None) if (params.model is None): raise Sorry( "A PDB file is required if F(model) is not pre-calculated.") make_sub_header("Calculating F(model)", out=out) pdb_in = file_reader.any_file(params.model, force_type="pdb") pdb_in.check_file_type("pdb") pdb_symm = pdb_in.file_object.crystal_symmetry() if (pdb_symm is None): pdb_symm = f_obs else: if (f_obs.crystal_symmetry() is None): f_obs = f_obs.customized_copy(crystal_symmetry=pdb_symm) elif (not pdb_symm.is_similar_symmetry(f_obs)): mmtbx.command_line.show_symmetry_error(file1="PDB file", file2="data file", symm1=pdb_symm, symm2=f_obs) xray_structure = pdb_in.file_object.xray_structure_simple( crystal_symmetry=pdb_symm) from mmtbx.utils import fmodel_simple # XXX this gets done anyway later, but they need to be consistent before # creating the fmodel manager if (f_obs.anomalous_flag()): f_obs = f_obs.average_bijvoet_mates() f_obs = f_obs.eliminate_sys_absent() f_obs, r_free_flags = f_obs.map_to_asu().common_sets( other=r_free_flags.map_to_asu()) fmodel = fmodel_simple(f_obs=f_obs, r_free_flags=r_free_flags, xray_structures=[xray_structure], skip_twin_detection=True, scattering_table="n_gaussian") fmodel.show(log=out) f_model = fmodel.f_model() f_obs = fmodel.f_obs() r_free_flags = fmodel.r_free_flags() else: if (f_model.anomalous_flag()): f_model = f_model.average_bijvoet_mates() stats = mmtbx.validation.experimental.merging_and_model_statistics( f_model=f_model, f_obs=f_obs, r_free_flags=r_free_flags, unmerged_i_obs=unmerged_i_obs, n_bins=params.n_bins, sigma_filtering=params.sigma_filtering) stats.show_cc_star(out=out) if (params.loggraph): stats.show_loggraph(out=out) print("", file=out) print("Reference:", file=out) print(" Karplus PA & Diederichs K (2012) Science 336:1030-3.", file=out) print("", file=out) return stats
def run(args, out=sys.stdout): from cctbx import french_wilson from iotbx import file_reader hkl_file = None sources = [] interpreter = master_phil.command_line_argument_interpreter() for arg in args : if os.path.isfile(arg): input_file = file_reader.any_file(arg) if (input_file.file_type == "hkl"): hkl_file = input_file sources.append(interpreter.process(arg="file_name=\"%s\"" % arg)) elif (input_file.file_type == "phil"): sources.append(input_file.file_object) else : arg_phil = interpreter.process(arg=arg) sources.append(arg_phil) work_phil = master_phil.fetch(sources=sources) work_params = work_phil.extract() if (work_params.french_wilson.file_name is None): if (hkl_file is None): raise Usage("phenix.french_wilson data.mtz [params.eff] [options ...]") else : work_params.french_wilson.file_name = hkl_file.file_name elif (hkl_file is None): hkl_file = file_reader.any_file(work_params.french_wilson.file_name) params = work_params.french_wilson xray_data_server = hkl_file.file_server crystal_symmetry = xray_data_server.miller_arrays[0].crystal_symmetry() if (crystal_symmetry is None): raise Sorry("No crystal symmetry found. This program requires an input "+ "format with complete symmetry information.") unit_cell = xray_data_server.miller_arrays[0].unit_cell() if (unit_cell is None): raise Sorry("No unit cell found. This program requires an input "+ "format with complete unit cell information.") i_obs = None i_obs = xray_data_server.get_xray_data( file_name = params.file_name, labels = params.intensity_labels, ignore_all_zeros = True, parameter_scope = 'french_wilson', parameter_name = 'intensity_labels') import cStringIO xray_data_server.err = cStringIO.StringIO() try : r_free_flags, test_flag_value = xray_data_server.get_r_free_flags( file_name = params.file_name, label = params.r_free_flags.label, test_flag_value = None, disable_suitability_test = False, parameter_scope = "french_wilson.r_free_flags") except Sorry as e : r_free_flags = None if (i_obs is None): raise Sorry("Couldn't find intensities!") wavelength = params.wavelength if (wavelength is None): info = i_obs.info() if (info is not None): wavelength = info.wavelength if (wavelength is not None): print("Using wavelength=%g from input file" % wavelength, file=out) sigma_iobs_rejection_criterion = work_params.french_wilson.\ sigma_iobs_rejection_criterion if (not i_obs.is_unique_set_under_symmetry()): print("Merging symmetry-equivalent reflections", file=out) i_obs = i_obs.merge_equivalents().array() f_obs = french_wilson.french_wilson_scale(miller_array=i_obs, params=params, sigma_iobs_rejection_criterion=sigma_iobs_rejection_criterion, log=out) if f_obs is None: raise Sorry("Not enough data to accurately apply the French-Wilson method."+\ " Exiting.") if params.output_file == None: output_file = "french_wilson.mtz" else: output_file = params.output_file mtz_dataset = i_obs.as_mtz_dataset( column_root_label = "I", wavelength = wavelength) mtz_dataset.add_miller_array( miller_array = f_obs, column_root_label = "F") if (r_free_flags is not None) and (params.keep_r_free_flags): mtz_dataset.add_miller_array( miller_array = r_free_flags, column_root_label = "R-free-flags") mtz_object = mtz_dataset.mtz_object() mtz_object.write(file_name = output_file) print("Wrote %s" % output_file, file=out) return output_file
raise Sorry("Couldn't find intensities!") wavelength = params.wavelength if (wavelength is None): info = i_obs.info() if (info is not None): wavelength = info.wavelength if (wavelength is not None): print >> out, "Using wavelength=%g from input file" % wavelength sigma_iobs_rejection_criterion = work_params.french_wilson.\ sigma_iobs_rejection_criterion if (not i_obs.is_unique_set_under_symmetry()): print >> out, "Merging symmetry-equivalent reflections" i_obs = i_obs.merge_equivalents().array() f_obs = french_wilson.french_wilson_scale( miller_array=i_obs, params=params, sigma_iobs_rejection_criterion=sigma_iobs_rejection_criterion, log=out) if f_obs is None: raise Sorry("Not enough data to accurately apply the French-Wilson method."+\ " Exiting.") if params.output_file == None: output_file = "french_wilson.mtz" else: output_file = params.output_file mtz_dataset = i_obs.as_mtz_dataset(column_root_label="I", wavelength=wavelength) mtz_dataset.add_miller_array(miller_array=f_obs, column_root_label="F") if (r_free_flags is not None) and (params.keep_r_free_flags): mtz_dataset.add_miller_array(miller_array=r_free_flags, column_root_label="R-free-flags")
def __init__ (self, obs, r_free_flags, test_flag_value, phases=None, d_min=None, d_max=None, r_free_flags_params=None, merge_anomalous=False, log=sys.stdout, verbose=True) : assert (log is not None) and (obs is not None) if (r_free_flags_params is None) : from cctbx.r_free_utils import generate_r_free_params_str r_free_flags_params = libtbx.phil.parse( generate_r_free_params_str).extract() obs_info = obs.info() r_free_flags_info = phases_info = None sg = obs.space_group_info() obs = obs.map_to_asu().merge_equivalents().array() obs = obs.eliminate_sys_absent(log=log) obs = obs.resolution_filter(d_min=d_min, d_max=d_max) if (obs.is_xray_intensity_array()) : from cctbx import french_wilson if (verbose) : fw_out = log else : fw_out = null_out() obs = french_wilson.french_wilson_scale( miller_array=obs, params=None, log=fw_out) assert (obs is not None) merged_obs = obs.average_bijvoet_mates() if (merged_obs.completeness() < 0.9) : print >> log, """ WARNING: data are incomplete (%.1f%% of possible reflections measured to %.2fA). This may cause problems if you plan to use the maps for building and/or ligand fitting! """ % (100*merged_obs.completeness(), merged_obs.d_min()) # XXX this is kind of a hack (the reconstructed arrays break some of my # assumptions about labels) if (merge_anomalous) : obs = obs.average_bijvoet_mates() if (r_free_flags is not None) : r_free_flags_info = r_free_flags.info() format = "cns" if (test_flag_value == 0) : format = "ccp4" elif (test_flag_value == -1) : format = "shelx" if (r_free_flags.anomalous_flag()) : r_free_flags = r_free_flags.average_bijvoet_mates() is_compatible_symmetry = False obs_pg = obs.space_group().build_derived_point_group() flags_pg = r_free_flags.space_group().build_derived_point_group() if (obs_pg.type().number() == flags_pg.type().number()) : is_compatible_symmetry = True else : pass # TODO unit cell comparison? if (is_compatible_symmetry) : r_free_flags = r_free_flags.map_to_asu().merge_equivalents().array() r_free_flags = r_free_flags.eliminate_sys_absent(log=log) if (format == "cns") : r_free_flags = r_free_flags.customized_copy( crystal_symmetry=obs.crystal_symmetry(), data=(r_free_flags.data() == test_flag_value)) test_flag_value = True obs_tmp = obs.deep_copy() if (obs.anomalous_flag()) : obs_tmp = obs.average_bijvoet_mates() r_free_flags = r_free_flags.common_set(other=obs_tmp) n_r_free = r_free_flags.indices().size() n_obs = obs_tmp.indices().size() if ((test_flag_value is None) or (r_free_flags.data().all_eq(r_free_flags.data()[0]))) : print >> log, """ WARNING: uniform R-free flags detected; a new test set will be generated, but this will bias the refinement statistics. """ r_free_flags = None elif (n_r_free != n_obs) : missing_set = obs_tmp.lone_set(other=r_free_flags) n_missing = missing_set.indices().size() if (n_missing > 0) : print >> log, """ WARNING: R-free flags are incomplete relative to experimental data (%d vs. %d reflections). The flags will be extended to complete the set, but we recommend supplying flags that are already generated to the maximum expected resolution. """ % (n_r_free, n_obs) if (n_missing < 20) : # FIXME if (format == "cns") : missing_flags = missing_set.array(data=flex.bool(n_missing, False)) else : missing_flags = missing_set.array(data=flex.int(n_missing, 1)) else : missing_flags = missing_set.generate_r_free_flags( fraction=(r_free_flags.data().count(test_flag_value)/n_r_free), max_free=None, use_lattice_symmetry=True, format=format) r_free_flags = r_free_flags.concatenate(other=missing_flags) if (r_free_flags is not None) : assert (r_free_flags.indices().size() == obs_tmp.indices().size()) else : print >> log, """ NOTE: incompatible symmetry between the data and the R-free flags: Data : %s %s Flags : %s %s A new test set will be generated. """ % (str(obs.space_group_info()), " ".join([ "%g" % x for x in obs.unit_cell().parameters() ]), str(r_free_flags.space_group_info()), " ".join(["%g" % x for x in r_free_flags.unit_cell().parameters()])) else : print >> log, """ WARNING: R-free flags not supplied. This may bias the refinement if the structures are very nearly isomorphous! """ self._generate_new = False if (r_free_flags is None) : r_free_flags = obs.generate_r_free_flags( fraction=r_free_flags_params.fraction, max_free=r_free_flags_params.max_free, use_lattice_symmetry=r_free_flags_params.use_lattice_symmetry, use_dataman_shells=r_free_flags_params.use_dataman_shells, n_shells=r_free_flags_params.n_shells, format="ccp4") test_flag_value = 0 self._generate_new = True if (r_free_flags.anomalous_flag()) : r_free_flags = r_free_flags.average_bijvoet_mates() if (phases is not None) : phases_info = phases.info() phases = phases.map_to_asu().resolution_filter(d_min=d_min, d_max=d_max) assert (obs.is_xray_amplitude_array()) self.f_obs = obs.set_info(obs_info) self.r_free_flags = r_free_flags.set_info(r_free_flags_info) self.test_flag_value = test_flag_value self.phases = None if (phases is not None) : self.phases = phases.set_info(phases_info)
def __init__(self, obs, r_free_flags, test_flag_value, phases=None, d_min=None, d_max=None, r_free_flags_params=None, merge_anomalous=False, log=sys.stdout, verbose=True): assert (log is not None) and (obs is not None) if (r_free_flags_params is None): from cctbx.r_free_utils import generate_r_free_params_str r_free_flags_params = libtbx.phil.parse( generate_r_free_params_str).extract() obs_info = obs.info() r_free_flags_info = phases_info = None sg = obs.space_group_info() obs = obs.map_to_asu().merge_equivalents().array() obs = obs.eliminate_sys_absent(log=log) obs = obs.resolution_filter(d_min=d_min, d_max=d_max) if (obs.is_xray_intensity_array()): from cctbx import french_wilson if (verbose): fw_out = log else : fw_out = null_out() obs = french_wilson.french_wilson_scale( miller_array=obs, params=None, log=fw_out) assert (obs is not None) merged_obs = obs.average_bijvoet_mates() if (merged_obs.completeness() < 0.9): print >> log, """ WARNING: data are incomplete (%.1f%% of possible reflections measured to %.2fA). This may cause problems if you plan to use the maps for building and/or ligand fitting! """ % (100*merged_obs.completeness(), merged_obs.d_min()) # XXX this is kind of a hack (the reconstructed arrays break some of my # assumptions about labels) if (merge_anomalous): obs = obs.average_bijvoet_mates() if (r_free_flags is not None): r_free_flags_info = r_free_flags.info() format = "cns" if (test_flag_value == 0): format = "ccp4" elif (test_flag_value == -1): format = "shelx" if (r_free_flags.anomalous_flag()): r_free_flags = r_free_flags.average_bijvoet_mates() is_compatible_symmetry = False obs_pg = obs.space_group().build_derived_point_group() flags_pg = r_free_flags.space_group().build_derived_point_group() if (obs_pg.type().number() == flags_pg.type().number()): is_compatible_symmetry = True else : pass # TODO unit cell comparison? if (is_compatible_symmetry): r_free_flags = r_free_flags.map_to_asu().merge_equivalents().array() r_free_flags = r_free_flags.eliminate_sys_absent(log=log) if (format == "cns"): r_free_flags = r_free_flags.customized_copy( crystal_symmetry=obs.crystal_symmetry(), data=(r_free_flags.data() == test_flag_value)) test_flag_value = True obs_tmp = obs.deep_copy() if (obs.anomalous_flag()): obs_tmp = obs.average_bijvoet_mates() r_free_flags = r_free_flags.common_set(other=obs_tmp) n_r_free = r_free_flags.indices().size() n_obs = obs_tmp.indices().size() if ((test_flag_value is None) or (r_free_flags.data().all_eq(r_free_flags.data()[0]))): print >> log, """ WARNING: uniform R-free flags detected; a new test set will be generated, but this will bias the refinement statistics. """ r_free_flags = None elif (n_r_free != n_obs): missing_set = obs_tmp.lone_set(other=r_free_flags) n_missing = missing_set.indices().size() if (n_missing > 0): print >> log, """ WARNING: R-free flags are incomplete relative to experimental data (%d vs. %d reflections). The flags will be extended to complete the set, but we recommend supplying flags that are already generated to the maximum expected resolution. """ % (n_r_free, n_obs) if (n_missing < 20) : # FIXME if (format == "cns"): missing_flags = missing_set.array(data=flex.bool(n_missing, False)) else : missing_flags = missing_set.array(data=flex.int(n_missing, 1)) else : missing_flags = missing_set.generate_r_free_flags( fraction=(r_free_flags.data().count(test_flag_value)/n_r_free), max_free=None, use_lattice_symmetry=True, format=format) r_free_flags = r_free_flags.concatenate(other=missing_flags) if (r_free_flags is not None): assert (r_free_flags.indices().size() == obs_tmp.indices().size()) else : print >> log, """ NOTE: incompatible symmetry between the data and the R-free flags: Data : %s %s Flags : %s %s A new test set will be generated. """ % (str(obs.space_group_info()), " ".join([ "%g" % x for x in obs.unit_cell().parameters() ]), str(r_free_flags.space_group_info()), " ".join(["%g" % x for x in r_free_flags.unit_cell().parameters()])) else : print >> log, """ WARNING: R-free flags not supplied. This may bias the refinement if the structures are very nearly isomorphous! """ self._generate_new = False if (r_free_flags is None): r_free_flags = obs.generate_r_free_flags( fraction=r_free_flags_params.fraction, max_free=r_free_flags_params.max_free, use_lattice_symmetry=r_free_flags_params.use_lattice_symmetry, use_dataman_shells=r_free_flags_params.use_dataman_shells, n_shells=r_free_flags_params.n_shells, format="ccp4") test_flag_value = 0 self._generate_new = True if (r_free_flags.anomalous_flag()): r_free_flags = r_free_flags.average_bijvoet_mates() if (phases is not None): phases_info = phases.info() phases = phases.map_to_asu().resolution_filter(d_min=d_min, d_max=d_max) assert (obs.is_xray_amplitude_array()) self.f_obs = obs.set_info(obs_info) self.r_free_flags = r_free_flags.set_info(r_free_flags_info) self.test_flag_value = test_flag_value self.phases = None if (phases is not None): self.phases = phases.set_info(phases_info)
def run (args, out=sys.stdout) : from iotbx import file_reader import iotbx.phil if (len(args) == 0) : raise Usage("""\ iotbx.simple_map_coefficients data_phases.mtz [options] Full parameters: %s""" % iotbx.phil.parse(master_phil).as_str(attributes_level=1, prefix=" ")) cmdline = iotbx.phil.process_command_line_with_files( args=args, master_phil_string=master_phil, reflection_file_def="file_name") params = cmdline.work.extract() if (params.file_name is None) : raise Sorry("No reflection file specified.") hkl_in = file_reader.any_file(params.file_name).check_file_type("hkl") hkl_server = hkl_in.file_server data = hkl_server.get_xray_data( file_name=params.file_name, labels=params.data_labels, ignore_all_zeros=False, parameter_name="data_labels", parameter_scope="", prefer_anomalous=True, prefer_amplitudes=True) data_labels = data.info().label_string() if (data.is_xray_intensity_array()) : from cctbx.french_wilson import french_wilson_scale data = french_wilson_scale(data, log=out) phases = hkl_server.get_phases_deg( file_name=params.file_name, labels=params.phase_labels, convert_to_phases_if_necessary=True, original_phase_units=None, parameter_scope="", parameter_name="phase_labels", minimum_score=2) assert (not phases.anomalous_flag()) deg = True # FIXME weights = None if (params.use_weights in [Auto, True]) : # FIXME centralize this in iotbx.reflection_file_utils for array in hkl_server.miller_arrays : if (array.is_real_array()) : label_string = array.info().label_string() if ((label_string == params.weight_labels) or ((params.weight_labels is None) and ("FOM" in label_string))) : weights = array break amplitudes = data if (params.map_type == "anom") : if (not data.anomalous_flag()) : raise Sorry("Anomalous map requested, but selected data are merged.") amplitudes = data.anomalous_differences() print >> out, "Using anomalous differences in %s" % data_labels else : print >> out, "Using amplitudes in %s" % data_labels if (data.anomalous_flag()) : amplitudes = data.average_bijvoet_mates() if (params.use_weights is Auto) and (weights is not None) : if (params.map_type != "anom") : params.use_weights = True elif (params.use_weights == True) and (weights is None) : raise Sorry("No weights (FOM, etc.) found in input file.") if (params.use_weights == True) : assert (not weights.anomalous_flag()) print >> out, "Applying weights in %s" % weights.info().label_string() amplitudes, weights = amplitudes.common_sets(other=weights) amplitudes = amplitudes.customized_copy( data=amplitudes.data()*weights.data()) amplitudes = amplitudes.customized_copy(sigmas=None) print >> out, "Applying phases in %s" % phases.info().label_string() amplitudes, phases = amplitudes.common_sets(phases) coeffs = amplitudes.phase_transfer(phases, deg=deg).set_observation_type(None) # FIXME if (params.map_type == "anom") : # apply 90-degree phase shift coeffs = coeffs.customized_copy(data=coeffs.data()/(2j)) assert (coeffs.is_complex_array()) column_root_label = "F" decorator = None if (params.map_type == "anom") : column_root_label = "ANOM" elif (params.use_weights == True) : column_root_label = "FWT" decorator = iotbx.mtz.ccp4_label_decorator() import iotbx.mtz mtz_dataset = coeffs.as_mtz_dataset( column_root_label=column_root_label, label_decorator=decorator) if (params.output_file is None) : params.output_file = "map_coeffs.mtz" mtz_dataset.mtz_object().write(params.output_file) print >> out, "Wrote %s" % params.output_file return os.path.abspath(params.output_file)
def run (args=None, params=None, out=sys.stdout) : assert [args, params].count(None) == 1 if args is not None: if (len(args) == 0) or ("--help" in args) : raise Usage(""" phenix.cc_star model.pdb data.mtz unmerged_data=data.hkl [n_bins=X] [options] phenix.cc_star model_refine_001.mtz unmerged_data=data.hkl [...] Implementation of the method for assessing data and model quality described in: Karplus PA & Diederichs K (2012) Science 336:1030-3. Full parameters: %s """ % master_phil.as_str(prefix=" ", attributes_level=1)) import iotbx.phil cmdline = iotbx.phil.process_command_line_with_files( args=args, master_phil=master_phil, pdb_file_def="model", reflection_file_def="data") params = cmdline.work.extract() import mmtbx.command_line import mmtbx.validation.experimental from iotbx import merging_statistics from iotbx import file_reader if (params.data is None) : raise Sorry("Please specify a data file (usually MTZ format).") if (params.unmerged_data is None) : raise Sorry("Please specify unmerged_data file") hkl_in = file_reader.any_file(params.data, force_type="hkl") hkl_in.check_file_type("hkl") f_model = f_obs = r_free_flags = None f_models = [] data_arrays = [] f_model_labels = [] if (params.f_model_labels is None) : for array in hkl_in.file_server.miller_arrays : labels = array.info().label_string() if (array.is_complex_array()) : if (labels.startswith("F-model") or labels.startswith("FMODEL")) : f_models.append(array) f_model_labels.append(labels) if (len(f_models) > 1) : raise Sorry(("Multiple F(model) arrays found:\n%s\nPlease specify the "+ "'labels' parameter.") % "\n".join(f_model_labels)) elif (len(f_models) == 1) : f_model = f_models[0] if (f_model.anomalous_flag()) : info = f_model.info() f_model = f_model.average_bijvoet_mates().set_info(info) print >> out, "F(model):" f_model.show_summary(f=out, prefix=" ") else : data_array = hkl_in.file_server.get_xray_data( file_name=params.data, labels=params.f_obs_labels, ignore_all_zeros=True, parameter_scope="") if (data_array.is_xray_intensity_array()) : from cctbx import french_wilson f_obs = french_wilson.french_wilson_scale( miller_array=data_array, out=out) else : f_obs = data_array else : for array in hkl_in.file_server.miller_arrays : array_labels = array.info().label_string() if (array_labels == params.f_model_labels) : if (array.is_complex_array()) : f_model = array break else : raise Sorry("The data in %s are not of the required type." % array_labels) if (f_model is not None) : assert (f_obs is None) for array in hkl_in.file_server.miller_arrays : labels = array.info().label_string() if (labels == params.f_obs_labels) : f_obs = array break else : try : f_obs = hkl_in.file_server.get_amplitudes( file_name=params.f_obs_labels, labels=None, convert_to_amplitudes_if_necessary=False, parameter_name="f_obs_labels", parameter_scope="", strict=True) except Sorry : raise Sorry("You must supply a file containing both F-obs and F-model "+ "if you want to use a pre-calculated F-model array.") assert (f_obs.is_xray_amplitude_array()) if (f_obs.anomalous_flag()) : info = f_obs.info() f_obs = f_obs.average_bijvoet_mates().set_info(info) print >> out, "F(obs):" f_obs.show_summary(f=out, prefix=" ") print >> out, "" r_free_flags, test_flag_value = hkl_in.file_server.get_r_free_flags( file_name=params.data, label=params.r_free_flags.label, test_flag_value=params.r_free_flags.test_flag_value, disable_suitability_test=False, parameter_scope="") info = r_free_flags.info() r_free_flags = r_free_flags.customized_copy( data=r_free_flags.data()==test_flag_value).set_info(info) if (r_free_flags.anomalous_flag()) : r_free_flags = r_free_flags.average_bijvoet_mates().set_info(info) print >> out, "R-free flags:" r_free_flags.show_summary(f=out, prefix=" ") print >> out, "" unmerged_i_obs = mmtbx.command_line.load_and_validate_unmerged_data( f_obs=f_obs, file_name=params.unmerged_data, data_labels=params.unmerged_labels, log=out) print >> out, "Unmerged intensities:" unmerged_i_obs.show_summary(f=out, prefix=" ") print >> out, "" if (f_model is None) : assert (f_obs is not None) if (params.model is None) : raise Sorry("A PDB file is required if F(model) is not pre-calculated.") make_sub_header("Calculating F(model)", out=out) pdb_in = file_reader.any_file(params.model, force_type="pdb") pdb_in.check_file_type("pdb") pdb_symm = pdb_in.file_object.crystal_symmetry() if (pdb_symm is None) : pdb_symm = f_obs else : if (f_obs.crystal_symmetry() is None) : f_obs = f_obs.customized_copy(crystal_symmetry=pdb_symm) elif (not pdb_symm.is_similar_symmetry(f_obs)) : mmtbx.command_line.show_symmetry_error( file1="PDB file", file2="data file", symm1=pdb_symm, symm2=f_obs) xray_structure = pdb_in.file_object.xray_structure_simple( crystal_symmetry=pdb_symm) from mmtbx.utils import fmodel_simple # XXX this gets done anyway later, but they need to be consistent before # creating the fmodel manager if (f_obs.anomalous_flag()) : f_obs = f_obs.average_bijvoet_mates() f_obs = f_obs.eliminate_sys_absent() f_obs, r_free_flags = f_obs.map_to_asu().common_sets( other=r_free_flags.map_to_asu()) fmodel = fmodel_simple( f_obs=f_obs, r_free_flags=r_free_flags, xray_structures=[xray_structure], skip_twin_detection=True, scattering_table="n_gaussian") fmodel.show(log=out) f_model = fmodel.f_model() r_free_flags = f_model.customized_copy(data=fmodel.arrays.free_sel) else : if (f_model.anomalous_flag()) : f_model = f_model.average_bijvoet_mates() f_model, r_free_flags = f_model.common_sets(other=r_free_flags) stats = mmtbx.validation.experimental.merging_and_model_statistics( f_model=f_model, f_obs=f_obs, r_free_flags=r_free_flags, unmerged_i_obs=unmerged_i_obs, n_bins=params.n_bins, sigma_filtering=params.sigma_filtering) stats.show_cc_star(out=out) if (params.loggraph) : stats.show_loggraph(out=out) print >> out, "" print >> out, "Reference:" print >> out, " Karplus PA & Diederichs K (2012) Science 336:1030-3." print >> out, "" return stats
if (i_obs is None) : raise Sorry("Couldn't find intensities!") wavelength = params.wavelength if (wavelength is None) : info = i_obs.info() if (info is not None) : wavelength = info.wavelength if (wavelength is not None) : print >> out, "Using wavelength=%g from input file" % wavelength sigma_iobs_rejection_criterion = work_params.french_wilson.\ sigma_iobs_rejection_criterion if (not i_obs.is_unique_set_under_symmetry()) : print >> out, "Merging symmetry-equivalent reflections" i_obs = i_obs.merge_equivalents().array() f_obs = french_wilson.french_wilson_scale(miller_array=i_obs, params=params, sigma_iobs_rejection_criterion=sigma_iobs_rejection_criterion, log=out) if f_obs is None: raise Sorry("Not enough data to accurately apply the French-Wilson method."+\ " Exiting.") if params.output_file == None: output_file = "french_wilson.mtz" else: output_file = params.output_file mtz_dataset = i_obs.as_mtz_dataset( column_root_label = "I", wavelength = wavelength) mtz_dataset.add_miller_array( miller_array = f_obs, column_root_label = "F") if (r_free_flags is not None) and (params.keep_r_free_flags):
def run(args, out=sys.stdout): from iotbx import file_reader import iotbx.phil if (len(args) == 0): raise Usage("""\ iotbx.simple_map_coefficients data_phases.mtz [options] Full parameters: %s""" % iotbx.phil.parse(master_phil).as_str(attributes_level=1, prefix=" ")) cmdline = iotbx.phil.process_command_line_with_files( args=args, master_phil_string=master_phil, reflection_file_def="file_name") params = cmdline.work.extract() if (params.file_name is None): raise Sorry("No reflection file specified.") hkl_in = file_reader.any_file(params.file_name).check_file_type("hkl") hkl_server = hkl_in.file_server data = hkl_server.get_xray_data(file_name=params.file_name, labels=params.data_labels, ignore_all_zeros=False, parameter_name="data_labels", parameter_scope="", prefer_anomalous=True, prefer_amplitudes=True) data_labels = data.info().label_string() if (data.is_xray_intensity_array()): from cctbx.french_wilson import french_wilson_scale data = french_wilson_scale(data, log=out) phases = hkl_server.get_phases_deg(file_name=params.file_name, labels=params.phase_labels, convert_to_phases_if_necessary=True, original_phase_units=None, parameter_scope="", parameter_name="phase_labels", minimum_score=2) assert (not phases.anomalous_flag()) deg = True # FIXME weights = None if (params.use_weights in [Auto, True]): # FIXME centralize this in iotbx.reflection_file_utils for array in hkl_server.miller_arrays: if (array.is_real_array()): label_string = array.info().label_string() if ((label_string == params.weight_labels) or ((params.weight_labels is None) and ("FOM" in label_string))): weights = array break amplitudes = data if (params.map_type == "anom"): if (not data.anomalous_flag()): raise Sorry( "Anomalous map requested, but selected data are merged.") amplitudes = data.anomalous_differences() print >> out, "Using anomalous differences in %s" % data_labels else: print >> out, "Using amplitudes in %s" % data_labels if (data.anomalous_flag()): amplitudes = data.average_bijvoet_mates() if (params.use_weights is Auto) and (weights is not None): if (params.map_type != "anom"): params.use_weights = True elif (params.use_weights == True) and (weights is None): raise Sorry("No weights (FOM, etc.) found in input file.") if (params.use_weights == True): assert (not weights.anomalous_flag()) print >> out, "Applying weights in %s" % weights.info().label_string() amplitudes, weights = amplitudes.common_sets(other=weights) amplitudes = amplitudes.customized_copy(data=amplitudes.data() * weights.data()) amplitudes = amplitudes.customized_copy(sigmas=None) print >> out, "Applying phases in %s" % phases.info().label_string() amplitudes, phases = amplitudes.common_sets(phases) coeffs = amplitudes.phase_transfer(phases, deg=deg).set_observation_type( None) # FIXME if (params.map_type == "anom"): # apply 90-degree phase shift coeffs = coeffs.customized_copy(data=coeffs.data() / (2j)) assert (coeffs.is_complex_array()) column_root_label = "F" decorator = None if (params.map_type == "anom"): column_root_label = "ANOM" elif (params.use_weights == True): column_root_label = "FWT" decorator = iotbx.mtz.ccp4_label_decorator() import iotbx.mtz mtz_dataset = coeffs.as_mtz_dataset(column_root_label=column_root_label, label_decorator=decorator) if (params.output_file is None): params.output_file = "map_coeffs.mtz" mtz_dataset.mtz_object().write(params.output_file) print >> out, "Wrote %s" % params.output_file return os.path.abspath(params.output_file)