def exercise_01():
"""
Sanity check - don't crash when mean intensity for a bin is zero.
"""
xrs = random_structure.xray_structure(unit_cell=(50, 50, 50, 90, 90, 90),
space_group_symbol="P1",
n_scatterers=1200,
elements="random")
fc = abs(xrs.structure_factors(d_min=1.5).f_calc())
fc = fc.set_observation_type_xray_amplitude()
cs = fc.complete_set(d_min=1.4)
ls = cs.lone_set(other=fc)
f_zero = ls.array(data=flex.double(ls.size(), 0))
f_zero.set_observation_type_xray_amplitude()
fc = fc.concatenate(other=f_zero)
sigf = flex.double(fc.size(), 0.1) + (fc.data() * 0.03)
fc = fc.customized_copy(sigmas=sigf)
try:
fc_fc = french_wilson.french_wilson_scale(miller_array=fc,
log=null_out())
except Sorry:
pass
else:
raise Exception_expected
ic = fc.f_as_f_sq().set_observation_type_xray_intensity()
fc_fc = french_wilson.french_wilson_scale(miller_array=ic, log=null_out())
def exercise_01 () :
"""
Sanity check - don't crash when mean intensity for a bin is zero.
"""
xrs = random_structure.xray_structure(
unit_cell=(50,50,50,90,90,90),
space_group_symbol="P1",
n_scatterers=1200,
elements="random")
fc = abs(xrs.structure_factors(d_min=1.5).f_calc())
fc = fc.set_observation_type_xray_amplitude()
cs = fc.complete_set(d_min=1.4)
ls = cs.lone_set(other=fc)
f_zero = ls.array(data=flex.double(ls.size(), 0))
f_zero.set_observation_type_xray_amplitude()
fc = fc.concatenate(other=f_zero)
sigf = flex.double(fc.size(), 0.1) + (fc.data() * 0.03)
fc = fc.customized_copy(sigmas=sigf)
try :
fc_fc = french_wilson.french_wilson_scale(miller_array=fc, log=null_out())
except Sorry :
pass
else :
raise Exception_expected
ic = fc.f_as_f_sq().set_observation_type_xray_intensity()
fc_fc = french_wilson.french_wilson_scale(miller_array=ic, log=null_out())
def data_as_f_obs(self):
"""
Convert input data array to amplitudes, adjusting the data type and
applying additional filters if necessary.
:param f_obs: selected input data
:returns: :py:class:`cctbx.miller.array` of real numbers with observation
type set to amplitudes
"""
if (self.raw_data_truncated is None): return None
f_obs = self.raw_data_truncated.deep_copy()
# Convert to non-anomalous if requested
if (self.force_non_anomalous):
f_obs = f_obs.average_bijvoet_mates()
#
d_min = f_obs.d_min()
if (d_min < 0.25):
raise Sorry("Resolution of data is too high: %-6.4f A" % d_min)
if (f_obs.is_complex_array()): f_obs = abs(f_obs)
if (f_obs.is_xray_intensity_array()):
if (self.parameters.french_wilson_scale):
try:
sigI_cutoff = self.parameters.sigma_iobs_rejection_criterion
f_obs = french_wilson.french_wilson_scale(
miller_array=f_obs,
params=self.parameters.french_wilson,
sigma_iobs_rejection_criterion=sigI_cutoff,
log=self.log)
except Exception as e:
print(str(e), file=self.log)
print("Using alternative Iobs->Fobs conversion.",
file=self.log)
f_obs = f_obs.f_sq_as_f()
#
f_obs.set_observation_type_xray_amplitude()
if (self.parameters.force_anomalous_flag_to_be_equal_to is not None):
if (not self.parameters.force_anomalous_flag_to_be_equal_to):
if (f_obs.anomalous_flag()):
merged = f_obs.as_non_anomalous_array().merge_equivalents()
f_obs = merged.array().set_observation_type(f_obs)
elif (not f_obs.anomalous_flag()):
observation_type = f_obs.observation_type()
f_obs = f_obs.generate_bijvoet_mates()
f_obs.set_observation_type(observation_type)
else:
f_obs = f_obs.convert_to_non_anomalous_if_ratio_pairs_lone_less_than(
threshold=self.parameters.
convert_to_non_anomalous_if_ratio_pairs_lone_less_than_threshold
)
f_obs.set_info(self.raw_data.info())
return f_obs
def run(args=None, params=None, out=sys.stdout):
assert [args, params].count(None) == 1
if args is not None:
if (len(args) == 0) or ("--help" in args):
raise Usage("""
phenix.cc_star model.pdb data.mtz unmerged_data=data.hkl [n_bins=X] [options]
phenix.cc_star model_refine_001.mtz unmerged_data=data.hkl [...]
Implementation of the method for assessing data and model quality described in:
Karplus PA & Diederichs K (2012) Science 336:1030-3.
Full parameters:
%s
""" % master_phil.as_str(prefix=" ", attributes_level=1))
import iotbx.phil
cmdline = iotbx.phil.process_command_line_with_files(
args=args,
master_phil=master_phil,
pdb_file_def="model",
reflection_file_def="data")
params = cmdline.work.extract()
import mmtbx.command_line
import mmtbx.validation.experimental
from iotbx import merging_statistics
from iotbx import file_reader
if (params.data is None):
raise Sorry("Please specify a data file (usually MTZ format).")
if (params.unmerged_data is None):
raise Sorry("Please specify unmerged_data file")
hkl_in = file_reader.any_file(params.data, force_type="hkl")
hkl_in.check_file_type("hkl")
f_model = f_obs = r_free_flags = None
f_models = []
data_arrays = []
f_model_labels = []
if (params.f_model_labels is None):
for array in hkl_in.file_server.miller_arrays:
labels = array.info().label_string()
if (array.is_complex_array()):
if (labels.startswith("F-model")
or labels.startswith("FMODEL")):
f_models.append(array)
f_model_labels.append(labels)
if (len(f_models) > 1):
raise Sorry(
("Multiple F(model) arrays found:\n%s\nPlease specify the " +
"'labels' parameter.") % "\n".join(f_model_labels))
elif (len(f_models) == 1):
f_model = f_models[0]
if (f_model.anomalous_flag()):
info = f_model.info()
f_model = f_model.average_bijvoet_mates().set_info(info)
print("F(model):", file=out)
f_model.show_summary(f=out, prefix=" ")
else:
data_array = hkl_in.file_server.get_xray_data(
file_name=params.data,
labels=params.f_obs_labels,
ignore_all_zeros=True,
parameter_scope="")
if (data_array.is_xray_intensity_array()):
from cctbx import french_wilson
f_obs = french_wilson.french_wilson_scale(
miller_array=data_array, out=out)
else:
f_obs = data_array
else:
for array in hkl_in.file_server.miller_arrays:
array_labels = array.info().label_string()
if (array_labels == params.f_model_labels):
if (array.is_complex_array()):
f_model = array
break
else:
raise Sorry(
"The data in %s are not of the required type." %
array_labels)
if (f_model is not None):
assert (f_obs is None)
for array in hkl_in.file_server.miller_arrays:
labels = array.info().label_string()
if (labels == params.f_obs_labels):
f_obs = array
break
else:
try:
f_obs = hkl_in.file_server.get_amplitudes(
file_name=params.f_obs_labels,
labels=None,
convert_to_amplitudes_if_necessary=False,
parameter_name="f_obs_labels",
parameter_scope="",
strict=True)
except Sorry:
raise Sorry(
"You must supply a file containing both F-obs and F-model "
+ "if you want to use a pre-calculated F-model array.")
assert (f_obs.is_xray_amplitude_array())
if (f_obs.anomalous_flag()):
info = f_obs.info()
f_obs = f_obs.average_bijvoet_mates().set_info(info)
print("F(obs):", file=out)
f_obs.show_summary(f=out, prefix=" ")
print("", file=out)
r_free_flags, test_flag_value = hkl_in.file_server.get_r_free_flags(
file_name=params.data,
label=params.r_free_flags.label,
test_flag_value=params.r_free_flags.test_flag_value,
disable_suitability_test=False,
parameter_scope="")
info = r_free_flags.info()
r_free_flags = r_free_flags.customized_copy(
data=r_free_flags.data() == test_flag_value).set_info(info)
if (r_free_flags.anomalous_flag()):
r_free_flags = r_free_flags.average_bijvoet_mates().set_info(info)
print("R-free flags:", file=out)
r_free_flags.show_summary(f=out, prefix=" ")
print("", file=out)
unmerged_i_obs = mmtbx.command_line.load_and_validate_unmerged_data(
f_obs=f_obs,
file_name=params.unmerged_data,
data_labels=params.unmerged_labels,
log=out)
print("Unmerged intensities:", file=out)
unmerged_i_obs.show_summary(f=out, prefix=" ")
print("", file=out)
if (f_model is None):
assert (f_obs is not None)
if (params.model is None):
raise Sorry(
"A PDB file is required if F(model) is not pre-calculated.")
make_sub_header("Calculating F(model)", out=out)
pdb_in = file_reader.any_file(params.model, force_type="pdb")
pdb_in.check_file_type("pdb")
pdb_symm = pdb_in.file_object.crystal_symmetry()
if (pdb_symm is None):
pdb_symm = f_obs
else:
if (f_obs.crystal_symmetry() is None):
f_obs = f_obs.customized_copy(crystal_symmetry=pdb_symm)
elif (not pdb_symm.is_similar_symmetry(f_obs)):
mmtbx.command_line.show_symmetry_error(file1="PDB file",
file2="data file",
symm1=pdb_symm,
symm2=f_obs)
xray_structure = pdb_in.file_object.xray_structure_simple(
crystal_symmetry=pdb_symm)
from mmtbx.utils import fmodel_simple
# XXX this gets done anyway later, but they need to be consistent before
# creating the fmodel manager
if (f_obs.anomalous_flag()):
f_obs = f_obs.average_bijvoet_mates()
f_obs = f_obs.eliminate_sys_absent()
f_obs, r_free_flags = f_obs.map_to_asu().common_sets(
other=r_free_flags.map_to_asu())
fmodel = fmodel_simple(f_obs=f_obs,
r_free_flags=r_free_flags,
xray_structures=[xray_structure],
skip_twin_detection=True,
scattering_table="n_gaussian")
fmodel.show(log=out)
f_model = fmodel.f_model()
f_obs = fmodel.f_obs()
r_free_flags = fmodel.r_free_flags()
else:
if (f_model.anomalous_flag()):
f_model = f_model.average_bijvoet_mates()
stats = mmtbx.validation.experimental.merging_and_model_statistics(
f_model=f_model,
f_obs=f_obs,
r_free_flags=r_free_flags,
unmerged_i_obs=unmerged_i_obs,
n_bins=params.n_bins,
sigma_filtering=params.sigma_filtering)
stats.show_cc_star(out=out)
if (params.loggraph):
stats.show_loggraph(out=out)
print("", file=out)
print("Reference:", file=out)
print(" Karplus PA & Diederichs K (2012) Science 336:1030-3.", file=out)
print("", file=out)
return stats
def run(args, out=sys.stdout):
from cctbx import french_wilson
from iotbx import file_reader
hkl_file = None
sources = []
interpreter = master_phil.command_line_argument_interpreter()
for arg in args :
if os.path.isfile(arg):
input_file = file_reader.any_file(arg)
if (input_file.file_type == "hkl"):
hkl_file = input_file
sources.append(interpreter.process(arg="file_name=\"%s\"" % arg))
elif (input_file.file_type == "phil"):
sources.append(input_file.file_object)
else :
arg_phil = interpreter.process(arg=arg)
sources.append(arg_phil)
work_phil = master_phil.fetch(sources=sources)
work_params = work_phil.extract()
if (work_params.french_wilson.file_name is None):
if (hkl_file is None):
raise Usage("phenix.french_wilson data.mtz [params.eff] [options ...]")
else :
work_params.french_wilson.file_name = hkl_file.file_name
elif (hkl_file is None):
hkl_file = file_reader.any_file(work_params.french_wilson.file_name)
params = work_params.french_wilson
xray_data_server = hkl_file.file_server
crystal_symmetry = xray_data_server.miller_arrays[0].crystal_symmetry()
if (crystal_symmetry is None):
raise Sorry("No crystal symmetry found. This program requires an input "+
"format with complete symmetry information.")
unit_cell = xray_data_server.miller_arrays[0].unit_cell()
if (unit_cell is None):
raise Sorry("No unit cell found. This program requires an input "+
"format with complete unit cell information.")
i_obs = None
i_obs = xray_data_server.get_xray_data(
file_name = params.file_name,
labels = params.intensity_labels,
ignore_all_zeros = True,
parameter_scope = 'french_wilson',
parameter_name = 'intensity_labels')
import cStringIO
xray_data_server.err = cStringIO.StringIO()
try :
r_free_flags, test_flag_value = xray_data_server.get_r_free_flags(
file_name = params.file_name,
label = params.r_free_flags.label,
test_flag_value = None,
disable_suitability_test = False,
parameter_scope = "french_wilson.r_free_flags")
except Sorry as e :
r_free_flags = None
if (i_obs is None):
raise Sorry("Couldn't find intensities!")
wavelength = params.wavelength
if (wavelength is None):
info = i_obs.info()
if (info is not None):
wavelength = info.wavelength
if (wavelength is not None):
print("Using wavelength=%g from input file" % wavelength, file=out)
sigma_iobs_rejection_criterion = work_params.french_wilson.\
sigma_iobs_rejection_criterion
if (not i_obs.is_unique_set_under_symmetry()):
print("Merging symmetry-equivalent reflections", file=out)
i_obs = i_obs.merge_equivalents().array()
f_obs = french_wilson.french_wilson_scale(miller_array=i_obs,
params=params,
sigma_iobs_rejection_criterion=sigma_iobs_rejection_criterion,
log=out)
if f_obs is None:
raise Sorry("Not enough data to accurately apply the French-Wilson method."+\
" Exiting.")
if params.output_file == None:
output_file = "french_wilson.mtz"
else:
output_file = params.output_file
mtz_dataset = i_obs.as_mtz_dataset(
column_root_label = "I",
wavelength = wavelength)
mtz_dataset.add_miller_array(
miller_array = f_obs,
column_root_label = "F")
if (r_free_flags is not None) and (params.keep_r_free_flags):
mtz_dataset.add_miller_array(
miller_array = r_free_flags,
column_root_label = "R-free-flags")
mtz_object = mtz_dataset.mtz_object()
mtz_object.write(file_name = output_file)
print("Wrote %s" % output_file, file=out)
return output_file
raise Sorry("Couldn't find intensities!")
wavelength = params.wavelength
if (wavelength is None):
info = i_obs.info()
if (info is not None):
wavelength = info.wavelength
if (wavelength is not None):
print >> out, "Using wavelength=%g from input file" % wavelength
sigma_iobs_rejection_criterion = work_params.french_wilson.\
sigma_iobs_rejection_criterion
if (not i_obs.is_unique_set_under_symmetry()):
print >> out, "Merging symmetry-equivalent reflections"
i_obs = i_obs.merge_equivalents().array()
f_obs = french_wilson.french_wilson_scale(
miller_array=i_obs,
params=params,
sigma_iobs_rejection_criterion=sigma_iobs_rejection_criterion,
log=out)
if f_obs is None:
raise Sorry("Not enough data to accurately apply the French-Wilson method."+\
" Exiting.")
if params.output_file == None:
output_file = "french_wilson.mtz"
else:
output_file = params.output_file
mtz_dataset = i_obs.as_mtz_dataset(column_root_label="I",
wavelength=wavelength)
mtz_dataset.add_miller_array(miller_array=f_obs, column_root_label="F")
if (r_free_flags is not None) and (params.keep_r_free_flags):
mtz_dataset.add_miller_array(miller_array=r_free_flags,
column_root_label="R-free-flags")
def __init__ (self,
obs,
r_free_flags,
test_flag_value,
phases=None,
d_min=None,
d_max=None,
r_free_flags_params=None,
merge_anomalous=False,
log=sys.stdout,
verbose=True) :
assert (log is not None) and (obs is not None)
if (r_free_flags_params is None) :
from cctbx.r_free_utils import generate_r_free_params_str
r_free_flags_params = libtbx.phil.parse(
generate_r_free_params_str).extract()
obs_info = obs.info()
r_free_flags_info = phases_info = None
sg = obs.space_group_info()
obs = obs.map_to_asu().merge_equivalents().array()
obs = obs.eliminate_sys_absent(log=log)
obs = obs.resolution_filter(d_min=d_min, d_max=d_max)
if (obs.is_xray_intensity_array()) :
from cctbx import french_wilson
if (verbose) :
fw_out = log
else :
fw_out = null_out()
obs = french_wilson.french_wilson_scale(
miller_array=obs,
params=None,
log=fw_out)
assert (obs is not None)
merged_obs = obs.average_bijvoet_mates()
if (merged_obs.completeness() < 0.9) :
print >> log, """
WARNING: data are incomplete (%.1f%% of possible reflections measured to
%.2fA). This may cause problems if you plan to use the maps for building
and/or ligand fitting!
""" % (100*merged_obs.completeness(), merged_obs.d_min())
# XXX this is kind of a hack (the reconstructed arrays break some of my
# assumptions about labels)
if (merge_anomalous) :
obs = obs.average_bijvoet_mates()
if (r_free_flags is not None) :
r_free_flags_info = r_free_flags.info()
format = "cns"
if (test_flag_value == 0) :
format = "ccp4"
elif (test_flag_value == -1) :
format = "shelx"
if (r_free_flags.anomalous_flag()) :
r_free_flags = r_free_flags.average_bijvoet_mates()
is_compatible_symmetry = False
obs_pg = obs.space_group().build_derived_point_group()
flags_pg = r_free_flags.space_group().build_derived_point_group()
if (obs_pg.type().number() == flags_pg.type().number()) :
is_compatible_symmetry = True
else :
pass # TODO unit cell comparison?
if (is_compatible_symmetry) :
r_free_flags = r_free_flags.map_to_asu().merge_equivalents().array()
r_free_flags = r_free_flags.eliminate_sys_absent(log=log)
if (format == "cns") :
r_free_flags = r_free_flags.customized_copy(
crystal_symmetry=obs.crystal_symmetry(),
data=(r_free_flags.data() == test_flag_value))
test_flag_value = True
obs_tmp = obs.deep_copy()
if (obs.anomalous_flag()) :
obs_tmp = obs.average_bijvoet_mates()
r_free_flags = r_free_flags.common_set(other=obs_tmp)
n_r_free = r_free_flags.indices().size()
n_obs = obs_tmp.indices().size()
if ((test_flag_value is None) or
(r_free_flags.data().all_eq(r_free_flags.data()[0]))) :
print >> log, """
WARNING: uniform R-free flags detected; a new test set will be generated,
but this will bias the refinement statistics.
"""
r_free_flags = None
elif (n_r_free != n_obs) :
missing_set = obs_tmp.lone_set(other=r_free_flags)
n_missing = missing_set.indices().size()
if (n_missing > 0) :
print >> log, """
WARNING: R-free flags are incomplete relative to experimental
data (%d vs. %d reflections). The flags will be extended to
complete the set, but we recommend supplying flags that are already
generated to the maximum expected resolution.
""" % (n_r_free, n_obs)
if (n_missing < 20) : # FIXME
if (format == "cns") :
missing_flags = missing_set.array(data=flex.bool(n_missing,
False))
else :
missing_flags = missing_set.array(data=flex.int(n_missing, 1))
else :
missing_flags = missing_set.generate_r_free_flags(
fraction=(r_free_flags.data().count(test_flag_value)/n_r_free),
max_free=None,
use_lattice_symmetry=True,
format=format)
r_free_flags = r_free_flags.concatenate(other=missing_flags)
if (r_free_flags is not None) :
assert (r_free_flags.indices().size() == obs_tmp.indices().size())
else :
print >> log, """
NOTE: incompatible symmetry between the data and the R-free flags:
Data : %s %s
Flags : %s %s
A new test set will be generated.
""" % (str(obs.space_group_info()),
" ".join([ "%g" % x for x in obs.unit_cell().parameters() ]),
str(r_free_flags.space_group_info()),
" ".join(["%g" % x for x in r_free_flags.unit_cell().parameters()]))
else :
print >> log, """
WARNING: R-free flags not supplied. This may bias the refinement if the
structures are very nearly isomorphous!
"""
self._generate_new = False
if (r_free_flags is None) :
r_free_flags = obs.generate_r_free_flags(
fraction=r_free_flags_params.fraction,
max_free=r_free_flags_params.max_free,
use_lattice_symmetry=r_free_flags_params.use_lattice_symmetry,
use_dataman_shells=r_free_flags_params.use_dataman_shells,
n_shells=r_free_flags_params.n_shells,
format="ccp4")
test_flag_value = 0
self._generate_new = True
if (r_free_flags.anomalous_flag()) :
r_free_flags = r_free_flags.average_bijvoet_mates()
if (phases is not None) :
phases_info = phases.info()
phases = phases.map_to_asu().resolution_filter(d_min=d_min, d_max=d_max)
assert (obs.is_xray_amplitude_array())
self.f_obs = obs.set_info(obs_info)
self.r_free_flags = r_free_flags.set_info(r_free_flags_info)
self.test_flag_value = test_flag_value
self.phases = None
if (phases is not None) :
self.phases = phases.set_info(phases_info)
def __init__(self,
obs,
r_free_flags,
test_flag_value,
phases=None,
d_min=None,
d_max=None,
r_free_flags_params=None,
merge_anomalous=False,
log=sys.stdout,
verbose=True):
assert (log is not None) and (obs is not None)
if (r_free_flags_params is None):
from cctbx.r_free_utils import generate_r_free_params_str
r_free_flags_params = libtbx.phil.parse(
generate_r_free_params_str).extract()
obs_info = obs.info()
r_free_flags_info = phases_info = None
sg = obs.space_group_info()
obs = obs.map_to_asu().merge_equivalents().array()
obs = obs.eliminate_sys_absent(log=log)
obs = obs.resolution_filter(d_min=d_min, d_max=d_max)
if (obs.is_xray_intensity_array()):
from cctbx import french_wilson
if (verbose):
fw_out = log
else :
fw_out = null_out()
obs = french_wilson.french_wilson_scale(
miller_array=obs,
params=None,
log=fw_out)
assert (obs is not None)
merged_obs = obs.average_bijvoet_mates()
if (merged_obs.completeness() < 0.9):
print >> log, """
WARNING: data are incomplete (%.1f%% of possible reflections measured to
%.2fA). This may cause problems if you plan to use the maps for building
and/or ligand fitting!
""" % (100*merged_obs.completeness(), merged_obs.d_min())
# XXX this is kind of a hack (the reconstructed arrays break some of my
# assumptions about labels)
if (merge_anomalous):
obs = obs.average_bijvoet_mates()
if (r_free_flags is not None):
r_free_flags_info = r_free_flags.info()
format = "cns"
if (test_flag_value == 0):
format = "ccp4"
elif (test_flag_value == -1):
format = "shelx"
if (r_free_flags.anomalous_flag()):
r_free_flags = r_free_flags.average_bijvoet_mates()
is_compatible_symmetry = False
obs_pg = obs.space_group().build_derived_point_group()
flags_pg = r_free_flags.space_group().build_derived_point_group()
if (obs_pg.type().number() == flags_pg.type().number()):
is_compatible_symmetry = True
else :
pass # TODO unit cell comparison?
if (is_compatible_symmetry):
r_free_flags = r_free_flags.map_to_asu().merge_equivalents().array()
r_free_flags = r_free_flags.eliminate_sys_absent(log=log)
if (format == "cns"):
r_free_flags = r_free_flags.customized_copy(
crystal_symmetry=obs.crystal_symmetry(),
data=(r_free_flags.data() == test_flag_value))
test_flag_value = True
obs_tmp = obs.deep_copy()
if (obs.anomalous_flag()):
obs_tmp = obs.average_bijvoet_mates()
r_free_flags = r_free_flags.common_set(other=obs_tmp)
n_r_free = r_free_flags.indices().size()
n_obs = obs_tmp.indices().size()
if ((test_flag_value is None) or
(r_free_flags.data().all_eq(r_free_flags.data()[0]))):
print >> log, """
WARNING: uniform R-free flags detected; a new test set will be generated,
but this will bias the refinement statistics.
"""
r_free_flags = None
elif (n_r_free != n_obs):
missing_set = obs_tmp.lone_set(other=r_free_flags)
n_missing = missing_set.indices().size()
if (n_missing > 0):
print >> log, """
WARNING: R-free flags are incomplete relative to experimental
data (%d vs. %d reflections). The flags will be extended to
complete the set, but we recommend supplying flags that are already
generated to the maximum expected resolution.
""" % (n_r_free, n_obs)
if (n_missing < 20) : # FIXME
if (format == "cns"):
missing_flags = missing_set.array(data=flex.bool(n_missing,
False))
else :
missing_flags = missing_set.array(data=flex.int(n_missing, 1))
else :
missing_flags = missing_set.generate_r_free_flags(
fraction=(r_free_flags.data().count(test_flag_value)/n_r_free),
max_free=None,
use_lattice_symmetry=True,
format=format)
r_free_flags = r_free_flags.concatenate(other=missing_flags)
if (r_free_flags is not None):
assert (r_free_flags.indices().size() == obs_tmp.indices().size())
else :
print >> log, """
NOTE: incompatible symmetry between the data and the R-free flags:
Data : %s %s
Flags : %s %s
A new test set will be generated.
""" % (str(obs.space_group_info()),
" ".join([ "%g" % x for x in obs.unit_cell().parameters() ]),
str(r_free_flags.space_group_info()),
" ".join(["%g" % x for x in r_free_flags.unit_cell().parameters()]))
else :
print >> log, """
WARNING: R-free flags not supplied. This may bias the refinement if the
structures are very nearly isomorphous!
"""
self._generate_new = False
if (r_free_flags is None):
r_free_flags = obs.generate_r_free_flags(
fraction=r_free_flags_params.fraction,
max_free=r_free_flags_params.max_free,
use_lattice_symmetry=r_free_flags_params.use_lattice_symmetry,
use_dataman_shells=r_free_flags_params.use_dataman_shells,
n_shells=r_free_flags_params.n_shells,
format="ccp4")
test_flag_value = 0
self._generate_new = True
if (r_free_flags.anomalous_flag()):
r_free_flags = r_free_flags.average_bijvoet_mates()
if (phases is not None):
phases_info = phases.info()
phases = phases.map_to_asu().resolution_filter(d_min=d_min, d_max=d_max)
assert (obs.is_xray_amplitude_array())
self.f_obs = obs.set_info(obs_info)
self.r_free_flags = r_free_flags.set_info(r_free_flags_info)
self.test_flag_value = test_flag_value
self.phases = None
if (phases is not None):
self.phases = phases.set_info(phases_info)
def run (args, out=sys.stdout) :
from iotbx import file_reader
import iotbx.phil
if (len(args) == 0) :
raise Usage("""\
iotbx.simple_map_coefficients data_phases.mtz [options]
Full parameters:
%s""" % iotbx.phil.parse(master_phil).as_str(attributes_level=1, prefix=" "))
cmdline = iotbx.phil.process_command_line_with_files(
args=args,
master_phil_string=master_phil,
reflection_file_def="file_name")
params = cmdline.work.extract()
if (params.file_name is None) :
raise Sorry("No reflection file specified.")
hkl_in = file_reader.any_file(params.file_name).check_file_type("hkl")
hkl_server = hkl_in.file_server
data = hkl_server.get_xray_data(
file_name=params.file_name,
labels=params.data_labels,
ignore_all_zeros=False,
parameter_name="data_labels",
parameter_scope="",
prefer_anomalous=True,
prefer_amplitudes=True)
data_labels = data.info().label_string()
if (data.is_xray_intensity_array()) :
from cctbx.french_wilson import french_wilson_scale
data = french_wilson_scale(data, log=out)
phases = hkl_server.get_phases_deg(
file_name=params.file_name,
labels=params.phase_labels,
convert_to_phases_if_necessary=True,
original_phase_units=None,
parameter_scope="",
parameter_name="phase_labels",
minimum_score=2)
assert (not phases.anomalous_flag())
deg = True # FIXME
weights = None
if (params.use_weights in [Auto, True]) :
# FIXME centralize this in iotbx.reflection_file_utils
for array in hkl_server.miller_arrays :
if (array.is_real_array()) :
label_string = array.info().label_string()
if ((label_string == params.weight_labels) or
((params.weight_labels is None) and ("FOM" in label_string))) :
weights = array
break
amplitudes = data
if (params.map_type == "anom") :
if (not data.anomalous_flag()) :
raise Sorry("Anomalous map requested, but selected data are merged.")
amplitudes = data.anomalous_differences()
print >> out, "Using anomalous differences in %s" % data_labels
else :
print >> out, "Using amplitudes in %s" % data_labels
if (data.anomalous_flag()) :
amplitudes = data.average_bijvoet_mates()
if (params.use_weights is Auto) and (weights is not None) :
if (params.map_type != "anom") :
params.use_weights = True
elif (params.use_weights == True) and (weights is None) :
raise Sorry("No weights (FOM, etc.) found in input file.")
if (params.use_weights == True) :
assert (not weights.anomalous_flag())
print >> out, "Applying weights in %s" % weights.info().label_string()
amplitudes, weights = amplitudes.common_sets(other=weights)
amplitudes = amplitudes.customized_copy(
data=amplitudes.data()*weights.data())
amplitudes = amplitudes.customized_copy(sigmas=None)
print >> out, "Applying phases in %s" % phases.info().label_string()
amplitudes, phases = amplitudes.common_sets(phases)
coeffs = amplitudes.phase_transfer(phases,
deg=deg).set_observation_type(None) # FIXME
if (params.map_type == "anom") : # apply 90-degree phase shift
coeffs = coeffs.customized_copy(data=coeffs.data()/(2j))
assert (coeffs.is_complex_array())
column_root_label = "F"
decorator = None
if (params.map_type == "anom") :
column_root_label = "ANOM"
elif (params.use_weights == True) :
column_root_label = "FWT"
decorator = iotbx.mtz.ccp4_label_decorator()
import iotbx.mtz
mtz_dataset = coeffs.as_mtz_dataset(
column_root_label=column_root_label,
label_decorator=decorator)
if (params.output_file is None) :
params.output_file = "map_coeffs.mtz"
mtz_dataset.mtz_object().write(params.output_file)
print >> out, "Wrote %s" % params.output_file
return os.path.abspath(params.output_file)
def run (args=None, params=None, out=sys.stdout) :
assert [args, params].count(None) == 1
if args is not None:
if (len(args) == 0) or ("--help" in args) :
raise Usage("""
phenix.cc_star model.pdb data.mtz unmerged_data=data.hkl [n_bins=X] [options]
phenix.cc_star model_refine_001.mtz unmerged_data=data.hkl [...]
Implementation of the method for assessing data and model quality described in:
Karplus PA & Diederichs K (2012) Science 336:1030-3.
Full parameters:
%s
""" % master_phil.as_str(prefix=" ", attributes_level=1))
import iotbx.phil
cmdline = iotbx.phil.process_command_line_with_files(
args=args,
master_phil=master_phil,
pdb_file_def="model",
reflection_file_def="data")
params = cmdline.work.extract()
import mmtbx.command_line
import mmtbx.validation.experimental
from iotbx import merging_statistics
from iotbx import file_reader
if (params.data is None) :
raise Sorry("Please specify a data file (usually MTZ format).")
if (params.unmerged_data is None) :
raise Sorry("Please specify unmerged_data file")
hkl_in = file_reader.any_file(params.data, force_type="hkl")
hkl_in.check_file_type("hkl")
f_model = f_obs = r_free_flags = None
f_models = []
data_arrays = []
f_model_labels = []
if (params.f_model_labels is None) :
for array in hkl_in.file_server.miller_arrays :
labels = array.info().label_string()
if (array.is_complex_array()) :
if (labels.startswith("F-model") or labels.startswith("FMODEL")) :
f_models.append(array)
f_model_labels.append(labels)
if (len(f_models) > 1) :
raise Sorry(("Multiple F(model) arrays found:\n%s\nPlease specify the "+
"'labels' parameter.") % "\n".join(f_model_labels))
elif (len(f_models) == 1) :
f_model = f_models[0]
if (f_model.anomalous_flag()) :
info = f_model.info()
f_model = f_model.average_bijvoet_mates().set_info(info)
print >> out, "F(model):"
f_model.show_summary(f=out, prefix=" ")
else :
data_array = hkl_in.file_server.get_xray_data(
file_name=params.data,
labels=params.f_obs_labels,
ignore_all_zeros=True,
parameter_scope="")
if (data_array.is_xray_intensity_array()) :
from cctbx import french_wilson
f_obs = french_wilson.french_wilson_scale(
miller_array=data_array,
out=out)
else :
f_obs = data_array
else :
for array in hkl_in.file_server.miller_arrays :
array_labels = array.info().label_string()
if (array_labels == params.f_model_labels) :
if (array.is_complex_array()) :
f_model = array
break
else :
raise Sorry("The data in %s are not of the required type." %
array_labels)
if (f_model is not None) :
assert (f_obs is None)
for array in hkl_in.file_server.miller_arrays :
labels = array.info().label_string()
if (labels == params.f_obs_labels) :
f_obs = array
break
else :
try :
f_obs = hkl_in.file_server.get_amplitudes(
file_name=params.f_obs_labels,
labels=None,
convert_to_amplitudes_if_necessary=False,
parameter_name="f_obs_labels",
parameter_scope="",
strict=True)
except Sorry :
raise Sorry("You must supply a file containing both F-obs and F-model "+
"if you want to use a pre-calculated F-model array.")
assert (f_obs.is_xray_amplitude_array())
if (f_obs.anomalous_flag()) :
info = f_obs.info()
f_obs = f_obs.average_bijvoet_mates().set_info(info)
print >> out, "F(obs):"
f_obs.show_summary(f=out, prefix=" ")
print >> out, ""
r_free_flags, test_flag_value = hkl_in.file_server.get_r_free_flags(
file_name=params.data,
label=params.r_free_flags.label,
test_flag_value=params.r_free_flags.test_flag_value,
disable_suitability_test=False,
parameter_scope="")
info = r_free_flags.info()
r_free_flags = r_free_flags.customized_copy(
data=r_free_flags.data()==test_flag_value).set_info(info)
if (r_free_flags.anomalous_flag()) :
r_free_flags = r_free_flags.average_bijvoet_mates().set_info(info)
print >> out, "R-free flags:"
r_free_flags.show_summary(f=out, prefix=" ")
print >> out, ""
unmerged_i_obs = mmtbx.command_line.load_and_validate_unmerged_data(
f_obs=f_obs,
file_name=params.unmerged_data,
data_labels=params.unmerged_labels,
log=out)
print >> out, "Unmerged intensities:"
unmerged_i_obs.show_summary(f=out, prefix=" ")
print >> out, ""
if (f_model is None) :
assert (f_obs is not None)
if (params.model is None) :
raise Sorry("A PDB file is required if F(model) is not pre-calculated.")
make_sub_header("Calculating F(model)", out=out)
pdb_in = file_reader.any_file(params.model, force_type="pdb")
pdb_in.check_file_type("pdb")
pdb_symm = pdb_in.file_object.crystal_symmetry()
if (pdb_symm is None) :
pdb_symm = f_obs
else :
if (f_obs.crystal_symmetry() is None) :
f_obs = f_obs.customized_copy(crystal_symmetry=pdb_symm)
elif (not pdb_symm.is_similar_symmetry(f_obs)) :
mmtbx.command_line.show_symmetry_error(
file1="PDB file",
file2="data file",
symm1=pdb_symm,
symm2=f_obs)
xray_structure = pdb_in.file_object.xray_structure_simple(
crystal_symmetry=pdb_symm)
from mmtbx.utils import fmodel_simple
# XXX this gets done anyway later, but they need to be consistent before
# creating the fmodel manager
if (f_obs.anomalous_flag()) :
f_obs = f_obs.average_bijvoet_mates()
f_obs = f_obs.eliminate_sys_absent()
f_obs, r_free_flags = f_obs.map_to_asu().common_sets(
other=r_free_flags.map_to_asu())
fmodel = fmodel_simple(
f_obs=f_obs,
r_free_flags=r_free_flags,
xray_structures=[xray_structure],
skip_twin_detection=True,
scattering_table="n_gaussian")
fmodel.show(log=out)
f_model = fmodel.f_model()
r_free_flags = f_model.customized_copy(data=fmodel.arrays.free_sel)
else :
if (f_model.anomalous_flag()) :
f_model = f_model.average_bijvoet_mates()
f_model, r_free_flags = f_model.common_sets(other=r_free_flags)
stats = mmtbx.validation.experimental.merging_and_model_statistics(
f_model=f_model,
f_obs=f_obs,
r_free_flags=r_free_flags,
unmerged_i_obs=unmerged_i_obs,
n_bins=params.n_bins,
sigma_filtering=params.sigma_filtering)
stats.show_cc_star(out=out)
if (params.loggraph) :
stats.show_loggraph(out=out)
print >> out, ""
print >> out, "Reference:"
print >> out, " Karplus PA & Diederichs K (2012) Science 336:1030-3."
print >> out, ""
return stats
if (i_obs is None) :
raise Sorry("Couldn't find intensities!")
wavelength = params.wavelength
if (wavelength is None) :
info = i_obs.info()
if (info is not None) :
wavelength = info.wavelength
if (wavelength is not None) :
print >> out, "Using wavelength=%g from input file" % wavelength
sigma_iobs_rejection_criterion = work_params.french_wilson.\
sigma_iobs_rejection_criterion
if (not i_obs.is_unique_set_under_symmetry()) :
print >> out, "Merging symmetry-equivalent reflections"
i_obs = i_obs.merge_equivalents().array()
f_obs = french_wilson.french_wilson_scale(miller_array=i_obs,
params=params,
sigma_iobs_rejection_criterion=sigma_iobs_rejection_criterion,
log=out)
if f_obs is None:
raise Sorry("Not enough data to accurately apply the French-Wilson method."+\
" Exiting.")
if params.output_file == None:
output_file = "french_wilson.mtz"
else:
output_file = params.output_file
mtz_dataset = i_obs.as_mtz_dataset(
column_root_label = "I",
wavelength = wavelength)
mtz_dataset.add_miller_array(
miller_array = f_obs,
column_root_label = "F")
if (r_free_flags is not None) and (params.keep_r_free_flags):
def run(args, out=sys.stdout):
from iotbx import file_reader
import iotbx.phil
if (len(args) == 0):
raise Usage("""\
iotbx.simple_map_coefficients data_phases.mtz [options]
Full parameters:
%s""" % iotbx.phil.parse(master_phil).as_str(attributes_level=1, prefix=" "))
cmdline = iotbx.phil.process_command_line_with_files(
args=args,
master_phil_string=master_phil,
reflection_file_def="file_name")
params = cmdline.work.extract()
if (params.file_name is None):
raise Sorry("No reflection file specified.")
hkl_in = file_reader.any_file(params.file_name).check_file_type("hkl")
hkl_server = hkl_in.file_server
data = hkl_server.get_xray_data(file_name=params.file_name,
labels=params.data_labels,
ignore_all_zeros=False,
parameter_name="data_labels",
parameter_scope="",
prefer_anomalous=True,
prefer_amplitudes=True)
data_labels = data.info().label_string()
if (data.is_xray_intensity_array()):
from cctbx.french_wilson import french_wilson_scale
data = french_wilson_scale(data, log=out)
phases = hkl_server.get_phases_deg(file_name=params.file_name,
labels=params.phase_labels,
convert_to_phases_if_necessary=True,
original_phase_units=None,
parameter_scope="",
parameter_name="phase_labels",
minimum_score=2)
assert (not phases.anomalous_flag())
deg = True # FIXME
weights = None
if (params.use_weights in [Auto, True]):
# FIXME centralize this in iotbx.reflection_file_utils
for array in hkl_server.miller_arrays:
if (array.is_real_array()):
label_string = array.info().label_string()
if ((label_string == params.weight_labels)
or ((params.weight_labels is None) and
("FOM" in label_string))):
weights = array
break
amplitudes = data
if (params.map_type == "anom"):
if (not data.anomalous_flag()):
raise Sorry(
"Anomalous map requested, but selected data are merged.")
amplitudes = data.anomalous_differences()
print >> out, "Using anomalous differences in %s" % data_labels
else:
print >> out, "Using amplitudes in %s" % data_labels
if (data.anomalous_flag()):
amplitudes = data.average_bijvoet_mates()
if (params.use_weights is Auto) and (weights is not None):
if (params.map_type != "anom"):
params.use_weights = True
elif (params.use_weights == True) and (weights is None):
raise Sorry("No weights (FOM, etc.) found in input file.")
if (params.use_weights == True):
assert (not weights.anomalous_flag())
print >> out, "Applying weights in %s" % weights.info().label_string()
amplitudes, weights = amplitudes.common_sets(other=weights)
amplitudes = amplitudes.customized_copy(data=amplitudes.data() *
weights.data())
amplitudes = amplitudes.customized_copy(sigmas=None)
print >> out, "Applying phases in %s" % phases.info().label_string()
amplitudes, phases = amplitudes.common_sets(phases)
coeffs = amplitudes.phase_transfer(phases, deg=deg).set_observation_type(
None) # FIXME
if (params.map_type == "anom"): # apply 90-degree phase shift
coeffs = coeffs.customized_copy(data=coeffs.data() / (2j))
assert (coeffs.is_complex_array())
column_root_label = "F"
decorator = None
if (params.map_type == "anom"):
column_root_label = "ANOM"
elif (params.use_weights == True):
column_root_label = "FWT"
decorator = iotbx.mtz.ccp4_label_decorator()
import iotbx.mtz
mtz_dataset = coeffs.as_mtz_dataset(column_root_label=column_root_label,
label_decorator=decorator)
if (params.output_file is None):
params.output_file = "map_coeffs.mtz"
mtz_dataset.mtz_object().write(params.output_file)
print >> out, "Wrote %s" % params.output_file
return os.path.abspath(params.output_file)