def run(processed_args, params):
if (params.scattering_table
not in ["n_gaussian", "wk1995", "it1992", "neutron"]):
raise Sorry("Incorrect scattering_table.")
crystal_symmetry = None
crystal_symmetries = []
for f in [str(params.pdb_file_name), str(params.reflection_file_name)]:
cs = crystal_symmetry_from_any.extract_from(f)
if (cs is not None): crystal_symmetries.append(cs)
if (len(crystal_symmetries) == 1): crystal_symmetry = crystal_symmetries[0]
elif (len(crystal_symmetries) == 0):
raise Sorry("No crystal symmetry found.")
else:
if (not crystal_symmetries[0].is_similar_symmetry(
crystal_symmetries[1])):
raise Sorry("Crystal symmetry mismatch between different files.")
crystal_symmetry = crystal_symmetries[0]
f_obs = None
r_free_flags = None
if (params.reflection_file_name is not None):
reflection_file = reflection_file_reader.any_reflection_file(
file_name=params.reflection_file_name, ensure_read_access=True)
rfs = reflection_file_server(crystal_symmetry=crystal_symmetry,
reflection_files=[reflection_file])
parameters = utils.data_and_flags_master_params().extract()
if (params.data_labels is not None):
parameters.labels = [processed_args.data_labels]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server=rfs,
parameters=parameters,
keep_going=True,
log=StringIO())
f_obs = determine_data_and_flags_result.f_obs
r_free_flags = determine_data_and_flags_result.r_free_flags
if (r_free_flags is None):
r_free_flags = f_obs.array(
data=flex.bool(f_obs.data().size(), False))
test_flag_value = None
xray_structure = iotbx.pdb.input(
file_name=params.pdb_file_name).xray_structure_simple()
xray_structure_da = None
if (params.external_da_pdb_file_name is not None):
xray_structure_da = iotbx.pdb.input(
file_name=params.external_da_pdb_file_name).xray_structure_simple(
)
if (f_obs is not None):
f_obs = f_obs.resolution_filter(d_min=params.high_resolution,
d_max=params.low_resolution)
r_free_flags = r_free_flags.resolution_filter(
d_min=params.high_resolution, d_max=params.low_resolution)
#
assert params.mode in ["build", "build_and_refine"]
grow_density(f_obs=f_obs,
r_free_flags=r_free_flags,
xray_structure=xray_structure,
xray_structure_da=xray_structure_da,
params=params)
def run(processed_args, params):
if(params.scattering_table not in ["n_gaussian","wk1995",
"it1992","neutron"]):
raise Sorry("Incorrect scattering_table.")
crystal_symmetry = None
crystal_symmetries = []
for f in [str(params.pdb_file_name), str(params.reflection_file_name)]:
cs = crystal_symmetry_from_any.extract_from(f)
if(cs is not None): crystal_symmetries.append(cs)
if(len(crystal_symmetries) == 1): crystal_symmetry = crystal_symmetries[0]
elif(len(crystal_symmetries) == 0):
raise Sorry("No crystal symmetry found.")
else:
if(not crystal_symmetries[0].is_similar_symmetry(crystal_symmetries[1])):
raise Sorry("Crystal symmetry mismatch between different files.")
crystal_symmetry = crystal_symmetries[0]
f_obs = None
r_free_flags = None
if(params.reflection_file_name is not None):
reflection_file = reflection_file_reader.any_reflection_file(
file_name = params.reflection_file_name, ensure_read_access = True)
rfs = reflection_file_server(
crystal_symmetry = crystal_symmetry,
reflection_files = [reflection_file])
parameters = utils.data_and_flags_master_params().extract()
if(params.data_labels is not None):
parameters.labels = [processed_args.data_labels]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server = rfs,
parameters = parameters,
keep_going = True,
log = StringIO())
f_obs = determine_data_and_flags_result.f_obs
r_free_flags = determine_data_and_flags_result.r_free_flags
if(r_free_flags is None):
r_free_flags=f_obs.array(data=flex.bool(f_obs.data().size(), False))
test_flag_value=None
xray_structure = iotbx.pdb.input(file_name =
params.pdb_file_name).xray_structure_simple()
xray_structure_da = None
if(params.external_da_pdb_file_name is not None):
xray_structure_da = iotbx.pdb.input(file_name =
params.external_da_pdb_file_name).xray_structure_simple()
if(f_obs is not None):
f_obs = f_obs.resolution_filter(d_min = params.high_resolution,
d_max = params.low_resolution)
r_free_flags = r_free_flags.resolution_filter(d_min = params.high_resolution,
d_max = params.low_resolution)
#
assert params.mode in ["build", "build_and_refine"]
grow_density(f_obs = f_obs,
r_free_flags = r_free_flags,
xray_structure = xray_structure,
xray_structure_da = xray_structure_da,
params = params)
def _get_f_obs_r_free(self):
""" Get f_obs and r_free_flags from pdb and mtz via self.inputs
Returns
-------
f_obs: cctbx.miller.array
observed data
r_free_flags: cctbx.miller.array
r free flags
"""
rfs = reflection_file_utils.reflection_file_server(
crystal_symmetry=self.inputs.crystal_symmetry,
force_symmetry=True,
reflection_files=self.inputs.reflection_files,
err=StringIO(),
)
# Parameter object descrinign the may to process data
# From mmtbx.utils
data_flags_params = data_and_flags_master_params().extract()
data_flags_params.labels = self.params.exhaustive.options.column_type
determined_data_and_flags = mmtbx.utils.determine_data_and_flags(
reflection_file_server=rfs,
parameters=data_flags_params,
keep_going=True,
log=StringIO(),
)
# Extract the need parts of determined_data_and_flags
f_obs = determined_data_and_flags.f_obs
r_free_flags = determined_data_and_flags.r_free_flags
return f_obs, r_free_flags
def run(args,
command_name = "mmtbx.model_vs_data",
show_geometry_statistics = True,
model_size_max_atoms = 80000,
data_size_max_reflections= 1000000,
unit_cell_max_dimension = 800.,
return_fmodel_and_pdb = False,
out = None,
log = sys.stdout):
import mmtbx.f_model_info
if(len(args)==0) or (args == ["--help"]) :
print >> log, msg
defaults(log=log, silent=False)
return
parsed = defaults(log=log, silent=True)
#
mvd_obj = mvd()
#
processed_args = utils.process_command_line_args(args = args,
log = log, master_params = parsed)
params = processed_args.params.extract()
#
reflection_files = processed_args.reflection_files
if(len(reflection_files) == 0):
raise Sorry("No reflection file found.")
crystal_symmetry = processed_args.crystal_symmetry
if(crystal_symmetry is None):
raise Sorry("No crystal symmetry found.")
if(len(processed_args.pdb_file_names) == 0):
raise Sorry("No PDB file found.")
pdb_file_names = processed_args.pdb_file_names
#
rfs = reflection_file_server(
crystal_symmetry = crystal_symmetry,
reflection_files = reflection_files)
parameters = utils.data_and_flags_master_params().extract()
if(params.f_obs_label is not None):
parameters.labels = params.f_obs_label
if(params.r_free_flags_label is not None):
parameters.r_free_flags.label = params.r_free_flags_label
if (params.high_resolution is not None) :
parameters.high_resolution = params.high_resolution
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server = rfs,
parameters = parameters,
data_parameter_scope = "refinement.input.xray_data",
flags_parameter_scope = "refinement.input.xray_data.r_free_flags",
data_description = "X-ray data",
keep_going = True,
log = StringIO())
f_obs = determine_data_and_flags_result.f_obs
number_of_reflections = f_obs.indices().size()
if(params.ignore_giant_models_and_datasets and
number_of_reflections > data_size_max_reflections):
raise Sorry("Too many reflections: %d"%number_of_reflections)
#
max_unit_cell_dimension = max(f_obs.unit_cell().parameters()[:3])
if(params.ignore_giant_models_and_datasets and
max_unit_cell_dimension > unit_cell_max_dimension):
raise Sorry("Too large unit cell (max dimension): %s"%
str(max_unit_cell_dimension))
#
r_free_flags = determine_data_and_flags_result.r_free_flags
test_flag_value = determine_data_and_flags_result.test_flag_value
if(r_free_flags is None):
r_free_flags=f_obs.array(data=flex.bool(f_obs.data().size(), False))
test_flag_value=None
#
mmtbx_pdb_file = mmtbx.utils.pdb_file(
pdb_file_names = pdb_file_names,
cif_objects = processed_args.cif_objects,
crystal_symmetry = crystal_symmetry,
use_neutron_distances = (params.scattering_table=="neutron"),
ignore_unknown_nonbonded_energy_types = not show_geometry_statistics,
log = log)
mmtbx_pdb_file.set_ppf(stop_if_duplicate_labels = False)
processed_pdb_file = mmtbx_pdb_file.processed_pdb_file
pdb_raw_records = mmtbx_pdb_file.pdb_raw_records
pdb_inp = mmtbx_pdb_file.pdb_inp
#
# just to avoid going any further with bad PDB file....
pdb_inp.xray_structures_simple()
#
acp = processed_pdb_file.all_chain_proxies
atom_selections = group_args(
all = acp.selection(string = "all"),
macromolecule = acp.selection(string = "protein or dna or rna"),
solvent = acp.selection(string = "water"), # XXX single_atom_residue
ligand = acp.selection(string = "not (protein or dna or rna or water)"),
backbone = acp.selection(string = "backbone"),
sidechain = acp.selection(string = "sidechain"))
#
scattering_table = params.scattering_table
exptl_method = pdb_inp.get_experiment_type()
if (exptl_method is not None) and ("NEUTRON" in exptl_method) :
scattering_table = "neutron"
xsfppf = mmtbx.utils.xray_structures_from_processed_pdb_file(
processed_pdb_file = processed_pdb_file,
scattering_table = scattering_table,
d_min = f_obs.d_min())
xray_structures = xsfppf.xray_structures
if(0): #XXX normalize occupancies if all models have occ=1 so the total=1
n_models = len(xray_structures)
for xrs in xray_structures:
occ = xrs.scatterers().extract_occupancies()
occ = occ/n_models
xrs.set_occupancies(occ)
model_selections = xsfppf.model_selections
mvd_obj.collect(crystal = group_args(
uc = f_obs.unit_cell(),
sg = f_obs.crystal_symmetry().space_group_info().symbol_and_number(),
n_sym_op = f_obs.crystal_symmetry().space_group_info().type().group().order_z(),
uc_vol = f_obs.unit_cell().volume()))
#
hierarchy = pdb_inp.construct_hierarchy()
pdb_atoms = hierarchy.atoms()
pdb_atoms.reset_i_seq()
#
# Extract TLS
pdb_tls = None
pdb_inp_tls = pdb_inp.extract_tls_params(hierarchy)
pdb_tls = group_args(pdb_inp_tls = pdb_inp_tls,
tls_selections = [],
tls_selection_strings = [])
# XXX no TLS + multiple models
if(pdb_inp_tls.tls_present and pdb_inp_tls.error_string is None and
len(xray_structures)==1):
pdb_tls = mmtbx.tls.tools.extract_tls_from_pdb(
pdb_inp_tls = pdb_inp_tls,
all_chain_proxies = mmtbx_pdb_file.processed_pdb_file.all_chain_proxies,
xray_structure = xsfppf.xray_structure_all)
if(len(pdb_tls.tls_selections)==len(pdb_inp_tls.tls_params) and
len(pdb_inp_tls.tls_params) > 0):
xray_structures = [utils.extract_tls_and_u_total_from_pdb(
f_obs = f_obs,
r_free_flags = r_free_flags,
xray_structure = xray_structures[0], # XXX no TLS + multiple models
tls_selections = pdb_tls.tls_selections,
tls_groups = pdb_inp_tls.tls_params)]
###########################
geometry_statistics = show_geometry(
xray_structures = xray_structures,
processed_pdb_file = processed_pdb_file,
scattering_table = scattering_table,
hierarchy = hierarchy,
model_selections = model_selections,
show_geometry_statistics = show_geometry_statistics,
mvd_obj = mvd_obj,
atom_selections = atom_selections)
###########################
mp = mmtbx.masks.mask_master_params.extract()
f_obs_labels = f_obs.info().label_string()
f_obs = f_obs.sort(reverse=True, by_value="packed_indices")
r_free_flags = r_free_flags.sort(reverse=True, by_value="packed_indices")
fmodel = utils.fmodel_simple(
xray_structures = xray_structures,
scattering_table = scattering_table,
mask_params = mp,
f_obs = f_obs,
r_free_flags = r_free_flags,
skip_twin_detection = params.skip_twin_detection)
n_outl = f_obs.data().size() - fmodel.f_obs().data().size()
mvd_obj.collect(model_vs_data = show_model_vs_data(fmodel))
# Extract information from PDB file header and output (if any)
pub_r_work = None
pub_r_free = None
pub_high = None
pub_low = None
pub_sigma = None
pub_program_name = None
pub_solv_cont = None
pub_matthews = None
published_results = pdb_inp.get_r_rfree_sigma(file_name=pdb_file_names[0])
if(published_results is not None):
pub_r_work = published_results.r_work
pub_r_free = published_results.r_free
pub_high = published_results.high
pub_low = published_results.low
pub_sigma = published_results.sigma
pub_program_name = pdb_inp.get_program_name()
pub_solv_cont = pdb_inp.get_solvent_content()
pub_matthews = pdb_inp.get_matthews_coeff()
mvd_obj.collect(pdb_header = group_args(
program_name = pub_program_name,
year = pdb_inp.extract_header_year(),
r_work = pub_r_work,
r_free = pub_r_free,
high_resolution = pub_high,
low_resolution = pub_low,
sigma_cutoff = pub_sigma,
matthews_coeff = pub_matthews,
solvent_cont = pub_solv_cont,
tls = pdb_tls,
exptl_method = exptl_method))
#
# Recompute R-factors using published cutoffs
fmodel_cut = fmodel
tmp_sel = flex.bool(fmodel.f_obs().data().size(), True)
if(pub_sigma is not None and fmodel.f_obs().sigmas() is not None):
tmp_sel &= fmodel.f_obs().data() > fmodel.f_obs().sigmas()*pub_sigma
if(pub_high is not None and abs(pub_high-fmodel.f_obs().d_min()) > 0.03):
tmp_sel &= fmodel.f_obs().d_spacings().data() > pub_high
if(pub_low is not None and abs(pub_low-fmodel.f_obs().d_max_min()[0]) > 0.03):
tmp_sel &= fmodel.f_obs().d_spacings().data() < pub_low
if(tmp_sel.count(True) != tmp_sel.size() and tmp_sel.count(True) > 0):
fmodel_cut = utils.fmodel_simple(
xray_structures = xray_structures,
scattering_table = scattering_table,
f_obs = fmodel.f_obs().select(tmp_sel),
r_free_flags = fmodel.r_free_flags().select(tmp_sel),
skip_twin_detection = params.skip_twin_detection)
mvd_obj.collect(misc = group_args(
r_work_cutoff = fmodel_cut.r_work(),
r_free_cutoff = fmodel_cut.r_free(),
n_refl_cutoff = fmodel_cut.f_obs().data().size()))
mvd_obj.collect(data =
show_data(fmodel = fmodel,
n_outl = n_outl,
test_flag_value = test_flag_value,
f_obs_labels = f_obs_labels,
fmodel_cut = fmodel_cut))
# map statistics
if(len(xray_structures)==1): # XXX no multi-model support yet
mvd_obj.collect(maps = maps(fmodel = fmodel, mvd_obj = mvd_obj))
# CC* and friends
cc_star_stats = None
if (params.unmerged_data is not None) :
import mmtbx.validation.experimental
import mmtbx.command_line
f_obs = fmodel.f_obs().average_bijvoet_mates()
unmerged_i_obs = mmtbx.command_line.load_and_validate_unmerged_data(
f_obs=f_obs,
file_name=params.unmerged_data,
data_labels=params.unmerged_labels,
log=null_out())
cc_star_stats = mmtbx.validation.experimental.merging_and_model_statistics(
f_model=fmodel.f_model().average_bijvoet_mates(),
f_obs=f_obs,
r_free_flags=fmodel.r_free_flags().average_bijvoet_mates(),
unmerged_i_obs=unmerged_i_obs,
n_bins=params.n_bins)
mvd_obj.show(log=out)
if (cc_star_stats is not None) :
cc_star_stats.show_model_vs_data(out=out, prefix=" ")
if return_fmodel_and_pdb :
mvd_obj.pdb_file = processed_pdb_file
mvd_obj.fmodel = fmodel
if(len(params.map) > 0):
for map_name_string in params.map:
map_type_obj = mmtbx.map_names(map_name_string = map_name_string)
map_params = mmtbx.maps.map_and_map_coeff_master_params().fetch(
mmtbx.maps.cast_map_coeff_params(map_type_obj)).extract()
maps_obj = mmtbx.maps.compute_map_coefficients(fmodel = fmodel_cut, params =
map_params.map_coefficients)
fn = os.path.basename(processed_args.reflection_file_names[0])
if(fn.count(".")):
prefix = fn[:fn.index(".")]
else: prefix= fn
file_name = prefix+"_%s_map_coeffs.mtz"%map_type_obj.format()
maps_obj.write_mtz_file(file_name = file_name)
# statistics in bins
if(not fmodel.twin):
print >> log, "Statistics in resolution bins:"
mmtbx.f_model_info.r_work_and_completeness_in_resolution_bins(
fmodel = fmodel, out = log, prefix=" ")
# report map cc
if(params.comprehensive and not fmodel_cut.twin and
fmodel_cut.xray_structure is not None):
rsc_params = real_space_correlation.master_params().extract()
rsc_params.scattering_table = scattering_table
real_space_correlation.simple(
fmodel = fmodel_cut,
pdb_hierarchy = hierarchy,
params = rsc_params,
log = log,
show_results = True)
#
if(params.dump_result_object_as_pickle):
output_prefixes = []
for op in processed_args.pdb_file_names+processed_args.reflection_file_names:
op = os.path.basename(op)
try: op = op[:op.index(".")]
except Exception: pass
if(not op in output_prefixes): output_prefixes.append(op)
output_prefix = "_".join(output_prefixes)
easy_pickle.dump("%s.pickle"%output_prefix, mvd_obj)
return mvd_obj
def run(args, command_name="phenix.twin_map_utils"):
log = sys.stdout
params = None
if (len(args) == 0 or "--help" in args or "--h" in args or "-h" in args):
print_help(command_name=command_name)
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="map_coefs")
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
if not arg_is_processed:
print("##----------------------------------------------##",
file=log)
print("## Unknown file or keyword:", arg, file=log)
print("##----------------------------------------------##",
file=log)
print(file=log)
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
"""
new_params = master_params.format(python_object=params)
new_params.show(out=log)
"""
# now get the unit cell from the pdb file
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.twin_utils.input.xray_data.file_name)
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.twin_utils.input.model.file_name)
phil_xs = None
if ([
params.twin_utils.input.unit_cell,
params.twin_utils.input.space_group
]).count(None) < 2:
phil_xs = crystal.symmetry(
unit_cell=params.twin_utils.input.unit_cell,
space_group_info=params.twin_utils.input.space_group)
combined_xs = crystal.select_crystal_symmetry(None, phil_xs, [pdb_xs],
[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.twin_utils.input.unit_cell = combined_xs.unit_cell()
params.twin_utils.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
new_params = master_params.format(python_object=params)
new_params.show(out=log)
if params.twin_utils.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.twin_utils.input.space_group is None:
raise Sorry("space group not specified")
if params.twin_utils.input.xray_data.file_name is None:
raise Sorry("Xray data not specified")
if params.twin_utils.input.model.file_name is None:
raise Sorry("pdb file with model not specified")
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
phil_xs = crystal.symmetry(
unit_cell=params.twin_utils.input.unit_cell,
space_group_info=params.twin_utils.input.space_group)
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=phil_xs, force_symmetry=True, reflection_files=[])
miller_array = None
free_flags = None
tmp_params = utils.data_and_flags_master_params().extract()
# insert proper values please
tmp_params.file_name = params.twin_utils.input.xray_data.file_name
tmp_params.labels = params.twin_utils.input.xray_data.obs_labels
tmp_params.r_free_flags.file_name = params.twin_utils.input.xray_data.file_name
tmp_params.r_free_flags.label = params.twin_utils.input.xray_data.free_flag
tmp_object = utils.determine_data_and_flags(
reflection_file_server=xray_data_server,
parameters=tmp_params,
log=log)
miller_array = tmp_object.extract_data()
if miller_array.is_xray_intensity_array():
miller_array = miller_array.f_sq_as_f()
assert miller_array.is_xray_amplitude_array()
free_flags = tmp_object.extract_flags(data=miller_array)
print(file=log)
print("Attempting to extract Free R flags", file=log)
free_flags = free_flags.customized_copy(data=flex.bool(
free_flags.data() == 1))
if free_flags is None:
free_flags = miller_array.generate_r_free_flags(
use_lattice_symmetry=True)
assert miller_array.observation_type() is not None
print(file=log)
print("Summary info of observed data", file=log)
print("=============================", file=log)
miller_array.show_summary(f=log)
print(file=log)
if miller_array.indices().size() == 0:
raise Sorry("No data available")
#----------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
model = pdb.input(file_name=params.twin_utils.input.model.file_name
).xray_structure_simple(crystal_symmetry=phil_xs)
print("Atomic model summary", file=log)
print("====================", file=log)
model.show_summary(f=log)
print(file=log)
#----------------------------------------------------------------
# step 3: get the twin laws for this xs
twin_laws = twin_analyses.twin_laws(
miller_array,
lattice_symmetry_max_delta=\
params.twin_utils.parameters.twinning.max_delta,
out=log)
print(file=log)
print("Preliminary data analyses", file=log)
print("==========================", file=log)
twin_laws.show(out=log)
#---------
# step 3:
# make twin model managers for all twin laws
print(file=log)
print(
"Overall and bulk solvent scale paranmeters and twin fraction estimation",
file=log)
print(
"=======================================================================",
file=log)
twin_models = []
operator_count = 0
if params.twin_utils.parameters.twinning.twin_law is not None:
tmp_law = sgtbx.rt_mx(
params.twin_utils.parameters.twinning.twin_law)
tmp_law = twin_analyses.twin_law(tmp_law, None, None, None, None,
None)
twin_laws.operators = [tmp_law]
for twin_law in twin_laws.operators:
operator_count += 1
operator_hkl = sgtbx.change_of_basis_op(twin_law.operator).as_hkl()
twin_model = twin_f_model.twin_model_manager(
f_obs=miller_array,
r_free_flags=free_flags,
xray_structure=model,
twin_law=twin_law.operator,
detwin_mode=params.twin_utils.parameters.twinning.detwin_mode,
out=log)
print("--- bulk solvent scaling ---", file=log)
twin_model.update_all_scales()
twin_model.r_values()
twin_model.target()
twin_model.show_k_sol_b_sol_b_cart_target()
twin_model.show_essential()
wfofc = twin_model.map_coefficients(map_type="mFo-DFc")
wtfofc = twin_model.map_coefficients(map_type="2mFo-DFc")
grad = twin_model.map_coefficients(map_type="gradient")
mtz_dataset = wtfofc.as_mtz_dataset(column_root_label="FWT")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array=wfofc, column_root_label="DFWT")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array=grad, column_root_label="GRAD")
name = params.twin_utils.output.map_coeffs_root + "_" + str(
operator_count) + ".mtz"
print(file=log)
print("Writing %s for twin law %s" % (name, operator_hkl),
file=log)
print(file=log)
mtz_dataset.mtz_object().write(file_name=name)
if params.twin_utils.output.obs_and_calc is not None:
# i want also a Fobs and Fmodel combined dataset please
mtz_dataset = miller_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array=twin_model.f_model(),
column_root_label="FMODEL")
name = params.twin_utils.output.obs_and_calc
mtz_dataset.mtz_object().write(file_name=name)
if len(twin_laws.operators) == 0:
print(file=log)
print("No twin laws were found", file=log)
print("Performing maximum likelihood based bulk solvent scaling",
file=log)
f_model_object = f_model.manager(f_obs=miller_array,
r_free_flags=free_flags,
xray_structure=model)
f_model_object.update_all_scales(log=log)
tfofc = f_model_object.map_coefficients(map_type="2mFobs-DFmodel")
fofc = f_model_object.map_coefficients(map_type="mFobs-DFmodel")
mtz_dataset = tfofc.as_mtz_dataset(column_root_label="FWT")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array=fofc, column_root_label="DELFWT")
name = params.twin_utils.output.map_coeffs_root + "_ML.mtz"
mtz_dataset.mtz_object().write(file_name=name)
if params.twin_utils.output.obs_and_calc is not None:
# i want also a Fobs and Fmodel combined dataset please
mtz_dataset = miller_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array=f_model_object.f_model(),
column_root_label="FMODEL")
name = params.twin_utils.output.obs_and_calc
mtz_dataset.mtz_object().write(file_name=name)
print(file=log)
print(file=log)
print("All done \n", file=log)
logfile = open(params.twin_utils.output.logfile, 'w')
print(string_buffer.getvalue(), file=logfile)
print(file=log)
def run(args, command_name="phenix.fobs_minus_fobs_map", log=None):
if (len(args) == 0): args = ["--help"]
examples = """Examples:
phenix.fobs_minus_fobs_map f_obs_1_file=data1.mtz f_obs_2_file=data2.sca \
f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 model.pdb
phenix.fobs_minus_fobs_map f_obs_1_file=data.mtz f_obs_2_file=data.mtz \
f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 phase_source=model.pdb \
high_res=2.0 sigma_cutoff=2 scattering_table=neutron"""
command_line = (iotbx_option_parser(
usage="%s [options]" % command_name,
description=examples).option("--silent",
action="store_true",
help="Suppress output to the screen.").
enable_symmetry_comprehensive()).process(args=args)
#
if (log is None):
log = sys.stdout
if (not command_line.options.silent):
utils.print_header("phenix.fobs_minus_fobs_map", out=log)
print("Command line arguments: ", file=log)
print(args, file=log)
print(file=log)
#
processed_args = utils.process_command_line_args(
args=command_line.args,
cmd_cs=command_line.symmetry,
master_params=fo_minus_fo_master_params(),
absolute_angle_tolerance=5,
absolute_length_tolerance=1,
log=log,
suppress_symmetry_related_errors=True)
working_phil = processed_args.params
if (not command_line.options.silent):
print("*** Parameters:", file=log)
working_phil.show(out=log)
print(file=log)
params = working_phil.extract()
consensus_symmetry = None
if (params.ignore_non_isomorphous_unit_cells):
if (None in [
params.f_obs_1_file_name, params.f_obs_2_file_name,
params.phase_source
]):
raise Sorry(
"The file parameters (f_obs_1_file_name, f_obs_2_file_name, " +
"phase_source) must be specified explicitly when " +
"ignore_non_isomorphous_unit_cells=True.")
symm_manager = iotbx.symmetry.manager()
pdb_in = iotbx.file_reader.any_file(params.phase_source,
force_type="pdb")
symm_manager.process_pdb_file(pdb_in)
hkl_in_1 = iotbx.file_reader.any_file(params.f_obs_1_file_name,
force_type="hkl")
sg_err_1, uc_err_1 = symm_manager.process_reflections_file(hkl_in_1)
hkl_in_2 = iotbx.file_reader.any_file(params.f_obs_2_file_name,
force_type="hkl")
sg_err_2, uc_err_2 = symm_manager.process_reflections_file(hkl_in_2)
out = StringIO()
symm_manager.show(out=out)
if (sg_err_1) or (sg_err_2):
raise Sorry((
"Incompatible space groups in input files:\n%s\nAll files " +
"must have the same point group (and ideally the same space group). "
+
"Please note that any symmetry information in the PDB file will be "
+ "used first.") % out.getvalue())
elif (uc_err_1) or (uc_err_2):
libtbx.call_back(
message="warn",
data=
("Crystal symmetry mismatch:\n%s\nCalculations will continue "
+
"using the symmetry in the PDB file (or if not available, the "
+
"first reflection file), but the maps should be treated with "
+ "extreme suspicion.") % out.getvalue())
crystal_symmetry = symm_manager.as_symmetry_object()
else:
processed_args = utils.process_command_line_args(
args=command_line.args,
cmd_cs=command_line.symmetry,
master_params=fo_minus_fo_master_params(),
suppress_symmetry_related_errors=False,
absolute_angle_tolerance=5,
absolute_length_tolerance=1,
log=StringIO())
crystal_symmetry = processed_args.crystal_symmetry
#
pdb_file_names = processed_args.pdb_file_names
if (len(processed_args.pdb_file_names) == 0):
if (params.phase_source is not None):
pdb_file_names = [params.phase_source]
else:
raise Sorry("No PDB file found.")
# Extaract Fobs1, Fobs2
f_obss = []
if (len(processed_args.reflection_files) == 2):
for reflection_file in processed_args.reflection_files:
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=[reflection_file],
err=null_out())
# XXX UGLY !!!
try:
parameters = utils.data_and_flags_master_params().extract()
if (params.f_obs_1_label is not None):
parameters.labels = [params.f_obs_1_label]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server=reflection_file_server,
keep_going=True,
parameters=parameters,
log=null_out())
except: # intentional
parameters = utils.data_and_flags_master_params().extract()
if (params.f_obs_2_label is not None):
parameters.labels = [params.f_obs_2_label]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server=reflection_file_server,
keep_going=True,
parameters=parameters,
log=null_out())
f_obss.append(determine_data_and_flags_result.f_obs)
else:
if ([params.f_obs_1_file_name,
params.f_obs_2_file_name].count(None) == 2):
raise Sorry("No reflection data file found.")
for file_name, label in zip(
[params.f_obs_1_file_name, params.f_obs_2_file_name],
[params.f_obs_1_label, params.f_obs_2_label]):
reflection_file = reflection_file_reader.any_reflection_file(
file_name=file_name, ensure_read_access=False)
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=[reflection_file],
err=null_out())
parameters = utils.data_and_flags_master_params().extract()
if (label is not None):
parameters.labels = [label]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server=reflection_file_server,
parameters=parameters,
keep_going=True,
log=null_out())
f_obss.append(determine_data_and_flags_result.f_obs)
if (len(f_obss) != 2):
raise Sorry(" ".join(errors))
if (not command_line.options.silent):
for ifobs, fobs in enumerate(f_obss):
print("*** Summary for data set %d:" % ifobs, file=log)
fobs.show_comprehensive_summary(f=log)
print(file=log)
pdb_combined = combine_unique_pdb_files(file_names=pdb_file_names)
pdb_combined.report_non_unique(out=log)
if (len(pdb_combined.unique_file_names) == 0):
raise Sorry("No coordinate file given.")
#
raw_recs = flex.std_string()
for rec in pdb_combined.raw_records:
if (rec.upper().count("CRYST1") == 0):
raw_recs.append(rec)
raw_recs.append(
iotbx.pdb.format_cryst1_record(crystal_symmetry=crystal_symmetry))
#
pdb_in = iotbx.pdb.input(source_info=None, lines=raw_recs)
model = mmtbx.model.manager(model_input=pdb_in)
d_min = min(f_obss[0].d_min(), f_obss[1].d_min())
model.setup_scattering_dictionaries(
scattering_table=params.scattering_table, d_min=d_min)
xray_structure = model.get_xray_structure()
hierarchy = model.get_hierarchy()
#
omit_sel = flex.bool(hierarchy.atoms_size(), False)
if (params.advanced.omit_selection is not None):
print("Will omit selection from phasing model:", file=log)
print(" " + params.advanced.omit_selection, file=log)
omit_sel = hierarchy.atom_selection_cache().selection(
params.advanced.omit_selection)
print("%d atoms selected for removal" % omit_sel.count(True), file=log)
del hierarchy
xray_structure = xray_structure.select(~omit_sel)
if (not command_line.options.silent):
print("*** Model summary:", file=log)
xray_structure.show_summary(f=log)
print(file=log)
info0 = f_obss[0].info()
info1 = f_obss[1].info()
f_obss[0] = f_obss[0].resolution_filter(
d_min=params.high_resolution,
d_max=params.low_resolution).set_info(info0)
f_obss[1] = f_obss[1].resolution_filter(
d_min=params.high_resolution,
d_max=params.low_resolution).set_info(info1)
if (params.sigma_cutoff is not None):
for i in [0, 1]:
if (f_obss[i].sigmas() is not None):
sel = f_obss[i].data(
) > f_obss[i].sigmas() * params.sigma_cutoff
f_obss[i] = f_obss[i].select(sel).set_info(info0)
for k, f_obs in enumerate(f_obss):
if (f_obs.indices().size() == 0):
raise Sorry(
"No data left in array %d (labels=%s) after filtering!" %
(k + 1, f_obs.info().label_string()))
output_file_name = params.output_file
if (output_file_name is None) and (params.file_name_prefix is not None):
output_file_name = "%s_%s.mtz" % (params.file_name_prefix,
params.job_id)
output_files = compute_fo_minus_fo_map(
data_arrays=f_obss,
xray_structure=xray_structure,
log=log,
silent=command_line.options.silent,
output_file=output_file_name,
peak_search=params.find_peaks_holes,
map_cutoff=params.map_cutoff,
peak_search_params=params.peak_search,
multiscale=params.advanced.multiscale,
anomalous=params.advanced.anomalous).file_names
return output_files
def run(args, command_name = "phenix.fobs_minus_fobs_map", log=None):
if(len(args) == 0): args = ["--help"]
examples = """Examples:
phenix.fobs_minus_fobs_map f_obs_1_file=data1.mtz f_obs_2_file=data2.sca \
f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 model.pdb
phenix.fobs_minus_fobs_map f_obs_1_file=data.mtz f_obs_2_file=data.mtz \
f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 phase_source=model.pdb \
high_res=2.0 sigma_cutoff=2 scattering_table=neutron"""
command_line = (iotbx_option_parser(
usage="%s [options]" % command_name,
description=examples)
.option("--silent",
action="store_true",
help="Suppress output to the screen.")
.enable_symmetry_comprehensive()
).process(args=args)
#
if (log is None) :
log = sys.stdout
if(not command_line.options.silent):
utils.print_header("phenix.fobs_minus_fobs_map", out = log)
print >> log, "Command line arguments: "
print >> log, args
print >> log
#
processed_args = utils.process_command_line_args(
args=command_line.args,
cmd_cs=command_line.symmetry,
master_params=fo_minus_fo_master_params(),
absolute_angle_tolerance=5,
absolute_length_tolerance=1,
log=log,
suppress_symmetry_related_errors=True)
working_phil = processed_args.params
if(not command_line.options.silent):
print >> log, "*** Parameters:"
working_phil.show(out = log)
print >> log
params = working_phil.extract()
consensus_symmetry = None
if (params.ignore_non_isomorphous_unit_cells) :
if (None in [params.f_obs_1_file_name, params.f_obs_2_file_name,
params.phase_source]):
raise Sorry("The file parameters (f_obs_1_file_name, f_obs_2_file_name, "+
"phase_source) must be specified explicitly when "+
"ignore_non_isomorphous_unit_cells=True.")
symm_manager = iotbx.symmetry.manager()
pdb_in = iotbx.file_reader.any_file(params.phase_source, force_type="pdb")
symm_manager.process_pdb_file(pdb_in)
hkl_in_1 = iotbx.file_reader.any_file(params.f_obs_1_file_name,
force_type="hkl")
sg_err_1, uc_err_1 = symm_manager.process_reflections_file(hkl_in_1)
hkl_in_2 = iotbx.file_reader.any_file(params.f_obs_2_file_name,
force_type="hkl")
sg_err_2, uc_err_2 = symm_manager.process_reflections_file(hkl_in_2)
out = StringIO()
symm_manager.show(out=out)
if (sg_err_1) or (sg_err_2) :
raise Sorry(("Incompatible space groups in input files:\n%s\nAll files "+
"must have the same point group (and ideally the same space group). "+
"Please note that any symmetry information in the PDB file will be "+
"used first.") % out.getvalue())
elif (uc_err_1) or (uc_err_2) :
libtbx.call_back(message="warn",
data=("Crystal symmetry mismatch:\n%s\nCalculations will continue "+
"using the symmetry in the PDB file (or if not available, the "+
"first reflection file), but the maps should be treated with "+
"extreme suspicion.") % out.getvalue())
crystal_symmetry = symm_manager.as_symmetry_object()
else :
processed_args = utils.process_command_line_args(
args=command_line.args,
cmd_cs=command_line.symmetry,
master_params=fo_minus_fo_master_params(),
absolute_angle_tolerance=5,
absolute_length_tolerance=1,
log=StringIO())
crystal_symmetry = processed_args.crystal_symmetry
#
pdb_file_names = processed_args.pdb_file_names
if(len(processed_args.pdb_file_names) == 0):
if(params.phase_source is not None):
pdb_file_names = [params.phase_source]
else:
raise Sorry("No PDB file found.")
# Extaract Fobs1, Fobs2
f_obss = []
if(len(processed_args.reflection_files)==2):
for reflection_file in processed_args.reflection_files:
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = crystal_symmetry,
force_symmetry = True,
reflection_files = [reflection_file],
err = null_out())
# XXX UGLY !!!
try:
parameters = utils.data_and_flags_master_params().extract()
if(params.f_obs_1_label is not None):
parameters.labels = [params.f_obs_1_label]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server = reflection_file_server,
keep_going = True,
parameters = parameters,
log = null_out())
except: # intentional
parameters = utils.data_and_flags_master_params().extract()
if(params.f_obs_2_label is not None):
parameters.labels = [params.f_obs_2_label]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server = reflection_file_server,
keep_going = True,
parameters = parameters,
log = null_out())
f_obss.append(determine_data_and_flags_result.f_obs)
else:
if([params.f_obs_1_file_name,params.f_obs_2_file_name].count(None)==2):
raise Sorry("No reflection data file found.")
for file_name, label in zip([params.f_obs_1_file_name,params.f_obs_2_file_name],
[params.f_obs_1_label,params.f_obs_2_label]):
reflection_file = reflection_file_reader.any_reflection_file(
file_name = file_name, ensure_read_access = False)
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = crystal_symmetry,
force_symmetry = True,
reflection_files = [reflection_file],
err = null_out())
parameters = utils.data_and_flags_master_params().extract()
if(label is not None):
parameters.labels = [label]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server = reflection_file_server,
parameters = parameters,
keep_going = True,
log = null_out())
f_obss.append(determine_data_and_flags_result.f_obs)
if(len(f_obss)!=2):
raise Sorry(" ".join(errors))
if(not command_line.options.silent):
for ifobs, fobs in enumerate(f_obss):
print >> log, "*** Summary for data set %d:"%ifobs
fobs.show_comprehensive_summary(f = log)
print >> log
pdb_combined = combine_unique_pdb_files(file_names = pdb_file_names)
pdb_combined.report_non_unique(out = log)
if(len(pdb_combined.unique_file_names) == 0):
raise Sorry("No coordinate file given.")
#
raw_recs = flex.std_string()
for rec in pdb_combined.raw_records:
if(rec.upper().count("CRYST1")==0):
raw_recs.append(rec)
raw_recs.append(iotbx.pdb.format_cryst1_record(
crystal_symmetry = crystal_symmetry))
#
pdb_in = iotbx.pdb.input(source_info = None, lines = raw_recs)
hierarchy = pdb_in.construct_hierarchy()
omit_sel = flex.bool(hierarchy.atoms_size(), False)
if (params.advanced.omit_selection is not None) :
print >> log, "Will omit selection from phasing model:"
print >> log, " " + params.advanced.omit_selection
omit_sel = hierarchy.atom_selection_cache().selection(
params.advanced.omit_selection)
print >> log, "%d atoms selected for removal" % omit_sel.count(True)
xray_structure = pdb_in.xray_structure_simple()
xray_structure = xray_structure.select(~omit_sel)
if(not command_line.options.silent):
print >> log, "*** Model summary:"
xray_structure.show_summary(f = log)
print >> log
info0 = f_obss[0].info()
info1 = f_obss[1].info()
f_obss[0] = f_obss[0].resolution_filter(d_min = params.high_resolution,
d_max = params.low_resolution).set_info(info0)
f_obss[1] = f_obss[1].resolution_filter(d_min = params.high_resolution,
d_max = params.low_resolution).set_info(info1)
if(params.sigma_cutoff is not None):
for i in [0,1]:
if(f_obss[i].sigmas() is not None):
sel = f_obss[i].data() > f_obss[i].sigmas()*params.sigma_cutoff
f_obss[i] = f_obss[i].select(sel).set_info(info0)
for k, f_obs in enumerate(f_obss) :
if (f_obs.indices().size() == 0) :
raise Sorry("No data left in array %d (labels=%s) after filtering!" % (k+1,
f_obs.info().label_string()))
output_file_name = params.output_file
if (output_file_name is None) and (params.file_name_prefix is not None) :
output_file_name = "%s_%s.mtz" % (params.file_name_prefix, params.job_id)
output_files = compute_fo_minus_fo_map(
data_arrays = f_obss,
xray_structure = xray_structure,
log = log,
silent = command_line.options.silent,
output_file = output_file_name,
peak_search=params.find_peaks_holes,
map_cutoff=params.map_cutoff,
peak_search_params=params.peak_search,
multiscale=params.advanced.multiscale,
anomalous=params.advanced.anomalous).file_names
return output_files
def run(args, out=None, log=sys.stdout):
if (len(args) == 0) or (args == ["--help"]):
print(msg, file=log)
defaults(log=log, silent=False)
return
parsed = defaults(log=log, silent=True)
#
processed_args = utils.process_command_line_args(args=args,
log=log,
master_params=parsed)
params = processed_args.params.extract()
#
reflection_files = processed_args.reflection_files
if (len(reflection_files) == 0):
raise Sorry("No reflection file found.")
crystal_symmetry = processed_args.crystal_symmetry
if (crystal_symmetry is None):
raise Sorry("No crystal symmetry found.")
if (len(processed_args.pdb_file_names) == 0):
raise Sorry("No PDB file found.")
pdb_file_names = processed_args.pdb_file_names
#
rfs = reflection_file_server(crystal_symmetry=crystal_symmetry,
reflection_files=reflection_files)
parameters = utils.data_and_flags_master_params().extract()
if (params.f_obs_label is not None):
parameters.labels = params.f_obs_label
if (params.r_free_flags_label is not None):
parameters.r_free_flags.label = params.r_free_flags_label
if (params.high_resolution is not None):
parameters.high_resolution = params.high_resolution
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server=rfs,
parameters=parameters,
data_parameter_scope="refinement.input.xray_data",
flags_parameter_scope="refinement.input.xray_data.r_free_flags",
data_description="X-ray data",
keep_going=True,
log=StringIO())
f_obs = determine_data_and_flags_result.f_obs
# Data
show_header(l="Data:", log=log)
f_obs.show_comprehensive_summary(prefix=" ", f=log)
# R-free-flags
show_header(l="R-free-flags:", log=log)
r_free_flags = determine_data_and_flags_result.r_free_flags
test_flag_value = determine_data_and_flags_result.test_flag_value
if (r_free_flags is None):
r_free_flags = f_obs.array(data=flex.bool(f_obs.data().size(), False))
test_flag_value = None
print(" not available", file=log)
else:
print(" flag value:", test_flag_value, file=log)
# Model
pdb_combined = iotbx.pdb.combine_unique_pdb_files(
file_names=processed_args.pdb_file_names)
pdb_combined.report_non_unique(out=log)
if (len(pdb_combined.unique_file_names) == 0):
raise Sorry("No coordinate file given.")
raw_records = pdb_combined.raw_records
try:
pdb_inp = iotbx.pdb.input(source_info=None,
lines=flex.std_string(raw_records))
except ValueError as e:
raise Sorry("Model format (PDB or mmCIF) error:\n%s" % str(e))
model = mmtbx.model.manager(model_input=pdb_inp,
crystal_symmetry=crystal_symmetry,
log=StringIO())
#
scattering_table = params.scattering_table
exptl_method = pdb_inp.get_experiment_type()
if (exptl_method is not None) and ("NEUTRON" in exptl_method):
scattering_table = "neutron"
model.setup_scattering_dictionaries(scattering_table=scattering_table,
d_min=f_obs.d_min())
#
# Model vs data
#
show_header(l="Model vs Data:", log=log)
fmodel = mmtbx.f_model.manager(xray_structure=model.get_xray_structure(),
f_obs=f_obs,
r_free_flags=r_free_flags,
twin_law=params.twin_law)
fmodel.update_all_scales(update_f_part1=True)
fmodel.show(log=log, show_header=False, show_approx=False)
print(" r_work: %6.4f" % fmodel.r_work(), file=log)
if (test_flag_value is not None):
print(" r_free: %6.4f" % fmodel.r_free(), file=log)
else:
print(" r_free: None", file=log)
print(file=log)
n_outl = f_obs.data().size() - fmodel.f_obs().data().size()
print(" Number of F-obs outliers:", n_outl, file=log)
#
# Extract information from PDB file header and output (if any)
#
pub_r_work = None
pub_r_free = None
pub_high = None
pub_low = None
pub_sigma = None
pub_program_name = None
pub_solv_cont = None
pub_matthews = None
published_results = pdb_inp.get_r_rfree_sigma(file_name=pdb_file_names[0])
if (published_results is not None):
pub_r_work = published_results.r_work
pub_r_free = published_results.r_free
pub_high = published_results.high
pub_low = published_results.low
pub_sigma = published_results.sigma
pub_program_name = pdb_inp.get_program_name()
pub_solv_cont = pdb_inp.get_solvent_content()
pub_matthews = pdb_inp.get_matthews_coeff()
#
show_header(l="Information extracted from PDB file header:", log=log)
print(" program_name : %-s" % format_value("%s", pub_program_name),
file=log)
print(" year : %-s" %
format_value("%s", pdb_inp.extract_header_year()),
file=log)
print(" r_work : %-s" % format_value("%s", pub_r_work), file=log)
print(" r_free : %-s" % format_value("%s", pub_r_free), file=log)
print(" high_resolution : %-s" % format_value("%s", pub_high), file=log)
print(" low_resolution : %-s" % format_value("%s", pub_low), file=log)
print(" sigma_cutoff : %-s" % format_value("%s", pub_sigma), file=log)
print(" matthews_coeff : %-s" % format_value("%s", pub_matthews),
file=log)
print(" solvent_cont : %-s" % format_value("%s", pub_solv_cont),
file=log)
if (exptl_method is not None):
print(" exptl_method : %-s" % format_value("%s", exptl_method),
file=log)
#
# Recompute R-factors using published cutoffs
fmodel_cut = fmodel
tmp_sel = flex.bool(fmodel.f_obs().data().size(), True)
if (pub_sigma is not None and fmodel.f_obs().sigmas() is not None):
tmp_sel &= fmodel.f_obs().data() > fmodel.f_obs().sigmas() * pub_sigma
if (pub_high is not None
and abs(pub_high - fmodel.f_obs().d_min()) > 0.03):
tmp_sel &= fmodel.f_obs().d_spacings().data() > pub_high
if (pub_low is not None
and abs(pub_low - fmodel.f_obs().d_max_min()[0]) > 0.03):
tmp_sel &= fmodel.f_obs().d_spacings().data() < pub_low
if (tmp_sel.count(True) != tmp_sel.size() and tmp_sel.count(True) > 0):
show_header(l="After applying resolution and sigma cutoffs:", log=log)
fmodel = mmtbx.f_model.manager(
xray_structure=model.get_xray_structure(),
f_obs=fmodel.f_obs().select(tmp_sel),
r_free_flags=fmodel.r_free_flags().select(tmp_sel),
twin_law=params.twin_law)
fmodel.update_all_scales(update_f_part1=True)
fmodel.show(log=log, show_header=False, show_approx=False)
print(" r_work: %6.4f" % fmodel.r_work(), file=log)
if (test_flag_value is not None):
print(" r_free: %6.4f" % fmodel.r_free(), file=log)
else:
print(" r_free: None", file=log)
print(file=log)
n_outl = f_obs.data().size() - fmodel.f_obs().data().size()
print(" Number of F-obs outliers:", n_outl, file=log)
def run(args, out=None, log=sys.stdout):
if (len(args) == 0) or (args == ["--help"]):
print >> log, msg
defaults(log=log, silent=False)
return
parsed = defaults(log=log, silent=True)
#
processed_args = utils.process_command_line_args(args=args,
log=log,
master_params=parsed)
params = processed_args.params.extract()
#
reflection_files = processed_args.reflection_files
if (len(reflection_files) == 0):
raise Sorry("No reflection file found.")
crystal_symmetry = processed_args.crystal_symmetry
if (crystal_symmetry is None):
raise Sorry("No crystal symmetry found.")
if (len(processed_args.pdb_file_names) == 0):
raise Sorry("No PDB file found.")
pdb_file_names = processed_args.pdb_file_names
#
rfs = reflection_file_server(crystal_symmetry=crystal_symmetry,
reflection_files=reflection_files)
parameters = utils.data_and_flags_master_params().extract()
if (params.f_obs_label is not None):
parameters.labels = params.f_obs_label
if (params.r_free_flags_label is not None):
parameters.r_free_flags.label = params.r_free_flags_label
if (params.high_resolution is not None):
parameters.high_resolution = params.high_resolution
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server=rfs,
parameters=parameters,
data_parameter_scope="refinement.input.xray_data",
flags_parameter_scope="refinement.input.xray_data.r_free_flags",
data_description="X-ray data",
keep_going=True,
log=StringIO())
f_obs = determine_data_and_flags_result.f_obs
# Data
show_header(l="Data:", log=log)
f_obs.show_comprehensive_summary(prefix=" ", f=log)
# R-free-flags
show_header(l="R-free-flags:", log=log)
r_free_flags = determine_data_and_flags_result.r_free_flags
test_flag_value = determine_data_and_flags_result.test_flag_value
if (r_free_flags is None):
r_free_flags = f_obs.array(data=flex.bool(f_obs.data().size(), False))
test_flag_value = None
print >> log, " not available"
else:
print >> log, " flag value:", test_flag_value
# Model
pdb_combined = iotbx.pdb.combine_unique_pdb_files(
file_names=processed_args.pdb_file_names)
pdb_combined.report_non_unique(out=log)
if (len(pdb_combined.unique_file_names) == 0):
raise Sorry("No coordinate file given.")
raw_records = pdb_combined.raw_records
try:
pdb_inp = iotbx.pdb.input(source_info=None,
lines=flex.std_string(raw_records))
except ValueError, e:
raise Sorry("Model format (PDB or mmCIF) error:\n%s" % str(e))
def run(args,
command_name="mmtbx.model_vs_data",
show_geometry_statistics=True,
model_size_max_atoms=80000,
data_size_max_reflections=1000000,
unit_cell_max_dimension=800.,
return_fmodel_and_pdb=False,
out=None,
log=sys.stdout):
import mmtbx.f_model.f_model_info
if (len(args) == 0) or (args == ["--help"]):
print >> log, msg
defaults(log=log, silent=False)
return
parsed = defaults(log=log, silent=True)
#
mvd_obj = mvd()
#
processed_args = utils.process_command_line_args(args=args,
log=log,
master_params=parsed)
params = processed_args.params.extract()
#
reflection_files = processed_args.reflection_files
if (len(reflection_files) == 0):
raise Sorry("No reflection file found.")
crystal_symmetry = processed_args.crystal_symmetry
if (crystal_symmetry is None):
raise Sorry("No crystal symmetry found.")
if (len(processed_args.pdb_file_names) == 0):
raise Sorry("No PDB file found.")
pdb_file_names = processed_args.pdb_file_names
#
rfs = reflection_file_server(crystal_symmetry=crystal_symmetry,
reflection_files=reflection_files)
parameters = utils.data_and_flags_master_params().extract()
if (params.f_obs_label is not None):
parameters.labels = params.f_obs_label
if (params.r_free_flags_label is not None):
parameters.r_free_flags.label = params.r_free_flags_label
if (params.high_resolution is not None):
parameters.high_resolution = params.high_resolution
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server=rfs,
parameters=parameters,
data_parameter_scope="refinement.input.xray_data",
flags_parameter_scope="refinement.input.xray_data.r_free_flags",
data_description="X-ray data",
keep_going=True,
log=StringIO())
f_obs = determine_data_and_flags_result.f_obs
number_of_reflections = f_obs.indices().size()
if (params.ignore_giant_models_and_datasets
and number_of_reflections > data_size_max_reflections):
raise Sorry("Too many reflections: %d" % number_of_reflections)
#
max_unit_cell_dimension = max(f_obs.unit_cell().parameters()[:3])
if (params.ignore_giant_models_and_datasets
and max_unit_cell_dimension > unit_cell_max_dimension):
raise Sorry("Too large unit cell (max dimension): %s" %
str(max_unit_cell_dimension))
#
r_free_flags = determine_data_and_flags_result.r_free_flags
test_flag_value = determine_data_and_flags_result.test_flag_value
if (r_free_flags is None):
r_free_flags = f_obs.array(data=flex.bool(f_obs.data().size(), False))
test_flag_value = None
#
mmtbx_pdb_file = mmtbx.utils.pdb_file(
pdb_file_names=pdb_file_names,
cif_objects=processed_args.cif_objects,
crystal_symmetry=crystal_symmetry,
use_neutron_distances=(params.scattering_table == "neutron"),
ignore_unknown_nonbonded_energy_types=not show_geometry_statistics,
log=log)
mmtbx_pdb_file.set_ppf(stop_if_duplicate_labels=False)
processed_pdb_file = mmtbx_pdb_file.processed_pdb_file
pdb_raw_records = mmtbx_pdb_file.pdb_raw_records
pdb_inp = mmtbx_pdb_file.pdb_inp
#
# just to avoid going any further with bad PDB file....
pdb_inp.xray_structures_simple()
#
acp = processed_pdb_file.all_chain_proxies
atom_selections = group_args(
all=acp.selection(string="all"),
macromolecule=acp.selection(string="protein or dna or rna"),
solvent=acp.selection(string="water"), # XXX single_atom_residue
ligand=acp.selection(string="not (protein or dna or rna or water)"),
backbone=acp.selection(string="backbone"),
sidechain=acp.selection(string="sidechain"))
#
scattering_table = params.scattering_table
exptl_method = pdb_inp.get_experiment_type()
if (exptl_method is not None) and ("NEUTRON" in exptl_method):
scattering_table = "neutron"
xsfppf = mmtbx.utils.xray_structures_from_processed_pdb_file(
processed_pdb_file=processed_pdb_file,
scattering_table=scattering_table,
d_min=f_obs.d_min())
xray_structures = xsfppf.xray_structures
if (0): #XXX normalize occupancies if all models have occ=1 so the total=1
n_models = len(xray_structures)
for xrs in xray_structures:
occ = xrs.scatterers().extract_occupancies()
occ = occ / n_models
xrs.set_occupancies(occ)
model_selections = xsfppf.model_selections
mvd_obj.collect(crystal=group_args(
uc=f_obs.unit_cell(),
sg=f_obs.crystal_symmetry().space_group_info().symbol_and_number(),
n_sym_op=f_obs.crystal_symmetry().space_group_info().type().group(
).order_z(),
uc_vol=f_obs.unit_cell().volume()))
#
hierarchy = pdb_inp.construct_hierarchy()
pdb_atoms = hierarchy.atoms()
pdb_atoms.reset_i_seq()
#
# Extract TLS
pdb_tls = None
pdb_inp_tls = pdb_inp.extract_tls_params(hierarchy)
pdb_tls = group_args(pdb_inp_tls=pdb_inp_tls,
tls_selections=[],
tls_selection_strings=[])
# XXX no TLS + multiple models
if (pdb_inp_tls.tls_present and pdb_inp_tls.error_string is None
and len(xray_structures) == 1):
pdb_tls = mmtbx.tls.tools.extract_tls_from_pdb(
pdb_inp_tls=pdb_inp_tls,
all_chain_proxies=mmtbx_pdb_file.processed_pdb_file.
all_chain_proxies,
xray_structure=xsfppf.xray_structure_all)
if (len(pdb_tls.tls_selections) == len(pdb_inp_tls.tls_params)
and len(pdb_inp_tls.tls_params) > 0):
xray_structures = [
utils.extract_tls_and_u_total_from_pdb(
f_obs=f_obs,
r_free_flags=r_free_flags,
xray_structure=xray_structures[
0], # XXX no TLS + multiple models
tls_selections=pdb_tls.tls_selections,
tls_groups=pdb_inp_tls.tls_params)
]
###########################
geometry_statistics = show_geometry(
xray_structures=xray_structures,
processed_pdb_file=processed_pdb_file,
scattering_table=scattering_table,
hierarchy=hierarchy,
model_selections=model_selections,
show_geometry_statistics=show_geometry_statistics,
mvd_obj=mvd_obj,
atom_selections=atom_selections)
###########################
mp = mmtbx.masks.mask_master_params.extract()
f_obs_labels = f_obs.info().label_string()
f_obs = f_obs.sort(reverse=True, by_value="packed_indices")
r_free_flags = r_free_flags.sort(reverse=True, by_value="packed_indices")
fmodel = utils.fmodel_simple(
xray_structures=xray_structures,
scattering_table=scattering_table,
mask_params=mp,
f_obs=f_obs,
r_free_flags=r_free_flags,
skip_twin_detection=params.skip_twin_detection)
n_outl = f_obs.data().size() - fmodel.f_obs().data().size()
mvd_obj.collect(model_vs_data=show_model_vs_data(fmodel))
# Extract information from PDB file header and output (if any)
pub_r_work = None
pub_r_free = None
pub_high = None
pub_low = None
pub_sigma = None
pub_program_name = None
pub_solv_cont = None
pub_matthews = None
published_results = pdb_inp.get_r_rfree_sigma(file_name=pdb_file_names[0])
if (published_results is not None):
pub_r_work = published_results.r_work
pub_r_free = published_results.r_free
pub_high = published_results.high
pub_low = published_results.low
pub_sigma = published_results.sigma
pub_program_name = pdb_inp.get_program_name()
pub_solv_cont = pdb_inp.get_solvent_content()
pub_matthews = pdb_inp.get_matthews_coeff()
mvd_obj.collect(pdb_header=group_args(program_name=pub_program_name,
year=pdb_inp.extract_header_year(),
r_work=pub_r_work,
r_free=pub_r_free,
high_resolution=pub_high,
low_resolution=pub_low,
sigma_cutoff=pub_sigma,
matthews_coeff=pub_matthews,
solvent_cont=pub_solv_cont,
tls=pdb_tls,
exptl_method=exptl_method))
#
# Recompute R-factors using published cutoffs
fmodel_cut = fmodel
tmp_sel = flex.bool(fmodel.f_obs().data().size(), True)
if (pub_sigma is not None and fmodel.f_obs().sigmas() is not None):
tmp_sel &= fmodel.f_obs().data() > fmodel.f_obs().sigmas() * pub_sigma
if (pub_high is not None
and abs(pub_high - fmodel.f_obs().d_min()) > 0.03):
tmp_sel &= fmodel.f_obs().d_spacings().data() > pub_high
if (pub_low is not None
and abs(pub_low - fmodel.f_obs().d_max_min()[0]) > 0.03):
tmp_sel &= fmodel.f_obs().d_spacings().data() < pub_low
if (tmp_sel.count(True) != tmp_sel.size() and tmp_sel.count(True) > 0):
fmodel_cut = utils.fmodel_simple(
xray_structures=xray_structures,
scattering_table=scattering_table,
f_obs=fmodel.f_obs().select(tmp_sel),
r_free_flags=fmodel.r_free_flags().select(tmp_sel),
skip_twin_detection=params.skip_twin_detection)
mvd_obj.collect(
misc=group_args(r_work_cutoff=fmodel_cut.r_work(),
r_free_cutoff=fmodel_cut.r_free(),
n_refl_cutoff=fmodel_cut.f_obs().data().size()))
mvd_obj.collect(data=show_data(fmodel=fmodel,
n_outl=n_outl,
test_flag_value=test_flag_value,
f_obs_labels=f_obs_labels,
fmodel_cut=fmodel_cut))
# CC* and friends
cc_star_stats = None
if (params.unmerged_data is not None):
import mmtbx.validation.experimental
import mmtbx.command_line
f_obs = fmodel.f_obs().average_bijvoet_mates()
unmerged_i_obs = mmtbx.command_line.load_and_validate_unmerged_data(
f_obs=f_obs,
file_name=params.unmerged_data,
data_labels=params.unmerged_labels,
log=null_out())
cc_star_stats = mmtbx.validation.experimental.merging_and_model_statistics(
f_model=fmodel.f_model().average_bijvoet_mates(),
f_obs=f_obs,
r_free_flags=fmodel.r_free_flags().average_bijvoet_mates(),
unmerged_i_obs=unmerged_i_obs,
n_bins=params.n_bins)
mvd_obj.show(log=out)
if (cc_star_stats is not None):
cc_star_stats.show_model_vs_data(out=out, prefix=" ")
if return_fmodel_and_pdb:
mvd_obj.pdb_file = processed_pdb_file
mvd_obj.fmodel = fmodel
if (len(params.map) > 0):
for map_name_string in params.map:
map_type_obj = mmtbx.map_names(map_name_string=map_name_string)
map_params = mmtbx.maps.map_and_map_coeff_master_params().fetch(
mmtbx.maps.cast_map_coeff_params(map_type_obj)).extract()
maps_obj = mmtbx.maps.compute_map_coefficients(
fmodel=fmodel_cut, params=map_params.map_coefficients)
fn = os.path.basename(processed_args.reflection_file_names[0])
if (fn.count(".")):
prefix = fn[:fn.index(".")]
else:
prefix = fn
file_name = prefix + "_%s_map_coeffs.mtz" % map_type_obj.format()
maps_obj.write_mtz_file(file_name=file_name)
# statistics in bins
if (not fmodel.twin):
print >> log, "Statistics in resolution bins:"
mmtbx.f_model.f_model_info.r_work_and_completeness_in_resolution_bins(
fmodel=fmodel, out=log, prefix=" ")
# report map cc
if (params.comprehensive and not fmodel_cut.twin
and fmodel_cut.xray_structure is not None):
rsc_params = real_space_correlation.master_params().extract()
rsc_params.scattering_table = scattering_table
real_space_correlation.simple(fmodel=fmodel_cut,
pdb_hierarchy=hierarchy,
params=rsc_params,
log=log,
show_results=True)
#
if (params.dump_result_object_as_pickle):
output_prefixes = []
for op in processed_args.pdb_file_names + processed_args.reflection_file_names:
op = os.path.basename(op)
try:
op = op[:op.index(".")]
except Exception:
pass
if (not op in output_prefixes): output_prefixes.append(op)
output_prefix = "_".join(output_prefixes)
easy_pickle.dump("%s.pickle" % output_prefix, mvd_obj)
return mvd_obj
def run(args, command_name="phenix.twin_map_utils"):
log=sys.stdout
params=None
if (len(args) == 0 or "--help" in args or "--h" in args or "-h" in args):
print_help(command_name=command_name)
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="map_coefs")
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown file or keyword:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
"""
new_params = master_params.format(python_object=params)
new_params.show(out=log)
"""
# now get the unit cell from the pdb file
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.twin_utils.input.xray_data.file_name)
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.twin_utils.input.model.file_name)
phil_xs=None
if ([params.twin_utils.input.unit_cell,
params.twin_utils.input.space_group]).count(None)<2:
phil_xs = crystal.symmetry(
unit_cell=params.twin_utils.input.unit_cell,
space_group_info=params.twin_utils.input.space_group )
combined_xs = crystal.select_crystal_symmetry(
None,phil_xs, [pdb_xs],[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.twin_utils.input.unit_cell = combined_xs.unit_cell()
params.twin_utils.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
new_params = master_params.format(python_object=params)
new_params.show(out=log)
if params.twin_utils.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.twin_utils.input.space_group is None:
raise Sorry("space group not specified")
if params.twin_utils.input.xray_data.file_name is None:
raise Sorry("Xray data not specified")
if params.twin_utils.input.model.file_name is None:
raise Sorry("pdb file with model not specified")
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
phil_xs = crystal.symmetry(
unit_cell=params.twin_utils.input.unit_cell,
space_group_info=params.twin_utils.input.space_group )
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = phil_xs,
force_symmetry = True,
reflection_files=[])
miller_array = None
free_flags = None
tmp_params = utils.data_and_flags_master_params().extract()
# insert proper values please
tmp_params.file_name = params.twin_utils.input.xray_data.file_name
tmp_params.labels = params.twin_utils.input.xray_data.obs_labels
tmp_params.r_free_flags.file_name=params.twin_utils.input.xray_data.file_name
tmp_params.r_free_flags.label=params.twin_utils.input.xray_data.free_flag
tmp_object = utils.determine_data_and_flags( reflection_file_server = xray_data_server,
parameters = tmp_params, log=log )
miller_array = tmp_object.extract_data()
if miller_array.is_xray_intensity_array():
miller_array = miller_array.f_sq_as_f()
assert miller_array.is_xray_amplitude_array()
free_flags = tmp_object.extract_flags(data = miller_array)
print >> log
print >> log, "Attempting to extract Free R flags"
free_flags = free_flags.customized_copy( data = flex.bool( free_flags.data()==1 ) )
if free_flags is None:
free_flags = miller_array.generate_r_free_flags(use_lattice_symmetry=True)
assert miller_array.observation_type() is not None
print >> log
print >> log, "Summary info of observed data"
print >> log, "============================="
miller_array.show_summary(f=log)
print >> log
if miller_array.indices().size() == 0:
raise Sorry("No data available")
#----------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
model = pdb.input(file_name=params.twin_utils.input.model.file_name).xray_structure_simple(
crystal_symmetry=phil_xs)
print >> log, "Atomic model summary"
print >> log, "===================="
model.show_summary(f=log)
print >> log
#----------------------------------------------------------------
# step 3: get the twin laws for this xs
twin_laws = twin_analyses.twin_laws(
miller_array,
lattice_symmetry_max_delta=\
params.twin_utils.parameters.twinning.max_delta,
out=log)
print >> log
print >> log, "Preliminary data analyses"
print >> log, "=========================="
twin_laws.show(out=log)
#---------
# step 3:
# make twin model managers for all twin laws
print >> log
print >> log, "Overall and bulk solvent scale paranmeters and twin fraction estimation"
print >> log, "======================================================================="
twin_models = []
operator_count = 0
if params.twin_utils.parameters.twinning.twin_law is not None:
tmp_law = sgtbx.rt_mx( params.twin_utils.parameters.twinning.twin_law )
tmp_law = twin_analyses.twin_law(tmp_law,None,None,None,None,None)
twin_laws.operators = [ tmp_law ]
for twin_law in twin_laws.operators:
operator_count += 1
operator_hkl = sgtbx.change_of_basis_op( twin_law.operator ).as_hkl()
twin_model = twin_f_model.twin_model_manager(
f_obs=miller_array,
r_free_flags = free_flags,
xray_structure=model,
twin_law = twin_law.operator,
detwin_mode = params.twin_utils.parameters.twinning.detwin_mode,
out=log)
print >> log, "--- bulk solvent scaling ---"
twin_model.update_solvent_and_scale(update_f_part1=False)
twin_model.r_values()
twin_model.target()
twin_model.show_k_sol_b_sol_b_cart_target()
twin_model.show_essential()
wfofc = twin_model.map_coefficients(map_type="mFo-DFc" )
wtfofc = twin_model.map_coefficients(map_type="2mFo-DFc" )
grad = twin_model.map_coefficients(map_type="gradient" )
mtz_dataset = wtfofc.as_mtz_dataset(
column_root_label="FWT")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = wfofc,
column_root_label = "DFWT"
)
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = grad,
column_root_label = "GRAD"
)
name = params.twin_utils.output.map_coeffs_root+"_"+str(operator_count)+".mtz"
print >> log
print >> log, "Writing %s for twin law %s"%(name,operator_hkl)
print >> log
mtz_dataset.mtz_object().write(
file_name=name)
if params.twin_utils.output.obs_and_calc is not None:
# i want also a Fobs and Fmodel combined dataset please
mtz_dataset = miller_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = twin_model.f_model(),
column_root_label="FMODEL")
name = params.twin_utils.output.obs_and_calc
mtz_dataset.mtz_object().write(
file_name=name)
if len(twin_laws.operators)==0:
print >> log
print >> log, "No twin laws were found"
print >> log, "Performing maximum likelihood based bulk solvent scaling"
f_model_object = f_model.manager(
f_obs = miller_array,
r_free_flags = free_flags,
xray_structure = model )
f_model_object.update_solvent_and_scale(out=log)
tfofc = f_model_object.map_coefficients(map_type="2mFobs-DFmodel")
fofc = f_model_object.map_coefficients(map_type="mFobs-DFmodel")
mtz_dataset = tfofc.as_mtz_dataset(
column_root_label="FWT")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = fofc,
column_root_label = "DELFWT"
)
name = params.twin_utils.output.map_coeffs_root+"_ML.mtz"
mtz_dataset.mtz_object().write(
file_name=name)
if params.twin_utils.output.obs_and_calc is not None:
# i want also a Fobs and Fmodel combined dataset please
mtz_dataset = miller_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = f_model_object.f_model(),
column_root_label="FMODEL")
name = params.twin_utils.output.obs_and_calc
mtz_dataset.mtz_object().write(
file_name=name)
print >> log
print >> log
print >> log, "All done \n"
logfile = open(params.twin_utils.output.logfile,'w')
print >> logfile, string_buffer.getvalue()
print >> log
def run(self, args, command_name, out=sys.stdout):
command_line = (iotbx_option_parser(
usage="%s [options]" % command_name,
description='Example: %s data.mtz data.mtz ref_model.pdb'%command_name)
.option(None, "--show_defaults",
action="store_true",
help="Show list of parameters.")
).process(args=args)
cif_file = None
processed_args = utils.process_command_line_args(
args = args,
log = sys.stdout,
master_params = master_phil)
params = processed_args.params
if(params is None): params = master_phil
self.params = params.extract().ensemble_probability
pdb_file_names = processed_args.pdb_file_names
if len(pdb_file_names) != 1 :
raise Sorry("Only one PDB structure may be used")
pdb_file = file_reader.any_file(pdb_file_names[0])
self.log = multi_out()
self.log.register(label="stdout", file_object=sys.stdout)
self.log.register(
label="log_buffer",
file_object=StringIO(),
atexit_send_to=None)
sys.stderr = self.log
log_file = open(pdb_file_names[0].split('/')[-1].replace('.pdb','') + '_pensemble.log', "w")
self.log.replace_stringio(
old_label="log_buffer",
new_label="log",
new_file_object=log_file)
utils.print_header(command_name, out = self.log)
params.show(out = self.log)
#
f_obs = None
r_free_flags = None
reflection_files = processed_args.reflection_files
if self.params.fobs_vs_fcalc_post_nll:
if len(reflection_files) == 0:
raise Sorry("Fobs from input MTZ required for fobs_vs_fcalc_post_nll")
if len(reflection_files) > 0:
crystal_symmetry = processed_args.crystal_symmetry
print >> self.log, 'Reflection file : ', processed_args.reflection_file_names[0]
utils.print_header("Model and data statistics", out = self.log)
rfs = reflection_file_server(
crystal_symmetry = crystal_symmetry,
reflection_files = processed_args.reflection_files,
log = self.log)
parameters = utils.data_and_flags_master_params().extract()
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server = rfs,
parameters = parameters,
data_parameter_scope = "refinement.input.xray_data",
flags_parameter_scope = "refinement.input.xray_data.r_free_flags",
data_description = "X-ray data",
keep_going = True,
log = self.log)
f_obs = determine_data_and_flags_result.f_obs
number_of_reflections = f_obs.indices().size()
r_free_flags = determine_data_and_flags_result.r_free_flags
test_flag_value = determine_data_and_flags_result.test_flag_value
if(r_free_flags is None):
r_free_flags=f_obs.array(data=flex.bool(f_obs.data().size(), False))
# process PDB
pdb_file.assert_file_type("pdb")
#
pdb_in = hierarchy.input(file_name=pdb_file.file_name)
ens_pdb_hierarchy = pdb_in.construct_hierarchy()
ens_pdb_hierarchy.atoms().reset_i_seq()
ens_pdb_xrs_s = pdb_in.input.xray_structures_simple()
number_structures = len(ens_pdb_xrs_s)
print >> self.log, 'Number of structure in ensemble : ', number_structures
# Calculate sigmas from input map only
if self.params.assign_sigma_from_map and self.params.ensemble_sigma_map_input is not None:
# process MTZ
input_file = file_reader.any_file(self.params.ensemble_sigma_map_input)
if input_file.file_type == "hkl" :
if input_file.file_object.file_type() != "ccp4_mtz" :
raise Sorry("Only MTZ format accepted for map input")
else:
mtz_file = input_file
else:
raise Sorry("Only MTZ format accepted for map input")
miller_arrays = mtz_file.file_server.miller_arrays
map_coeffs_1 = miller_arrays[0]
#
xrs_list = []
for n, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
# get sigma levels from ensemble fc for each structure
xrs = get_map_sigma(ens_pdb_hierarchy = ens_pdb_hierarchy,
ens_pdb_xrs = ens_pdb_xrs,
map_coeffs_1 = map_coeffs_1,
residue_detail = self.params.residue_detail,
ignore_hd = self.params.ignore_hd,
log = self.log)
xrs_list.append(xrs)
# write ensemble pdb file, occupancies as sigma level
filename = pdb_file_names[0].split('/')[-1].replace('.pdb','') + '_vs_' + self.params.ensemble_sigma_map_input.replace('.mtz','') + '_pensemble.pdb'
write_ensemble_pdb(filename = filename,
xrs_list = xrs_list,
ens_pdb_hierarchy = ens_pdb_hierarchy
)
# Do full analysis vs Fobs
else:
model_map_coeffs = []
fmodel = None
# Get <fcalc>
for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
ens_pdb_xrs.set_occupancies(1.0)
if model == 0:
# If mtz not supplied get fobs from xray structure...
# Use input Fobs for scoring against nll
if self.params.fobs_vs_fcalc_post_nll:
dummy_fobs = f_obs
else:
if f_obs == None:
if self.params.fcalc_high_resolution == None:
raise Sorry("Please supply high resolution limit or input mtz file.")
dummy_dmin = self.params.fcalc_high_resolution
dummy_dmax = self.params.fcalc_low_resolution
else:
print >> self.log, 'Supplied mtz used to determine high and low resolution cuttoffs'
dummy_dmax, dummy_dmin = f_obs.d_max_min()
#
dummy_fobs = abs(ens_pdb_xrs.structure_factors(d_min = dummy_dmin).f_calc())
dummy_fobs.set_observation_type_xray_amplitude()
# If mtz supplied, free flags are over written to prevent array size error
r_free_flags = dummy_fobs.array(data=flex.bool(dummy_fobs.data().size(),False))
#
fmodel = utils.fmodel_simple(
scattering_table = "wk1995",
xray_structures = [ens_pdb_xrs],
f_obs = dummy_fobs,
target_name = 'ls',
bulk_solvent_and_scaling = False,
r_free_flags = r_free_flags
)
f_calc_ave = fmodel.f_calc().array(data = fmodel.f_calc().data()*0).deep_copy()
# XXX Important to ensure scale is identical for each model and <model>
fmodel.set_scale_switch = 1.0
f_calc_ave_total = fmodel.f_calc().data().deep_copy()
else:
fmodel.update_xray_structure(xray_structure = ens_pdb_xrs,
update_f_calc = True,
update_f_mask = False)
f_calc_ave_total += fmodel.f_calc().data().deep_copy()
print >> self.log, 'Model :', model+1
print >> self.log, "\nStructure vs real Fobs (no bulk solvent or scaling)"
print >> self.log, 'Rwork : %5.4f '%fmodel.r_work()
print >> self.log, 'Rfree : %5.4f '%fmodel.r_free()
print >> self.log, 'K1 : %5.4f '%fmodel.scale_k1()
fcalc_edm = fmodel.electron_density_map()
fcalc_map_coeffs = fcalc_edm.map_coefficients(map_type = 'Fc')
fcalc_mtz_dataset = fcalc_map_coeffs.as_mtz_dataset(column_root_label ='Fc')
if self.params.output_model_and_model_ave_mtz:
fcalc_mtz_dataset.mtz_object().write(file_name = str(model+1)+"_Fc.mtz")
model_map_coeffs.append(fcalc_map_coeffs.deep_copy())
fmodel.update(f_calc = f_calc_ave.array(f_calc_ave_total / number_structures))
print >> self.log, "\nEnsemble vs real Fobs (no bulk solvent or scaling)"
print >> self.log, 'Rwork : %5.4f '%fmodel.r_work()
print >> self.log, 'Rfree : %5.4f '%fmodel.r_free()
print >> self.log, 'K1 : %5.4f '%fmodel.scale_k1()
# Get <Fcalc> map
fcalc_ave_edm = fmodel.electron_density_map()
fcalc_ave_map_coeffs = fcalc_ave_edm.map_coefficients(map_type = 'Fc').deep_copy()
fcalc_ave_mtz_dataset = fcalc_ave_map_coeffs.as_mtz_dataset(column_root_label ='Fc')
if self.params.output_model_and_model_ave_mtz:
fcalc_ave_mtz_dataset.mtz_object().write(file_name = "aveFc.mtz")
fcalc_ave_map_coeffs = fcalc_ave_map_coeffs.fft_map()
fcalc_ave_map_coeffs.apply_volume_scaling()
fcalc_ave_map_data = fcalc_ave_map_coeffs.real_map_unpadded()
fcalc_ave_map_stats = maptbx.statistics(fcalc_ave_map_data)
print >> self.log, "<Fcalc> Map Stats :"
fcalc_ave_map_stats.show_summary(f = self.log)
offset = fcalc_ave_map_stats.min()
model_neg_ll = []
number_previous_scatters = 0
# Run through structure list again and get probability
xrs_list = []
for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
if self.params.verbose:
print >> self.log, '\n\nModel : ', model+1
# Get model atom sigmas vs Fcalc
fcalc_map = model_map_coeffs[model].fft_map()
fcalc_map.apply_volume_scaling()
fcalc_map_data = fcalc_map.real_map_unpadded()
fcalc_map_stats = maptbx.statistics(fcalc_map_data)
if self.params.verbose:
print >> self.log, "Fcalc map stats :"
fcalc_map_stats.show_summary(f = self.log)
xrs = get_map_sigma(ens_pdb_hierarchy = ens_pdb_hierarchy,
ens_pdb_xrs = ens_pdb_xrs,
fft_map_1 = fcalc_map,
model_i = model,
residue_detail = self.params.residue_detail,
ignore_hd = self.params.ignore_hd,
number_previous_scatters = number_previous_scatters,
log = self.log)
fcalc_sigmas = xrs.scatterers().extract_occupancies()
del fcalc_map
# Get model atom sigmas vs <Fcalc>
xrs = get_map_sigma(ens_pdb_hierarchy = ens_pdb_hierarchy,
ens_pdb_xrs = ens_pdb_xrs,
fft_map_1 = fcalc_ave_map_coeffs,
model_i = model,
residue_detail = self.params.residue_detail,
ignore_hd = self.params.ignore_hd,
number_previous_scatters = number_previous_scatters,
log = self.log)
### For testing other residue averaging options
#print xrs.residue_selections
fcalc_ave_sigmas = xrs.scatterers().extract_occupancies()
# Probability of model given <model>
prob = fcalc_ave_sigmas / fcalc_sigmas
# XXX debug option
if False:
for n,p in enumerate(prob):
print >> self.log, ' {0:5d} {1:5.3f}'.format(n,p)
# Set probabilty between 0 and 1
# XXX Make Histogram / more stats
prob_lss_zero = flex.bool(prob <= 0)
prob_grt_one = flex.bool(prob > 1)
prob.set_selected(prob_lss_zero, 0.001)
prob.set_selected(prob_grt_one, 1.0)
xrs.set_occupancies(prob)
xrs_list.append(xrs)
sum_neg_ll = sum(-flex.log(prob))
model_neg_ll.append((sum_neg_ll, model))
if self.params.verbose:
print >> self.log, 'Model probability stats :'
print >> self.log, prob.min_max_mean().show()
print >> self.log, ' Count < 0.0 : ', prob_lss_zero.count(True)
print >> self.log, ' Count > 1.0 : ', prob_grt_one.count(True)
# For averaging by residue
number_previous_scatters += ens_pdb_xrs.sites_cart().size()
# write ensemble pdb file, occupancies as sigma level
write_ensemble_pdb(filename = pdb_file_names[0].split('/')[-1].replace('.pdb','') + '_pensemble.pdb',
xrs_list = xrs_list,
ens_pdb_hierarchy = ens_pdb_hierarchy
)
# XXX Test ordering models by nll
# XXX Test removing nth percentile atoms
if self.params.sort_ensemble_by_nll_score or self.params.fobs_vs_fcalc_post_nll:
for percentile in [1.0,0.975,0.95,0.9,0.8,0.6,0.2]:
model_neg_ll = sorted(model_neg_ll)
f_calc_ave_total_reordered = None
print_list = []
for i_neg_ll in model_neg_ll:
xrs = xrs_list[i_neg_ll[1]]
nll_occ = xrs.scatterers().extract_occupancies()
# Set q=0 nth percentile atoms
sorted_nll_occ = sorted(nll_occ, reverse=True)
number_atoms = len(sorted_nll_occ)
percentile_prob_cutoff = sorted_nll_occ[int(number_atoms * percentile)-1]
cutoff_selections = flex.bool(nll_occ < percentile_prob_cutoff)
cutoff_nll_occ = flex.double(nll_occ.size(), 1.0).set_selected(cutoff_selections, 0.0)
#XXX Debug
if False:
print '\nDebug'
for x in xrange(len(cutoff_selections)):
print cutoff_selections[x], nll_occ[x], cutoff_nll_occ[x]
print percentile
print percentile_prob_cutoff
print cutoff_selections.count(True)
print cutoff_selections.size()
print cutoff_nll_occ.count(0.0)
print 'Count q = 1 : ', cutoff_nll_occ.count(1.0)
print 'Count scatterers size : ', cutoff_nll_occ.size()
xrs.set_occupancies(cutoff_nll_occ)
fmodel.update_xray_structure(xray_structure = xrs,
update_f_calc = True,
update_f_mask = True)
if f_calc_ave_total_reordered == None:
f_calc_ave_total_reordered = fmodel.f_calc().data().deep_copy()
f_mask_ave_total_reordered = fmodel.f_masks()[0].data().deep_copy()
cntr = 1
else:
f_calc_ave_total_reordered += fmodel.f_calc().data().deep_copy()
f_mask_ave_total_reordered += fmodel.f_masks()[0].data().deep_copy()
cntr+=1
fmodel.update(f_calc = f_calc_ave.array(f_calc_ave_total_reordered / cntr).deep_copy(),
f_mask = f_calc_ave.array(f_mask_ave_total_reordered / cntr).deep_copy()
)
# Update solvent and scale
# XXX Will need to apply_back_trace on latest version
fmodel.set_scale_switch = 0
fmodel.update_all_scales()
# Reset occ for outout
xrs.set_occupancies(nll_occ)
# k1 updated vs Fobs
if self.params.fobs_vs_fcalc_post_nll:
print_list.append([cntr, i_neg_ll[0], i_neg_ll[1], fmodel.r_work(), fmodel.r_free()])
# Order models by nll and print summary
print >> self.log, '\nModels ranked by nll <Fcalc> R-factors recalculated'
print >> self.log, 'Percentile cutoff : {0:5.3f}'.format(percentile)
xrs_list_sorted_nll = []
print >> self.log, ' | NLL <Rw> <Rf> Ens Model'
for info in print_list:
print >> self.log, ' {0:4d} | {1:8.1f} {2:8.4f} {3:8.4f} {4:12d}'.format(
info[0],
info[1],
info[3],
info[4],
info[2]+1,
)
xrs_list_sorted_nll.append(xrs_list[info[2]])
# Output nll ordered ensemble
write_ensemble_pdb(filename = 'nll_ordered_' + pdb_file_names[0].split('/')[-1].replace('.pdb','') + '_pensemble.pdb',
xrs_list = xrs_list_sorted_nll,
ens_pdb_hierarchy = ens_pdb_hierarchy
)
phase_source = '/mnt/data4/XFEL/LR23/DED_tests/dat/Dark.pdb'
f_obs_1_file_name = 'a/ligh_old_truncate.mtz'
f_obs_2_file_name = 'a/dark_old_truncate.mtz'
def multiscale(self, other, reflections_per_bin=None):
if (reflections_per_bin is None):
reflections_per_bin = other.indices().size()
assert self.indices().all_eq(other.indices())
assert self.is_similar_symmetry(other)
self.setup_binner(reflections_per_bin=reflections_per_bin)
other.use_binning_of(self)
scale = flex.double(self.indices().size(), -1)
for i_bin in self.binner().range_used():
sel = self.binner().selection(i_bin)
f1 = self.select(sel)
f2 = other.select(sel)
scale_ = 1.0
den = flex.sum(flex.abs(f2.data()) * flex.abs(f2.data()))
if (den != 0):
scale_ = flex.sum(flex.abs(f1.data()) * flex.abs(f2.data())) / den
scale.set_selected(sel, scale_)
assert (scale > 0).count(True) == scale.size()
return other.array(data=other.data() * scale)
parameters = utils.data_and_flags_master_params().extract()
print(parameters.labels)
inputs = mmtbx.utils.process_command_line_args(args=args)
rfs = reflection_file_utils.reflection_file_server(
crystal_symmetry=inputs.crystal_symmetry,
force_symmetry=True,
reflection_files=inputs.reflection_files,
err=StringIO(),
)
pdb_inp = iotbx.pdb.input(file_name=inputs.pdb_file_names[0])
ph = pdb_inp.construct_hierarchy()
xrs = ph.extract_xray_structure(crystal_symmetry=inputs.crystal_symmetry)
xrs.show_summary()
data_flags_params = data_and_flags_master_params().extract()
data_flags_params.labels = params.exhaustive.options.column_type
determined_data_and_flags = mmtbx.utils.determine_data_and_flags(
reflection_file_server=rfs,
parameters=data_flags_params,
keep_going=True,
log=StringIO(),
)
f_obs = determined_data_and_flags.f_obs
r_free_flags = determined_data_and_flags.r_free_flags
sites_frac = xrs.sites_frac()
f_obs.show_summary()