def run():
data_dir = os.path.dirname(os.path.abspath(__file__))
data_mtz = os.path.join(data_dir, 'data', 'phaser_1.mtz')
params = ed.data_and_flags_master_params().extract()
params.file_name = data_mtz
params.labels=['FP,SIGFP']
phases_params = ed.experimental_phases_master_params().extract()
phases_params.labels="HLA,HLB,HLC,HLD"
reflection_files = [
iotbx.reflection_file_reader.any_reflection_file(data_mtz)]
server = iotbx.reflection_file_utils.reflection_file_server(
crystal_symmetry = None,
reflection_files = reflection_files,
err = sys.stdout)
fo = ed.run(
reflection_file_server = server,
parameters = params,
experimental_phases_params = phases_params,
extract_r_free_flags = False).f_obs
assert cctbx.miller.array(fo, cctbx.miller.array)
def get_data(pdbf, mtzf):
pdb_inp = iotbx.pdb.input(file_name=pdbf)
xrs = pdb_inp.xray_structure_simple()
#
selection = xrs.scatterers().extract_occupancies() > 0
xrs = xrs.select(selection)
selection = ~xrs.hd_selection()
xrs = xrs.select(selection)
#
#xrs.switch_to_neutron_scattering_dictionary()
#
reflection_file = reflection_file_reader.any_reflection_file(
file_name=mtzf, ensure_read_access=False)
rfs = reflection_file_utils.reflection_file_server(
crystal_symmetry=xrs.crystal_symmetry(),
force_symmetry=True,
reflection_files=[reflection_file],
err=null_out())
determine_data_and_flags_result = extract_xtal_data.run(
reflection_file_server=rfs,
keep_going=True,
extract_r_free_flags=True,
force_non_anomalous=True,
allow_mismatch_flags=True)
f_obs = determine_data_and_flags_result.f_obs
r_free_flags = determine_data_and_flags_result.r_free_flags
fmodel = mmtbx.f_model.manager(f_obs=f_obs,
r_free_flags=r_free_flags,
xray_structure=xrs)
fmodel.update_all_scales(remove_outliers=True,
apply_scale_k1_to_f_obs=True)
# Skip
rw = fmodel.r_work()
rf = fmodel.r_free()
if (fmodel.f_obs().d_min() > 3): return None
if (fmodel.f_obs().completeness() < 0.95): return None
if (rw > 0.25): return None
if (rf <= rw): return None
if (abs(rf - rw) * 100 < 2): return None
if (fmodel.f_obs().resolution_filter(d_min=8).completeness() < 0.95):
return None
#
def f_obs():
return fmodel.f_obs()
def r_free_flags():
return fmodel.r_free_flags()
def f_calc():
return fmodel.f_calc()
def flags():
return fmodel.r_free_flags()
return group_args(f_obs=f_obs,
r_free_flags=r_free_flags,
xray_structure=fmodel.xray_structure,
f_calc=f_calc,
flags=flags)
def get_f_obs_and_flags(reflection_file_name,
crystal_symmetry,
f_obs_label=None,
r_free_flags_label=None,
log=None):
reflection_files = []
reflection_files.append(
reflection_file_reader.any_reflection_file(
file_name=reflection_file_name, ensure_read_access=False))
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=log)
parameters = extract_xtal_data.data_and_flags_master_params().extract()
if (f_obs_label is not None):
parameters.labels = command_line.options.f_obs_label
if (r_free_flags_label is not None):
parameters.r_free_flags.label = command_line.options.r_free_flags_label
determine_data_and_flags_result = extract_xtal_data.run(
reflection_file_server=reflection_file_server,
parameters=parameters,
keep_going=True)
f_obs = determine_data_and_flags_result.f_obs
r_free_flags = determine_data_and_flags_result.r_free_flags
return f_obs, r_free_flags
def get_fobs_rfree(self, crystal_symmetry):
f_obs, r_free_flags = None, None
rfs = self.data_manager.get_reflection_file_server(
filenames=self.data_manager.get_miller_array_names(),
crystal_symmetry=crystal_symmetry,
logger=null_out())
parameters = extract_xtal_data.data_and_flags_master_params().extract()
if (self.params.data_labels is not None):
parameters.labels = self.params.data_labels
if (self.params.r_free_flags_labels is not None):
parameters.r_free_flags.label = self.params.r_free_flags_labels
determined_data_and_flags = extract_xtal_data.run(
reflection_file_server=rfs,
parameters=parameters,
keep_going=True,
working_point_group=crystal_symmetry.space_group(
).build_derived_point_group())
f_obs = determined_data_and_flags.f_obs
r_free_flags = determined_data_and_flags.r_free_flags
assert (f_obs is not None)
if (self.params.data_labels is None):
self.params.data_labels = f_obs.info().label_string()
if (r_free_flags is not None):
self.params.r_free_flags_labels = r_free_flags.info().label_string(
)
return f_obs, r_free_flags
def run(processed_args, params):
if (params.scattering_table
not in ["n_gaussian", "wk1995", "it1992", "neutron"]):
raise Sorry("Incorrect scattering_table.")
crystal_symmetry = None
crystal_symmetries = []
for f in [str(params.pdb_file_name), str(params.reflection_file_name)]:
cs = crystal_symmetry_from_any.extract_from(f)
if (cs is not None): crystal_symmetries.append(cs)
if (len(crystal_symmetries) == 1): crystal_symmetry = crystal_symmetries[0]
elif (len(crystal_symmetries) == 0):
raise Sorry("No crystal symmetry found.")
else:
if (not crystal_symmetries[0].is_similar_symmetry(
crystal_symmetries[1])):
raise Sorry("Crystal symmetry mismatch between different files.")
crystal_symmetry = crystal_symmetries[0]
f_obs = None
r_free_flags = None
if (params.reflection_file_name is not None):
reflection_file = reflection_file_reader.any_reflection_file(
file_name=params.reflection_file_name, ensure_read_access=True)
rfs = reflection_file_server(crystal_symmetry=crystal_symmetry,
reflection_files=[reflection_file])
parameters = extract_xtal_data.data_and_flags_master_params().extract()
if (params.data_labels is not None):
parameters.labels = [processed_args.data_labels]
determine_data_and_flags_result = extract_xtal_data.run(
reflection_file_server=rfs,
parameters=parameters,
keep_going=True,
log=StringIO())
f_obs = determine_data_and_flags_result.f_obs
r_free_flags = determine_data_and_flags_result.r_free_flags
if (r_free_flags is None):
r_free_flags = f_obs.array(
data=flex.bool(f_obs.data().size(), False))
test_flag_value = None
xray_structure = iotbx.pdb.input(
file_name=params.pdb_file_name).xray_structure_simple()
xray_structure_da = None
if (params.external_da_pdb_file_name is not None):
xray_structure_da = iotbx.pdb.input(
file_name=params.external_da_pdb_file_name).xray_structure_simple(
)
if (f_obs is not None):
f_obs = f_obs.resolution_filter(d_min=params.high_resolution,
d_max=params.low_resolution)
r_free_flags = r_free_flags.resolution_filter(
d_min=params.high_resolution, d_max=params.low_resolution)
#
assert params.mode in ["build", "build_and_refine"]
grow_density(f_obs=f_obs,
r_free_flags=r_free_flags,
xray_structure=xray_structure,
xray_structure_da=xray_structure_da,
params=params)
def run(args, external_params=None, out=sys.stdout):
from mmtbx.refinement import select_best_starting_model
from iotbx.file_reader import any_file
import iotbx.phil
cmdline = iotbx.phil.process_command_line_with_files(
args=args,
master_phil=master_params(),
pdb_file_def="input.model",
reflection_file_def="input.xray_data.file_name",
usage_string="""\
mmtbx.select_best_starting_model data.mtz model1.pdb model2.pdb [...]
Given experimental data and a set of models, determines whether any of the
models can be refined directly (i.e. if isomorphous), and picks the one
with the best R-free below a specified cutoff. Will optionally perform
rigid-body refinement on suitable models if requested.""")
params = cmdline.work.extract()
validate_params(params)
hkl_in = any_file(params.input.xray_data.file_name)
data_and_flags = extract_xtal_data.run(
reflection_file_server=hkl_in.file_server,
parameters=params.input.xray_data,
data_parameter_scope="input.data",
flags_parameter_scope="input.data.r_free_flags",
log=out)
model_data = []
for file_name in params.input.model:
model_in = cmdline.get_file(file_name=file_name,
force_type="pdb").file_object
pdb_hierarchy = model_in.hierarchy
xray_structure = model_in.xray_structure_simple()
model_data.append((pdb_hierarchy, xray_structure))
# we can optionally pass a parameter block from elsewhere (e.g. phenix ligand
# pipeline) and just use this run() method to load files
params_ = external_params
if (params_ is None):
params_ = params
result = select_best_starting_model.select_model(
model_names=params.input.model,
model_data=model_data,
f_obs=data_and_flags.f_obs,
r_free_flags=data_and_flags.r_free_flags,
params=params_,
nproc=params.nproc,
skip_twin_detection=params.input.skip_twin_detection,
log=out)
if result.success() and params.output.write_files:
result.save_best_model(file_name=params.output.model_file_name)
print("", file=out)
print("Wrote best model to %s" % params.output.model_file_name,
file=out)
result.save_updated_data(file_name=params.output.data_file_name)
print("Wrote updated data to %s" % params.output.data_file_name,
file=out)
return result
def get_fmodel(
self,
array_type,
crystal_symmetry=None,
parameters=None, # XXX Replace with what DataManager uses
experimental_phases_params=None, # XXX Need to be part of 'parameters'
scattering_table=None # XXX Make part of parameters?
):
#
if (array_type == "x_ray"):
assert scattering_table in ["wk1995", "it1992", "n_gaussian"]
elif (array_type == "neutron"):
assert scattering_table == "neutron"
elif (array_type == "electron"):
assert scattering_table == "electron"
# Gather models of apropriate type
models = []
for filename in self.get_model_names(model_type=array_type):
models.append(self.get_model(filename))
if (len(models) == 0):
raise Sorry("No model of '%s' type found to make fmodel." %
array_type)
if (len(models) > 1):
raise Sorry("More than one model of '%s' type found." % array_type)
model = models[0]
# Get reflection file server
rfs = self.get_reflection_file_server(array_type=array_type)
# Resolve symmetry issues (nplace)
self._resolve_symmetry_conflicts(params=crystal_symmetry,
model=model,
reflection_file_server=rfs)
# Get reflection data
data = extract_xtal_data.run(
reflection_file_server=rfs,
parameters=parameters,
experimental_phases_params=experimental_phases_params,
working_point_group=model.crystal_symmetry().space_group(
).build_derived_point_group(),
remark_r_free_flags_md5_hexdigest=model.
get_header_r_free_flags_md5_hexdigest()).result()
if (len(data.err) > 0):
raise Sorry("\n".join(data.err))
if (data.f_obs is None):
raise Sorry("Diffraction date are not available to make fmodel.")
# Setup scattering table of xray_structure
model.setup_scattering_dictionaries(scattering_table=scattering_table,
d_min=data.f_obs.d_min())
# Create and return fmodel
fmodel = mmtbx.f_model.manager(
f_obs=data.f_obs,
r_free_flags=data.r_free_flags,
abcd=data.experimental_phases,
xray_structure=model.get_xray_structure(),
origin=data.mtz_object)
return fmodel
def get_map_from_hkl(hkl_file_object, params, xrs, log):
print("Processing input hkl file...", file=log)
crystal_symmetry = hkl_file_object.crystal_symmetry()
rfs = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=[hkl_file_object.file_content],
err=StringIO())
parameters = extract_xtal_data.data_and_flags_master_params().extract()
if (params.data_labels is not None):
parameters.labels = params.data_labels
if (params.r_free_flags_labels is not None):
parameters.r_free_flags.label = params.r_free_flags_labels
determined_data_and_flags = extract_xtal_data.run(
reflection_file_server=rfs,
parameters=parameters,
keep_going=True,
working_point_group=crystal_symmetry.space_group(
).build_derived_point_group(),
log=StringIO(),
symmetry_safety_check=True)
f_obs = determined_data_and_flags.f_obs
if (params.data_labels is None):
params.data_labels = f_obs.info().label_string()
r_free_flags = determined_data_and_flags.r_free_flags
assert f_obs is not None
print("Input data:", file=log)
print(" Iobs or Fobs:", f_obs.info().labels, file=log)
if (r_free_flags is not None):
print(" Free-R flags:", r_free_flags.info().labels, file=log)
params.r_free_flags_labels = r_free_flags.info().label_string()
else:
print(" Free-R flags: Not present", file=log)
fmodel = mmtbx.f_model.manager(f_obs=f_obs,
r_free_flags=r_free_flags,
xray_structure=xrs)
fmodel.update_all_scales()
fft_map = fmodel.electron_density_map().fft_map(
resolution_factor=0.25, map_type="2mFo-DFc",
use_all_data=False) # Exclude free reflections
fft_map.apply_sigma_scaling()
map_data = fft_map.real_map_unpadded(in_place=False)
if params.debug:
fft_map.as_xplor_map(file_name="%s_21.map" % params.output_prefix)
iotbx.mrcfile.write_ccp4_map(
file_name="%s_21.ccp4" % params.output_prefix,
unit_cell=crystal_symmetry.unit_cell(),
space_group=crystal_symmetry.space_group(),
map_data=map_data,
labels=flex.std_string([""]))
return map_data, crystal_symmetry
def get_fobs_rfree(self, crystal_symmetry):
rfs = self.data_manager.get_reflection_file_server(
filenames=[self.data_manager.get_default_miller_array_name()],
crystal_symmetry=crystal_symmetry,
logger=null_out())
parameters = extract_xtal_data.data_and_flags_master_params().extract()
determined_data_and_flags = extract_xtal_data.run(
reflection_file_server=rfs,
parameters=parameters,
keep_going=True,
working_point_group=crystal_symmetry.space_group(
).build_derived_point_group())
f_obs = determined_data_and_flags.f_obs
r_free_flags = determined_data_and_flags.r_free_flags
return f_obs, r_free_flags
def extract_data_and_flags(params, crystal_symmetry=None):
data_and_flags = None
if (params.reflection_file_name is not None):
reflection_file = reflection_file_reader.any_reflection_file(
file_name=params.reflection_file_name)
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=[reflection_file])
parameters = extract_xtal_data.data_and_flags_master_params().extract()
parameters.force_anomalous_flag_to_be_equal_to = False
if (params.data_labels is not None):
parameters.labels = [params.data_labels]
if (params.high_resolution is not None):
parameters.high_resolution = params.high_resolution
if (params.low_resolution is not None):
parameters.low_resolution = params.low_resolution
data_and_flags = extract_xtal_data.run(
reflection_file_server=reflection_file_server,
parameters=parameters,
extract_r_free_flags=False) #XXX
return data_and_flags
def run(self, args, command_name, out=sys.stdout):
command_line = (iotbx_option_parser(
usage="%s [options]" % command_name,
description='Example: %s data.mtz data.mtz ref_model.pdb' %
command_name).option(
None,
"--show_defaults",
action="store_true",
help="Show list of parameters.")).process(args=args)
cif_file = None
processed_args = utils.process_command_line_args(
args=args, log=sys.stdout, master_params=master_phil)
params = processed_args.params
if (params is None): params = master_phil
self.params = params.extract().ensemble_probability
pdb_file_names = processed_args.pdb_file_names
if len(pdb_file_names) != 1:
raise Sorry("Only one PDB structure may be used")
pdb_file = file_reader.any_file(pdb_file_names[0])
self.log = multi_out()
self.log.register(label="stdout", file_object=sys.stdout)
self.log.register(label="log_buffer",
file_object=StringIO(),
atexit_send_to=None)
sys.stderr = self.log
log_file = open(
pdb_file_names[0].split('/')[-1].replace('.pdb', '') +
'_pensemble.log', "w")
self.log.replace_stringio(old_label="log_buffer",
new_label="log",
new_file_object=log_file)
utils.print_header(command_name, out=self.log)
params.show(out=self.log)
#
f_obs = None
r_free_flags = None
reflection_files = processed_args.reflection_files
if self.params.fobs_vs_fcalc_post_nll:
if len(reflection_files) == 0:
raise Sorry(
"Fobs from input MTZ required for fobs_vs_fcalc_post_nll")
if len(reflection_files) > 0:
crystal_symmetry = processed_args.crystal_symmetry
print('Reflection file : ',
processed_args.reflection_file_names[0],
file=self.log)
utils.print_header("Model and data statistics", out=self.log)
rfs = reflection_file_server(
crystal_symmetry=crystal_symmetry,
reflection_files=processed_args.reflection_files,
log=self.log)
parameters = extract_xtal_data.data_and_flags_master_params(
).extract()
determine_data_and_flags_result = extract_xtal_data.run(
reflection_file_server=rfs,
parameters=parameters,
data_parameter_scope="refinement.input.xray_data",
flags_parameter_scope="refinement.input.xray_data.r_free_flags",
data_description="X-ray data",
keep_going=True,
log=self.log)
f_obs = determine_data_and_flags_result.f_obs
number_of_reflections = f_obs.indices().size()
r_free_flags = determine_data_and_flags_result.r_free_flags
test_flag_value = determine_data_and_flags_result.test_flag_value
if (r_free_flags is None):
r_free_flags = f_obs.array(
data=flex.bool(f_obs.data().size(), False))
# process PDB
pdb_file.assert_file_type("pdb")
#
pdb_in = hierarchy.input(file_name=pdb_file.file_name)
ens_pdb_hierarchy = pdb_in.construct_hierarchy()
ens_pdb_hierarchy.atoms().reset_i_seq()
ens_pdb_xrs_s = pdb_in.input.xray_structures_simple()
number_structures = len(ens_pdb_xrs_s)
print('Number of structure in ensemble : ',
number_structures,
file=self.log)
# Calculate sigmas from input map only
if self.params.assign_sigma_from_map and self.params.ensemble_sigma_map_input is not None:
# process MTZ
input_file = file_reader.any_file(
self.params.ensemble_sigma_map_input)
if input_file.file_type == "hkl":
if input_file.file_object.file_type() != "ccp4_mtz":
raise Sorry("Only MTZ format accepted for map input")
else:
mtz_file = input_file
else:
raise Sorry("Only MTZ format accepted for map input")
miller_arrays = mtz_file.file_server.miller_arrays
map_coeffs_1 = miller_arrays[0]
#
xrs_list = []
for n, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
# get sigma levels from ensemble fc for each structure
xrs = get_map_sigma(ens_pdb_hierarchy=ens_pdb_hierarchy,
ens_pdb_xrs=ens_pdb_xrs,
map_coeffs_1=map_coeffs_1,
residue_detail=self.params.residue_detail,
ignore_hd=self.params.ignore_hd,
log=self.log)
xrs_list.append(xrs)
# write ensemble pdb file, occupancies as sigma level
filename = pdb_file_names[0].split('/')[-1].replace(
'.pdb',
'') + '_vs_' + self.params.ensemble_sigma_map_input.replace(
'.mtz', '') + '_pensemble.pdb'
write_ensemble_pdb(filename=filename,
xrs_list=xrs_list,
ens_pdb_hierarchy=ens_pdb_hierarchy)
# Do full analysis vs Fobs
else:
model_map_coeffs = []
fmodel = None
# Get <fcalc>
for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
ens_pdb_xrs.set_occupancies(1.0)
if model == 0:
# If mtz not supplied get fobs from xray structure...
# Use input Fobs for scoring against nll
if self.params.fobs_vs_fcalc_post_nll:
dummy_fobs = f_obs
else:
if f_obs == None:
if self.params.fcalc_high_resolution == None:
raise Sorry(
"Please supply high resolution limit or input mtz file."
)
dummy_dmin = self.params.fcalc_high_resolution
dummy_dmax = self.params.fcalc_low_resolution
else:
print(
'Supplied mtz used to determine high and low resolution cuttoffs',
file=self.log)
dummy_dmax, dummy_dmin = f_obs.d_max_min()
#
dummy_fobs = abs(
ens_pdb_xrs.structure_factors(
d_min=dummy_dmin).f_calc())
dummy_fobs.set_observation_type_xray_amplitude()
# If mtz supplied, free flags are over written to prevent array size error
r_free_flags = dummy_fobs.array(
data=flex.bool(dummy_fobs.data().size(), False))
#
fmodel = utils.fmodel_simple(
scattering_table="wk1995",
xray_structures=[ens_pdb_xrs],
f_obs=dummy_fobs,
target_name='ls',
bulk_solvent_and_scaling=False,
r_free_flags=r_free_flags)
f_calc_ave = fmodel.f_calc().array(
data=fmodel.f_calc().data() * 0).deep_copy()
# XXX Important to ensure scale is identical for each model and <model>
fmodel.set_scale_switch = 1.0
f_calc_ave_total = fmodel.f_calc().data().deep_copy()
else:
fmodel.update_xray_structure(xray_structure=ens_pdb_xrs,
update_f_calc=True,
update_f_mask=False)
f_calc_ave_total += fmodel.f_calc().data().deep_copy()
print('Model :', model + 1, file=self.log)
print("\nStructure vs real Fobs (no bulk solvent or scaling)",
file=self.log)
print('Rwork : %5.4f ' % fmodel.r_work(),
file=self.log)
print('Rfree : %5.4f ' % fmodel.r_free(),
file=self.log)
print('K1 : %5.4f ' % fmodel.scale_k1(),
file=self.log)
fcalc_edm = fmodel.electron_density_map()
fcalc_map_coeffs = fcalc_edm.map_coefficients(map_type='Fc')
fcalc_mtz_dataset = fcalc_map_coeffs.as_mtz_dataset(
column_root_label='Fc')
if self.params.output_model_and_model_ave_mtz:
fcalc_mtz_dataset.mtz_object().write(
file_name=str(model + 1) + "_Fc.mtz")
model_map_coeffs.append(fcalc_map_coeffs.deep_copy())
fmodel.update(f_calc=f_calc_ave.array(f_calc_ave_total /
number_structures))
print("\nEnsemble vs real Fobs (no bulk solvent or scaling)",
file=self.log)
print('Rwork : %5.4f ' % fmodel.r_work(), file=self.log)
print('Rfree : %5.4f ' % fmodel.r_free(), file=self.log)
print('K1 : %5.4f ' % fmodel.scale_k1(), file=self.log)
# Get <Fcalc> map
fcalc_ave_edm = fmodel.electron_density_map()
fcalc_ave_map_coeffs = fcalc_ave_edm.map_coefficients(
map_type='Fc').deep_copy()
fcalc_ave_mtz_dataset = fcalc_ave_map_coeffs.as_mtz_dataset(
column_root_label='Fc')
if self.params.output_model_and_model_ave_mtz:
fcalc_ave_mtz_dataset.mtz_object().write(file_name="aveFc.mtz")
fcalc_ave_map_coeffs = fcalc_ave_map_coeffs.fft_map()
fcalc_ave_map_coeffs.apply_volume_scaling()
fcalc_ave_map_data = fcalc_ave_map_coeffs.real_map_unpadded()
fcalc_ave_map_stats = maptbx.statistics(fcalc_ave_map_data)
print("<Fcalc> Map Stats :", file=self.log)
fcalc_ave_map_stats.show_summary(f=self.log)
offset = fcalc_ave_map_stats.min()
model_neg_ll = []
number_previous_scatters = 0
# Run through structure list again and get probability
xrs_list = []
for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
if self.params.verbose:
print('\n\nModel : ',
model + 1,
file=self.log)
# Get model atom sigmas vs Fcalc
fcalc_map = model_map_coeffs[model].fft_map()
fcalc_map.apply_volume_scaling()
fcalc_map_data = fcalc_map.real_map_unpadded()
fcalc_map_stats = maptbx.statistics(fcalc_map_data)
if self.params.verbose:
print("Fcalc map stats :", file=self.log)
fcalc_map_stats.show_summary(f=self.log)
xrs = get_map_sigma(
ens_pdb_hierarchy=ens_pdb_hierarchy,
ens_pdb_xrs=ens_pdb_xrs,
fft_map_1=fcalc_map,
model_i=model,
residue_detail=self.params.residue_detail,
ignore_hd=self.params.ignore_hd,
number_previous_scatters=number_previous_scatters,
log=self.log)
fcalc_sigmas = xrs.scatterers().extract_occupancies()
del fcalc_map
# Get model atom sigmas vs <Fcalc>
xrs = get_map_sigma(
ens_pdb_hierarchy=ens_pdb_hierarchy,
ens_pdb_xrs=ens_pdb_xrs,
fft_map_1=fcalc_ave_map_coeffs,
model_i=model,
residue_detail=self.params.residue_detail,
ignore_hd=self.params.ignore_hd,
number_previous_scatters=number_previous_scatters,
log=self.log)
### For testing other residue averaging options
#print xrs.residue_selections
fcalc_ave_sigmas = xrs.scatterers().extract_occupancies()
# Probability of model given <model>
prob = fcalc_ave_sigmas / fcalc_sigmas
# XXX debug option
if False:
for n, p in enumerate(prob):
print(' {0:5d} {1:5.3f}'.format(n, p), file=self.log)
# Set probabilty between 0 and 1
# XXX Make Histogram / more stats
prob_lss_zero = flex.bool(prob <= 0)
prob_grt_one = flex.bool(prob > 1)
prob.set_selected(prob_lss_zero, 0.001)
prob.set_selected(prob_grt_one, 1.0)
xrs.set_occupancies(prob)
xrs_list.append(xrs)
sum_neg_ll = sum(-flex.log(prob))
model_neg_ll.append((sum_neg_ll, model))
if self.params.verbose:
print('Model probability stats :', file=self.log)
print(prob.min_max_mean().show(), file=self.log)
print(' Count < 0.0 : ',
prob_lss_zero.count(True),
file=self.log)
print(' Count > 1.0 : ',
prob_grt_one.count(True),
file=self.log)
# For averaging by residue
number_previous_scatters += ens_pdb_xrs.sites_cart().size()
# write ensemble pdb file, occupancies as sigma level
write_ensemble_pdb(
filename=pdb_file_names[0].split('/')[-1].replace('.pdb', '') +
'_pensemble.pdb',
xrs_list=xrs_list,
ens_pdb_hierarchy=ens_pdb_hierarchy)
# XXX Test ordering models by nll
# XXX Test removing nth percentile atoms
if self.params.sort_ensemble_by_nll_score or self.params.fobs_vs_fcalc_post_nll:
for percentile in [1.0, 0.975, 0.95, 0.9, 0.8, 0.6, 0.2]:
model_neg_ll = sorted(model_neg_ll)
f_calc_ave_total_reordered = None
print_list = []
for i_neg_ll in model_neg_ll:
xrs = xrs_list[i_neg_ll[1]]
nll_occ = xrs.scatterers().extract_occupancies()
# Set q=0 nth percentile atoms
sorted_nll_occ = sorted(nll_occ, reverse=True)
number_atoms = len(sorted_nll_occ)
percentile_prob_cutoff = sorted_nll_occ[
int(number_atoms * percentile) - 1]
cutoff_selections = flex.bool(
nll_occ < percentile_prob_cutoff)
cutoff_nll_occ = flex.double(nll_occ.size(),
1.0).set_selected(
cutoff_selections,
0.0)
#XXX Debug
if False:
print('\nDebug')
for x in range(len(cutoff_selections)):
print(cutoff_selections[x], nll_occ[x],
cutoff_nll_occ[x])
print(percentile)
print(percentile_prob_cutoff)
print(cutoff_selections.count(True))
print(cutoff_selections.size())
print(cutoff_nll_occ.count(0.0))
print('Count q = 1 : ',
cutoff_nll_occ.count(1.0))
print('Count scatterers size : ',
cutoff_nll_occ.size())
xrs.set_occupancies(cutoff_nll_occ)
fmodel.update_xray_structure(xray_structure=xrs,
update_f_calc=True,
update_f_mask=True)
if f_calc_ave_total_reordered == None:
f_calc_ave_total_reordered = fmodel.f_calc().data(
).deep_copy()
f_mask_ave_total_reordered = fmodel.f_masks(
)[0].data().deep_copy()
cntr = 1
else:
f_calc_ave_total_reordered += fmodel.f_calc().data(
).deep_copy()
f_mask_ave_total_reordered += fmodel.f_masks(
)[0].data().deep_copy()
cntr += 1
fmodel.update(
f_calc=f_calc_ave.array(
f_calc_ave_total_reordered / cntr).deep_copy(),
f_mask=f_calc_ave.array(
f_mask_ave_total_reordered / cntr).deep_copy())
# Update solvent and scale
# XXX Will need to apply_back_trace on latest version
fmodel.set_scale_switch = 0
fmodel.update_all_scales()
# Reset occ for outout
xrs.set_occupancies(nll_occ)
# k1 updated vs Fobs
if self.params.fobs_vs_fcalc_post_nll:
print_list.append([
cntr, i_neg_ll[0], i_neg_ll[1],
fmodel.r_work(),
fmodel.r_free()
])
# Order models by nll and print summary
print(
'\nModels ranked by nll <Fcalc> R-factors recalculated',
file=self.log)
print('Percentile cutoff : {0:5.3f}'.format(percentile),
file=self.log)
xrs_list_sorted_nll = []
print(' | NLL <Rw> <Rf> Ens Model',
file=self.log)
for info in print_list:
print(' {0:4d} | {1:8.1f} {2:8.4f} {3:8.4f} {4:12d}'.
format(
info[0],
info[1],
info[3],
info[4],
info[2] + 1,
),
file=self.log)
xrs_list_sorted_nll.append(xrs_list[info[2]])
# Output nll ordered ensemble
write_ensemble_pdb(
filename='nll_ordered_' +
pdb_file_names[0].split('/')[-1].replace('.pdb', '') +
'_pensemble.pdb',
xrs_list=xrs_list_sorted_nll,
ens_pdb_hierarchy=ens_pdb_hierarchy)
def run(args, command_name="phenix.twin_map_utils"):
log=sys.stdout
params=None
if (len(args) == 0 or "--help" in args or "--h" in args or "-h" in args):
print_help(command_name=command_name)
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="map_coefs")
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print("##----------------------------------------------##", file=log)
print("## Unknown file or keyword:", arg, file=log)
print("##----------------------------------------------##", file=log)
print(file=log)
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
"""
new_params = master_params.format(python_object=params)
new_params.show(out=log)
"""
# now get the unit cell from the pdb file
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.twin_utils.input.xray_data.file_name)
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.twin_utils.input.model.file_name)
phil_xs=None
if ([params.twin_utils.input.unit_cell,
params.twin_utils.input.space_group]).count(None)<2:
phil_xs = crystal.symmetry(
unit_cell=params.twin_utils.input.unit_cell,
space_group_info=params.twin_utils.input.space_group )
combined_xs = crystal.select_crystal_symmetry(
None,phil_xs, [pdb_xs],[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.twin_utils.input.unit_cell = combined_xs.unit_cell()
params.twin_utils.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
new_params = master_params.format(python_object=params)
new_params.show(out=log)
if params.twin_utils.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.twin_utils.input.space_group is None:
raise Sorry("space group not specified")
if params.twin_utils.input.xray_data.file_name is None:
raise Sorry("Xray data not specified")
if params.twin_utils.input.model.file_name is None:
raise Sorry("pdb file with model not specified")
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
phil_xs = crystal.symmetry(
unit_cell=params.twin_utils.input.unit_cell,
space_group_info=params.twin_utils.input.space_group )
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = phil_xs,
force_symmetry = True,
reflection_files=[])
miller_array = None
free_flags = None
tmp_params = extract_xtal_data.data_and_flags_master_params().extract()
# insert proper values please
tmp_params.file_name = params.twin_utils.input.xray_data.file_name
tmp_params.labels = params.twin_utils.input.xray_data.obs_labels
tmp_params.r_free_flags.file_name=params.twin_utils.input.xray_data.file_name
tmp_params.r_free_flags.label=params.twin_utils.input.xray_data.free_flag
tmp_object = extract_xtal_data.run( reflection_file_server = xray_data_server,
parameters = tmp_params, log=log )
miller_array = tmp_object.extract_data()
if miller_array.is_xray_intensity_array():
miller_array = miller_array.f_sq_as_f()
assert miller_array.is_xray_amplitude_array()
free_flags = tmp_object.extract_flags(data = miller_array)
print(file=log)
print("Attempting to extract Free R flags", file=log)
free_flags = free_flags.customized_copy( data = flex.bool( free_flags.data()==1 ) )
if free_flags is None:
free_flags = miller_array.generate_r_free_flags(use_lattice_symmetry=True)
assert miller_array.observation_type() is not None
print(file=log)
print("Summary info of observed data", file=log)
print("=============================", file=log)
miller_array.show_summary(f=log)
print(file=log)
if miller_array.indices().size() == 0:
raise Sorry("No data available")
#----------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
model = pdb.input(file_name=params.twin_utils.input.model.file_name).xray_structure_simple(
crystal_symmetry=phil_xs)
print("Atomic model summary", file=log)
print("====================", file=log)
model.show_summary(f=log)
print(file=log)
#----------------------------------------------------------------
# step 3: get the twin laws for this xs
twin_laws = twin_analyses.twin_laws(
miller_array,
lattice_symmetry_max_delta=\
params.twin_utils.parameters.twinning.max_delta,
out=log)
print(file=log)
print("Preliminary data analyses", file=log)
print("==========================", file=log)
twin_laws.show(out=log)
#---------
# step 3:
# make twin model managers for all twin laws
print(file=log)
print("Overall and bulk solvent scale paranmeters and twin fraction estimation", file=log)
print("=======================================================================", file=log)
twin_models = []
operator_count = 0
if params.twin_utils.parameters.twinning.twin_law is not None:
tmp_law = sgtbx.rt_mx( params.twin_utils.parameters.twinning.twin_law )
tmp_law = twin_analyses.twin_law(tmp_law,None,None,None,None,None)
twin_laws.operators = [ tmp_law ]
for twin_law in twin_laws.operators:
operator_count += 1
operator_hkl = sgtbx.change_of_basis_op( twin_law.operator ).as_hkl()
twin_model = twin_f_model.twin_model_manager(
f_obs=miller_array,
r_free_flags = free_flags,
xray_structure=model,
twin_law = twin_law.operator,
detwin_mode = params.twin_utils.parameters.twinning.detwin_mode,
out=log)
print("--- bulk solvent scaling ---", file=log)
twin_model.update_all_scales()
twin_model.r_values()
twin_model.target()
twin_model.show_k_sol_b_sol_b_cart_target()
twin_model.show_essential()
wfofc = twin_model.map_coefficients(map_type="mFo-DFc" )
wtfofc = twin_model.map_coefficients(map_type="2mFo-DFc" )
grad = twin_model.map_coefficients(map_type="gradient" )
mtz_dataset = wtfofc.as_mtz_dataset(
column_root_label="FWT")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = wfofc,
column_root_label = "DFWT"
)
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = grad,
column_root_label = "GRAD"
)
name = params.twin_utils.output.map_coeffs_root+"_"+str(operator_count)+".mtz"
print(file=log)
print("Writing %s for twin law %s"%(name,operator_hkl), file=log)
print(file=log)
mtz_dataset.mtz_object().write(
file_name=name)
if params.twin_utils.output.obs_and_calc is not None:
# i want also a Fobs and Fmodel combined dataset please
mtz_dataset = miller_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = twin_model.f_model(),
column_root_label="FMODEL")
name = params.twin_utils.output.obs_and_calc
mtz_dataset.mtz_object().write(
file_name=name)
if len(twin_laws.operators)==0:
print(file=log)
print("No twin laws were found", file=log)
print("Performing maximum likelihood based bulk solvent scaling", file=log)
f_model_object = f_model.manager(
f_obs = miller_array,
r_free_flags = free_flags,
xray_structure = model )
f_model_object.update_all_scales(log=log)
tfofc = f_model_object.map_coefficients(map_type="2mFobs-DFmodel")
fofc = f_model_object.map_coefficients(map_type="mFobs-DFmodel")
mtz_dataset = tfofc.as_mtz_dataset(
column_root_label="FWT")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = fofc,
column_root_label = "DELFWT"
)
name = params.twin_utils.output.map_coeffs_root+"_ML.mtz"
mtz_dataset.mtz_object().write(
file_name=name)
if params.twin_utils.output.obs_and_calc is not None:
# i want also a Fobs and Fmodel combined dataset please
mtz_dataset = miller_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = f_model_object.f_model(),
column_root_label="FMODEL")
name = params.twin_utils.output.obs_and_calc
mtz_dataset.mtz_object().write(
file_name=name)
print(file=log)
print(file=log)
print("All done \n", file=log)
logfile = open(params.twin_utils.output.logfile,'w')
print(string_buffer.getvalue(), file=logfile)
print(file=log)
def run(args, out=None, log=sys.stdout):
if (len(args) == 0) or (args == ["--help"]):
print(msg, file=log)
defaults(log=log, silent=False)
return
parsed = defaults(log=log, silent=True)
#
processed_args = utils.process_command_line_args(args=args,
log=log,
master_params=parsed)
params = processed_args.params.extract()
#
reflection_files = processed_args.reflection_files
if (len(reflection_files) == 0):
raise Sorry("No reflection file found.")
crystal_symmetry = processed_args.crystal_symmetry
if (crystal_symmetry is None):
raise Sorry("No crystal symmetry found.")
if (len(processed_args.pdb_file_names) == 0):
raise Sorry("No PDB file found.")
pdb_file_names = processed_args.pdb_file_names
#
rfs = reflection_file_server(crystal_symmetry=crystal_symmetry,
reflection_files=reflection_files)
parameters = extract_xtal_data.data_and_flags_master_params().extract()
if (params.f_obs_label is not None):
parameters.labels = params.f_obs_label
if (params.r_free_flags_label is not None):
parameters.r_free_flags.label = params.r_free_flags_label
if (params.high_resolution is not None):
parameters.high_resolution = params.high_resolution
determine_data_and_flags_result = extract_xtal_data.run(
reflection_file_server=rfs,
parameters=parameters,
data_parameter_scope="refinement.input.xray_data",
flags_parameter_scope="refinement.input.xray_data.r_free_flags",
data_description="X-ray data",
keep_going=True,
log=StringIO())
f_obs = determine_data_and_flags_result.f_obs
# Data
show_header(l="Data:", log=log)
f_obs.show_comprehensive_summary(prefix=" ", f=log)
# R-free-flags
show_header(l="R-free-flags:", log=log)
r_free_flags = determine_data_and_flags_result.r_free_flags
test_flag_value = determine_data_and_flags_result.test_flag_value
if (r_free_flags is None):
r_free_flags = f_obs.array(data=flex.bool(f_obs.data().size(), False))
test_flag_value = None
print(" not available", file=log)
else:
print(" flag value:", test_flag_value, file=log)
# Model
pdb_combined = iotbx.pdb.combine_unique_pdb_files(
file_names=processed_args.pdb_file_names)
pdb_combined.report_non_unique(out=log)
if (len(pdb_combined.unique_file_names) == 0):
raise Sorry("No coordinate file given.")
raw_records = pdb_combined.raw_records
try:
pdb_inp = iotbx.pdb.input(source_info=None,
lines=flex.std_string(raw_records))
except ValueError as e:
raise Sorry("Model format (PDB or mmCIF) error:\n%s" % str(e))
model = mmtbx.model.manager(model_input=pdb_inp,
crystal_symmetry=crystal_symmetry,
log=StringIO())
#
scattering_table = params.scattering_table
exptl_method = pdb_inp.get_experiment_type()
if (exptl_method is not None) and ("NEUTRON" in exptl_method):
scattering_table = "neutron"
model.setup_scattering_dictionaries(scattering_table=scattering_table,
d_min=f_obs.d_min())
#
# Model vs data
#
show_header(l="Model vs Data:", log=log)
fmodel = mmtbx.f_model.manager(xray_structure=model.get_xray_structure(),
f_obs=f_obs,
r_free_flags=r_free_flags,
twin_law=params.twin_law)
fmodel.update_all_scales(update_f_part1=True)
fmodel.show(log=log, show_header=False, show_approx=False)
print(" r_work: %6.4f" % fmodel.r_work(), file=log)
if (test_flag_value is not None):
print(" r_free: %6.4f" % fmodel.r_free(), file=log)
else:
print(" r_free: None", file=log)
print(file=log)
n_outl = f_obs.data().size() - fmodel.f_obs().data().size()
print(" Number of F-obs outliers:", n_outl, file=log)
#
# Extract information from PDB file header and output (if any)
#
pub_r_work = None
pub_r_free = None
pub_high = None
pub_low = None
pub_sigma = None
pub_program_name = None
pub_solv_cont = None
pub_matthews = None
published_results = pdb_inp.get_r_rfree_sigma(file_name=pdb_file_names[0])
if (published_results is not None):
pub_r_work = published_results.r_work
pub_r_free = published_results.r_free
pub_high = published_results.high
pub_low = published_results.low
pub_sigma = published_results.sigma
pub_program_name = pdb_inp.get_program_name()
pub_solv_cont = pdb_inp.get_solvent_content()
pub_matthews = pdb_inp.get_matthews_coeff()
#
show_header(l="Information extracted from PDB file header:", log=log)
print(" program_name : %-s" % format_value("%s", pub_program_name),
file=log)
print(" year : %-s" %
format_value("%s", pdb_inp.extract_header_year()),
file=log)
print(" r_work : %-s" % format_value("%s", pub_r_work), file=log)
print(" r_free : %-s" % format_value("%s", pub_r_free), file=log)
print(" high_resolution : %-s" % format_value("%s", pub_high), file=log)
print(" low_resolution : %-s" % format_value("%s", pub_low), file=log)
print(" sigma_cutoff : %-s" % format_value("%s", pub_sigma), file=log)
print(" matthews_coeff : %-s" % format_value("%s", pub_matthews),
file=log)
print(" solvent_cont : %-s" % format_value("%s", pub_solv_cont),
file=log)
if (exptl_method is not None):
print(" exptl_method : %-s" % format_value("%s", exptl_method),
file=log)
#
# Recompute R-factors using published cutoffs
fmodel_cut = fmodel
tmp_sel = flex.bool(fmodel.f_obs().data().size(), True)
if (pub_sigma is not None and fmodel.f_obs().sigmas() is not None):
tmp_sel &= fmodel.f_obs().data() > fmodel.f_obs().sigmas() * pub_sigma
if (pub_high is not None
and abs(pub_high - fmodel.f_obs().d_min()) > 0.03):
tmp_sel &= fmodel.f_obs().d_spacings().data() > pub_high
if (pub_low is not None
and abs(pub_low - fmodel.f_obs().d_max_min()[0]) > 0.03):
tmp_sel &= fmodel.f_obs().d_spacings().data() < pub_low
if (tmp_sel.count(True) != tmp_sel.size() and tmp_sel.count(True) > 0):
show_header(l="After applying resolution and sigma cutoffs:", log=log)
fmodel = mmtbx.f_model.manager(
xray_structure=model.get_xray_structure(),
f_obs=fmodel.f_obs().select(tmp_sel),
r_free_flags=fmodel.r_free_flags().select(tmp_sel),
twin_law=params.twin_law)
fmodel.update_all_scales(update_f_part1=True)
fmodel.show(log=log, show_header=False, show_approx=False)
print(" r_work: %6.4f" % fmodel.r_work(), file=log)
if (test_flag_value is not None):
print(" r_free: %6.4f" % fmodel.r_free(), file=log)
else:
print(" r_free: None", file=log)
print(file=log)
n_outl = f_obs.data().size() - fmodel.f_obs().data().size()
print(" Number of F-obs outliers:", n_outl, file=log)
def run(args, log = sys.stdout):
if(len(args)==0):
print(legend, file=log)
defaults(log=log)
return
#
parsed = defaults(log=log)
processed_args = mmtbx.utils.process_command_line_args(args = args,
log = sys.stdout, master_params = parsed)
params = processed_args.params.extract()
reflection_files = processed_args.reflection_files
if(len(reflection_files) == 0):
if (params.hkl_file is None):
raise Sorry("No reflection file found.")
else :
hkl_in = file_reader.any_file(params.hkl_file, force_type="hkl")
hkl_in.assert_file_type("hkl")
reflection_files = [ hkl_in.file_object ]
crystal_symmetry = processed_args.crystal_symmetry
if(crystal_symmetry is None):
if (params.space_group is not None) and (params.unit_cell is not None):
from cctbx import crystal
crystal_symmetry = crystal.symmetry(
space_group_info=params.space_group,
unit_cell=params.unit_cell)
else :
raise Sorry("No crystal symmetry found.")
if(len(processed_args.pdb_file_names) == 0):
if (params.pdb_file is None):
raise Sorry("No model file found.")
else :
pdb_file_names = [ params.pdb_file ]
else :
pdb_file_names = processed_args.pdb_file_names
#
rfs = reflection_file_utils.reflection_file_server(
crystal_symmetry = crystal_symmetry,
force_symmetry = True,
reflection_files = reflection_files,
err = StringIO())
parameters = extract_xtal_data.data_and_flags_master_params().extract()
parameters.labels = params.f_obs_label
parameters.r_free_flags.label = params.r_free_flags_label
determine_data_and_flags_result = extract_xtal_data.run(
reflection_file_server = rfs,
parameters = parameters,
keep_going = True,
log = StringIO())
f_obs = determine_data_and_flags_result.f_obs
print("Input data:")
print(" Iobs or Fobs:", f_obs.info().labels)
r_free_flags = determine_data_and_flags_result.r_free_flags
print(" Free-R flags:", r_free_flags.info().labels)
#
experimental_phases = determine_data_and_flags_result.experimental_phases
#
if(r_free_flags is None):
r_free_flags=f_obs.array(data=flex.bool(f_obs.data().size(), False))
#
pdb_inp = mmtbx.utils.pdb_inp_from_multiple_files(pdb_file_names, log=sys.stdout)
model = mmtbx.model.manager(
model_input = pdb_inp,
crystal_symmetry = crystal_symmetry,
log = sys.stdout)
if(model.get_number_of_models()>1): #XXX support multi-models
raise Sorry("Multiple model file not supported in this tool.")
# XXX Twining not supported
xray_structure = model.get_xray_structure()
if (not xray_structure.unit_cell().is_similar_to(f_obs.unit_cell())):
raise Sorry("The unit cells in the model and reflections files are not "+
"isomorphous.")
print("Input model:")
print(" number of atoms:", xray_structure.scatterers().size())
fmodel = mmtbx.f_model.manager(
xray_structure = xray_structure,
r_free_flags = r_free_flags,
f_obs = f_obs,
abcd = experimental_phases)
fmodel.update_all_scales(
update_f_part1 = True,
remove_outliers = params.remove_f_obs_outliers,
bulk_solvent_and_scaling = params.bulk_solvent_and_scaling)
print("Overall statistics:")
fmodel.info().show_all()
#
print("Output data:")
if(params.output_file_name is not None):
output_file_name = params.output_file_name
else:
pdb_file_bn = os.path.basename(pdb_file_names[0])
hkl_file_bn = os.path.basename(reflection_files[0].file_name())
try: pdb_file_prefix = pdb_file_bn[:pdb_file_bn.index(".")]
except ValueError: pdb_file_prefix = pdb_file_bn
try:
hkl_file_prefix = hkl_file_bn[:hkl_file_bn.index(".")]
except ValueError: hkl_file_prefix = hkl_file_bn
output_file_name = "%s_%s.mtz"%(pdb_file_prefix, hkl_file_prefix)
print(" file name:", output_file_name)
print(" to see the contnt of %s:"%output_file_name)
print(" phenix.mtz.dump %s"%output_file_name)
out = open(output_file_name,"w")
fmodel.export(out = out)
out.close()
print("All done.")
return output_file_name
def run(args, log=sys.stdout, as_gui_program=False):
if (len(args) == 0):
parsed = defaults(log=log)
parsed.show(prefix=" ", out=log)
return
command_line = (option_parser().enable_symmetry_comprehensive().option(
"-q",
"--quiet",
action="store_true",
default=False,
help="suppress output").option("--output_plots",
action="store_true",
default=False)).process(args=args)
parsed = defaults(log=log)
processed_args = mmtbx.utils.process_command_line_args(
args=command_line.args,
cmd_cs=command_line.symmetry,
master_params=parsed,
log=log,
suppress_symmetry_related_errors=True)
processed_args.params.show(out=log)
params = processed_args.params.extract().density_modification
output_plots = command_line.options.output_plots
crystal_symmetry = crystal.symmetry(
unit_cell=params.input.unit_cell,
space_group_info=params.input.space_group)
reflection_files = {}
for rfn in (params.input.reflection_data.file_name,
params.input.experimental_phases.file_name,
params.input.map_coefficients.file_name):
if os.path.isfile(str(rfn)) and rfn not in reflection_files:
reflection_files.setdefault(
rfn,
iotbx.reflection_file_reader.any_reflection_file(
file_name=rfn, ensure_read_access=False))
# TODO is reflection_files a dict ?
server = iotbx.reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
reflection_files=list(reflection_files.values()))
o = extract_xtal_data.run(
server,
parameters=params.input.reflection_data,
experimental_phases_params=params.input.experimental_phases,
extract_r_free_flags=False)
fo = o.f_obs
hl_coeffs = o.experimental_phases
if params.input.map_coefficients.file_name is not None:
map_coeffs = server.get_phases_deg(
file_name=params.input.map_coefficients.file_name,
labels=params.input.map_coefficients.labels,
convert_to_phases_if_necessary=False,
original_phase_units=None,
parameter_scope="",
parameter_name="labels").map_to_asu()
else:
map_coeffs = None
ncs_object = None
if params.input.ncs_file_name is not None:
ncs_object = ncs.ncs()
ncs_object.read_ncs(params.input.ncs_file_name)
ncs_object.display_all(log=log)
fo = fo.map_to_asu()
hl_coeffs = hl_coeffs.map_to_asu()
fo = fo.eliminate_sys_absent().average_bijvoet_mates()
hl_coeffs = hl_coeffs.eliminate_sys_absent().average_bijvoet_mates()
model_map = None
model_map_coeffs = None
if len(processed_args.pdb_file_names):
pdb_inp = mmtbx.utils.pdb_inp_from_multiple_files(
pdb_files=processed_args.pdb_file_names, log=log)
xs = pdb_inp.xray_structure_simple()
fo_, hl_ = fo, hl_coeffs
if params.change_basis_to_niggli_cell:
change_of_basis_op = xs.change_of_basis_op_to_niggli_cell()
xs = xs.change_basis(change_of_basis_op)
fo_ = fo_.change_basis(change_of_basis_op).map_to_asu()
hl_ = hl_.change_basis(change_of_basis_op).map_to_asu()
#fo_, hl_ = fo_.common_sets(hl_)
fmodel_refined = mmtbx.utils.fmodel_simple(
f_obs=fo_,
scattering_table=
"wk1995", #XXX pva: 1) neutrons? 2) move up as a parameter.
xray_structures=[xs],
bulk_solvent_correction=True,
anisotropic_scaling=True,
r_free_flags=fo_.array(data=flex.bool(fo_.size(), False)))
fmodel_refined.update(abcd=hl_)
master_phil = mmtbx.maps.map_and_map_coeff_master_params()
map_params = master_phil.fetch(
iotbx.phil.parse("""\
map_coefficients {
map_type = 2mFo-DFc
isotropize = True
}
""")).extract().map_coefficients[0]
model_map_coeffs = mmtbx.maps.map_coefficients_from_fmodel(
fmodel=fmodel_refined, params=map_params)
model_map = model_map_coeffs.fft_map(
resolution_factor=params.grid_resolution_factor).real_map_unpadded(
)
import time
t0 = time.time()
dm = density_modify(params,
fo,
hl_coeffs,
ncs_object=ncs_object,
map_coeffs=map_coeffs,
model_map_coeffs=model_map_coeffs,
log=log,
as_gui_program=as_gui_program)
time_dm = time.time() - t0
print("Time taken for density modification: %.2fs" % time_dm, file=log)
# run cns
if 0:
from cctbx.development import cns_density_modification
cns_result = cns_density_modification.run(params, fo, hl_coeffs)
print(cns_result.modified_map.all())
print(dm.map.all())
dm_map_coeffs = dm.map_coeffs_in_original_setting
from cctbx import maptbx, miller
crystal_gridding = maptbx.crystal_gridding(
dm_map_coeffs.unit_cell(),
space_group_info=dm_map_coeffs.space_group().info(),
pre_determined_n_real=cns_result.modified_map.all())
dm_map = miller.fft_map(crystal_gridding,
dm_map_coeffs).apply_sigma_scaling()
corr = flex.linear_correlation(cns_result.modified_map.as_1d(),
dm_map.real_map_unpadded().as_1d())
print("CNS dm/mmtbx dm correlation:")
corr.show_summary()
if dm.model_map_coeffs is not None:
model_map = miller.fft_map(
crystal_gridding,
dm.miller_array_in_original_setting(
dm.model_map_coeffs)).apply_sigma_scaling()
corr = flex.linear_correlation(
cns_result.modified_map.as_1d(),
model_map.real_map_unpadded().as_1d())
print("CNS dm/model correlation:")
corr.show_summary()
if output_plots:
plots_to_make = (
"fom",
"skewness",
"r1_factor",
"r1_factor_fom",
"mean_solvent_density",
"mean_protein_density",
"f000_over_v",
"k_flip",
"rms_solvent_density",
"rms_protein_density",
"standard_deviation_local_rms",
"mean_delta_phi",
"mean_delta_phi_initial",
)
from matplotlib.backends.backend_pdf import PdfPages
from libtbx import pyplot
stats = dm.get_stats()
pdf = PdfPages("density_modification.pdf")
if len(dm.correlation_coeffs) > 1:
if 0:
start_coeffs, model_coeffs = dm.map_coeffs_start.common_sets(
model_map_coeffs)
model_phases = model_coeffs.phases(deg=True).data()
exptl_phases = nearest_phase(
model_phases,
start_coeffs.phases(deg=True).data(),
deg=True)
corr = flex.linear_correlation(exptl_phases, model_phases)
corr.show_summary()
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.set_title("phases start")
ax.set_xlabel("Experimental phases")
ax.set_ylabel("Phases from refined model")
ax.scatter(exptl_phases, model_phases, marker="x", s=10)
pdf.savefig(fig)
#
dm_coeffs, model_coeffs = dm.map_coeffs.common_sets(
model_map_coeffs)
model_phases = model_coeffs.phases(deg=True).data()
dm_phases = nearest_phase(model_phases,
dm_coeffs.phases(deg=True).data(),
deg=True)
corr = flex.linear_correlation(dm_phases, model_phases)
corr.show_summary()
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.set_title("phases dm")
ax.set_xlabel("Phases from density modification")
ax.set_ylabel("Phases from refined model")
ax.scatter(dm_phases, model_phases, marker="x", s=10)
pdf.savefig(fig)
#
data = dm.correlation_coeffs
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.set_title("correlation coefficient")
ax.plot(list(range(1, dm.i_cycle + 2)), data)
pdf.savefig(fig)
#
data = dm.mean_phase_errors
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.set_title("Mean effective phase errors")
ax.plot(list(range(1, dm.i_cycle + 2)), data)
pdf.savefig(fig)
for plot in plots_to_make:
data = [
getattr(stats.get_cycle_stats(i), plot)
for i in range(1, dm.i_cycle + 2)
]
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.set_title(plot.replace("_", " "))
ax.plot(list(range(1, dm.i_cycle + 2)), data)
pdf.savefig(fig)
data = [
stats.get_cycle_stats(i).rms_solvent_density /
stats.get_cycle_stats(i).rms_protein_density
for i in range(1, dm.i_cycle + 2)
]
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.set_title("RMS solvent/protein density ratio")
ax.plot(list(range(1, dm.i_cycle + 2)), data)
pdf.savefig(fig)
pdf.close()
dm_map_coeffs = dm.map_coeffs_in_original_setting
dm_hl_coeffs = dm.hl_coeffs_in_original_setting
# output map if requested
map_params = params.output.map
if map_params.file_name is not None:
fft_map = dm_map_coeffs.fft_map(
resolution_factor=params.grid_resolution_factor)
if map_params.scale == "sigma":
fft_map.apply_sigma_scaling()
else:
fft_map.apply_volume_scaling()
gridding_first = gridding_last = None
title_lines = []
if map_params.format == "xplor":
fft_map.as_xplor_map(file_name=map_params.file_name,
title_lines=title_lines,
gridding_first=gridding_first,
gridding_last=gridding_last)
else:
fft_map.as_ccp4_map(file_name=map_params.file_name,
gridding_first=gridding_first,
gridding_last=gridding_last,
labels=title_lines)
# output map coefficients if requested
mtz_params = params.output.mtz
# Decide if we are going to actually write the mtz
if mtz_params.file_name is not None:
orig_fom, final_fom = dm.start_and_end_fom()
if mtz_params.skip_output_if_worse and final_fom < orig_fom:
ok_to_write_mtz = False
print(
"Not writing out mtz. Final FOM (%7.3f) worse than start (%7.3f)"
% (final_fom, orig_fom))
else: # usual
ok_to_write_mtz = True
else:
ok_to_write_mtz = True
if mtz_params.file_name is not None and ok_to_write_mtz:
label_decorator = iotbx.mtz.ccp4_label_decorator()
fo = dm.miller_array_in_original_setting(
dm.f_obs_complete).common_set(dm_map_coeffs)
mtz_dataset = fo.as_mtz_dataset(column_root_label="F",
label_decorator=label_decorator)
mtz_dataset.add_miller_array(dm_map_coeffs,
column_root_label="FWT",
label_decorator=label_decorator)
phase_source = dm.miller_array_in_original_setting(
dm.phase_source).common_set(dm_map_coeffs)
mtz_dataset.add_miller_array(
phase_source.array(data=flex.abs(phase_source.data())),
column_root_label="FOM",
column_types='W',
label_decorator=label_decorator)
mtz_dataset.add_miller_array(
phase_source.array(data=phase_source.phases(deg=True).data()),
column_root_label="PHIB",
column_types='P',
label_decorator=None)
if mtz_params.output_hendrickson_lattman_coefficients:
mtz_dataset.add_miller_array(dm_hl_coeffs,
column_root_label="HL",
label_decorator=label_decorator)
mtz_dataset.mtz_object().write(mtz_params.file_name)
return result(map_file=map_params.file_name,
mtz_file=mtz_params.file_name,
stats=dm.get_stats())
def __init__(self,
args,
master_phil,
out=sys.stdout,
process_pdb_file=True,
require_data=True,
create_fmodel=True,
prefer_anomalous=None,
force_non_anomalous=False,
set_wavelength_from_model_header=False,
set_inelastic_form_factors=None,
usage_string=None,
create_log_buffer=False,
remove_unknown_scatterers=False,
generate_input_phil=False):
import mmtbx.monomer_library.pdb_interpretation
import mmtbx.monomer_library.server
import mmtbx.utils
import mmtbx.model
from iotbx import crystal_symmetry_from_any
import iotbx.phil
if generate_input_phil:
from six import string_types
assert isinstance(master_phil, string_types)
master_phil = generate_master_phil_with_inputs(
phil_string=master_phil)
if isinstance(master_phil, str):
master_phil = iotbx.phil.parse(master_phil)
if (usage_string is not None):
if (len(args) == 0) or ("--help" in args):
raise Usage("""%s\n\nFull parameters:\n%s""" %
(usage_string, master_phil.as_str(prefix=" ")))
if (force_non_anomalous):
assert (not prefer_anomalous)
assert (set_inelastic_form_factors in [None, "sasaki", "henke"])
self.args = args
self.master_phil = master_phil
self.processed_pdb_file = self.pdb_inp = None
self.pdb_hierarchy = self.xray_structure = None
self.geometry = None
self.sequence = None
self.fmodel = None
self.f_obs = None
self.r_free_flags = None
self.intensity_flag = None
self.raw_data = None
self.raw_flags = None
self.test_flag_value = None
self.miller_arrays = None
self.hl_coeffs = None
self.cif_objects = []
self.log = out
if ("--quiet" in args) or ("quiet=True" in args):
self.log = null_out()
elif create_log_buffer:
self.log = multi_out()
self.log.register(label="stdout", file_object=out)
self.log.register(label="log_buffer", file_object=StringIO())
make_header("Collecting inputs", out=self.log)
cmdline = iotbx.phil.process_command_line_with_files(
args=args,
master_phil=master_phil,
pdb_file_def="input.pdb.file_name",
reflection_file_def="input.xray_data.file_name",
cif_file_def="input.monomers.file_name",
seq_file_def="input.sequence")
self.working_phil = cmdline.work
params = self.working_phil.extract()
if len(params.input.pdb.file_name) == 0:
raise Sorry("At least one PDB file is required as input.")
self.cif_file_names = params.input.monomers.file_name
self.pdb_file_names = params.input.pdb.file_name
# SYMMETRY HANDLING - PDB FILES
self.crystal_symmetry = pdb_symm = None
for pdb_file_name in params.input.pdb.file_name:
pdb_symm = crystal_symmetry_from_any.extract_from(pdb_file_name)
if (pdb_symm is not None):
break
# DATA INPUT
data_and_flags = hkl_symm = hkl_in = None
if (params.input.xray_data.file_name is None):
if (require_data):
raise Sorry(
"At least one reflections file is required as input.")
else:
# FIXME this may still require that the data file has full crystal
# symmetry defined (although for MTZ input this will not be a problem)
make_sub_header("Processing X-ray data", out=self.log)
hkl_in = file_reader.any_file(params.input.xray_data.file_name)
hkl_in.check_file_type("hkl")
hkl_server = hkl_in.file_server
symm = hkl_server.miller_arrays[0].crystal_symmetry()
if ((symm is None) or (symm.space_group() is None)
or (symm.unit_cell() is None)):
if (pdb_symm is not None):
from iotbx.reflection_file_utils import reflection_file_server
print(
"No symmetry in X-ray data file - using PDB symmetry:",
file=self.log)
pdb_symm.show_summary(f=out, prefix=" ")
hkl_server = reflection_file_server(
crystal_symmetry=pdb_symm,
reflection_files=[hkl_in.file_object])
else:
raise Sorry(
"No crystal symmetry information found in input files."
)
if (hkl_server is None):
hkl_server = hkl_in.file_server
try:
pp = params.input.experimental_phases
except AttributeError:
pp = None
data_and_flags = extract_xtal_data.run(
reflection_file_server=hkl_server,
parameters=params.input.xray_data,
experimental_phases_params=pp,
prefer_anomalous=prefer_anomalous,
force_non_anomalous=force_non_anomalous)
self.intensity_flag = data_and_flags.f_obs.is_xray_intensity_array(
)
self.raw_data = data_and_flags.raw_data
self.raw_flags = data_and_flags.raw_flags
self.test_flag_value = data_and_flags.test_flag_value
self.f_obs = data_and_flags.f_obs
self.r_free_flags = data_and_flags.r_free_flags
self.miller_arrays = hkl_in.file_server.miller_arrays
self.hl_coeffs = None
target_name = "ml"
if (data_and_flags.experimental_phases is not None):
target_name = "mlhl"
self.hl_coeffs = data_and_flags.experimental_phases
hkl_symm = self.raw_data.crystal_symmetry()
if len(self.cif_file_names) > 0:
for file_name in self.cif_file_names:
cif_obj = mmtbx.monomer_library.server.read_cif(
file_name=file_name)
self.cif_objects.append((file_name, cif_obj))
# SYMMETRY HANDLING - COMBINED
if (hkl_symm is not None):
use_symmetry = hkl_symm
# check for weird crystal symmetry
# modified from mmtbx.command_line.secondary_structure_restraints
# plan to centralize functionality in another location
# -------------------------------------------------------------------------
cs = pdb_symm
corrupted_cs = False
if cs is not None:
if [cs.unit_cell(), cs.space_group()].count(None) > 0:
corrupted_cs = True
cs = None
elif cs.unit_cell().volume() < 10:
corrupted_cs = True
cs = None
if cs is None:
if corrupted_cs:
print("Symmetry information is corrupted,", end=' ', file=out)
else:
print("Symmetry information was not found,", end=' ', file=out)
if (hkl_symm is not None):
print("using symmetry from data.", file=out)
cs = hkl_symm
else:
print("putting molecule in P1 box.", file=out)
pdb_combined = iotbx.pdb.combine_unique_pdb_files(
file_names=self.pdb_file_names)
pdb_structure = iotbx.pdb.input(source_info=None,
lines=flex.std_string(
pdb_combined.raw_records))
atoms = pdb_structure.atoms()
box = uctbx.non_crystallographic_unit_cell_with_the_sites_in_its_center(
sites_cart=atoms.extract_xyz(), buffer_layer=3)
atoms.set_xyz(new_xyz=box.sites_cart)
cs = box.crystal_symmetry()
pdb_symm = cs
# -------------------------------------------------------------------------
from iotbx.symmetry import combine_model_and_data_symmetry
self.crystal_symmetry = combine_model_and_data_symmetry(
model_symmetry=pdb_symm, data_symmetry=hkl_symm)
if (self.crystal_symmetry is not None) and (self.f_obs is not None):
self.f_obs = self.f_obs.customized_copy(
crystal_symmetry=self.crystal_symmetry).eliminate_sys_absent(
).set_info(self.f_obs.info())
self.r_free_flags = self.r_free_flags.customized_copy(
crystal_symmetry=self.crystal_symmetry).eliminate_sys_absent(
).set_info(self.r_free_flags.info())
# PDB INPUT
self.unknown_residues_flag = False
self.unknown_residues_error_message = False
pdb_combined = mmtbx.utils.combine_unique_pdb_files(
file_names=params.input.pdb.file_name, )
pdb_combined.report_non_unique(out=self.log)
pdb_raw_records = pdb_combined.raw_records
try:
self.pdb_inp = iotbx.pdb.input(
source_info=None, lines=flex.std_string(pdb_raw_records))
except ValueError as e:
raise Sorry("Model format (PDB or mmCIF) error:\n%s" % str(e))
if (remove_unknown_scatterers):
h = self.pdb_inp.construct_hierarchy()
known_sel = h.atom_selection_cache().selection("not element X")
if known_sel.count(False) > 0:
self.pdb_inp = iotbx.pdb.input(
source_info=None,
lines=h.select(known_sel).as_pdb_string())
model_params = mmtbx.model.manager.get_default_pdb_interpretation_params(
)
pdb_interp_params = getattr(params, "pdb_interpretation", None)
if pdb_interp_params is None:
pdb_interp_params = iotbx.phil.parse(
input_string=mmtbx.monomer_library.pdb_interpretation.
grand_master_phil_str,
process_includes=True).extract()
pdb_interp_params = pdb_interp_params.pdb_interpretation
model_params.pdb_interpretation = pdb_interp_params
stop_for_unknowns = getattr(pdb_interp_params, "stop_for_unknowns",
False) or remove_unknown_scatterers
if not process_pdb_file:
stop_for_unknowns = True and not remove_unknown_scatterers
self.model = mmtbx.model.manager(
model_input=self.pdb_inp,
crystal_symmetry=self.crystal_symmetry,
restraint_objects=self.cif_objects,
stop_for_unknowns=stop_for_unknowns,
log=self.log)
if process_pdb_file:
make_sub_header("Processing PDB file(s)", out=self.log)
self.model.process(pdb_interpretation_params=model_params,
make_restraints=True)
full_grm = self.model.get_restraints_manager()
self.geometry = full_grm.geometry
self.processed_pdb_file = self.model._processed_pdb_file # to remove later XXX
self.xray_structure = self.model.get_xray_structure()
self.pdb_hierarchy = self.model.get_hierarchy()
self.pdb_hierarchy.atoms().reset_i_seq()
# wavelength
if (params.input.energy is not None):
if (params.input.wavelength is not None):
raise Sorry("Both wavelength and energy have been specified!")
params.input.wavelength = 12398.424468024265 / params.input.energy
if (set_wavelength_from_model_header
and params.input.wavelength is None):
wavelength = self.pdb_inp.extract_wavelength()
if (wavelength is not None):
print("", file=self.log)
print("Using wavelength = %g from PDB header" % wavelength,
file=self.log)
params.input.wavelength = wavelength
# set scattering table
if (data_and_flags is not None):
self.model.setup_scattering_dictionaries(
scattering_table=params.input.scattering_table,
d_min=self.f_obs.d_min(),
log=self.log,
set_inelastic_form_factors=set_inelastic_form_factors,
iff_wavelength=params.input.wavelength)
self.xray_structure.show_summary(f=self.log)
# FMODEL SETUP
if (create_fmodel) and (data_and_flags is not None):
make_sub_header("F(model) initialization", out=self.log)
skip_twin_detection = getattr(params.input, "skip_twin_detection",
True)
twin_law = getattr(params.input, "twin_law", None)
if (twin_law is Auto):
if (self.hl_coeffs is not None):
raise Sorry(
"Automatic twin law determination not supported when "
+ "experimental phases are used.")
elif (not skip_twin_detection):
twin_law = Auto
if (twin_law is Auto):
print("Twinning will be detected automatically.",
file=self.log)
self.fmodel = mmtbx.utils.fmodel_simple(
xray_structures=[self.xray_structure],
scattering_table=params.input.scattering_table,
f_obs=self.f_obs,
r_free_flags=self.r_free_flags,
skip_twin_detection=skip_twin_detection,
target_name=target_name,
log=self.log)
else:
if ((twin_law is not None) and (self.hl_coeffs is not None)):
raise Sorry(
"Automatic twin law determination not supported when "
+ "experimental phases are used.")
self.fmodel = mmtbx.utils.fmodel_manager(
f_obs=self.f_obs,
xray_structure=self.xray_structure,
r_free_flags=self.r_free_flags,
twin_law=params.input.twin_law,
hl_coeff=self.hl_coeffs,
target_name=target_name)
self.fmodel.update_all_scales(params=None,
log=self.log,
optimize_mask=True,
show=True)
self.fmodel.info().show_rfactors_targets_scales_overall(
out=self.log)
# SEQUENCE
if (params.input.sequence is not None):
seq_file = file_reader.any_file(params.input.sequence,
force_type="seq",
raise_sorry_if_errors=True)
self.sequence = seq_file.file_object
# UNMERGED DATA
self.unmerged_i_obs = None
if hasattr(params.input, "unmerged_data"):
if (params.input.unmerged_data.file_name is not None):
self.unmerged_i_obs = load_and_validate_unmerged_data(
f_obs=self.f_obs,
file_name=params.input.unmerged_data.file_name,
data_labels=params.input.unmerged_data.labels,
log=self.log)
self.params = params
print("", file=self.log)
print("End of input processing", file=self.log)
def run(args,
log=sys.stdout,
use_output_directory=True,
suppress_fmodel_output=False):
print(legend, file=log)
print("-" * 79, file=log)
master_params = mmtbx.maps.maps_including_IO_master_params()
if (len(args) == 0 or (len(args) == 1 and args[0] == "NO_PARAMETER_FILE")):
if (not (len(args) == 1 and args[0] == "NO_PARAMETER_FILE")):
parameter_file_name = "maps.params"
print(
"Creating parameter file '%s' in the following directory:\n%s"
% (parameter_file_name, os.path.abspath('.')),
file=log)
if (os.path.isfile(parameter_file_name)):
msg = "File '%s' exists already. Re-name it or move and run the command again."
raise Sorry(msg % parameter_file_name)
pfo = open(parameter_file_name, "w")
else:
pfo = log
print("\nAll phenix.maps parameters::\n", file=pfo)
master_params = master_params.fetch(iotbx.phil.parse(default_params))
master_params.show(out=pfo, prefix=" ", expert_level=1)
return
processed_args = mmtbx.utils.process_command_line_args(
args=args, log=log, master_params=master_params)
working_phil = processed_args.params
params = working_phil.extract()
fmodel_data_file_format = params.maps.output.fmodel_data_file_format
if (len(params.maps.map_coefficients) == 0) and (len(params.maps.map)
== 0):
print("No map input specified - using default map types", file=log)
working_phil = master_params.fetch(
sources=[working_phil,
iotbx.phil.parse(default_params)])
params = working_phil.extract()
# XXX BUG - the extra fetch will always set fmodel_data_file_format to
# mtz; this is probaby a low-level phil problem
if (fmodel_data_file_format is None) or (suppress_fmodel_output):
params.maps.output.fmodel_data_file_format = None
analyze_input_params(params=params)
have_phil_file_input = len(processed_args.phil_file_names) > 0
if (len(processed_args.pdb_file_names) > 1):
raise Sorry("Only one model file is allowed as input.")
if ((params.maps.input.pdb_file_name is None)
and (len(processed_args.pdb_file_names) == 1)):
params.maps.input.pdb_file_name = processed_args.pdb_file_names[0]
if (not os.path.isfile(str(params.maps.input.pdb_file_name))):
raise Sorry(
"model file is not given: maps.input.pdb_file_name=%s is not a file"%\
str(params.maps.input.pdb_file_name))
if ((params.maps.input.reflection_data.file_name is None) and
(params.maps.input.reflection_data.r_free_flags.file_name is None)
and (len(processed_args.reflection_file_names) == 1)):
params.maps.input.reflection_data.file_name = \
processed_args.reflection_file_names[0]
print("FORMAT:", params.maps.output.fmodel_data_file_format, file=log)
working_phil = master_params.format(python_object=params)
print("-" * 79, file=log)
print("\nParameters to compute maps::\n", file=log)
working_phil.show(out=log, prefix=" ")
pdb_inp = iotbx.pdb.input(file_name=params.maps.input.pdb_file_name)
# get all crystal symmetries
cs_from_coordinate_files = [pdb_inp.crystal_symmetry_from_cryst1()]
cs_from_reflection_files = []
for rfn in [
params.maps.input.reflection_data.file_name,
params.maps.input.reflection_data.r_free_flags.file_name
]:
if (os.path.isfile(str(rfn))):
try:
cs_from_reflection_files.append(
crystal_symmetry_from_any.extract_from(rfn))
except KeyboardInterrupt:
raise
except RuntimeError:
pass
crystal_symmetry = None
try:
crystal_symmetry = crystal.select_crystal_symmetry(
from_coordinate_files=cs_from_coordinate_files,
from_reflection_files=cs_from_reflection_files)
except AssertionError as e:
if ("No unit cell and symmetry information supplied" in str(e)):
raise Sorry(
"Missing or incomplete symmetry information. This program " +
"will only work with reflection file formats that contain both "
+ "unit cell and space group records, such as MTZ files.")
#
reflection_files = []
reflection_file_names = []
for rfn in [
params.maps.input.reflection_data.file_name,
params.maps.input.reflection_data.r_free_flags.file_name
]:
if (os.path.isfile(str(rfn))) and (not rfn in reflection_file_names):
reflection_files.append(
reflection_file_reader.any_reflection_file(
file_name=rfn, ensure_read_access=False))
reflection_file_names.append(rfn)
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files, #[],
err=log)
#
reflection_data_master_params = extract_xtal_data.data_and_flags_master_params(
master_scope_name="reflection_data")
reflection_data_input_params = processed_args.params.get(
"maps.input.reflection_data")
reflection_data_params = reflection_data_master_params.fetch(
reflection_data_input_params).extract().reflection_data
#
determine_data_and_flags_result = extract_xtal_data.run(
reflection_file_server=reflection_file_server,
parameters=reflection_data_params,
keep_going=True)
f_obs = determine_data_and_flags_result.f_obs
r_free_flags = determine_data_and_flags_result.r_free_flags
test_flag_value = determine_data_and_flags_result.test_flag_value
if (r_free_flags is None):
r_free_flags = f_obs.array(data=flex.bool(f_obs.data().size(), False))
test_flag_value = None
print("-" * 79, file=log)
print("\nInput model file:", params.maps.input.pdb_file_name, file=log)
pdb_hierarchy = pdb_inp.construct_hierarchy(set_atom_i_seq=True)
atom_selection_manager = pdb_hierarchy.atom_selection_cache()
xray_structure = pdb_hierarchy.extract_xray_structure(
crystal_symmetry=crystal_symmetry)
# apply omit selection
if (params.maps.omit.selection is not None):
omit_selection = atom_selection_manager.selection(
string=params.maps.omit.selection)
keep_selection = ~omit_selection
xray_structure = xray_structure.select(selection=keep_selection)
pdb_hierarchy = pdb_hierarchy.select(keep_selection)
atom_selection_manager = pdb_hierarchy.atom_selection_cache()
#
mmtbx.utils.setup_scattering_dictionaries(
scattering_table=params.maps.scattering_table,
xray_structure=xray_structure,
d_min=f_obs.d_min(),
log=log)
if (params.maps.wavelength is not None):
if (params.maps.scattering_table == "neutron"):
raise Sorry(
"Wavelength parameter not supported when the neutron " +
"scattering table is used.")
xray_structure.set_inelastic_form_factors(
photon=params.maps.wavelength, table="sasaki")
xray_structure.show_summary(f=log, prefix=" ")
print("-" * 79, file=log)
print("Bulk solvent correction and anisotropic scaling:", file=log)
fmodel = mmtbx.utils.fmodel_simple(
xray_structures=[xray_structure],
scattering_table=params.maps.scattering_table,
f_obs=f_obs,
r_free_flags=r_free_flags,
outliers_rejection=params.maps.input.reflection_data.
outliers_rejection,
skip_twin_detection=params.maps.skip_twin_detection,
bulk_solvent_correction=params.maps.bulk_solvent_correction,
anisotropic_scaling=params.maps.anisotropic_scaling)
fmodel_info = fmodel.info()
fmodel_info.show_rfactors_targets_scales_overall(out=log)
print("-" * 79, file=log)
print("Compute maps.", file=log)
# XXX if run from the Phenix GUI, the output directory parameter is actually
# one level up from the current directory, and use_output_directory=False
if (params.maps.output.directory is not None) and (use_output_directory):
assert os.path.isdir(params.maps.output.directory)
output_dir = params.maps.output.directory
else:
output_dir = os.getcwd()
if params.maps.output.prefix is not None:
file_name_base = os.path.join(
output_dir, os.path.basename(params.maps.output.prefix))
else:
file_name_base = params.maps.input.pdb_file_name
if (file_name_base.count(".") > 0):
file_name_base = file_name_base[:file_name_base.index(".")]
xplor_maps = mmtbx.maps.compute_xplor_maps(
fmodel=fmodel,
params=params.maps.map,
atom_selection_manager=atom_selection_manager,
file_name_prefix=None,
file_name_base=file_name_base,
pdb_hierarchy=pdb_hierarchy)
cmo = mmtbx.maps.compute_map_coefficients(
fmodel=fmodel,
params=params.maps.map_coefficients,
pdb_hierarchy=pdb_hierarchy,
log=log)
map_coeff_file_name = file_name_base + "_map_coeffs.mtz"
r_free_flags_output = None
if (params.maps.output.include_r_free_flags):
r_free_flags_output = fmodel.r_free_flags().average_bijvoet_mates()
write_mtz_file_result = cmo.write_mtz_file(
file_name=map_coeff_file_name, r_free_flags=r_free_flags_output)
if (params.maps.output.fmodel_data_file_format is not None):
fmodel_file_name = file_name_base + "_fmodel." + \
params.maps.output.fmodel_data_file_format
print("Writing fmodel arrays (Fobs, Fcalc, m, ...) to %s file."%\
fmodel_file_name, file=log)
fmodel_file_object = open(fmodel_file_name, "w")
fmodel.export(out=fmodel_file_object,
format=params.maps.output.fmodel_data_file_format)
fmodel_file_object.close()
print("All done.", file=log)
if (write_mtz_file_result):
print("Map coefficients: %s" % map_coeff_file_name, file=log)
for file_name in xplor_maps:
print("CCP4 or XPLOR map: %s" % file_name, file=log)
print("-" * 79, file=log)
return (map_coeff_file_name, xplor_maps)
def run(args, log):
cmdline = mmtbx.utils.process_command_line_args(
args = args,
master_params = get_master_phil())
if(len(args)==0 or '--help' in args):
get_help()
return
elif('--defaults' in args or '--show' in args):
print_legend_and_usage(log)
return
elif('--version' in args):
print(__version__)
return
print("Running refinement", file=log)
cmdline.params.show(out=log, prefix=" ")
params = cmdline.params.extract()
if len(cmdline.pdb_file_names)==0:
raise Sorry('PDB model files not provided and/or not found')
model = qr.process_model_file(
pdb_file_name = cmdline.pdb_file_names[0],
cif_objects = cmdline.cif_objects,
crystal_symmetry = cmdline.crystal_symmetry,
scattering_table = params.scattering_table)
map_data = None
fmodel = None
if(params.refine.mode=="opt" or params.refine.mode=='gtest'):
if (len(cmdline.reflection_files)>0 or cmdline.ccp4_map is not None):
print("WARNING: data files not used in optimization or gradient test! ", file=log)
elif(len(cmdline.reflection_files)>0 and params.refine.mode=="refine"):
# Read reflection data
rfs = reflection_file_server(
crystal_symmetry = cmdline.crystal_symmetry,
reflection_files = cmdline.reflection_files)
determine_data_and_flags_result = extract_xtal_data.run(
reflection_file_server = rfs,
keep_going = True)
f_obs = determine_data_and_flags_result.f_obs
number_of_reflections = f_obs.indices().size()
r_free_flags = determine_data_and_flags_result.r_free_flags
test_flag_value = determine_data_and_flags_result.test_flag_value
if(r_free_flags is None):
r_free_flags=f_obs.generate_r_free_flags()
print("WARNING: no free-R flags available in inputs. ", file=log)
fmodel = mmtbx.f_model.manager(
f_obs = f_obs,
r_free_flags = r_free_flags,
xray_structure = model.get_xray_structure(),
target_name = params.refine.refinement_target_name)
if(params.refine.update_all_scales):
fmodel.update_all_scales(remove_outliers=False)
fmodel.show(show_header=False, show_approx=False)
print("Initial r_work=%6.4f r_free=%6.4f" % (fmodel.r_work(),
fmodel.r_free()), file=log)
elif(cmdline.ccp4_map is not None and params.refine.mode=="refine"):
# Read map
map_data = cmdline.ccp4_map.map_data()
# Normalize map
map_data = map_data - flex.mean(map_data)
sd = map_data.sample_standard_deviation()
assert sd != 0
map_data = map_data/sd
else:
raise Sorry("Refinement requested (refine.mode==refine) but no data provided.")
log.flush()
qr.run(
model = model, # XXX This is not mmtbx.model.manager !!! (see above).
fmodel = fmodel,
map_data = map_data,
params = params,
rst_file = params.rst_file,
prefix = os.path.basename(cmdline.pdb_file_names[0])[:-4],
log = log)
def run(args, command_name = "phenix.fobs_minus_fobs_map", log=None):
if(len(args) == 0): args = ["--help"]
examples = """Examples:
phenix.fobs_minus_fobs_map f_obs_1_file=data1.mtz f_obs_2_file=data2.sca \
f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 model.pdb
phenix.fobs_minus_fobs_map f_obs_1_file=data.mtz f_obs_2_file=data.mtz \
f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 phase_source=model.pdb \
high_res=2.0 sigma_cutoff=2 scattering_table=neutron"""
command_line = (iotbx_option_parser(
usage="%s [options]" % command_name,
description=examples)
.option("--silent",
action="store_true",
help="Suppress output to the screen.")
.enable_symmetry_comprehensive()
).process(args=args)
#
if (log is None):
log = sys.stdout
if(not command_line.options.silent):
utils.print_header("phenix.fobs_minus_fobs_map", out = log)
print("Command line arguments: ", file=log)
print(args, file=log)
print(file=log)
#
processed_args = utils.process_command_line_args(
args=command_line.args,
cmd_cs=command_line.symmetry,
master_params=fo_minus_fo_master_params(),
absolute_angle_tolerance=5,
absolute_length_tolerance=1,
log=log,
suppress_symmetry_related_errors=True)
working_phil = processed_args.params
if(not command_line.options.silent):
print("*** Parameters:", file=log)
working_phil.show(out = log)
print(file=log)
params = working_phil.extract()
consensus_symmetry = None
if (params.ignore_non_isomorphous_unit_cells):
if (None in [params.f_obs_1_file_name, params.f_obs_2_file_name,
params.phase_source]):
raise Sorry("The file parameters (f_obs_1_file_name, f_obs_2_file_name, "+
"phase_source) must be specified explicitly when "+
"ignore_non_isomorphous_unit_cells=True.")
symm_manager = iotbx.symmetry.manager()
pdb_in = iotbx.file_reader.any_file(params.phase_source, force_type="pdb")
symm_manager.process_pdb_file(pdb_in)
hkl_in_1 = iotbx.file_reader.any_file(params.f_obs_1_file_name,
force_type="hkl")
sg_err_1, uc_err_1 = symm_manager.process_reflections_file(hkl_in_1)
hkl_in_2 = iotbx.file_reader.any_file(params.f_obs_2_file_name,
force_type="hkl")
sg_err_2, uc_err_2 = symm_manager.process_reflections_file(hkl_in_2)
out = StringIO()
symm_manager.show(out=out)
if (sg_err_1) or (sg_err_2):
raise Sorry(("Incompatible space groups in input files:\n%s\nAll files "+
"must have the same point group (and ideally the same space group). "+
"Please note that any symmetry information in the PDB file will be "+
"used first.") % out.getvalue())
elif (uc_err_1) or (uc_err_2):
libtbx.call_back(message="warn",
data=("Crystal symmetry mismatch:\n%s\nCalculations will continue "+
"using the symmetry in the PDB file (or if not available, the "+
"first reflection file), but the maps should be treated with "+
"extreme suspicion.") % out.getvalue())
crystal_symmetry = symm_manager.as_symmetry_object()
else :
processed_args = utils.process_command_line_args(
args=command_line.args,
cmd_cs=command_line.symmetry,
master_params=fo_minus_fo_master_params(),
suppress_symmetry_related_errors = False,
absolute_angle_tolerance=5,
absolute_length_tolerance=1,
log=StringIO())
crystal_symmetry = processed_args.crystal_symmetry
#
pdb_file_names = processed_args.pdb_file_names
if(len(processed_args.pdb_file_names) == 0):
if(params.phase_source is not None):
pdb_file_names = [params.phase_source]
else:
raise Sorry("No PDB file found.")
# Extaract Fobs1, Fobs2
f_obss = []
if(len(processed_args.reflection_files)==2):
for reflection_file in processed_args.reflection_files:
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = crystal_symmetry,
force_symmetry = True,
reflection_files = [reflection_file],
err = null_out())
# XXX UGLY !!!
try:
parameters = extract_xtal_data.data_and_flags_master_params().extract()
if(params.f_obs_1_label is not None):
parameters.labels = [params.f_obs_1_label]
determine_data_and_flags_result = extract_xtal_data.run(
reflection_file_server = reflection_file_server,
keep_going = True,
parameters = parameters)
except: # intentional
parameters = extract_xtal_data.data_and_flags_master_params().extract()
if(params.f_obs_2_label is not None):
parameters.labels = [params.f_obs_2_label]
determine_data_and_flags_result = extract_xtal_data.run(
reflection_file_server = reflection_file_server,
keep_going = True,
parameters = parameters)
f_obss.append(determine_data_and_flags_result.f_obs)
else:
if([params.f_obs_1_file_name,params.f_obs_2_file_name].count(None)==2):
raise Sorry("No reflection data file found.")
for file_name, label in zip([params.f_obs_1_file_name,params.f_obs_2_file_name],
[params.f_obs_1_label,params.f_obs_2_label]):
reflection_file = reflection_file_reader.any_reflection_file(
file_name = file_name, ensure_read_access = False)
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = crystal_symmetry,
force_symmetry = True,
reflection_files = [reflection_file],
err = null_out())
parameters = extract_xtal_data.data_and_flags_master_params().extract()
if(label is not None):
parameters.labels = [label]
determine_data_and_flags_result = extract_xtal_data.run(
reflection_file_server = reflection_file_server,
parameters = parameters,
keep_going = True)
f_obss.append(determine_data_and_flags_result.f_obs)
if(len(f_obss)!=2):
raise Sorry(" ".join(errors))
if(not command_line.options.silent):
for ifobs, fobs in enumerate(f_obss):
print("*** Summary for data set %d:"%ifobs, file=log)
fobs.show_comprehensive_summary(f = log)
print(file=log)
pdb_combined = combine_unique_pdb_files(file_names = pdb_file_names)
pdb_combined.report_non_unique(out = log)
if(len(pdb_combined.unique_file_names) == 0):
raise Sorry("No coordinate file given.")
#
raw_recs = flex.std_string()
for rec in pdb_combined.raw_records:
if(rec.upper().count("CRYST1")==0):
raw_recs.append(rec)
raw_recs.append(iotbx.pdb.format_cryst1_record(
crystal_symmetry = crystal_symmetry))
#
pdb_in = iotbx.pdb.input(source_info = None, lines = raw_recs)
model = mmtbx.model.manager(model_input = pdb_in)
d_min = min(f_obss[0].d_min(), f_obss[1].d_min())
model.setup_scattering_dictionaries(
scattering_table = params.scattering_table,
d_min = d_min)
xray_structure = model.get_xray_structure()
hierarchy = model.get_hierarchy()
#
omit_sel = flex.bool(hierarchy.atoms_size(), False)
if (params.advanced.omit_selection is not None):
print("Will omit selection from phasing model:", file=log)
print(" " + params.advanced.omit_selection, file=log)
omit_sel = hierarchy.atom_selection_cache().selection(
params.advanced.omit_selection)
print("%d atoms selected for removal" % omit_sel.count(True), file=log)
del hierarchy
xray_structure = xray_structure.select(~omit_sel)
if(not command_line.options.silent):
print("*** Model summary:", file=log)
xray_structure.show_summary(f = log)
print(file=log)
info0 = f_obss[0].info()
info1 = f_obss[1].info()
f_obss[0] = f_obss[0].resolution_filter(d_min = params.high_resolution,
d_max = params.low_resolution).set_info(info0)
f_obss[1] = f_obss[1].resolution_filter(d_min = params.high_resolution,
d_max = params.low_resolution).set_info(info1)
if(params.sigma_cutoff is not None):
for i in [0,1]:
if(f_obss[i].sigmas() is not None):
sel = f_obss[i].data() > f_obss[i].sigmas()*params.sigma_cutoff
f_obss[i] = f_obss[i].select(sel).set_info(info0)
for k, f_obs in enumerate(f_obss):
if (f_obs.indices().size() == 0):
raise Sorry("No data left in array %d (labels=%s) after filtering!" % (k+1,
f_obs.info().label_string()))
output_file_name = params.output_file
if (output_file_name is None) and (params.file_name_prefix is not None):
output_file_name = "%s_%s.mtz" % (params.file_name_prefix, params.job_id)
output_files = compute_fo_minus_fo_map(
data_arrays = f_obss,
xray_structure = xray_structure,
log = log,
silent = command_line.options.silent,
output_file = output_file_name,
peak_search=params.find_peaks_holes,
map_cutoff=params.map_cutoff,
peak_search_params=params.peak_search,
multiscale=params.advanced.multiscale,
anomalous=params.advanced.anomalous).file_names
return output_files
