def reflection_file_server(crystal_symmetry, reflection_files, log):
from iotbx import reflection_file_utils
return reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=log)
def get_f_obs_and_flags(reflection_file_name,
crystal_symmetry,
f_obs_label=None,
r_free_flags_label=None,
log=None):
reflection_files = []
reflection_files.append(
reflection_file_reader.any_reflection_file(
file_name=reflection_file_name, ensure_read_access=False))
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=log)
parameters = extract_xtal_data.data_and_flags_master_params().extract()
if (f_obs_label is not None):
parameters.labels = command_line.options.f_obs_label
if (r_free_flags_label is not None):
parameters.r_free_flags.label = command_line.options.r_free_flags_label
determine_data_and_flags_result = extract_xtal_data.run(
reflection_file_server=reflection_file_server,
parameters=parameters,
keep_going=True)
f_obs = determine_data_and_flags_result.f_obs
r_free_flags = determine_data_and_flags_result.r_free_flags
return f_obs, r_free_flags
def exercise_get_experimental_phases():
crystal_symmetry = crystal.symmetry(
unit_cell=(30,31,32,85,95,100),
space_group_symbol="P 1")
miller_set = miller.build_set(
crystal_symmetry=crystal_symmetry,
anomalous_flag=False,
d_min=3)
input_array = miller_set.array(
data=flex.hendrickson_lattman(miller_set.indices().size(), (0,0,0,0)))
mtz_dataset = input_array.as_mtz_dataset(column_root_label="P")
mtz_dataset.mtz_object().write("tmp.mtz")
reflection_files = [reflection_file_reader.any_reflection_file(
file_name="tmp.mtz")]
err = StringIO()
reflection_file_srv = reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=err)
experimental_phases = reflection_file_srv.get_experimental_phases(
file_name=None,
labels=None,
ignore_all_zeros=False,
parameter_scope="experimental_phases")
assert str(experimental_phases.info()) == "tmp.mtz:PA,PB,PC,PD"
try:
reflection_file_srv.get_experimental_phases(
file_name=None,
labels=None,
ignore_all_zeros=True,
parameter_scope="experimental_phases")
except Sorry, e:
assert str(e) == "No array of experimental phases found."
assert err.getvalue() == """\
def exercise_get_experimental_phases():
crystal_symmetry = crystal.symmetry(unit_cell=(30, 31, 32, 85, 95, 100),
space_group_symbol="P 1")
miller_set = miller.build_set(crystal_symmetry=crystal_symmetry,
anomalous_flag=False,
d_min=3)
input_array = miller_set.array(
data=flex.hendrickson_lattman(miller_set.indices().size(), (0, 0, 0,
0)))
mtz_dataset = input_array.as_mtz_dataset(column_root_label="P")
mtz_dataset.mtz_object().write("tmp.mtz")
reflection_files = [
reflection_file_reader.any_reflection_file(file_name="tmp.mtz")
]
err = StringIO()
reflection_file_srv = reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=err)
experimental_phases = reflection_file_srv.get_experimental_phases(
file_name=None,
labels=None,
ignore_all_zeros=False,
parameter_scope="experimental_phases")
assert str(experimental_phases.info()) == "tmp.mtz:PA,PB,PC,PD"
try:
reflection_file_srv.get_experimental_phases(
file_name=None,
labels=None,
ignore_all_zeros=True,
parameter_scope="experimental_phases")
except Sorry, e:
assert str(e) == "No array of experimental phases found."
assert err.getvalue() == """\
def run(args):
cmdline = iotbx.phil.process_command_line_with_files(
args=args,
master_phil_string=master_phil_str,
reflection_file_def="data",
pdb_file_def="symmetry")
params = cmdline.work.extract()
if (params.data is None):
raise Sorry("Data file not defined")
hkl_file = cmdline.get_file(params.data)
crystal_symmetry = None
if (params.symmetry is not None):
from iotbx import crystal_symmetry_from_any
crystal_symmetry = crystal_symmetry_from_any.extract_from(
file_name=params.symmetry)
hkl_server = reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=[hkl_file.file_object],
err=sys.stderr)
data = hkl_server.get_xray_data(file_name=params.data,
labels=params.labels,
ignore_all_zeros=True,
parameter_scope="",
minimum_score=4,
prefer_anomalous=True)
if (not data.anomalous_flag()):
raise Sorry("Must provide anomalous data.")
if data.is_xray_intensity_array():
data = data.f_sq_as_f()
dano = abs(data.anomalous_differences())
dano.set_observation_type_xray_amplitude()
dano.as_mtz_dataset(column_root_label="F").mtz_object().write(
params.mtz_out)
print("Wrote DANO to %s" % params.mtz_out)
def get_data(pdbf, mtzf):
pdb_inp = iotbx.pdb.input(file_name=pdbf)
xrs = pdb_inp.xray_structure_simple()
#
selection = xrs.scatterers().extract_occupancies() > 0
xrs = xrs.select(selection)
selection = ~xrs.hd_selection()
xrs = xrs.select(selection)
#
#xrs.switch_to_neutron_scattering_dictionary()
#
reflection_file = reflection_file_reader.any_reflection_file(
file_name=mtzf, ensure_read_access=False)
rfs = reflection_file_utils.reflection_file_server(
crystal_symmetry=xrs.crystal_symmetry(),
force_symmetry=True,
reflection_files=[reflection_file],
err=null_out())
determine_data_and_flags_result = extract_xtal_data.run(
reflection_file_server=rfs,
keep_going=True,
extract_r_free_flags=True,
force_non_anomalous=True,
allow_mismatch_flags=True)
f_obs = determine_data_and_flags_result.f_obs
r_free_flags = determine_data_and_flags_result.r_free_flags
fmodel = mmtbx.f_model.manager(f_obs=f_obs,
r_free_flags=r_free_flags,
xray_structure=xrs)
fmodel.update_all_scales(remove_outliers=True,
apply_scale_k1_to_f_obs=True)
# Skip
rw = fmodel.r_work()
rf = fmodel.r_free()
if (fmodel.f_obs().d_min() > 3): return None
if (fmodel.f_obs().completeness() < 0.95): return None
if (rw > 0.25): return None
if (rf <= rw): return None
if (abs(rf - rw) * 100 < 2): return None
if (fmodel.f_obs().resolution_filter(d_min=8).completeness() < 0.95):
return None
#
def f_obs():
return fmodel.f_obs()
def r_free_flags():
return fmodel.r_free_flags()
def f_calc():
return fmodel.f_calc()
def flags():
return fmodel.r_free_flags()
return group_args(f_obs=f_obs,
r_free_flags=r_free_flags,
xray_structure=fmodel.xray_structure,
f_calc=f_calc,
flags=flags)
def get_f_obs_and_flags(reflection_file_name,
crystal_symmetry,
f_obs_label = None,
r_free_flags_label = None,
log = None):
reflection_files = []
reflection_files.append(reflection_file_reader.any_reflection_file(
file_name = reflection_file_name, ensure_read_access = False))
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = crystal_symmetry,
force_symmetry = True,
reflection_files = reflection_files,
err = log)
parameters = mmtbx.utils.data_and_flags_master_params().extract()
if(f_obs_label is not None):
parameters.labels = command_line.options.f_obs_label
if(r_free_flags_label is not None):
parameters.r_free_flags.label = command_line.options.r_free_flags_label
determine_data_and_flags_result = mmtbx.utils.determine_data_and_flags(
reflection_file_server = reflection_file_server,
parameters = parameters,
keep_going = True,
log = log)
f_obs = determine_data_and_flags_result.f_obs
r_free_flags = determine_data_and_flags_result.r_free_flags
return f_obs, r_free_flags
def run(args, command_name="iotbx.r_free_flags_accumulation"):
def raise_usage():
raise Usage("%s reflection_file [label=value]" % command_name)
if (len(args) == 0 or "--help" in args or "-h" in args):
raise_usage()
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="r_free_flags_accumulation")
reflection_files = []
for arg in args:
if (os.path.isfile(arg)):
refl_file = reflection_file_reader.any_reflection_file(
file_name=arg)
if (refl_file.file_type() is not None):
reflection_files.append(refl_file)
arg = None
if (arg is not None):
try: command_line_params = argument_interpreter.process(arg=arg)
except KeyboardInterrupt: raise
except Exception: raise Sorry("Unknown file or keyword: %s" % arg)
else: phil_objects.append(command_line_params)
params_scope = master_params.fetch(sources=phil_objects).extract()
params = params_scope.r_free_flags_accumulation
srv = reflection_file_utils.reflection_file_server(
reflection_files=reflection_files)
r_free_flags, test_flag_value = srv.get_r_free_flags(
file_name=params.file_name,
label=params.label,
test_flag_value=params.test_flag_value,
disable_suitability_test=params.disable_suitability_test,
parameter_scope="r_free_flags_accumulation")
params.file_name = r_free_flags.info().source
params.label = r_free_flags.info().label_string()
params.test_flag_value = test_flag_value
if (params.output is None):
params.output = os.path.basename(params.file_name) \
+ ".r_free_flags_accumulation"
working_params = master_params.format(python_object=params_scope)
working_params.show()
print
print "#phil __OFF__"
r_free_flags = r_free_flags.array(
data=r_free_flags.data()==params.test_flag_value)
r_free_flags.show_r_free_flags_info()
print
accu = r_free_flags \
.sort(by_value="resolution") \
.r_free_flags_accumulation()
print "Writing file: %s" % show_string(params.output)
print " 1. column: reflection counts, sorted by resolution"
print " 2. column: number of free reflections / total number of reflections"
sys.stdout.flush()
out = open(params.output, "w")
for line in params.plot_header:
print >> out, line
for c,f in zip(accu.reflection_counts, accu.free_fractions):
print >> out, c, f
out.close()
print
sys.stdout.flush()
def run (args) :
cmdline = iotbx.phil.process_command_line_with_files(
args=args,
master_phil_string=master_phil_str,
reflection_file_def="data",
pdb_file_def="symmetry")
params = cmdline.work.extract()
if (params.data is None) :
raise Sorry("Data file not defined")
hkl_file = cmdline.get_file(params.data)
crystal_symmetry = None
if (params.symmetry is not None) :
from iotbx import crystal_symmetry_from_any
crystal_symmetry = crystal_symmetry_from_any.extract_from(
file_name=params.symmetry)
hkl_server = reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=[hkl_file.file_object],
err=sys.stderr)
data = hkl_server.get_xray_data(
file_name=params.data,
labels=params.labels,
ignore_all_zeros=True,
parameter_scope="",
minimum_score=4,
prefer_anomalous=True)
if (not data.anomalous_flag()) :
raise Sorry("Must provide anomalous data.")
if data.is_xray_intensity_array() :
data = data.f_sq_as_f()
dano = abs(data.anomalous_differences())
dano.set_observation_type_xray_amplitude()
dano.as_mtz_dataset(column_root_label="F").mtz_object().write(params.mtz_out)
print "Wrote DANO to %s" % params.mtz_out
def reflection_file_server(crystal_symmetry, reflection_files, log):
from iotbx import reflection_file_utils
return reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=log)
def run(args):
pdb_file_name, hkl_file_name = args
pdb_inp = iotbx.pdb.input(file_name=pdb_file_name)
xrs = pdb_inp.xray_structure_simple()
xrs.scattering_type_registry(table="wk1995")
#
reflection_file = reflection_file_reader.any_reflection_file(
file_name=hkl_file_name, ensure_read_access=False)
rfs = reflection_file_utils.reflection_file_server(
crystal_symmetry=xrs.crystal_symmetry(),
force_symmetry=True,
reflection_files=[reflection_file],
err=StringIO())
determine_data_and_flags_result = mmtbx.utils.determine_data_and_flags(
reflection_file_server=rfs, keep_going=True, log=StringIO())
f_obs = determine_data_and_flags_result.f_obs
sel = f_obs.data() > 0
f_obs = f_obs.select(selection=sel)
#
r_free_flags = determine_data_and_flags_result.r_free_flags
if (r_free_flags is None):
print "No R-free flags available."
return
merged = f_obs.as_non_anomalous_array().merge_equivalents()
f_obs = merged.array().set_observation_type(f_obs)
#
merged = r_free_flags.as_non_anomalous_array().merge_equivalents()
r_free_flags = merged.array().set_observation_type(r_free_flags)
f_obs, r_free_flags = f_obs.common_sets(r_free_flags)
#
crystal_gridding = f_obs.crystal_gridding(
d_min=f_obs.d_min(),
symmetry_flags=maptbx.use_space_group_symmetry,
resolution_factor=1. / 4)
#
sel = xrs.hd_selection()
xrs = xrs.select(~sel)
sel = xrs.scatterers().extract_occupancies() > 0.
xrs = xrs.select(sel)
#
old = compute(f_obs=f_obs,
r_free_flags=r_free_flags,
xrs=xrs,
use_new=False)
new = compute(f_obs=f_obs,
r_free_flags=r_free_flags,
xrs=xrs,
use_new=True)
#
print "OLD:Rw_l/Rw_h/Rw/Rf %6.4f %6.4f %6.4f %6.4f" % (old.rwl, old.rwh,
old.rw, old.rf)
print "NEW:Rw_l/Rw_h/Rw/Rf %6.4f %6.4f %6.4f %6.4f" % (new.rwl, new.rwh,
new.rw, new.rf)
#
mtz_dataset = old.mc_diff.as_mtz_dataset(column_root_label="FoFc_old")
mtz_dataset.add_miller_array(miller_array=new.mc_diff,
column_root_label="FoFc_new")
mtz_object = mtz_dataset.mtz_object()
mtz_object.write(file_name="FoFc.mtz")
def setup_reflection_data(inputs, params, crystal_symmetry, reflection_files,
log):
#setup reflection data
f_obs, r_free_flags = None, None
rfs = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=StringIO())
parameters = mmtbx.utils.data_and_flags_master_params().extract()
if (params.input.reflection_data.labels is not None):
parameters.labels = params.params.input.reflection_data.labels
if (params.input.reflection_data.r_free_flags.label is not None):
parameters.r_free_flags.label = params.input.reflection_data.r_free_flags.label
try:
determined_data_and_flags = mmtbx.utils.determine_data_and_flags(
reflection_file_server=rfs,
parameters=parameters,
keep_going=True,
log=StringIO())
f_obs = determined_data_and_flags.f_obs
r_free_flags = determined_data_and_flags.r_free_flags
except:
print "DATA PROCESSING ERROR --> picking first miller array"
f_obs = rfs.miller_arrays[0]
if (params.input.reflection_data.labels is None):
params.input.reflection_data.labels = f_obs.info().label_string()
if (params.input.reflection_data.reflection_file_name is None):
params.input.reflection_data.reflection_file_name = parameters.file_name
assert f_obs is not None
print >> log, "_" * 79
print >> log, "Input data:"
print >> log, " Iobs or Fobs:", f_obs.info().labels
if (r_free_flags is not None):
print >> log, " Free-R flags:", r_free_flags.info().labels
params.input.reflection_data.r_free_flags.label = r_free_flags.info(
).label_string()
else:
print >> log, " Free-R flags Not present, generating ..."
r_free_flags = f_obs.generate_r_free_flags(fraction=0.05,
max_free=1000)
# Merge anomalous if needed
if (f_obs.anomalous_flag()):
sel = f_obs.data() > 0
f_obs = f_obs.select(sel)
merged = f_obs.as_non_anomalous_array().merge_equivalents()
f_obs = merged.array().set_observation_type(f_obs)
r_free_flags = r_free_flags.select(sel)
merged = r_free_flags.as_non_anomalous_array().merge_equivalents()
r_free_flags = merged.array().set_observation_type(r_free_flags)
f_obs, r_free_flags = f_obs.common_sets(r_free_flags)
if params.input.parameters.score_res is None:
params.input.parameters.score_res = f_obs.d_min()
print >> log, " Determined Resolution Limit: %.2f" % params.input.parameters.score_res
print >> log, " -->Override with \"score_res=XXX\""
return f_obs, r_free_flags
def get_map_from_hkl(hkl_file_object, params, xrs, log):
print("Processing input hkl file...", file=log)
crystal_symmetry = hkl_file_object.crystal_symmetry()
rfs = reflection_file_utils.reflection_file_server(
crystal_symmetry = crystal_symmetry,
force_symmetry = True,
reflection_files = [hkl_file_object.file_content],
err = StringIO())
parameters = mmtbx.utils.data_and_flags_master_params().extract()
if (params.data_labels is not None):
parameters.labels = params.data_labels
if (params.r_free_flags_labels is not None):
parameters.r_free_flags.label = params.r_free_flags_labels
determined_data_and_flags = mmtbx.utils.determine_data_and_flags(
reflection_file_server = rfs,
parameters = parameters,
keep_going = True,
working_point_group = crystal_symmetry.space_group().build_derived_point_group(),
log = StringIO(),
symmetry_safety_check = True)
f_obs = determined_data_and_flags.f_obs
if (params.data_labels is None):
params.data_labels = f_obs.info().label_string()
r_free_flags = determined_data_and_flags.r_free_flags
assert f_obs is not None
print("Input data:", file=log)
print(" Iobs or Fobs:", f_obs.info().labels, file=log)
if (r_free_flags is not None):
print(" Free-R flags:", r_free_flags.info().labels, file=log)
params.r_free_flags_labels = r_free_flags.info().label_string()
else:
print(" Free-R flags: Not present", file=log)
fmodel = mmtbx.f_model.manager(
f_obs = f_obs,
r_free_flags = r_free_flags,
xray_structure = xrs)
fmodel.update_all_scales()
fft_map = fmodel.electron_density_map().fft_map(
resolution_factor = 0.25,
map_type = "2mFo-DFc",
use_all_data = False) # Exclude free reflections
fft_map.apply_sigma_scaling()
map_data = fft_map.real_map_unpadded(in_place=False)
if params.debug:
fft_map.as_xplor_map(file_name="%s_21.map" % params.output_prefix)
iotbx.mrcfile.write_ccp4_map(
file_name="%s_21.ccp4" % params.output_prefix,
unit_cell=crystal_symmetry.unit_cell(),
space_group=crystal_symmetry.space_group(),
map_data=map_data,
labels=flex.std_string([""]))
return map_data, crystal_symmetry
def perturb_structure_factors_proportional_to_model_error(mtzin, mtzout, seed=0):
"""Perturb the observed structure factors proportional to the model error (FOBS-FCALC)"""
random.seed(seed)
assert os.path.exists(mtzin), 'MTZ File does not exist! {!s}'.format(mtzin)
assert not os.path.exists(mtzout), 'MTZ File already exists! {!s}'.format(mtzout)
output_column_label = 'F_PERTURBED'
# Create an mtz file object
mtz_file = reflection_file_reader.any_reflection_file(mtzin)
# Create file server to serve up the mtz file
mtz_server = reflection_file_server(crystal_symmetry=None, force_symmetry=True, reflection_files=[mtz_file], err=sys.stderr)
# At the moment hardcode a ccp4 label
sfobs_label = 'F,SIGF'
sfdif_label = 'DELFWT,PHDELWT'
rfree_label = 'FreeR_flag'
# Get the observed miller array, and the R-free flags
sfobs_miller = mtz_server.get_miller_array(sfobs_label)
sfdif_miller = mtz_server.get_miller_array(sfdif_label).amplitudes()
rfree_miller = mtz_server.get_miller_array(rfree_label)
# F column
sfobs_amplts = sfobs_miller.data()
# FOBS-FCALC column
sfobs_scales = sfdif_miller.data()
# Add to observed data
perturbed_data = [f + e*random.gauss(0,1) for f,e in zip(sfobs_amplts, sfobs_scales)]
# Zero any negative amplitudes
perturbed_data = [f if f>0.0 else 0.0 for f in perturbed_data]
# Generate new miller array
perturbed_array = sfobs_miller.customized_copy(data=flex.double(perturbed_data))
assert perturbed_array.is_real_array()
print 'CORRELATION:', sfobs_miller.correlation(perturbed_array).coefficient()
# Convert to mtz dataset
perturbed_mtz_dataset = perturbed_array.as_mtz_dataset(output_column_label)
# Copy across the R-Free
perturbed_mtz_dataset.add_miller_array(rfree_miller, rfree_label)
# Write out mtz file
perturbed_mtz_object = perturbed_mtz_dataset.mtz_object()
perturbed_mtz_object.write(file_name=mtzout)
return output_column_label
def run (args, out=sys.stdout) :
cmdline = iotbx.phil.process_command_line_with_files(
args=args,
master_phil_string=master_phil_str,
reflection_file_def="data",
pdb_file_def="symmetry",
space_group_def="space_group",
unit_cell_def="unit_cell",
usage_string="""\
iotbx.show_systematic_absences data.hkl [space_group] [unit_cell]""")
params = cmdline.work.extract()
if (params.data is None) :
raise Sorry("Data file not specified.")
hkl_file = cmdline.get_file(params.data)
hkl_server = reflection_file_server(
crystal_symmetry=None,
force_symmetry=True,
reflection_files=[hkl_file.file_object],
err=sys.stderr)
data = hkl_server.get_xray_data(
file_name=params.data,
labels=params.labels,
ignore_all_zeros=False,
parameter_scope="",
minimum_score=4,
prefer_amplitudes=False)
symm = data.crystal_symmetry()
space_group = unit_cell = None
if (params.symmetry is not None) :
from iotbx import crystal_symmetry_from_any
symm = crystal_symmetry_from_any.extract_from(file_name=params.symmetry)
if (symm is not None) :
space_group = symm.space_group_info()
unit_cell = symm.unit_cell()
if (space_group is None) :
if (params.space_group is not None) :
space_group = params.space_group
else :
raise Sorry("No space group defined.")
if (unit_cell is None) :
if (params.unit_cell is not None) :
unit_cell = params.unit_cell
else :
raise Sorry("No unit cell defined.")
symm = crystal.symmetry(
space_group_info=space_group,
unit_cell=unit_cell)
data = data.customized_copy(crystal_symmetry=symm)
if (data.sigmas() is None) :
raise Sorry("Input data are missing experimental sigmas.")
data.show_all_possible_systematic_absences(out=out)
def run(args, out=sys.stdout):
cmdline = iotbx.phil.process_command_line_with_files(
args=args,
master_phil_string=master_phil_str,
reflection_file_def="data",
pdb_file_def="symmetry",
space_group_def="space_group",
unit_cell_def="unit_cell",
usage_string="""\
iotbx.show_systematic_absences data.hkl [space_group] [unit_cell]""")
params = cmdline.work.extract()
if (params.data is None):
raise Sorry("Data file not specified.")
hkl_file = cmdline.get_file(params.data)
hkl_server = reflection_file_server(
crystal_symmetry=None,
force_symmetry=True,
reflection_files=[hkl_file.file_object],
err=sys.stderr)
data = hkl_server.get_xray_data(file_name=params.data,
labels=params.labels,
ignore_all_zeros=False,
parameter_scope="",
minimum_score=4,
prefer_amplitudes=False)
symm = data.crystal_symmetry()
space_group = unit_cell = None
if (params.symmetry is not None):
from iotbx import crystal_symmetry_from_any
symm = crystal_symmetry_from_any.extract_from(
file_name=params.symmetry)
if (symm is not None):
space_group = symm.space_group_info()
unit_cell = symm.unit_cell()
if (space_group is None):
if (params.space_group is not None):
space_group = params.space_group
else:
raise Sorry("No space group defined.")
if (unit_cell is None):
if (params.unit_cell is not None):
unit_cell = params.unit_cell
else:
raise Sorry("No unit cell defined.")
symm = crystal.symmetry(space_group_info=space_group, unit_cell=unit_cell)
data = data.customized_copy(crystal_symmetry=symm)
if (data.sigmas() is None):
raise Sorry("Input data are missing experimental sigmas.")
data.show_all_possible_systematic_absences(out=out)
def file_server(self):
"""
For reflection files only, returns an
:py:class:`iotbx.reflection_file_utils.reflection_file_server` object
containing the extracted Miller arrays. Note that this will implicitly
merge any non-unique observations.
"""
from iotbx.reflection_file_utils import reflection_file_server
if (self._file_server is None):
if (self._file_type == "hkl"):
self._file_server = reflection_file_server(
crystal_symmetry=None,
force_symmetry=True,
reflection_files=[self._file_object],
err=sys.stderr)
return self._file_server
def file_server (self) :
"""
For reflection files only, returns an
:py:class:`iotbx.reflection_file_utils.reflection_file_server` object
containing the extracted Miller arrays. Note that this will implicitly
merge any non-unique observations.
"""
from iotbx.reflection_file_utils import reflection_file_server
if (self._file_server is None) :
if (self._file_type == "hkl") :
self._file_server = reflection_file_server(
crystal_symmetry=None,
force_symmetry=True,
reflection_files=[self._file_object],
err=sys.stderr)
return self._file_server
def get_data(pdbf, mtzf):
pdb_inp = iotbx.pdb.input(file_name=pdbf)
xrs = pdb_inp.xray_structure_simple()
#
selection = xrs.scatterers().extract_occupancies() > 0
xrs = xrs.select(selection)
selection = ~xrs.hd_selection()
xrs = xrs.select(selection)
#
#xrs.switch_to_neutron_scattering_dictionary()
#
reflection_file = reflection_file_reader.any_reflection_file(
file_name=mtzf, ensure_read_access=False)
rfs = reflection_file_utils.reflection_file_server(
crystal_symmetry=xrs.crystal_symmetry(),
force_symmetry=True,
reflection_files=[reflection_file],
err=null_out())
determine_data_and_flags_result = mmtbx.utils.determine_data_and_flags(
reflection_file_server=rfs,
keep_going=True,
extract_r_free_flags=False,
force_non_anomalous=True,
log=null_out())
f_obs = determine_data_and_flags_result.f_obs
r_free_flags = determine_data_and_flags_result.r_free_flags
fmodel = mmtbx.f_model.manager(f_obs=f_obs,
r_free_flags=r_free_flags,
xray_structure=xrs)
fmodel.update_all_scales(remove_outliers=True,
apply_scale_k1_to_f_obs=True)
def f_obs():
return fmodel.f_obs()
def r_free_flags():
return fmodel.r_free_flags()
def f_calc():
return fmodel.f_calc()
return group_args(f_obs=f_obs,
r_free_flags=r_free_flags,
xray_structure=fmodel.xray_structure,
f_calc=f_calc)
def get_fmodel(xrs, hkl_file):
rfs = reflection_file_utils.reflection_file_server(
crystal_symmetry = xrs.crystal_symmetry(),
force_symmetry = True,
reflection_files = hkl_file,
err = null_out())
determine_data_and_flags_result = mmtbx.utils.determine_data_and_flags(
reflection_file_server = rfs,
keep_going = True,
log = null_out())
f_obs = determine_data_and_flags_result.f_obs
#r_free_flags = determine_data_and_flags_result.r_free_flags
print "f-obs labels:", f_obs.info().labels
#print "r-free-flags labels", r_free_flags.info().labels
fmodel = mmtbx.f_model.manager(
xray_structure = xrs,
f_obs = f_obs,
#r_free_flags = r_free_flags,
)
fmodel.update_all_scales()
print "r_work, r_free: %6.4f %6.4f" % (fmodel.r_work(), fmodel.r_free())
return fmodel
def extract_data_and_flags(params, crystal_symmetry=None):
data_and_flags = None
if (params.reflection_file_name is not None):
reflection_file = reflection_file_reader.any_reflection_file(
file_name=params.reflection_file_name)
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=[reflection_file])
parameters = extract_xtal_data.data_and_flags_master_params().extract()
parameters.force_anomalous_flag_to_be_equal_to = False
if (params.data_labels is not None):
parameters.labels = [params.data_labels]
if (params.high_resolution is not None):
parameters.high_resolution = params.high_resolution
if (params.low_resolution is not None):
parameters.low_resolution = params.low_resolution
data_and_flags = extract_xtal_data.run(
reflection_file_server=reflection_file_server,
parameters=parameters,
extract_r_free_flags=False) #XXX
return data_and_flags
def extract_data_and_flags(params, crystal_symmetry=None):
data_and_flags = None
if (params.reflection_file_name is not None):
reflection_file = reflection_file_reader.any_reflection_file(
file_name=params.reflection_file_name)
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=[reflection_file])
parameters = mmtbx.utils.data_and_flags_master_params().extract()
parameters.force_anomalous_flag_to_be_equal_to = False
if (params.data_labels is not None):
parameters.labels = [params.data_labels]
if (params.high_resolution is not None):
parameters.high_resolution = params.high_resolution
if (params.low_resolution is not None):
parameters.low_resolution = params.low_resolution
data_and_flags = mmtbx.utils.determine_data_and_flags(
reflection_file_server=reflection_file_server,
parameters=parameters,
data_description="X-ray data",
extract_r_free_flags=False, # XXX
log=StringIO())
return data_and_flags
def extract_data_and_flags(params, crystal_symmetry=None):
data_and_flags = None
if(params.reflection_file_name is not None):
reflection_file = reflection_file_reader.any_reflection_file(
file_name = params.reflection_file_name)
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = crystal_symmetry,
force_symmetry = True,
reflection_files = [reflection_file])
parameters = mmtbx.utils.data_and_flags_master_params().extract()
parameters.force_anomalous_flag_to_be_equal_to = False
if(params.data_labels is not None):
parameters.labels = [params.data_labels]
if(params.high_resolution is not None):
parameters.high_resolution = params.high_resolution
if(params.low_resolution is not None):
parameters.low_resolution = params.low_resolution
data_and_flags = mmtbx.utils.determine_data_and_flags(
reflection_file_server = reflection_file_server,
parameters = parameters,
data_description = "X-ray data",
extract_r_free_flags = False, # XXX
log = StringIO())
return data_and_flags
def _get_f_obs_r_free(self):
""" Get f_obs and r_free_flags from pdb and mtz via self.inputs
Returns
-------
f_obs: cctbx.miller.array
observed data
r_free_flags: cctbx.miller.array
r free flags
"""
rfs = reflection_file_utils.reflection_file_server(
crystal_symmetry=self.inputs.crystal_symmetry,
force_symmetry=True,
reflection_files=self.inputs.reflection_files,
err=StringIO(),
)
# Parameter object descrinign the may to process data
# From mmtbx.utils
data_flags_params = data_and_flags_master_params().extract()
data_flags_params.labels = self.params.exhaustive.options.column_type
determined_data_and_flags = mmtbx.utils.determine_data_and_flags(
reflection_file_server=rfs,
parameters=data_flags_params,
keep_going=True,
log=StringIO(),
)
# Extract the need parts of determined_data_and_flags
f_obs = determined_data_and_flags.f_obs
r_free_flags = determined_data_and_flags.r_free_flags
return f_obs, r_free_flags
test_flag_value=trial_test_flag_value)
assert (actual_test_flag_value_2 == actual_test_flag_value)
for second_label in ["test", "foo"]:
input_array = miller_set.array(data=exercise_flag_arrays[0])
mtz_dataset = input_array.as_mtz_dataset(
column_root_label="FreeRflags")
mtz_dataset.add_miller_array(
miller_array=input_array,
column_root_label=second_label)
mtz_dataset.mtz_object().write("tmp.mtz")
reflection_files = [reflection_file_reader.any_reflection_file(
file_name="tmp.mtz")]
err = StringIO()
reflection_file_srv = reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=err)
try:
reflection_file_srv.get_r_free_flags(
file_name=None,
label=None,
test_flag_value=None,
disable_suitability_test=False,
parameter_scope="r_free_flags")
except Sorry, e:
assert str(e)=="Multiple equally suitable arrays of R-free flags found."
assert err.getvalue() == """\
Multiple equally suitable arrays of R-free flags found.
def run(args, log=sys.stdout):
if len(args) == 0:
print >> log, legend
defaults(log=log)
return
#
parsed = defaults(log=log)
processed_args = mmtbx.utils.process_command_line_args(args=args, log=sys.stdout, master_params=parsed)
params = processed_args.params.extract()
reflection_files = processed_args.reflection_files
if len(reflection_files) == 0:
if params.hkl_file is None:
raise Sorry("No reflection file found.")
else:
hkl_in = file_reader.any_file(params.hkl_file, force_type="hkl")
hkl_in.assert_file_type("hkl")
reflection_files = [hkl_in.file_object]
crystal_symmetry = processed_args.crystal_symmetry
if crystal_symmetry is None:
if (params.space_group is not None) and (params.unit_cell is not None):
from cctbx import crystal
crystal_symmetry = crystal.symmetry(space_group_info=params.space_group, unit_cell=params.unit_cell)
else:
raise Sorry("No crystal symmetry found.")
if len(processed_args.pdb_file_names) == 0:
if params.pdb_file is None:
raise Sorry("No model file found.")
else:
pdb_file_names = [params.pdb_file]
else:
pdb_file_names = processed_args.pdb_file_names
#
rfs = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry, force_symmetry=True, reflection_files=reflection_files, err=StringIO()
)
parameters = mmtbx.utils.data_and_flags_master_params().extract()
parameters.labels = params.f_obs_label
parameters.r_free_flags.label = params.r_free_flags_label
determine_data_and_flags_result = mmtbx.utils.determine_data_and_flags(
reflection_file_server=rfs, parameters=parameters, keep_going=True, log=StringIO()
)
f_obs = determine_data_and_flags_result.f_obs
print "Input data:"
print " Iobs or Fobs:", f_obs.info().labels
r_free_flags = determine_data_and_flags_result.r_free_flags
print " Free-R flags:", r_free_flags.info().labels
#
parameters = mmtbx.utils.experimental_phases_params.extract()
parameters.labels = params.hendrickson_lattman_coefficients_label
experimental_phases_result = mmtbx.utils.determine_experimental_phases(
reflection_file_server=rfs,
parameters=parameters,
log=StringIO(),
parameter_scope="",
working_point_group=None,
symmetry_safety_check=True,
ignore_all_zeros=True,
)
if experimental_phases_result is not None:
print " HL coefficients:", experimental_phases_result.info().labels
experimental_phases = extract_experimental_phases(experimental_phases=experimental_phases_result, f_obs=f_obs)
#
if r_free_flags is None:
r_free_flags = f_obs.array(data=flex.bool(f_obs.data().size(), False))
#
mmtbx_pdb_file = mmtbx.utils.pdb_file(
pdb_file_names=pdb_file_names, crystal_symmetry=crystal_symmetry, log=sys.stdout
)
if len(mmtbx_pdb_file.pdb_inp.xray_structures_simple()) > 1: # XXX support multi-models
raise Sorry("Multiple model file not supported in this tool.")
# XXX Twining not supported
xray_structure = mmtbx_pdb_file.pdb_inp.xray_structure_simple(crystal_symmetry=crystal_symmetry, weak_symmetry=True)
if not xray_structure.unit_cell().is_similar_to(f_obs.unit_cell()):
raise Sorry("The unit cells in the model and reflections files are not " + "isomorphous.")
print "Input model:"
print " number of atoms:", xray_structure.scatterers().size()
fmodel = mmtbx.f_model.manager(
xray_structure=xray_structure, r_free_flags=r_free_flags, f_obs=f_obs, abcd=experimental_phases
)
fmodel.update_all_scales(
update_f_part1=True,
remove_outliers=params.remove_f_obs_outliers,
bulk_solvent_and_scaling=params.bulk_solvent_and_scaling,
)
print "Overall statistics:"
fmodel.info().show_all()
#
print "Output data:"
if params.output_file_name is not None:
output_file_name = params.output_file_name
else:
pdb_file_bn = os.path.basename(pdb_file_names[0])
hkl_file_bn = os.path.basename(reflection_files[0].file_name())
try:
pdb_file_prefix = pdb_file_bn[: pdb_file_bn.index(".")]
except ValueError:
pdb_file_prefix = pdb_file_bn
try:
hkl_file_prefix = hkl_file_bn[: hkl_file_bn.index(".")]
except ValueError:
hkl_file_prefix = hkl_file_bn
output_file_name = "%s_%s.mtz" % (pdb_file_prefix, hkl_file_prefix)
print " file name:", output_file_name
print " to see the contnt of %s:" % output_file_name
print " phenix.mtz.dump %s" % output_file_name
out = open(output_file_name, "w")
fmodel.export(out=out)
out.close()
print "All done."
return output_file_name
def run(args, command_name="phenix.remove_outliers"):
if (len(args) == 0 or "--help" in args or "--h" in args or "-h" in args):
print_help(command_name=command_name)
else:
log = multi_out()
plot_out = None
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="outlier_detection")
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if arg == "--quiet":
arg_is_processed = True
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown file or keyword:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
if not os.path.exists(params.outlier_utils.input.xray_data.file_name):
raise Sorry("File %s can not be found" %
(params.outlier_utils.input.xray_data.file_name))
if params.outlier_utils.input.model.file_name is not None:
if not os.path.exists(params.outlier_utils.input.model.file_name):
raise Sorry("File %s can not be found" %
(params.outlier_utils.input.model.file_name))
# now get the unit cell from the pdb file
hkl_xs = None
if params.outlier_utils.input.xray_data.file_name is not None:
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.outlier_utils.input.xray_data.file_name)
pdb_xs = None
if params.outlier_utils.input.model.file_name is not None:
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.outlier_utils.input.model.file_name)
phil_xs = crystal.symmetry(
unit_cell=params.outlier_utils.input.unit_cell,
space_group_info=params.outlier_utils.input.space_group)
phil_xs.show_summary()
hkl_xs.show_summary()
combined_xs = crystal.select_crystal_symmetry(None, phil_xs, [pdb_xs],
[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.outlier_utils.input.unit_cell = combined_xs.unit_cell()
params.outlier_utils.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
new_params = master_params.format(python_object=params)
new_params.show(out=log)
if params.outlier_utils.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.outlier_utils.input.space_group is None:
raise Sorry("space group not specified")
if params.outlier_utils.input.xray_data.file_name is None:
raise Sorry("Xray data not specified")
if params.outlier_utils.input.model.file_name is None:
print "PDB file not specified. Basic wilson outlier rejections only."
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
phil_xs = crystal.symmetry(
unit_cell=params.outlier_utils.input.unit_cell,
space_group_info=params.outlier_utils.input.space_group)
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=phil_xs, force_symmetry=True, reflection_files=[])
miller_array = None
miller_array = xray_data_server.get_xray_data(
file_name=params.outlier_utils.input.xray_data.file_name,
labels=params.outlier_utils.input.xray_data.obs_labels,
ignore_all_zeros=True,
parameter_scope='outlier_utils.input.xray_data',
parameter_name='obs_labels')
info = miller_array.info()
miller_array = miller_array.map_to_asu()
miller_array = miller_array.select(miller_array.indices() != (0, 0, 0))
#we have to check if the sigma's make any sense at all
if not miller_array.sigmas_are_sensible():
miller_array = miller_array.customized_copy(
data=miller_array.data(),
sigmas=None).set_observation_type(miller_array)
miller_array = miller_array.select(miller_array.data() > 0)
if miller_array.sigmas() is not None:
miller_array = miller_array.select(miller_array.sigmas() > 0)
if (miller_array.is_xray_intensity_array()):
miller_array = miller_array.f_sq_as_f()
elif (miller_array.is_complex_array()):
miller_array = abs(miller_array)
miller_array.set_info(info)
merged_anomalous = False
if miller_array.anomalous_flag():
miller_array = miller_array.average_bijvoet_mates(
).set_observation_type(miller_array)
merged_anomalous = True
miller_array = miller_array.map_to_asu()
# get the free reflections please
free_flags = None
if params.outlier_utils.input.xray_data.free_flags is None:
free_flags = miller_array.generate_r_free_flags(
fraction=params.outlier_utils.\
additional_parameters.free_flag_generation.fraction,
max_free=params.outlier_utils.\
additional_parameters.free_flag_generation.max_number,
lattice_symmetry_max_delta=params.outlier_utils.\
additional_parameters.free_flag_generation.lattice_symmetry_max_delta,
use_lattice_symmetry=params.outlier_utils.\
additional_parameters.free_flag_generation.use_lattice_symmetry
)
else:
free_flags = xray_data_server.get_xray_data(
file_name=params.outlier_utils.input.xray_data.file_name,
labels=params.outlier_utils.input.xray_data.free_flags,
ignore_all_zeros=True,
parameter_scope='outlier_utils.input.xray_data',
parameter_name='free_flags')
if free_flags.anomalous_flag():
free_flags = free_flags.average_bijvoet_mates()
merged_anomalous = True
free_flags = free_flags.customized_copy(data=flex.bool(
free_flags.data() == 1))
free_flags = free_flags.map_to_asu()
free_flags = free_flags.common_set(miller_array)
print >> log
print >> log, "Summary info of observed data"
print >> log, "============================="
miller_array.show_summary(f=log)
if merged_anomalous:
print >> log, "For outlier detection purposes, the Bijvoet pairs have been merged."
print >> log
print >> log, "Constructing an outlier manager"
print >> log, "==============================="
print >> log
outlier_manager = outlier_rejection.outlier_manager(miller_array,
free_flags,
out=log)
basic_array = None
extreme_array = None
model_based_array = None
basic_array = outlier_manager.basic_wilson_outliers(
p_basic_wilson = params.outlier_utils.outlier_detection.\
parameters.basic_wilson.level,
return_data = True)
extreme_array = outlier_manager.extreme_wilson_outliers(
p_extreme_wilson = params.outlier_utils.outlier_detection.parameters.\
extreme_wilson.level,
return_data = True)
beamstop_array = outlier_manager.beamstop_shadow_outliers(
level = params.outlier_utils.outlier_detection.parameters.\
beamstop.level,
d_min = params.outlier_utils.outlier_detection.parameters.\
beamstop.d_min,
return_data=True)
#----------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
if params.outlier_utils.input.model.file_name is not None:
model = pdb.input(file_name=params.outlier_utils.input.model.
file_name).xray_structure_simple(
crystal_symmetry=phil_xs)
print >> log, "Atomic model summary"
print >> log, "===================="
model.show_summary(f=log)
print >> log
# please make an f_model object for bulk solvent scaling etc etc
f_model_object = f_model.manager(f_obs=miller_array,
r_free_flags=free_flags,
xray_structure=model)
print >> log, "Bulk solvent scaling of the data"
print >> log, "================================"
print >> log, "Maximum likelihood bulk solvent scaling."
print >> log
f_model_object.update_all_scales(log=log, remove_outliers=False)
plot_out = StringIO()
model_based_array = outlier_manager.model_based_outliers(
f_model_object.f_model(),
level=params.outlier_utils.outlier_detection.parameters.
model_based.level,
return_data=True,
plot_out=plot_out)
#check what needs to be put out please
if params.outlier_utils.output.hklout is not None:
if params.outlier_utils.outlier_detection.protocol == "model":
if params.outlier_utils.input.model.file_name == None:
print >> log, "Model based rejections requested. No model was supplied."
print >> log, "Switching to writing out rejections based on extreme value Wilson statistics."
params.outlier_utils.outlier_detection.protocol = "extreme"
output_array = None
print >> log
if params.outlier_utils.outlier_detection.protocol == "basic":
print >> log, "Non-outliers found by the basic wilson statistics"
print >> log, "protocol will be written out."
output_array = basic_array
new_set_of_free_flags = free_flags.common_set(basic_array)
if params.outlier_utils.outlier_detection.protocol == "extreme":
print >> log, "Non-outliers found by the extreme value wilson statistics"
print >> log, "protocol will be written out."
output_array = extreme_array
new_set_of_free_flags = free_flags.common_set(extreme_array)
if params.outlier_utils.outlier_detection.protocol == "model":
print >> log, "Non-outliers found by the model based"
print >> log, "protocol will be written out to the file:"
print >> log, params.outlier_utils.output.hklout
print >> log
output_array = model_based_array
new_set_of_free_flags = free_flags.common_set(
model_based_array)
if params.outlier_utils.outlier_detection.protocol == "beamstop":
print >> log, "Outliers found for the beamstop shadow"
print >> log, "problems detection protocol will be written to the file:"
print >> log, params.outlier_utils.output.hklout
print >> log
output_array = model_based_array
new_set_of_free_flags = free_flags.common_set(
model_based_array)
mtz_dataset = output_array.as_mtz_dataset(column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array=new_set_of_free_flags,
column_root_label="Free_R_Flag")
mtz_dataset.mtz_object().write(
file_name=params.outlier_utils.output.hklout)
if (params.outlier_utils.output.logfile is not None):
final_log = StringIO()
print >> final_log, string_buffer.getvalue()
print >> final_log
if plot_out is not None:
print >> final_log, plot_out.getvalue()
outfile = open(params.outlier_utils.output.logfile, 'w')
outfile.write(final_log.getvalue())
print >> log
print >> log, "A logfile named %s was created." % (
params.outlier_utils.output.logfile)
print >> log, "This logfile contains the screen output and"
print >> log, "(possibly) some ccp4 style loggraph plots"
def run(args, out=sys.stdout):
import iotbx.phil
cmdline = iotbx.phil.process_command_line_with_files(
args=args,
master_phil_string=master_phil_str,
pdb_file_def="crystal_symmetry.symm_file",
reflection_file_def="input.data",
space_group_def="crystal_symmetry.space_group",
unit_cell_def="crystal_symmetry.unit_cell",
usage_string="""\
mmtbx.massage_data data.mtz [labels=I,SIGI] [options]
Modification of experimental data: remove anisotropy, apply B-factor, filter
negative intensities, add or remove twinning. For expert use (and extreme
cases) only.
""")
params = cmdline.work.extract()
if (params.input.data is None):
raise Sorry("No data file supplied.")
from mmtbx.scaling import massage_twin_detwin_data
from iotbx import crystal_symmetry_from_any
from iotbx import reflection_file_utils
from cctbx import crystal
crystal_symmetry = space_group = unit_cell = None
if (params.crystal_symmetry.space_group is not None):
space_group = params.crystal_symmetry.space_group
if (params.crystal_symmetry.unit_cell is not None):
unit_cell = params.crystal_symmetry.unit_cell
crystal_symmetry = None
if (params.crystal_symmetry.symm_file is not None):
crystal_symmetry = crystal_symmetry_from_any.extract_from(
file_name=params.crystal_symmetry.symm_file)
if (crystal_symmetry is None):
raise Sorry("No crystal symmetry defined in %s" %
params.crystal_symmetry.symm_file)
if (crystal_symmetry is None) and (not None in [space_group, unit_cell]):
crystal_symmetry = crystal.symmetry(space_group_info=space_group,
unit_cell=unit_cell)
hkl_in = cmdline.get_file(params.input.data)
hkl_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=[hkl_in.file_object],
err=sys.stderr)
data = hkl_server.get_xray_data(file_name=params.input.data,
labels=params.input.labels,
ignore_all_zeros=True,
parameter_scope="input",
prefer_anomalous=True,
prefer_amplitudes=False)
result = massage_twin_detwin_data.massage_data(miller_array=data,
parameters=params.options,
out=out)
if (params.output.hklout is None):
file_base = op.splitext(op.basename(params.input.data))[0]
if (params.output.hklout_type in ["Auto", "mtz"]):
params.output.hklout = file_base + ".mtz"
else:
params.output.hklout = file_base + ".sca"
result.write_data(file_name=params.output.hklout,
output_type=params.output.hklout_type,
label_extension=params.output.label_extension)
def run(args):
def vdw_radii_callback(option, opt_str, value, parser):
# create a dict from space separated string of element types and radii
radii = {}
items = value.split()
assert len(items) % 2 == 0
for i in range(int(len(items) / 2)):
radii.setdefault(items[i * 2], float(items[i * 2 + 1]))
setattr(parser.values, option.dest, radii)
command_line = (option_parser(
usage="smtbx.masks structure reflections [options]"
).enable_symmetry_comprehensive().option(
None, "--solvent_radius", action="store", type="float",
default=1.3).option(
None,
"--shrink_truncation_radius",
action="store",
type="float",
default=1.3).option(None, "--debug", action="store_true").option(
None, "--verbose", action="store_true").option(
None,
"--resolution_factor",
action="store",
type="float",
default=1 / 4).option(
None, "--grid_step", action="store",
type="float").option(
None, "--d_min", action="store",
type="float").option(
None,
"--two_theta_max",
action="store",
type="float").option(
None,
"--cb_op",
action="store",
type="string").option(
None,
"--vdw_radii",
action="callback",
callback=vdw_radii_callback,
type="string",
nargs=1).option(
None,
"--use_space_group_symmetry",
action="store_true")).process(
args=args)
structure_file = command_line.args[0]
ext = os.path.splitext(structure_file)[-1].lower()
if ext in ('.res', '.ins'):
xs = xray.structure.from_shelx(filename=structure_file)
elif ext == '.cif':
xs = xray.structure.from_cif(filename=structure_file)
else:
print "%s: unsupported structure file format {shelx|cif}" % ext
return
reflections_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=xs.crystal_symmetry(),
reflection_files=[
reflection_file_reader.any_reflection_file(command_line.args[1])
])
fo_sq = reflections_server.get_miller_arrays(None)[0]
if command_line.options.cb_op is not None:
cb_op = sgtbx.change_of_basis_op(
sgtbx.rt_mx(command_line.options.cb_op))
fo_sq = fo_sq.change_basis(cb_op).customized_copy(crystal_symmetry=xs)
print "structure file: %s" % command_line.args[0]
print "reflection file: %s" % command_line.args[1]
if command_line.options.debug:
print "debug: %s" % command_line.options.debug
print
xs.show_summary()
print
d_min = command_line.options.d_min
two_theta_max = command_line.options.two_theta_max
assert [d_min, two_theta_max].count(None) > 0
if two_theta_max is not None:
d_min = uctbx.two_theta_as_d(two_theta_max,
wavelength=0.71073,
deg=True)
exercise_masks(
xs,
fo_sq,
solvent_radius=command_line.options.solvent_radius,
shrink_truncation_radius=command_line.options.shrink_truncation_radius,
resolution_factor=command_line.options.resolution_factor,
grid_step=command_line.options.grid_step,
resolution_cutoff=d_min,
atom_radii_table=command_line.options.vdw_radii,
use_space_group_symmetry=command_line.options.use_space_group_symmetry,
debug=command_line.options.debug,
verbose=command_line.options.verbose)
print "OK"
def exercise_get_xtal_data():
crystal_symmetry = crystal.symmetry(
unit_cell=(10,11,12,85,95,100),
space_group_symbol="P 1")
miller_set = miller.build_set(
crystal_symmetry=crystal_symmetry,
anomalous_flag=False,
d_min=3)
input_arrays = [miller_set.array(
data=flex.random_double(size=miller_set.indices().size()),
sigmas=flex.random_double(size=miller_set.indices().size())/10)
.set_observation_type_xray_intensity()
for i in [0,1]]
mtz_dataset = input_arrays[0].as_mtz_dataset(column_root_label="F0")
mtz_dataset.mtz_object().write("tmp_rfu1.mtz")
reflection_files = [reflection_file_reader.any_reflection_file(
file_name="tmp_rfu1.mtz")]
reflection_file_srv = reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files)
f_obs = reflection_file_srv.get_xray_data(
file_name=None,
labels=None,
ignore_all_zeros=False,
parameter_scope="xray_data")
assert str(f_obs.info()) == "tmp_rfu1.mtz:F0,SIGF0"
mtz_dataset.add_miller_array(
miller_array=input_arrays[1], column_root_label="F1")
mtz_dataset.mtz_object().write("tmp_rfu2.mtz")
reflection_files = [reflection_file_reader.any_reflection_file(
file_name="tmp_rfu2.mtz")]
err = StringIO()
reflection_file_srv = reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=err)
try:
f_obs = reflection_file_srv.get_xray_data(
file_name=None,
labels=None,
ignore_all_zeros=True,
parameter_scope="xray_data")
except Sorry:
assert err.getvalue() == """\
Multiple equally suitable arrays of observed xray data found.
Possible choices:
tmp_rfu2.mtz:F0,SIGF0
tmp_rfu2.mtz:F1,SIGF1
Please use xray_data.labels
to specify an unambiguous substring of the target label.
"""
err = reflection_file_srv.err = StringIO()
else:
raise Exception_expected
f_obs_list = reflection_file_srv.get_xray_data(
file_name = None,
labels = None,
ignore_all_zeros=True,
parameter_scope="xray_data",
return_all_valid_arrays=True,
minimum_score=1)
assert len(f_obs_list) == 2
f_obs = reflection_file_srv.get_xray_data(
file_name=None,
labels=["F1", "SIGF1"],
ignore_all_zeros=True,
parameter_scope="xray_data")
assert str(f_obs.info()) == "tmp_rfu2.mtz:F1,SIGF1"
try:
f_obs = reflection_file_srv.get_xray_data(
file_name=None,
labels=["F1", "SIGF0"],
ignore_all_zeros=True,
parameter_scope="xray_data")
except Sorry:
assert err.getvalue() == """\
No matching array: xray_data.labels=F1 SIGF0
Possible choices:
tmp_rfu2.mtz:F0,SIGF0
tmp_rfu2.mtz:F1,SIGF1
Please use xray_data.labels
to specify an unambiguous substring of the target label.
"""
err = reflection_file_srv.err = StringIO()
else:
raise Exception_expected
assert len(reflection_file_srv.file_name_miller_arrays) == 1
f_obs = reflection_file_srv.get_xray_data(
file_name="tmp_rfu1.mtz",
labels=None,
ignore_all_zeros=True,
parameter_scope="xray_data")
assert len(reflection_file_srv.file_name_miller_arrays) == 2
assert str(f_obs.info()) == "tmp_rfu1.mtz:F0,SIGF0"
f_obs = reflection_file_srv.get_xray_data(
file_name=os.path.abspath("tmp_rfu1.mtz"),
labels=["sigf0"],
ignore_all_zeros=True,
parameter_scope="xray_data")
assert len(reflection_file_srv.file_name_miller_arrays) == 2
assert str(f_obs.info()) == "tmp_rfu1.mtz:F0,SIGF0"
try:
f_obs = reflection_file_srv.get_xray_data(
file_name=None,
labels=None,
ignore_all_zeros=True,
parameter_scope="xray_data")
except Sorry:
assert err.getvalue() == """\
Multiple equally suitable arrays of observed xray data found.
Possible choices:
tmp_rfu2.mtz:F0,SIGF0
tmp_rfu2.mtz:F1,SIGF1
Please use xray_data.labels
to specify an unambiguous substring of the target label.
"""
err = reflection_file_srv.err = StringIO()
else:
raise Exception_expected
# test preference for anomalous (or merged) data
miller_set = miller.build_set(
crystal_symmetry=crystal_symmetry,
anomalous_flag=True,
d_min=3)
i_obs = miller_set.array(
data=flex.random_double(size=miller_set.indices().size()),
sigmas=flex.random_double(size=miller_set.indices().size())/10
).set_observation_type_xray_intensity()
i_mean = i_obs.average_bijvoet_mates()
mtz_data = i_obs.as_mtz_dataset(column_root_label="I")
mtz_data.add_miller_array(i_mean, column_root_label="I")
mtz_data.mtz_object().write("tmp_rfu3.mtz")
reflection_files = [reflection_file_reader.any_reflection_file(
file_name="tmp_rfu3.mtz")]
reflection_file_srv = reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files)
err = reflection_file_srv.err = StringIO()
try :
i_obs = reflection_file_srv.get_xray_data(
file_name=None,
labels=None,
ignore_all_zeros=False,
parameter_scope="xray_data")
except Sorry :
pass
i_obs = reflection_file_srv.get_xray_data(
file_name=None,
labels=None,
ignore_all_zeros=False,
parameter_scope="xray_data",
prefer_anomalous=True)
assert (i_obs.info().label_string() == "I(+),SIGI(+),I(-),SIGI(-)")
i_obs = reflection_file_srv.get_xray_data(
file_name=None,
labels=None,
ignore_all_zeros=False,
parameter_scope="xray_data",
prefer_anomalous=False)
assert (i_obs.info().label_string() == "I,SIGI")
def __init__(self,
args,
master_phil,
out=sys.stdout,
process_pdb_file=True,
require_data=True,
create_fmodel=True,
prefer_anomalous=None,
force_non_anomalous=False,
set_wavelength_from_model_header=False,
set_inelastic_form_factors=None,
usage_string=None,
create_log_buffer=False,
remove_unknown_scatterers=False,
generate_input_phil=False):
import mmtbx.monomer_library.pdb_interpretation
import mmtbx.monomer_library.server
import mmtbx.utils
import mmtbx.model
from iotbx import crystal_symmetry_from_any
import iotbx.phil
if generate_input_phil:
from six import string_types
assert isinstance(master_phil, string_types)
master_phil = generate_master_phil_with_inputs(
phil_string=master_phil)
if isinstance(master_phil, str):
master_phil = iotbx.phil.parse(master_phil)
if (usage_string is not None):
if (len(args) == 0) or ("--help" in args):
raise Usage("""%s\n\nFull parameters:\n%s""" %
(usage_string, master_phil.as_str(prefix=" ")))
if (force_non_anomalous):
assert (not prefer_anomalous)
assert (set_inelastic_form_factors in [None, "sasaki", "henke"])
self.args = args
self.master_phil = master_phil
self.processed_pdb_file = self.pdb_inp = None
self.pdb_hierarchy = self.xray_structure = None
self.geometry = None
self.sequence = None
self.fmodel = None
self.f_obs = None
self.r_free_flags = None
self.intensity_flag = None
self.raw_data = None
self.raw_flags = None
self.test_flag_value = None
self.miller_arrays = None
self.hl_coeffs = None
self.cif_objects = []
self.log = out
if ("--quiet" in args) or ("quiet=True" in args):
self.log = null_out()
elif create_log_buffer:
self.log = multi_out()
self.log.register(label="stdout", file_object=out)
self.log.register(label="log_buffer", file_object=StringIO())
make_header("Collecting inputs", out=self.log)
cmdline = iotbx.phil.process_command_line_with_files(
args=args,
master_phil=master_phil,
pdb_file_def="input.pdb.file_name",
reflection_file_def="input.xray_data.file_name",
cif_file_def="input.monomers.file_name",
seq_file_def="input.sequence")
self.working_phil = cmdline.work
params = self.working_phil.extract()
if len(params.input.pdb.file_name) == 0:
raise Sorry("At least one PDB file is required as input.")
self.cif_file_names = params.input.monomers.file_name
self.pdb_file_names = params.input.pdb.file_name
# SYMMETRY HANDLING - PDB FILES
self.crystal_symmetry = pdb_symm = None
for pdb_file_name in params.input.pdb.file_name:
pdb_symm = crystal_symmetry_from_any.extract_from(pdb_file_name)
if (pdb_symm is not None):
break
# DATA INPUT
data_and_flags = hkl_symm = hkl_in = None
if (params.input.xray_data.file_name is None):
if (require_data):
raise Sorry(
"At least one reflections file is required as input.")
else:
# FIXME this may still require that the data file has full crystal
# symmetry defined (although for MTZ input this will not be a problem)
make_sub_header("Processing X-ray data", out=self.log)
hkl_in = file_reader.any_file(params.input.xray_data.file_name)
hkl_in.check_file_type("hkl")
hkl_server = hkl_in.file_server
symm = hkl_server.miller_arrays[0].crystal_symmetry()
if ((symm is None) or (symm.space_group() is None)
or (symm.unit_cell() is None)):
if (pdb_symm is not None):
from iotbx.reflection_file_utils import reflection_file_server
print(
"No symmetry in X-ray data file - using PDB symmetry:",
file=self.log)
pdb_symm.show_summary(f=out, prefix=" ")
hkl_server = reflection_file_server(
crystal_symmetry=pdb_symm,
reflection_files=[hkl_in.file_object])
else:
raise Sorry(
"No crystal symmetry information found in input files."
)
if (hkl_server is None):
hkl_server = hkl_in.file_server
data_and_flags = mmtbx.utils.determine_data_and_flags(
reflection_file_server=hkl_server,
parameters=params.input.xray_data,
data_parameter_scope="input.xray_data",
flags_parameter_scope="input.xray_data.r_free_flags",
prefer_anomalous=prefer_anomalous,
force_non_anomalous=force_non_anomalous,
log=self.log)
self.intensity_flag = data_and_flags.intensity_flag
self.raw_data = data_and_flags.raw_data
self.raw_flags = data_and_flags.raw_flags
self.test_flag_value = data_and_flags.test_flag_value
self.f_obs = data_and_flags.f_obs
self.r_free_flags = data_and_flags.r_free_flags
self.miller_arrays = hkl_in.file_server.miller_arrays
hkl_symm = self.raw_data.crystal_symmetry()
if len(self.cif_file_names) > 0:
for file_name in self.cif_file_names:
cif_obj = mmtbx.monomer_library.server.read_cif(
file_name=file_name)
self.cif_objects.append((file_name, cif_obj))
# SYMMETRY HANDLING - COMBINED
if (hkl_symm is not None):
use_symmetry = hkl_symm
# check for weird crystal symmetry
# modified from mmtbx.command_line.secondary_structure_restraints
# plan to centralize functionality in another location
# -------------------------------------------------------------------------
cs = pdb_symm
corrupted_cs = False
if cs is not None:
if [cs.unit_cell(), cs.space_group()].count(None) > 0:
corrupted_cs = True
cs = None
elif cs.unit_cell().volume() < 10:
corrupted_cs = True
cs = None
if cs is None:
if corrupted_cs:
print("Symmetry information is corrupted,", end=' ', file=out)
else:
print("Symmetry information was not found,", end=' ', file=out)
if (hkl_symm is not None):
print("using symmetry from data.", file=out)
cs = hkl_symm
else:
print("putting molecule in P1 box.", file=out)
pdb_combined = iotbx.pdb.combine_unique_pdb_files(
file_names=self.pdb_file_names)
pdb_structure = iotbx.pdb.input(source_info=None,
lines=flex.std_string(
pdb_combined.raw_records))
atoms = pdb_structure.atoms()
box = uctbx.non_crystallographic_unit_cell_with_the_sites_in_its_center(
sites_cart=atoms.extract_xyz(), buffer_layer=3)
atoms.set_xyz(new_xyz=box.sites_cart)
cs = box.crystal_symmetry()
pdb_symm = cs
# -------------------------------------------------------------------------
from iotbx.symmetry import combine_model_and_data_symmetry
self.crystal_symmetry = combine_model_and_data_symmetry(
model_symmetry=pdb_symm, data_symmetry=hkl_symm)
if (self.crystal_symmetry is not None) and (self.f_obs is not None):
self.f_obs = self.f_obs.customized_copy(
crystal_symmetry=self.crystal_symmetry).eliminate_sys_absent(
).set_info(self.f_obs.info())
self.r_free_flags = self.r_free_flags.customized_copy(
crystal_symmetry=self.crystal_symmetry).eliminate_sys_absent(
).set_info(self.r_free_flags.info())
# EXPERIMENTAL PHASES
target_name = "ml"
if hasattr(params.input, "experimental_phases"):
flag = params.input.use_experimental_phases
if (flag in [True, Auto]):
phases_file = params.input.experimental_phases.file_name
if (phases_file is None):
phases_file = params.input.xray_data.file_name
phases_in = hkl_in
else:
phases_in = file_reader.any_file(phases_file)
phases_in.check_file_type("hkl")
phases_in.file_server.err = self.log # redirect error output
space_group = self.crystal_symmetry.space_group()
point_group = space_group.build_derived_point_group()
hl_coeffs = mmtbx.utils.determine_experimental_phases(
reflection_file_server=phases_in.file_server,
parameters=params.input.experimental_phases,
log=self.log,
parameter_scope="input.experimental_phases",
working_point_group=point_group,
symmetry_safety_check=True)
if (hl_coeffs is not None):
hl_coeffs = hl_coeffs.map_to_asu()
if hl_coeffs.anomalous_flag():
if (not self.f_obs.anomalous_flag()):
hl_coeffs = hl_coeffs.average_bijvoet_mates()
elif self.f_obs.anomalous_flag():
hl_coeffs = hl_coeffs.generate_bijvoet_mates()
self.hl_coeffs = hl_coeffs.matching_set(
other=self.f_obs, data_substitute=(0, 0, 0, 0))
target_name = "mlhl"
# PDB INPUT
self.unknown_residues_flag = False
self.unknown_residues_error_message = False
pdb_combined = mmtbx.utils.combine_unique_pdb_files(
file_names=params.input.pdb.file_name, )
pdb_combined.report_non_unique(out=self.log)
pdb_raw_records = pdb_combined.raw_records
try:
self.pdb_inp = iotbx.pdb.input(
source_info=None, lines=flex.std_string(pdb_raw_records))
except ValueError as e:
raise Sorry("Model format (PDB or mmCIF) error:\n%s" % str(e))
if (remove_unknown_scatterers):
h = self.pdb_inp.construct_hierarchy()
known_sel = h.atom_selection_cache().selection("not element X")
if known_sel.count(False) > 0:
self.pdb_inp = iotbx.pdb.input(
source_info=None,
lines=h.select(known_sel).as_pdb_string())
model_params = mmtbx.model.manager.get_default_pdb_interpretation_params(
)
pdb_interp_params = getattr(params, "pdb_interpretation", None)
if pdb_interp_params is None:
pdb_interp_params = iotbx.phil.parse(
input_string=mmtbx.monomer_library.pdb_interpretation.
grand_master_phil_str,
process_includes=True).extract()
pdb_interp_params = pdb_interp_params.pdb_interpretation
model_params.pdb_interpretation = pdb_interp_params
stop_for_unknowns = getattr(pdb_interp_params, "stop_for_unknowns",
False) or remove_unknown_scatterers
if not process_pdb_file:
stop_for_unknowns = True and not remove_unknown_scatterers
self.model = mmtbx.model.manager(
model_input=self.pdb_inp,
crystal_symmetry=self.crystal_symmetry,
restraint_objects=self.cif_objects,
pdb_interpretation_params=model_params,
process_input=False,
stop_for_unknowns=stop_for_unknowns,
log=self.log)
if process_pdb_file:
make_sub_header("Processing PDB file(s)", out=self.log)
self.model.process_input_model(make_restraints=True)
full_grm = self.model.get_restraints_manager()
self.geometry = full_grm.geometry
self.processed_pdb_file = self.model._processed_pdb_file # to remove later XXX
self.xray_structure = self.model.get_xray_structure()
self.pdb_hierarchy = self.model.get_hierarchy()
self.pdb_hierarchy.atoms().reset_i_seq()
# wavelength
if (params.input.energy is not None):
if (params.input.wavelength is not None):
raise Sorry("Both wavelength and energy have been specified!")
params.input.wavelength = 12398.424468024265 / params.input.energy
if (set_wavelength_from_model_header
and params.input.wavelength is None):
wavelength = self.pdb_inp.extract_wavelength()
if (wavelength is not None):
print("", file=self.log)
print("Using wavelength = %g from PDB header" % wavelength,
file=self.log)
params.input.wavelength = wavelength
# set scattering table
if (data_and_flags is not None):
self.model.setup_scattering_dictionaries(
scattering_table=params.input.scattering_table,
d_min=self.f_obs.d_min(),
log=self.log,
set_inelastic_form_factors=set_inelastic_form_factors,
iff_wavelength=params.input.wavelength)
self.xray_structure.show_summary(f=self.log)
# FMODEL SETUP
if (create_fmodel) and (data_and_flags is not None):
make_sub_header("F(model) initialization", out=self.log)
skip_twin_detection = getattr(params.input, "skip_twin_detection",
True)
twin_law = getattr(params.input, "twin_law", None)
if (twin_law is Auto):
if (self.hl_coeffs is not None):
raise Sorry(
"Automatic twin law determination not supported when "
+ "experimental phases are used.")
elif (not skip_twin_detection):
twin_law = Auto
if (twin_law is Auto):
print("Twinning will be detected automatically.",
file=self.log)
self.fmodel = mmtbx.utils.fmodel_simple(
xray_structures=[self.xray_structure],
scattering_table=params.input.scattering_table,
f_obs=self.f_obs,
r_free_flags=self.r_free_flags,
skip_twin_detection=skip_twin_detection,
target_name=target_name,
log=self.log)
else:
if ((twin_law is not None) and (self.hl_coeffs is not None)):
raise Sorry(
"Automatic twin law determination not supported when "
+ "experimental phases are used.")
self.fmodel = mmtbx.utils.fmodel_manager(
f_obs=self.f_obs,
xray_structure=self.xray_structure,
r_free_flags=self.r_free_flags,
twin_law=params.input.twin_law,
hl_coeff=self.hl_coeffs,
target_name=target_name)
self.fmodel.update_all_scales(params=None,
log=self.log,
optimize_mask=True,
show=True)
self.fmodel.info().show_rfactors_targets_scales_overall(
out=self.log)
# SEQUENCE
if (params.input.sequence is not None):
seq_file = file_reader.any_file(params.input.sequence,
force_type="seq",
raise_sorry_if_errors=True)
self.sequence = seq_file.file_object
# UNMERGED DATA
self.unmerged_i_obs = None
if hasattr(params.input, "unmerged_data"):
if (params.input.unmerged_data.file_name is not None):
self.unmerged_i_obs = load_and_validate_unmerged_data(
f_obs=self.f_obs,
file_name=params.input.unmerged_data.file_name,
data_labels=params.input.unmerged_data.labels,
log=self.log)
self.params = params
print("", file=self.log)
print("End of input processing", file=self.log)
def run(args):
if len(args)==0:
master_params.show(expert_level=0)
elif ( "--help" in args ):
print "no help available"
elif ( "--h" in args ):
print "no help available"
elif ( "--show_defaults" in args ):
master_params.show(expert_level=0)
elif ( "--show_defaults_all" in args ):
master_params.show(expert_level=10)
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
log_plots = StringIO()
print >> log,"#phil __OFF__"
print >> log
print >> log, date_and_time()
print >> log
print >> log
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="scaling")
reflection_file = None
for arg in args:
command_line_params = None
arg_is_processed = False
if arg == '--quiet':
arg_is_processed = True
## The associated action with this keyword is implemented above
if (os.path.isfile(arg)): ## is this a file name?
## Check if this is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
except KeyboardInterrupt: raise
except Exception : pass
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
## Check if this file is a reflection file
if command_line_params is None:
reflection_file = reflection_file_reader.any_reflection_file(
file_name=arg, ensure_read_access=False)
if (reflection_file is not None):
reflection_file = arg
arg_is_processed = True
## If it is not a file, it must be a phil command
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown phil-file or phil-command:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file format or phil command: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
## Now please read in the reflections files
## get symmetry and cell data first please
## By default, the native cell and symmetry are used
## as reference
crystal_symmetry_nat = None
crystal_symmetry_nat = crystal_symmetry_from_any.extract_from(
file_name=params.scaling.input.xray_data.after_burn.file_name)
if params.scaling.input.xray_data.space_group is None:
params.scaling.input.xray_data.space_group =\
crystal_symmetry_nat.space_group_info()
print >> log, "Using symmetry of after_burn data"
if params.scaling.input.xray_data.unit_cell is None:
params.scaling.input.xray_data.unit_cell =\
crystal_symmetry_nat.unit_cell()
print >> log, "Using cell of after_burn data"
## Check if a unit cell is defined
if params.scaling.input.xray_data.space_group is None:
raise Sorry("No space group defined")
if params.scaling.input.xray_data.unit_cell is None:
raise Sorry("No unit cell defined")
crystal_symmetry = crystal_symmetry = crystal.symmetry(
unit_cell = params.scaling.input.xray_data.unit_cell,
space_group_symbol = str(
params.scaling.input.xray_data.space_group) )
effective_params = master_params.fetch(sources=phil_objects)
new_params = master_params.format(python_object=params)
print >> log, "Effective parameters"
print >> log, "#phil __ON__"
new_params.show(out=log,
expert_level=params.scaling.input.expert_level )
print >> log, "#phil __END__"
print >> log
## define a xray data server
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = crystal_symmetry,
force_symmetry = True,
reflection_files=[])
## Read in native data and make appropriatre selections
miller_array_native = None
miller_array_native = xray_data_server.get_xray_data(
file_name = params.scaling.input.xray_data.after_burn.file_name,
labels = params.scaling.input.xray_data.after_burn.labels,
ignore_all_zeros = True,
parameter_scope = 'scaling.input.SIR_scale.xray_data.after_burn'
)
info_native = miller_array_native.info()
miller_array_native=miller_array_native.map_to_asu().select(
miller_array_native.indices()!=(0,0,0) )
miller_array_native = miller_array_native.select(
miller_array_native.data() > 0 )
## Convert to amplitudes
if (miller_array_native.is_xray_intensity_array()):
miller_array_native = miller_array_native.f_sq_as_f()
elif (miller_array_native.is_complex_array()):
miller_array_native = abs(miller_array_native)
if not miller_array_native.is_real_array():
raise Sorry("miller_array_native is not a real array")
miller_array_native.set_info(info = info_native)
## Read in derivative data and make appropriate selections
miller_array_derivative = None
miller_array_derivative = xray_data_server.get_xray_data(
file_name = params.scaling.input.xray_data.before_burn.file_name,
labels = params.scaling.input.xray_data.before_burn.labels,
ignore_all_zeros = True,
parameter_scope = 'scaling.input.SIR_scale.xray_data.before_burn'
)
info_derivative = miller_array_derivative.info()
miller_array_derivative=miller_array_derivative.map_to_asu().select(
miller_array_derivative.indices()!=(0,0,0) )
miller_array_derivative = miller_array_derivative.select(
miller_array_derivative.data() > 0 )
## Convert to amplitudes
if (miller_array_derivative.is_xray_intensity_array()):
miller_array_derivative = miller_array_derivative.f_sq_as_f()
elif (miller_array_derivative.is_complex_array()):
miller_array_derivative = abs(miller_array_derivative)
if not miller_array_derivative.is_real_array():
raise Sorry("miller_array_derivative is not a real array")
miller_array_derivative.set_info(info = info_derivative)
## As this is a SIR case, we will remove any anomalous pairs
if miller_array_derivative.anomalous_flag():
miller_array_derivative = miller_array_derivative.average_bijvoet_mates()\
.set_observation_type( miller_array_derivative )
if miller_array_native.anomalous_flag():
miller_array_native = miller_array_native.average_bijvoet_mates()\
.set_observation_type( miller_array_native )
## Print info
print >> log
print >> log, "Native data"
print >> log, "==========="
miller_array_native.show_comprehensive_summary(f=log)
print >> log
native_pre_scale = pre_scale.pre_scaler(
miller_array_native,
params.scaling.input.scaling_strategy.pre_scaler_protocol,
params.scaling.input.basic)
miller_array_native = native_pre_scale.x1.deep_copy()
del native_pre_scale
print >> log
print >> log, "Derivative data"
print >> log, "==============="
miller_array_derivative.show_comprehensive_summary(f=log)
print >> log
derivative_pre_scale = pre_scale.pre_scaler(
miller_array_derivative,
params.scaling.input.scaling_strategy.pre_scaler_protocol,
params.scaling.input.basic)
miller_array_derivative = derivative_pre_scale.x1.deep_copy()
del derivative_pre_scale
scaler = fa_estimation.combined_scaling(
miller_array_native,
miller_array_derivative,
params.scaling.input.scaling_strategy.iso_protocol)
miller_array_native = scaler.x1.deep_copy()
miller_array_derivative = scaler.x2.deep_copy()
del scaler
print >> log
print >> log, "Making delta f's"
print >> log, "----------------"
print >> log
delta_gen = pair_analyses.delta_generator( miller_array_native,
miller_array_derivative,
params.scaling.input.scaling_strategy.iso_protocol.nsr_bias )
print >> log
print >> log, "writing mtz file"
print >> log, "----------------"
print >> log
## some assertions to make sure nothing went weerd
assert miller_array_native.observation_type() is not None
assert miller_array_derivative.observation_type() is not None
assert delta_gen.abs_delta_f.observation_type() is not None
## Please write out the abs_delta_f array
mtz_dataset = delta_gen.abs_delta_f.as_mtz_dataset(
column_root_label='F'+params.scaling.input.output.outlabel)
mtz_dataset.mtz_object().write(
file_name=params.scaling.input.output.hklout)
def simul_utils(args):
if len(args)==0:
print_help()
elif ( "--help" in args ):
print_help()
elif ( "--h" in args ):
print_help()
elif ("-h" in args ):
print_help()
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="map_coefs")
print >> log, "#phil __OFF__"
print >> log, "======================"
print >> log, " SIMUL "
print >> log, "A data simulation tool"
print >> log, "======================"
print >> log
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown file or keyword:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
"""
new_params = master_params.format(python_object=params)
new_params.show(out=log)
"""
# now get the unit cell from the pdb file
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.simul_utils.input.xray_data.file_name)
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.simul_utils.input.model.file_name)
phil_xs = crystal.symmetry(
unit_cell=params.simul_utils.input.unit_cell,
space_group_info=params.simul_utils.input.space_group )
combined_xs = select_crystal_symmetry(
None,phil_xs, [pdb_xs],[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.simul_utils.input.unit_cell = combined_xs.unit_cell()
params.simul_utils.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
print >> log, "#phil __ON__"
new_params = master_params.format(python_object=params)
new_params.show(out=log)
print >> log, "#phil __END__"
if params.simul_utils.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.simul_utils.input.space_group is None:
raise Sorry("space group not specified")
if params.simul_utils.input.xray_data.file_name is None:
raise Sorry("Xray data not specified")
if params.simul_utils.input.model.file_name is None:
raise Sorry("pdb file with model not specified")
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
phil_xs = crystal.symmetry(
unit_cell=params.simul_utils.input.unit_cell,
space_group_info=params.simul_utils.input.space_group )
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = phil_xs,
force_symmetry = True,
reflection_files=[])
miller_array = None
miller_array = xray_data_server.get_xray_data(
file_name = params.simul_utils.input.xray_data.file_name,
labels = params.simul_utils.input.xray_data.labels,
ignore_all_zeros = True,
parameter_scope = 'simul_utils.input.xray_data',
parameter_name = 'labels'
)
info = miller_array.info()
miller_array = miller_array.map_to_asu()
miller_array = miller_array.select(
miller_array.indices() != (0,0,0))
miller_array = miller_array.select(
miller_array.data() > 0 )
if miller_array.sigmas() is not None:
miller_array = miller_array.select(
miller_array.sigmas() > 0 )
if (miller_array.is_xray_intensity_array()):
miller_array = miller_array.f_sq_as_f()
elif (miller_array.is_complex_array()):
miller_array = abs(miller_array)
miller_array.set_info(info)
print >> log
print >> log, "Summary info of observed data"
print >> log, "============================="
miller_array.show_summary(f=log)
print >> log
free_flags = miller_array.generate_r_free_flags()
#--------------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
model = pdb.input(file_name=params.simul_utils.input.model.file_name).xray_structure_simple(
crystal_symmetry=phil_xs,
)
print >> log, "Atomic model summary"
print >> log, "===================="
model.show_summary()
print >> log
#-------------------------------------------------------------------
# Step 3: make an F_model object to get model phases and amplitudes
#
print >> log, "Performing bulk solvent scaling"
print >> log, "==============================="
print >> log
print >> log
f_model_object = f_model.manager(
f_obs = miller_array,
r_free_flags = free_flags,
xray_structure = model )
f_model_object.update_solvent_and_scale(out=log)
fmodel = abs( f_model_object.f_model() ).set_observation_type( miller_array )
mockfmodel = None
if params.simul_utils.input.mock_model.file_name is not None:
print >> log, "Reading in mock model"
print >> log, "====================="
print >> log
print >> log
mock_model = pdb.input(file_name=params.simul_utils.input.mock_model.file_name).xray_structure_simple(
crystal_symmetry=phil_xs)
mock_f_model = f_model.manager(
f_obs = miller_array,
r_free_flags = free_flags,
xray_structure = mock_model )
mock_f_model.update(
k_sol = f_model_object.k_sol() ,
b_sol = f_model_object.b_sol() ,
b_cart = f_model_object.b_cart()
)
mockfmodel = abs( mock_f_model.f_model() ).set_observation_type( miller_array )
else:
mockfmodel = fmodel.deep_copy()
print >> log, "Making new data"
print >> log, "==============="
print >> log
print >> log
new_data_builder = error_swap( miller_array,
fmodel,
mockfmodel )
new_data = new_data_builder.new_obs
# we now have to write the data actually
print >> log, "Writing new data set"
print >> log, "===================="
mtz_dataset = new_data.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset.mtz_object().write(
file_name=params.simul_utils.output.hklout)
def run(args, command_name="phenix.xmanip"):
if (len(args) == 0 or "--help" in args or "--h" in args or "-h" in args):
print_help(command_name=command_name)
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="map_coefs")
print >> log, "#phil __OFF__"
print >> log, "=========================="
print >> log, " XMANIP "
print >> log, "reindexing and other tasks"
print >> log, "=========================="
print >> log
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown file or keyword:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
# now get the unit cell from the files
hkl_xs = []
pdb_xs = None
#multiple file names are allowed
for xray_data in params.xmanip.input.xray_data:
if xray_data.file_name is not None:
hkl_xs.append( crystal_symmetry_from_any.extract_from(
file_name=xray_data.file_name) )
if params.xmanip.input.model.file_name is not None:
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.xmanip.input.model.file_name)
phil_xs = crystal.symmetry(
unit_cell=params.xmanip.input.unit_cell,
space_group_info=params.xmanip.input.space_group )
combined_xs = crystal.select_crystal_symmetry(
None,phil_xs, [pdb_xs],hkl_xs)
if combined_xs is not None:
# inject the unit cell and symmetry in the phil scope please
params.xmanip.input.unit_cell = combined_xs.unit_cell()
params.xmanip.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
print >> log, "#phil __ON__"
new_params = master_params.format(python_object=params)
new_params.show(out=log)
print >> log, "#phil __END__"
if params.xmanip.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.xmanip.input.space_group is None:
raise Sorry("space group not specified")
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
miller_arrays = []
labels = []
label_appendix = []
write_it = []
names = {}
if len(params.xmanip.input.xray_data)>0:
phil_xs = crystal.symmetry(
unit_cell=params.xmanip.input.unit_cell,
space_group_info=params.xmanip.input.space_group )
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = phil_xs,
force_symmetry = True,
reflection_files=[])
count=0
for xray_data in params.xmanip.input.xray_data:
if xray_data.file_name is not None:
miller_array = None
miller_array = read_data(xray_data.file_name,
xray_data.labels,
phil_xs)
print >> log
print >> log, "Summary info of observed data"
print >> log, "============================="
if miller_array is None:
raise Sorry("Failed to read data. see errors above" )
miller_array.show_summary(f=log)
print >> log
miller_arrays.append( miller_array )
labels.append( miller_array.info().labels )
label_appendix.append( xray_data.label_appendix )
this_name = "COL_"+str(count)
if xray_data.name is not None:
this_name = xray_data.name
#check if this name is allready used
if names.has_key( this_name ):
raise Sorry( "Non unique dataset name. Please change the input script" )
names.update( {this_name:count} )
count += 1
write_it.append( xray_data.write_out)
output_label_root = construct_output_labels( labels, label_appendix )
for ii in range(len(labels)):
test=0
for jj in range( ii+1,len(labels) ):
for lab_name1, lab_name2 in zip(labels[ii],labels[jj]):
if lab_name1==lab_name2:
test+=1
if test == 2:
print >> log, "\n***** You are trying to import the data with label(s) %s more then one time. ***** \n"%(str(labels[ii]))
for ii in range(len(output_label_root)):
for jj in range(ii+1,len(output_label_root)):
if output_label_root[ii]==output_label_root[jj]:
if write_it[ii]:
if write_it[jj]:
print >> log, "Output label roots:"
print >> log, output_label_root
raise Sorry( "Output labels are not unique. Modify input." )
#----------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
pdb_model = None
model = None
if params.xmanip.input.model.file_name is not None:
pdb_model = iotbx.pdb.input(
file_name=params.xmanip.input.model.file_name)
model = pdb_model.xray_structure_simple(crystal_symmetry=phil_xs)
print >> log, "Atomic model summary"
print >> log, "===================="
model.show_summary(f=log)
print >> log
write_miller_array = False
write_pdb_file = False
# define some output holder thingamebobs
new_miller_arrays = []
new_model = None
#manipulate miller arrays
if params.xmanip.parameters.action == "manipulate_miller":
write_miller_array = True
new_miller = xmanip_tasks.manipulate_miller(names,
miller_arrays,
model,
params.xmanip.parameters.manipulate_miller,
log )
miller_arrays.append( new_miller )
# not very smart to rely here on a phil defintion defined in another file
tmp_root = params.xmanip.parameters.manipulate_miller.output_label_root
if tmp_root is None:
tmp_root = "UNSPECIFIED"
output_label_root.append( tmp_root )
write_it.append(True)
if params.xmanip.parameters.action=="reindex":
write_miller_array = True
#----------------------------------------------------------------
# step 3: get the reindex laws
phil_xs.show_summary()
to_niggli = phil_xs.change_of_basis_op_to_niggli_cell()
to_reference = phil_xs.change_of_basis_op_to_reference_setting()
to_inverse = phil_xs.change_of_basis_op_to_inverse_hand()
to_primitive = phil_xs.change_of_basis_op_to_primitive_setting()
cb_op = None
if (params.xmanip.parameters.reindex.standard_laws == "niggli"):
cb_op = to_niggli
if (params.xmanip.parameters.reindex.standard_laws == "reference_setting"):
cb_op = to_reference
if (params.xmanip.parameters.reindex.standard_laws == "invert"):
cb_op = to_inverse
if (params.xmanip.parameters.reindex.standard_laws == "user_supplied"):
cb_op = sgtbx.change_of_basis_op( params.xmanip.parameters.reindex.user_supplied_law )
if (params.xmanip.parameters.reindex.standard_laws == "primitive_setting"):
cb_op = to_primitive
if cb_op is None:
raise Sorry("No change of basis operation is supplied.")
print >> log, "Supplied reindexing law:"
print >> log, "========================"
print >> log, "hkl notation: ", cb_op.as_hkl()
print >> log, "xyz notation: ", cb_op.as_xyz()
print >> log, "abc notation: ", cb_op.as_abc()
#----------------------------------------------------------------
# step 4: do the reindexing
#
# step 4a: first do the miller array object
#new_miller_arrays = []
for miller_array in miller_arrays:
new_miller_array = None
if miller_array is not None:
new_miller_array = miller_array.change_basis( cb_op )
new_miller_arrays.append( new_miller_array )
#
# step 4b: the xray structure
if pdb_model is not None:
write_pdb_file=True
new_model = model.change_basis( cb_op )
if write_miller_array:
if len(new_miller_arrays)==0:
new_miller_arrays = miller_arrays
#----------------------------------------------------------------
print >> log
print >> log, "The data has been reindexed/manipulated"
print >> log, "--------------------------------------"
print >> log
print >> log, "Writing output files...."
mtz_dataset=None
if len(new_miller_arrays)>0:
first=0
for item in range(len(write_it)):
if write_it[item]:
first=item
if new_miller_arrays[ first ] is not None:
break
if new_miller_arrays[first] is not None:
tmp = new_miller_arrays[first].map_to_asu()
mtz_dataset = tmp.as_mtz_dataset(
column_root_label=output_label_root[first])
if mtz_dataset is not None:
for miller_array, new_root in zip(new_miller_arrays[first+1:],
output_label_root[first+1:]):
if miller_array is not None:
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = miller_array,
column_root_label = new_root)
print >> log, "Writing mtz file with name %s"%(params.xmanip.output.hklout)
mtz_dataset.mtz_object().write(
file_name=params.xmanip.output.hklout)
#step 5b: write the new pdb file
if new_model is not None:
pdb_file = open( params.xmanip.output.xyzout, 'w')
print >> log, "Wring pdb file to: %s"%(params.xmanip.output.xyzout)
write_as_pdb_file(
input_pdb = pdb_model,
input_xray_structure = new_model,
out = pdb_file,
chain_id_increment= 0,
additional_remark = "Generated by %s" % command_name)
pdb_file.close()
if ( [miller_array,new_model]).count(None)==2:
print >>log, "No input reflection of coordinate files have been given"
if params.xmanip.parameters.action=="manipulate_pdb":
if params.xmanip.parameters.manipulate_pdb.task == "apply_operator":
rt_mx = None
if params.xmanip.parameters.manipulate_pdb.apply_operator.standard_operators == "user_supplied_operator":
rt_mx = sgtbx.rt_mx(
params.xmanip.parameters.manipulate_pdb.apply_operator.user_supplied_operator,t_den=12*8 )
print >> log, "Applied operator : ", rt_mx.as_xyz()
if params.xmanip.parameters.manipulate_pdb.apply_operator.standard_operators == \
"user_supplied_cartesian_rotation_matrix":
rt = params.xmanip.parameters.manipulate_pdb.apply_operator.user_supplied_cartesian_rotation_matrix
tmp_r=None
tmp_t=None
if "," in rt.r:
tmp_r = rt.r.split(',')
else:
tmp_r = rt.r.split(' ')
if "," in rt.r:
tmp_t = rt.t.split(',')
else:
tmp_t = rt.t.split(' ')
tmp_tmp_r=[]
tmp_tmp_t=[]
for item in tmp_r:
tmp_tmp_r.append( float(item) )
if len(tmp_tmp_r)!=9:
raise Sorry("Invalid rotation matrix. Please check input: %s"%(rt.r) )
for item in tmp_t:
tmp_tmp_t.append( float(item) )
if len(tmp_tmp_t)!=3:
raise Sorry("Invalid translational vector. Please check input: %s"%(rt.t) )
tmp_tmp_t = (tmp_tmp_t)
rt_mx = quick_rt_mx(tmp_tmp_r, tmp_tmp_t)
print >> log, "User supplied cartesian matrix and vector: "
rt_mx.show()
o = matrix.sqr(model.unit_cell().orthogonalization_matrix())
tmp_r = o.inverse()*rt_mx.r()*o
tmp_t = o.inverse()*matrix.col(list(rt_mx.t()))
print >> log
print >> log, "Operator in fractional coordinates: "
rt_mx = quick_rt_mx(r=tmp_r.as_float(), t=list(tmp_t))
rt_mx.show(out=log)
print >> log
if params.xmanip.parameters.manipulate_pdb.apply_operator.invert:
rt_mx = rt_mx.inverse()
print >> log
print >> log, "Taking inverse of given operator"
print >> log
sites = model.sites_frac()
new_sites = flex.vec3_double()
for site in sites:
new_site = rt_mx.r()*matrix.col(site)
new_site = flex.double(new_site)+flex.double( rt_mx.t().as_double() )
new_sites.append( tuple(new_site) )
new_model = model.deep_copy_scatterers()
new_model.set_sites_frac( new_sites )
# write the new [pdb file please
pdb_file = open( params.xmanip.output.xyzout, 'w')
print >> log, "Wring pdb file to: %s"%(params.xmanip.output.xyzout)
if params.xmanip.parameters.manipulate_pdb.apply_operator.concatenate_model:
write_as_pdb_file( input_pdb = pdb_model,
input_xray_structure = model,
out = pdb_file,
chain_id_increment = 0,
additional_remark = None,
print_cryst_and_scale=True )
write_as_pdb_file( input_pdb = pdb_model,
input_xray_structure = new_model,
out = pdb_file,
chain_id_increment = params.xmanip.parameters.manipulate_pdb.apply_operator.chain_id_increment,
additional_remark = None,
print_cryst_and_scale=False )
pdb_file.close()
if params.xmanip.parameters.manipulate_pdb.task =="set_b":
#rest all the b values
if params.xmanip.parameters.manipulate_pdb.set_b:
b_iso = params.xmanip.parameters.manipulate_pdb.b_iso
new_model = model.set_b_iso( value = b_iso )
print >> log
print >> log, "All B-values have been set to %5.3f"%(b_iso)
print >> log, "Writing PDB file %s"%(params.xmanip.output.xyzout)
print >> log
pdb_file = open( params.xmanip.output.xyzout, 'w')
write_as_pdb_file( input_pdb = pdb_model,
input_xray_structure = new_model,
out = pdb_file,
chain_id_increment = 0,
additional_remark = None,
print_cryst_and_scale=True)
pdb_file.close()
#write the logfile
logger = open( params.xmanip.output.logfile, 'w')
print >> log, "Writing log file with name %s "%(params.xmanip.output.logfile)
print >> logger, string_buffer.getvalue()[0:len(string_buffer.getvalue())-1] #avoid a newline at the end ...
logger.close()
def get_reflection_file_server(self,
filenames=None,
labels=None,
array_type=None,
crystal_symmetry=None,
force_symmetry=None,
logger=None):
'''
Return the file server for a single miller_array file or mulitple files
:param filenames: list of filenames or None
:param labels: list of lists of labels or None
:param array_type: "x_ray", "neutron", "electron", or None
:param crystal_symmetry: cctbx.crystal.symmetry object or None
:param force_symmetry: bool or None
:param logger: defaults to self.logger (multi_out)
The order in filenames and labels should match, e.g. labels[0] should be the
labels for filenames[0]. The lengths of filenames and labels should be equal
as well. If all the labels in a file are to be added, set the labels entry
to None, e.g. labels[0] = None.
If array_type is None, files of any type are allowed.
'''
# use default logger if no logger for reflection_file_server is provided
if logger is None:
logger = self.logger
# use default miller_array file if no filenames provided
if filenames is None:
default_filename = self._check_miller_array_default_filename(
'miller_array')
filenames = [default_filename]
# set labels
if labels is None:
labels = [None for filename in filenames]
assert len(filenames) == len(labels)
# force crystal symmetry if a crystal symmetry is provided
if crystal_symmetry is not None and force_symmetry is None:
force_symmetry = True
if force_symmetry is None:
force_symmetry = False
# determine crystal symmetry from file list, if not provided
if crystal_symmetry is None:
try:
from_reflection_files = []
for filename, file_labels in zip(filenames, labels):
miller_arrays = self.get_miller_arrays(filename=filename)
for miller_array in miller_arrays:
if ((file_labels is None)
or (miller_array.info().label_string()
in file_labels)):
from_reflection_files.append(
miller_array.crystal_symmetry())
crystal_symmetry = crystal.select_crystal_symmetry(
from_reflection_files=from_reflection_files)
except Exception:
raise Sorry(
'A unit cell and space group could not be determined from the "filenames" argument. Please make sure there are enough data arrays being selected.'
)
# crystal_symmetry and force_symmetry should be set by now
miller_arrays = []
for filename, file_labels in zip(filenames, labels):
file_arrays = self.get_miller_array(filename).\
as_miller_arrays(crystal_symmetry=crystal_symmetry,
force_symmetry=force_symmetry)
if file_labels is None:
file_labels = self.get_miller_array_labels(filename)
for miller_array in file_arrays:
label_name = miller_array.info().label_string()
# check array label
if label_name in file_labels:
# check array type
if (array_type is None
or array_type == self.get_miller_array_type(
filename, label_name)):
miller_arrays.append(miller_array)
file_server = reflection_file_server(crystal_symmetry=crystal_symmetry,
force_symmetry=force_symmetry,
miller_arrays=miller_arrays,
err=logger)
return file_server
def reindex_utils(args):
if len(args)==0:
print_help()
elif ( "--help" in args ):
print_help()
elif ( "--h" in args ):
print_help()
elif ("-h" in args ):
print_help()
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="map_coefs")
print >> log, "#phil __OFF__"
print >> log, "================="
print >> log, " REINDEX "
print >> log, "A reindexing tool"
print >> log, "================="
print >> log
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown file or keyword:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params_root = effective_params.extract()
params = params_root.reindex_utils
# now get the unit cell from the files
hkl_xs = None
pdb_xs = None
if params.input.xray_data.file_name is not None:
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.input.xray_data.file_name)
if params.input.model.file_name is not None:
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.input.model.file_name)
phil_xs = crystal.symmetry(
unit_cell=params.input.unit_cell,
space_group_info=params.input.space_group )
combined_xs = crystal.select_crystal_symmetry(
None,phil_xs, [pdb_xs],[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.input.unit_cell = combined_xs.unit_cell()
params.input.space_group = combined_xs.space_group_info()
print >> log, "#phil __ON__"
new_params = master_params.format(python_object=params_root)
new_params.show(out=log)
print >> log, "#phil __END__"
if params.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.input.space_group is None:
raise Sorry("space group not specified")
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
miller_array = None
if params.input.xray_data.file_name is not None:
phil_xs = crystal.symmetry(
unit_cell=params.input.unit_cell,
space_group_info=params.input.space_group )
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = phil_xs,
force_symmetry = True,
reflection_files=[])
miller_array = xray_data_server.get_xray_data(
file_name = params.input.xray_data.file_name,
labels = params.input.xray_data.labels,
ignore_all_zeros = True,
parameter_scope = 'reindex_utils.input.xray_data',
parameter_name = 'labels'
)
info = miller_array.info()
miller_array = miller_array.map_to_asu()
miller_array = miller_array.select(
miller_array.indices() != (0,0,0))
miller_array = miller_array.select(
miller_array.data() > 0 )
if miller_array.sigmas() is not None:
miller_array = miller_array.select(
miller_array.sigmas() > 0 )
if (miller_array.is_xray_intensity_array()):
miller_array = miller_array.f_sq_as_f()
elif (miller_array.is_complex_array()):
miller_array = abs(miller_array)
miller_array.set_info(info)
print >> log
print >> log, "Summary info of observed data"
print >> log, "============================="
miller_array.show_summary(f=log)
print >> log
#----------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
pdb_model = None
if params.input.model.file_name is not None:
pdb_model = iotbx.pdb.input(
file_name=params.input.model.file_name)
model = pdb_model.xray_structure_simple(crystal_symmetry=phil_xs)
print >> log, "Atomic model summary"
print >> log, "===================="
model.show_summary()
print >> log
if params.parameters.action=="reindex":
#----------------------------------------------------------------
# step 3: get the reindex laws
to_niggli = phil_xs.change_of_basis_op_to_niggli_cell()
to_reference = phil_xs.change_of_basis_op_to_reference_setting()
to_inverse = phil_xs.change_of_basis_op_to_inverse_hand()
cb_op = None
pr = params.parameters.reindex
if (pr.standard_laws == "niggli"):
cb_op = to_niggli
elif (pr.standard_laws == "reference_setting"):
cb_op = to_reference
elif (pr.standard_laws == "invert"):
cb_op = to_inverse
else:
cb_op = sgtbx.change_of_basis_op(pr.user_supplied_law)
if cb_op is None:
raise Sorry("No change of basis operation is supplied.")
if params.parameters.inverse:
cb_op = cb_op.inverse()
print >> log, "Supplied reindexing law:"
print >> log, "========================"
print >> log, "hkl notation: ", cb_op.as_hkl()
print >> log, "xyz notation: ", cb_op.as_xyz()
print >> log, "abc notation: ", cb_op.as_abc()
#----------------------------------------------------------------
# step 4: do the reindexing
#
# step 4a: first do the miller array object
new_miller_array = None
if miller_array is not None:
new_miller_array = miller_array.change_basis( cb_op )
#
# step 4b: the xray structure
new_model = None
if pdb_model is not None:
new_model = model.change_basis( cb_op )
#----------------------------------------------------------------
# step 5a: write the new mtz file
print >> log
print >> log, "The data and model have been reindexed"
print >> log, "--------------------------------------"
print >> log
print >> log, "Writing output files...."
if miller_array is not None:
print >> log, "writing mtz file with name %s" % (params.output.hklout)
mtz_dataset = new_miller_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset.mtz_object().write(file_name=params.output.hklout)
#step 5b: write the new pdb file
if new_model is not None:
pdb_file = open(params.output.xyzout, 'w')
print >> log, "Wring pdb file to: %s" % params.output.xyzout
write_as_pdb_file(
input_pdb = pdb_model,
input_xray_structure = new_model,
out = pdb_file,
chain_id_increment = params.parameters.chain_id_increment,
additional_remark = "Generated by mmtbx reindex")
print >> pdb_file, "END"
pdb_file.close()
if ( [miller_array,new_model]).count(None)==2:
print >>log, "No input reflection of coordinate files have been given"
if params.parameters.action=="operator":
rt_mx = sgtbx.rt_mx(
params.parameters.apply_operator.user_supplied_operator,t_den=12*8 )
if params.parameters.inverse:
rt_mx = rt_mx.inverse()
print >> log
print >> log, "Applied operator : ", rt_mx.as_xyz()
print >> log
sites = model.sites_frac()
new_sites = flex.vec3_double()
for site in sites:
new_site = rt_mx.r()*matrix.col(site)
new_site = flex.double(new_site)+flex.double( rt_mx.t().as_double() )
new_sites.append( tuple(new_site) )
new_model = model.deep_copy_scatterers()
new_model.set_sites_frac( new_sites )
# write the new [pdb file please
pdb_file = open(params.output.xyzout, 'w')
print >> log, "Wring pdb file to: %s" % params.output.xyzout
if params.parameters.apply_operator.concatenate_model:
write_as_pdb_file(
input_pdb = pdb_model,
input_xray_structure = model,
out = pdb_file,
chain_id_increment = 0,
additional_remark = None,
print_cryst_and_scale=True)
write_as_pdb_file(
input_pdb = pdb_model,
input_xray_structure = new_model,
out = pdb_file,
chain_id_increment = params.parameters.chain_id_increment,
additional_remark = None,
print_cryst_and_scale=False)
print >> pdb_file, "END"
pdb_file.close()
if params.parameters.action=="manipulate_pdb":
#rest all the b values
if params.parameters.manipulate_pdb.set_b:
b_iso = params.reindex_utils.parameters.manipulate_pdb.b_iso
new_model = model.set_b_iso( value = b_iso )
print >> log
print >> log, "All B-values have been set to %5.3f"%(b_iso)
print >> log, "Writing PDB file %s"%(params.output.xyzout)
print >> log
pdb_file = open(params.output.xyzout, 'w')
write_as_pdb_file(
input_pdb = pdb_model,
input_xray_structure = new_model,
out = pdb_file,
chain_id_increment = 0,
additional_remark = None,
print_cryst_and_scale=True)
print >> pdb_file, "END"
pdb_file.close()
#write the logfile
logger = open(params.output.logfile, 'w')
print >> log, "Writing log file with name %s" % params.output.logfile
print >> log
print >> logger, string_buffer.getvalue()
def exercise_get_r_free_flags():
crystal_symmetry = crystal.symmetry(
unit_cell=(30,31,32,85,95,100),
space_group_symbol="P 1")
miller_set = miller.build_set(
crystal_symmetry=crystal_symmetry,
anomalous_flag=False,
d_min=3)
n = miller_set.indices().size()
exercise_flag_arrays = []
exercise_flag_arrays.append(
flex.int(list(flex.random_permutation(size=n)%10)))
exercise_flag_arrays.append(flex.int(xrange(n)))
exercise_flag_arrays.append(flex.int(n, 0))
for style in ["ccp4", "cns", "shelx", "bool"]:
for i_exercise,exercise_flag_array in enumerate(exercise_flag_arrays):
for reversed in [False, True]:
if (style == "ccp4"):
if (reversed): break
data = exercise_flag_array
test_flag_value = 3
else:
if (not reversed):
data = (exercise_flag_array == 0)
test_flag_value = True
else:
data = (exercise_flag_array != 0)
test_flag_value = False
if (style == "cns"):
data = data.as_int()
test_flag_value = int(test_flag_value)
elif (style == "shelx"):
data = -data.as_int()
data.set_selected((data == 0), 1)
if (not reversed): test_flag_value = -1
else: test_flag_value = 1
input_array = miller_set.array(data=data)
mtz_dataset = input_array.as_mtz_dataset(
column_root_label="FreeRflags")
mtz_dataset.mtz_object().write("tmp.mtz")
reflection_files = [reflection_file_reader.any_reflection_file(
file_name="tmp.mtz")]
err = StringIO()
reflection_file_srv = reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=err)
for trial_test_flag_value in [None, test_flag_value]:
for trial_label in [None, "free", "foo"]:
try:
r_free_flags, actual_test_flag_value = \
reflection_file_srv.get_r_free_flags(
file_name=None,
label=trial_label,
test_flag_value=trial_test_flag_value,
disable_suitability_test=False,
parameter_scope="r_free_flags")
except Sorry, e:
if (trial_label != "foo"):
assert i_exercise > 0
if (trial_label is None):
assert str(e) == """\
No array of R-free flags found.
For manual selection define:
r_free_flags.label
r_free_flags.test_flag_value
r_free_flags.disable_suitability_test=True"""
else:
assert str(e) == \
"Not a suitable array of R-free flags:" \
+ " r_free_flags.label=free\n" \
+ "To override the suitability test define:" \
+ " r_free_flags.disable_suitability_test=True"
else:
assert str(e) == "No matching array: r_free_flags.label=foo"
if (i_exercise == 0):
assert err.getvalue() == """\
No matching array: r_free_flags.label=foo
Possible choices:
tmp.mtz:FreeRflags
Please use r_free_flags.label
to specify an unambiguous substring of the target label.
"""
else:
assert err.getvalue() == """\
No matching array: r_free_flags.label=foo
"""
err = reflection_file_srv.err = StringIO()
else:
assert i_exercise == 0
actual_test_flag_value_2 = guess_r_free_flag_value(
miller_array=r_free_flags,
test_flag_value=trial_test_flag_value)
assert (actual_test_flag_value_2 == actual_test_flag_value)
def run(args, log=sys.stdout):
if (len(args) == 0):
print >> log, legend
defaults(log=log)
return
#
parsed = defaults(log=log)
processed_args = mmtbx.utils.process_command_line_args(
args=args, log=sys.stdout, master_params=parsed)
params = processed_args.params.extract()
reflection_files = processed_args.reflection_files
if (len(reflection_files) == 0):
if (params.hkl_file is None):
raise Sorry("No reflection file found.")
else:
hkl_in = file_reader.any_file(params.hkl_file, force_type="hkl")
hkl_in.assert_file_type("hkl")
reflection_files = [hkl_in.file_object]
crystal_symmetry = processed_args.crystal_symmetry
if (crystal_symmetry is None):
if (params.space_group is not None) and (params.unit_cell is not None):
from cctbx import crystal
crystal_symmetry = crystal.symmetry(
space_group_info=params.space_group,
unit_cell=params.unit_cell)
else:
raise Sorry("No crystal symmetry found.")
if (len(processed_args.pdb_file_names) == 0):
if (params.pdb_file is None):
raise Sorry("No model file found.")
else:
pdb_file_names = [params.pdb_file]
else:
pdb_file_names = processed_args.pdb_file_names
#
rfs = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=StringIO())
parameters = mmtbx.utils.data_and_flags_master_params().extract()
parameters.labels = params.f_obs_label
parameters.r_free_flags.label = params.r_free_flags_label
determine_data_and_flags_result = mmtbx.utils.determine_data_and_flags(
reflection_file_server=rfs,
parameters=parameters,
keep_going=True,
log=StringIO())
f_obs = determine_data_and_flags_result.f_obs
print "Input data:"
print " Iobs or Fobs:", f_obs.info().labels
r_free_flags = determine_data_and_flags_result.r_free_flags
print " Free-R flags:", r_free_flags.info().labels
#
parameters = mmtbx.utils.experimental_phases_params.extract()
parameters.labels = params.hendrickson_lattman_coefficients_label
experimental_phases_result = mmtbx.utils.determine_experimental_phases(
reflection_file_server=rfs,
parameters=parameters,
log=StringIO(),
parameter_scope="",
working_point_group=None,
symmetry_safety_check=True,
ignore_all_zeros=True)
if (experimental_phases_result is not None):
print " HL coefficients:", experimental_phases_result.info().labels
experimental_phases = extract_experimental_phases(
experimental_phases=experimental_phases_result, f_obs=f_obs)
#
if (r_free_flags is None):
r_free_flags = f_obs.array(data=flex.bool(f_obs.data().size(), False))
#
pdb_inp = mmtbx.utils.pdb_inp_from_multiple_files(pdb_file_names,
log=sys.stdout)
model = mmtbx.model.manager(model_input=pdb_inp,
process_input=False,
crystal_symmetry=crystal_symmetry,
log=sys.stdout)
if (model.get_number_of_models() > 1): #XXX support multi-models
raise Sorry("Multiple model file not supported in this tool.")
# XXX Twining not supported
xray_structure = model.get_xray_structure()
if (not xray_structure.unit_cell().is_similar_to(f_obs.unit_cell())):
raise Sorry(
"The unit cells in the model and reflections files are not " +
"isomorphous.")
print "Input model:"
print " number of atoms:", xray_structure.scatterers().size()
fmodel = mmtbx.f_model.manager(xray_structure=xray_structure,
r_free_flags=r_free_flags,
f_obs=f_obs,
abcd=experimental_phases)
fmodel.update_all_scales(
update_f_part1=True,
remove_outliers=params.remove_f_obs_outliers,
bulk_solvent_and_scaling=params.bulk_solvent_and_scaling)
print "Overall statistics:"
fmodel.info().show_all()
#
print "Output data:"
if (params.output_file_name is not None):
output_file_name = params.output_file_name
else:
pdb_file_bn = os.path.basename(pdb_file_names[0])
hkl_file_bn = os.path.basename(reflection_files[0].file_name())
try:
pdb_file_prefix = pdb_file_bn[:pdb_file_bn.index(".")]
except ValueError:
pdb_file_prefix = pdb_file_bn
try:
hkl_file_prefix = hkl_file_bn[:hkl_file_bn.index(".")]
except ValueError:
hkl_file_prefix = hkl_file_bn
output_file_name = "%s_%s.mtz" % (pdb_file_prefix, hkl_file_prefix)
print " file name:", output_file_name
print " to see the contnt of %s:" % output_file_name
print " phenix.mtz.dump %s" % output_file_name
out = open(output_file_name, "w")
fmodel.export(out=out)
out.close()
print "All done."
return output_file_name
def run(args, command_name="phenix.twin_map_utils"):
log = sys.stdout
params = None
if (len(args) == 0 or "--help" in args or "--h" in args or "-h" in args):
print_help(command_name=command_name)
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="map_coefs")
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
if not arg_is_processed:
print("##----------------------------------------------##",
file=log)
print("## Unknown file or keyword:", arg, file=log)
print("##----------------------------------------------##",
file=log)
print(file=log)
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
"""
new_params = master_params.format(python_object=params)
new_params.show(out=log)
"""
# now get the unit cell from the pdb file
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.twin_utils.input.xray_data.file_name)
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.twin_utils.input.model.file_name)
phil_xs = None
if ([
params.twin_utils.input.unit_cell,
params.twin_utils.input.space_group
]).count(None) < 2:
phil_xs = crystal.symmetry(
unit_cell=params.twin_utils.input.unit_cell,
space_group_info=params.twin_utils.input.space_group)
combined_xs = crystal.select_crystal_symmetry(None, phil_xs, [pdb_xs],
[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.twin_utils.input.unit_cell = combined_xs.unit_cell()
params.twin_utils.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
new_params = master_params.format(python_object=params)
new_params.show(out=log)
if params.twin_utils.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.twin_utils.input.space_group is None:
raise Sorry("space group not specified")
if params.twin_utils.input.xray_data.file_name is None:
raise Sorry("Xray data not specified")
if params.twin_utils.input.model.file_name is None:
raise Sorry("pdb file with model not specified")
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
phil_xs = crystal.symmetry(
unit_cell=params.twin_utils.input.unit_cell,
space_group_info=params.twin_utils.input.space_group)
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=phil_xs, force_symmetry=True, reflection_files=[])
miller_array = None
free_flags = None
tmp_params = utils.data_and_flags_master_params().extract()
# insert proper values please
tmp_params.file_name = params.twin_utils.input.xray_data.file_name
tmp_params.labels = params.twin_utils.input.xray_data.obs_labels
tmp_params.r_free_flags.file_name = params.twin_utils.input.xray_data.file_name
tmp_params.r_free_flags.label = params.twin_utils.input.xray_data.free_flag
tmp_object = utils.determine_data_and_flags(
reflection_file_server=xray_data_server,
parameters=tmp_params,
log=log)
miller_array = tmp_object.extract_data()
if miller_array.is_xray_intensity_array():
miller_array = miller_array.f_sq_as_f()
assert miller_array.is_xray_amplitude_array()
free_flags = tmp_object.extract_flags(data=miller_array)
print(file=log)
print("Attempting to extract Free R flags", file=log)
free_flags = free_flags.customized_copy(data=flex.bool(
free_flags.data() == 1))
if free_flags is None:
free_flags = miller_array.generate_r_free_flags(
use_lattice_symmetry=True)
assert miller_array.observation_type() is not None
print(file=log)
print("Summary info of observed data", file=log)
print("=============================", file=log)
miller_array.show_summary(f=log)
print(file=log)
if miller_array.indices().size() == 0:
raise Sorry("No data available")
#----------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
model = pdb.input(file_name=params.twin_utils.input.model.file_name
).xray_structure_simple(crystal_symmetry=phil_xs)
print("Atomic model summary", file=log)
print("====================", file=log)
model.show_summary(f=log)
print(file=log)
#----------------------------------------------------------------
# step 3: get the twin laws for this xs
twin_laws = twin_analyses.twin_laws(
miller_array,
lattice_symmetry_max_delta=\
params.twin_utils.parameters.twinning.max_delta,
out=log)
print(file=log)
print("Preliminary data analyses", file=log)
print("==========================", file=log)
twin_laws.show(out=log)
#---------
# step 3:
# make twin model managers for all twin laws
print(file=log)
print(
"Overall and bulk solvent scale paranmeters and twin fraction estimation",
file=log)
print(
"=======================================================================",
file=log)
twin_models = []
operator_count = 0
if params.twin_utils.parameters.twinning.twin_law is not None:
tmp_law = sgtbx.rt_mx(
params.twin_utils.parameters.twinning.twin_law)
tmp_law = twin_analyses.twin_law(tmp_law, None, None, None, None,
None)
twin_laws.operators = [tmp_law]
for twin_law in twin_laws.operators:
operator_count += 1
operator_hkl = sgtbx.change_of_basis_op(twin_law.operator).as_hkl()
twin_model = twin_f_model.twin_model_manager(
f_obs=miller_array,
r_free_flags=free_flags,
xray_structure=model,
twin_law=twin_law.operator,
detwin_mode=params.twin_utils.parameters.twinning.detwin_mode,
out=log)
print("--- bulk solvent scaling ---", file=log)
twin_model.update_all_scales()
twin_model.r_values()
twin_model.target()
twin_model.show_k_sol_b_sol_b_cart_target()
twin_model.show_essential()
wfofc = twin_model.map_coefficients(map_type="mFo-DFc")
wtfofc = twin_model.map_coefficients(map_type="2mFo-DFc")
grad = twin_model.map_coefficients(map_type="gradient")
mtz_dataset = wtfofc.as_mtz_dataset(column_root_label="FWT")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array=wfofc, column_root_label="DFWT")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array=grad, column_root_label="GRAD")
name = params.twin_utils.output.map_coeffs_root + "_" + str(
operator_count) + ".mtz"
print(file=log)
print("Writing %s for twin law %s" % (name, operator_hkl),
file=log)
print(file=log)
mtz_dataset.mtz_object().write(file_name=name)
if params.twin_utils.output.obs_and_calc is not None:
# i want also a Fobs and Fmodel combined dataset please
mtz_dataset = miller_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array=twin_model.f_model(),
column_root_label="FMODEL")
name = params.twin_utils.output.obs_and_calc
mtz_dataset.mtz_object().write(file_name=name)
if len(twin_laws.operators) == 0:
print(file=log)
print("No twin laws were found", file=log)
print("Performing maximum likelihood based bulk solvent scaling",
file=log)
f_model_object = f_model.manager(f_obs=miller_array,
r_free_flags=free_flags,
xray_structure=model)
f_model_object.update_all_scales(log=log)
tfofc = f_model_object.map_coefficients(map_type="2mFobs-DFmodel")
fofc = f_model_object.map_coefficients(map_type="mFobs-DFmodel")
mtz_dataset = tfofc.as_mtz_dataset(column_root_label="FWT")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array=fofc, column_root_label="DELFWT")
name = params.twin_utils.output.map_coeffs_root + "_ML.mtz"
mtz_dataset.mtz_object().write(file_name=name)
if params.twin_utils.output.obs_and_calc is not None:
# i want also a Fobs and Fmodel combined dataset please
mtz_dataset = miller_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array=f_model_object.f_model(),
column_root_label="FMODEL")
name = params.twin_utils.output.obs_and_calc
mtz_dataset.mtz_object().write(file_name=name)
print(file=log)
print(file=log)
print("All done \n", file=log)
logfile = open(params.twin_utils.output.logfile, 'w')
print(string_buffer.getvalue(), file=logfile)
print(file=log)
def reflection_file_server(crystal_symmetry, reflection_files):
return reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=StringIO())
def run(args, command_name="phenix.remove_outliers"):
if (len(args)==0 or "--help" in args or "--h" in args or "-h" in args):
print_help(command_name=command_name)
else:
log = multi_out()
plot_out = None
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="outlier_detection")
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if arg=="--quiet":
arg_is_processed = True
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown file or keyword:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
if not os.path.exists( params.outlier_utils.input.xray_data.file_name ) :
raise Sorry("File %s can not be found"%(params.outlier_utils.input.xray_data.file_name) )
if params.outlier_utils.input.model.file_name is not None:
if not os.path.exists( params.outlier_utils.input.model.file_name ):
raise Sorry("File %s can not be found"%(params.outlier_utils.input.model.file_name) )
# now get the unit cell from the pdb file
hkl_xs = None
if params.outlier_utils.input.xray_data.file_name is not None:
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.outlier_utils.input.xray_data.file_name)
pdb_xs = None
if params.outlier_utils.input.model.file_name is not None:
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.outlier_utils.input.model.file_name)
phil_xs = crystal.symmetry(
unit_cell=params.outlier_utils.input.unit_cell,
space_group_info=params.outlier_utils.input.space_group )
phil_xs.show_summary()
hkl_xs.show_summary()
combined_xs = crystal.select_crystal_symmetry(
None,phil_xs, [pdb_xs],[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.outlier_utils.input.unit_cell = combined_xs.unit_cell()
params.outlier_utils.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
new_params = master_params.format(python_object=params)
new_params.show(out=log)
if params.outlier_utils.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.outlier_utils.input.space_group is None:
raise Sorry("space group not specified")
if params.outlier_utils.input.xray_data.file_name is None:
raise Sorry("Xray data not specified")
if params.outlier_utils.input.model.file_name is None:
print "PDB file not specified. Basic wilson outlier rejections only."
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
phil_xs = crystal.symmetry(
unit_cell=params.outlier_utils.input.unit_cell,
space_group_info=params.outlier_utils.input.space_group )
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = phil_xs,
force_symmetry = True,
reflection_files=[])
miller_array = None
miller_array = xray_data_server.get_xray_data(
file_name = params.outlier_utils.input.xray_data.file_name,
labels = params.outlier_utils.input.xray_data.obs_labels,
ignore_all_zeros = True,
parameter_scope = 'outlier_utils.input.xray_data',
parameter_name = 'obs_labels'
)
info = miller_array.info()
miller_array = miller_array.map_to_asu()
miller_array = miller_array.select(
miller_array.indices() != (0,0,0))
#we have to check if the sigma's make any sense at all
if not miller_array.sigmas_are_sensible():
miller_array = miller_array.customized_copy(
data = miller_array.data(),
sigmas=None).set_observation_type(miller_array)
miller_array = miller_array.select(
miller_array.data() > 0 )
if miller_array.sigmas() is not None:
miller_array = miller_array.select(
miller_array.sigmas() > 0 )
if (miller_array.is_xray_intensity_array()):
miller_array = miller_array.f_sq_as_f()
elif (miller_array.is_complex_array()):
miller_array = abs(miller_array)
miller_array.set_info(info)
merged_anomalous=False
if miller_array.anomalous_flag():
miller_array = miller_array.average_bijvoet_mates().set_observation_type(
miller_array )
merged_anomalous=True
miller_array = miller_array.map_to_asu()
# get the free reflections please
free_flags = None
if params.outlier_utils.input.xray_data.free_flags is None:
free_flags = miller_array.generate_r_free_flags(
fraction=params.outlier_utils.\
additional_parameters.free_flag_generation.fraction,
max_free=params.outlier_utils.\
additional_parameters.free_flag_generation.max_number,
lattice_symmetry_max_delta=params.outlier_utils.\
additional_parameters.free_flag_generation.lattice_symmetry_max_delta,
use_lattice_symmetry=params.outlier_utils.\
additional_parameters.free_flag_generation.use_lattice_symmetry
)
else:
free_flags = xray_data_server.get_xray_data(
file_name = params.outlier_utils.input.xray_data.file_name,
labels = params.outlier_utils.input.xray_data.free_flags,
ignore_all_zeros = True,
parameter_scope = 'outlier_utils.input.xray_data',
parameter_name = 'free_flags'
)
if free_flags.anomalous_flag():
free_flags = free_flags.average_bijvoet_mates()
merged_anomalous=True
free_flags = free_flags.customized_copy(
data = flex.bool( free_flags.data() == 1 ))
free_flags = free_flags.map_to_asu()
free_flags = free_flags.common_set( miller_array )
print >> log
print >> log, "Summary info of observed data"
print >> log, "============================="
miller_array.show_summary(f=log)
if merged_anomalous:
print >> log, "For outlier detection purposes, the Bijvoet pairs have been merged."
print >> log
print >> log, "Constructing an outlier manager"
print >> log, "==============================="
print >> log
outlier_manager = outlier_rejection.outlier_manager(
miller_array,
free_flags,
out=log)
basic_array = None
extreme_array = None
model_based_array = None
basic_array = outlier_manager.basic_wilson_outliers(
p_basic_wilson = params.outlier_utils.outlier_detection.\
parameters.basic_wilson.level,
return_data = True)
extreme_array = outlier_manager.extreme_wilson_outliers(
p_extreme_wilson = params.outlier_utils.outlier_detection.parameters.\
extreme_wilson.level,
return_data = True)
beamstop_array = outlier_manager.beamstop_shadow_outliers(
level = params.outlier_utils.outlier_detection.parameters.\
beamstop.level,
d_min = params.outlier_utils.outlier_detection.parameters.\
beamstop.d_min,
return_data=True)
#----------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
if params.outlier_utils.input.model.file_name is not None:
model = pdb.input(file_name=params.outlier_utils.input.model.file_name).xray_structure_simple(
crystal_symmetry=phil_xs)
print >> log, "Atomic model summary"
print >> log, "===================="
model.show_summary(f=log)
print >> log
# please make an f_model object for bulk solvent scaling etc etc
f_model_object = f_model.manager(
f_obs = miller_array,
r_free_flags = free_flags,
xray_structure = model )
print >> log, "Bulk solvent scaling of the data"
print >> log, "================================"
print >> log, "Maximum likelihood bulk solvent scaling."
print >> log
f_model_object.update_solvent_and_scale(out=log)
plot_out = StringIO()
model_based_array = outlier_manager.model_based_outliers(
f_model_object.f_model(),
level=params.outlier_utils.outlier_detection.parameters.model_based.level,
return_data=True,
plot_out=plot_out)
#check what needs to be put out please
if params.outlier_utils.output.hklout is not None:
if params.outlier_utils.outlier_detection.protocol == "model":
if params.outlier_utils.input.model.file_name == None:
print >> log, "Model based rejections requested. No model was supplied."
print >> log, "Switching to writing out rejections based on extreme value Wilson statistics."
params.outlier_utils.outlier_detection.protocol="extreme"
output_array = None
print >> log
if params.outlier_utils.outlier_detection.protocol == "basic":
print >> log, "Non-outliers found by the basic wilson statistics"
print >> log, "protocol will be written out."
output_array = basic_array
new_set_of_free_flags = free_flags.common_set( basic_array )
if params.outlier_utils.outlier_detection.protocol == "extreme":
print >> log, "Non-outliers found by the extreme value wilson statistics"
print >> log, "protocol will be written out."
output_array = extreme_array
new_set_of_free_flags = free_flags.common_set( extreme_array )
if params.outlier_utils.outlier_detection.protocol == "model":
print >> log, "Non-outliers found by the model based"
print >> log, "protocol will be written out to the file:"
print >> log, params.outlier_utils.output.hklout
print >> log
output_array = model_based_array
new_set_of_free_flags = free_flags.common_set( model_based_array )
if params.outlier_utils.outlier_detection.protocol == "beamstop":
print >> log, "Outliers found for the beamstop shadow"
print >> log, "problems detection protocol will be written to the file:"
print >> log, params.outlier_utils.output.hklout
print >> log
output_array = model_based_array
new_set_of_free_flags = free_flags.common_set( model_based_array )
mtz_dataset = output_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = new_set_of_free_flags,
column_root_label = "Free_R_Flag"
)
mtz_dataset.mtz_object().write(
file_name=params.outlier_utils.output.hklout)
if (params.outlier_utils.output.logfile is not None):
final_log = StringIO()
print >> final_log, string_buffer.getvalue()
print >> final_log
if plot_out is not None:
print >> final_log, plot_out.getvalue()
outfile = open( params.outlier_utils.output.logfile, 'w' )
outfile.write( final_log.getvalue() )
print >> log
print >> log, "A logfile named %s was created."%(
params.outlier_utils.output.logfile)
print >> log, "This logfile contains the screen output and"
print >> log, "(possibly) some ccp4 style loggraph plots"
def exercise_get_amplitudes_and_get_phases_deg():
crystal_symmetry = crystal.symmetry(
unit_cell=(10,11,12,85,95,100),
space_group_symbol="P 1")
miller_set = miller.build_set(
crystal_symmetry=crystal_symmetry,
anomalous_flag=False,
d_min=3)
input_arrays = [miller_set.array(
data=flex.random_double(size=miller_set.indices().size()))
.set_observation_type_xray_amplitude()
for i in [0,1]]
mtz_dataset = input_arrays[0].as_mtz_dataset(column_root_label="F0")
mtz_dataset.mtz_object().write("tmp_rfu1.mtz")
reflection_files = [reflection_file_reader.any_reflection_file(
file_name="tmp_rfu1.mtz")]
reflection_file_srv = reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files)
ampl = reflection_file_srv.get_amplitudes(
file_name=None,
labels=None,
convert_to_amplitudes_if_necessary=True,
parameter_scope="amplitudes",
parameter_name="labels")
assert str(ampl.info()) == "tmp_rfu1.mtz:F0"
ampl = reflection_file_srv.get_miller_array(labels="F0")
assert str(ampl.info()) == "tmp_rfu1.mtz:F0"
mtz_dataset.add_miller_array(
miller_array=input_arrays[1], column_root_label="F1")
mtz_dataset.mtz_object().write("tmp_rfu2.mtz")
reflection_files = [reflection_file_reader.any_reflection_file(
file_name="tmp_rfu2.mtz")]
err = StringIO()
reflection_file_srv = reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=err)
try:
reflection_file_srv.get_amplitudes(
file_name=None,
labels=None,
convert_to_amplitudes_if_necessary=True,
parameter_scope="amplitudes",
parameter_name="labels")
except Sorry:
assert not show_diff(err.getvalue(), """\
Multiple equally suitable arrays of amplitudes found.
Possible choices:
tmp_rfu2.mtz:F0
tmp_rfu2.mtz:F1
Please use amplitudes.labels
to specify an unambiguous substring of the target label.
""")
err = reflection_file_srv.err = StringIO()
else:
raise Exception_expected
ampl = reflection_file_srv.get_amplitudes(
file_name=None,
labels=["F1"],
convert_to_amplitudes_if_necessary=True,
parameter_scope="amplitudes",
parameter_name="labels")
assert str(ampl.info()) == "tmp_rfu2.mtz:F1"
try:
reflection_file_srv.get_amplitudes(
file_name=None,
labels=["F2"],
convert_to_amplitudes_if_necessary=True,
parameter_name="labels",
parameter_scope=None)
except Sorry:
assert not show_diff(err.getvalue(), """\
No matching array: labels=F2
Possible choices:
tmp_rfu2.mtz:F0
tmp_rfu2.mtz:F1
Please use labels
to specify an unambiguous substring of the target label.
""")
err = reflection_file_srv.err = StringIO()
else:
raise Exception_expected
assert len(reflection_file_srv.file_name_miller_arrays) == 1
ampl = reflection_file_srv.get_amplitudes(
file_name="tmp_rfu1.mtz",
labels=None,
convert_to_amplitudes_if_necessary=True,
parameter_scope="amplitudes",
parameter_name="labels")
assert len(reflection_file_srv.file_name_miller_arrays) == 2
assert str(ampl.info()) == "tmp_rfu1.mtz:F0"
ampl = reflection_file_srv.get_amplitudes(
file_name=os.path.abspath("tmp_rfu1.mtz"),
labels=["f0"],
convert_to_amplitudes_if_necessary=True,
parameter_scope="amplitudes",
parameter_name="labels")
assert len(reflection_file_srv.file_name_miller_arrays) == 2
assert str(ampl.info()) == "tmp_rfu1.mtz:F0"
try:
reflection_file_srv.get_amplitudes(
file_name=None,
labels=None,
convert_to_amplitudes_if_necessary=True,
parameter_scope=None,
parameter_name=None)
except Sorry:
assert not show_diff(err.getvalue(), """\
Multiple equally suitable arrays of amplitudes found.
Possible choices:
tmp_rfu2.mtz:F0
tmp_rfu2.mtz:F1
Please specify an unambiguous substring of the target label.
""")
err = reflection_file_srv.err = StringIO()
else:
raise Exception_expected
#
mtz_dataset.add_miller_array(
miller_array=miller_set.array(
data=flex.polar(
flex.random_double(size=miller_set.indices().size()),
flex.random_double(size=miller_set.indices().size()))),
column_root_label="F2")
mtz_dataset.add_miller_array(
miller_array=miller_set.array(
data=flex.random_double(size=miller_set.indices().size()),
sigmas=flex.random_double(size=miller_set.indices().size())/10)
.set_observation_type_xray_intensity(),
column_root_label="F3")
mtz_dataset.add_miller_array(
miller_array=miller_set.array(
data=flex.hendrickson_lattman(miller_set.indices().size(), (0,0,0,0))),
column_root_label="P")
mtz_dataset.mtz_object().write("tmp_rfu3.mtz")
reflection_files = [reflection_file_reader.any_reflection_file(
file_name="tmp_rfu3.mtz")]
err = StringIO()
reflection_file_srv = reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=err)
ampl = reflection_file_srv.get_amplitudes(
file_name=None,
labels=["f2"],
convert_to_amplitudes_if_necessary=False,
parameter_scope="amplitudes",
parameter_name="labels")
assert str(ampl.info()) == "tmp_rfu3.mtz:F2,PHIF2"
assert ampl.is_complex_array()
ampl = reflection_file_srv.get_amplitudes(
file_name=None,
labels=["f2"],
convert_to_amplitudes_if_necessary=True,
parameter_scope="amplitudes",
parameter_name="labels")
assert str(ampl.info()) == "tmp_rfu3.mtz:F2"
assert ampl.is_real_array()
ampl = reflection_file_srv.get_amplitudes(
file_name=None,
labels=["f3"],
convert_to_amplitudes_if_necessary=False,
parameter_scope="amplitudes",
parameter_name="labels")
assert str(ampl.info()) == "tmp_rfu3.mtz:F3,SIGF3"
assert ampl.is_xray_intensity_array()
ampl = reflection_file_srv.get_amplitudes(
file_name=None,
labels=["f3"],
convert_to_amplitudes_if_necessary=True,
parameter_scope="amplitudes",
parameter_name="labels")
assert str(ampl.info()) == "tmp_rfu3.mtz:F3,as_amplitude_array"
assert ampl.is_real_array()
ampl = reflection_file_srv.get_amplitudes(
file_name=None,
labels=None,
convert_to_amplitudes_if_necessary=False,
parameter_scope="amplitudes",
parameter_name="labels",
return_all_valid_arrays=True,
strict=True)
assert (len(ampl) == 2)
for f in ampl :
assert (not f.is_xray_intensity_array()) and (not f.is_complex_array())
#
phases = reflection_file_srv.get_phases_deg(
file_name=None,
labels=["f2"],
convert_to_phases_if_necessary=False,
original_phase_units=None,
parameter_scope="phases",
parameter_name="labels")
assert str(phases.info()) == "tmp_rfu3.mtz:F2,PHIF2"
assert phases.is_complex_array()
phases = reflection_file_srv.get_phases_deg(
file_name=None,
labels=["f2"],
convert_to_phases_if_necessary=True,
original_phase_units=None,
parameter_scope=None,
parameter_name="labels")
assert str(phases.info()) == "tmp_rfu3.mtz:PHIF2"
assert phases.is_real_array()
assert flex.mean(phases.data()) > 5
phases = reflection_file_srv.get_phases_deg(
file_name=None,
labels=["PA"],
convert_to_phases_if_necessary=False,
original_phase_units=None,
parameter_scope="phases",
parameter_name="labels")
assert str(phases.info()) == "tmp_rfu3.mtz:PA,PB,PC,PD"
phases = reflection_file_srv.get_phases_deg(
file_name=None,
labels=["PA"],
convert_to_phases_if_necessary=True,
original_phase_units=None,
parameter_scope="phases",
parameter_name="labels")
assert str(phases.info()) \
== "tmp_rfu3.mtz:PA,PB,PC,PD,converted_to_centroid_phases"
assert phases.is_real_array()
for original_phase_units in [None, "deg", "rad"]:
phases = reflection_file_srv.get_phases_deg(
file_name=None,
labels=["F0"],
convert_to_phases_if_necessary=False,
original_phase_units=original_phase_units,
parameter_scope=None,
parameter_name="labels")
if (original_phase_units != "rad"):
assert str(phases.info()) == "tmp_rfu3.mtz:F0"
else:
assert str(phases.info()) == "tmp_rfu3.mtz:F0,converted_to_deg"
def run (args, out=sys.stdout) :
import iotbx.phil
cmdline = iotbx.phil.process_command_line_with_files(
args=args,
master_phil_string=master_phil_str,
pdb_file_def="crystal_symmetry.symm_file",
reflection_file_def="input.data",
space_group_def="crystal_symmetry.space_group",
unit_cell_def="crystal_symmetry.unit_cell",
usage_string="""\
mmtbx.massage_data data.mtz [labels=I,SIGI] [options]
Modification of experimental data: remove anisotropy, apply B-factor, filter
negative intensities, add or remove twinning. For expert use (and extreme
cases) only.
""")
params = cmdline.work.extract()
if (params.input.data is None) :
raise Sorry("No data file supplied.")
from mmtbx.scaling import massage_twin_detwin_data
from iotbx import crystal_symmetry_from_any
from iotbx import reflection_file_utils
from cctbx import crystal
crystal_symmetry = space_group = unit_cell = None
if (params.crystal_symmetry.space_group is not None) :
space_group = params.crystal_symmetry.space_group
if (params.crystal_symmetry.unit_cell is not None) :
unit_cell = params.crystal_symmetry.unit_cell
crystal_symmetry = None
if (params.crystal_symmetry.symm_file is not None) :
crystal_symmetry = crystal_symmetry_from_any.extract_from(
file_name=params.crystal_symmetry.symm_file)
if (crystal_symmetry is None) :
raise Sorry("No crystal symmetry defined in %s" %
params.crystal_symmetry.symm_file)
if (crystal_symmetry is None) and (not None in [space_group, unit_cell]) :
crystal_symmetry = crystal.symmetry(
space_group_info=space_group,
unit_cell=unit_cell)
hkl_in = cmdline.get_file(params.input.data)
hkl_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=[hkl_in.file_object],
err=sys.stderr)
data = hkl_server.get_xray_data(
file_name=params.input.data,
labels=params.input.labels,
ignore_all_zeros=True,
parameter_scope="input",
prefer_anomalous=True,
prefer_amplitudes=False)
result = massage_twin_detwin_data.massage_data(
miller_array=data,
parameters=params.options,
out=out)
if (params.output.hklout is None) :
file_base = op.splitext(op.basename(params.input.data))[0]
if (params.output.hklout_type in ["Auto", "mtz"]) :
params.output.hklout = file_base + ".mtz"
else :
params.output.hklout = file_base + ".sca"
result.write_data(
file_name=params.output.hklout,
output_type=params.output.hklout_type,
label_extension=params.output.label_extension)
class GetMapCoeffs(refinement_base):
def __init__(self, args=None, out=sys.stderr):
super(GetMapCoeffs, self).__init__(args=args, out=out)
def run(self):
self.get_inputs()
self.extract_model()
self.extract_data()
# {{{ compute_map
def compute_map(self):
log = sys.stderr
# the mtz and pdb files are self.mtz_file self.pdb_file, respectively
#out = maps.run(args = [self.pdb_file, self.mtz_file])
#self.map_coeff_file, self.xplor_maps = out[0],out[1]
master_params = mmtbx.maps.maps_including_IO_master_params()
master_params = master_params.fetch(iotbx.phil.parse(default_params))
processed_args = mmtbx.utils.process_command_line_args(
args=[self.mtz_file, self.pdb_file],
log=log,
master_params=master_params)
working_phil = processed_args.params
params = working_phil.extract()
if params.maps.input.pdb_file_name is None:
params.maps.input.pdb_file_name = self.pdb_file
if params.maps.input.reflection_data.file_name is None:
params.maps.input.reflection_data.file_name = self.mtz_file
pdb_inp = iotbx.pdb.input(file_name=params.maps.input.pdb_file_name)
# get all crystal symmetries
cs_from_coordinate_files = [pdb_inp.crystal_symmetry_from_cryst1()]
csa = crystal_symmetry_from_any.extract_from(
params.maps.input.reflection_data.file_name)
cs_from_reflection_files = [csa]
crystal_symmetry = None
try:
crystal_symmetry = crystal.select_crystal_symmetry(
from_coordinate_files=cs_from_coordinate_files,
from_reflection_files=cs_from_reflection_files)
except AssertionError, e:
if ("No unit cell and symmetry information supplied" in str(e)):
raise Sorry(
"Missing or incomplete symmetry information. This program "
+
"will only work with reflection file formats that contain both "
+ "unit cell and space group records, such as MTZ files.")
#
rfn = params.maps.input.reflection_data.file_name
rf = reflection_file_reader.any_reflection_file(file_name=rfn)
reflection_files = [rf]
reflection_file_names = [rfn]
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files, #[],
err=log)
#
reflection_data_master_params = mmtbx.utils.data_and_flags_master_params(
master_scope_name="reflection_data")
reflection_data_input_params = processed_args.params.get(
"maps.input.reflection_data")
reflection_data_params = reflection_data_master_params.fetch(
reflection_data_input_params).extract().reflection_data
#
determine_data_and_flags_result = mmtbx.utils.determine_data_and_flags(
reflection_file_server=reflection_file_server,
parameters=reflection_data_params,
data_parameter_scope="maps.input.reflection_data",
flags_parameter_scope="maps.input.reflection_data.r_free_flags",
data_description="Reflection data",
keep_going=True,
log=log)
f_obs = determine_data_and_flags_result.f_obs
self.resolution_range = f_obs.resolution_range()
r_free_flags = determine_data_and_flags_result.r_free_flags
test_flag_value = determine_data_and_flags_result.test_flag_value
if (r_free_flags is None):
r_free_flags = f_obs.array(
data=flex.bool(f_obs.data().size(), False))
test_flag_value = None
print >> log, "-" * 79
print >> log, "\nInput model file:", params.maps.input.pdb_file_name
pdb_hierarchy = pdb_inp.construct_hierarchy(set_atom_i_seq=True)
atom_selection_manager = pdb_hierarchy.atom_selection_cache()
# get xray_structure
xray_structure = pdb_inp.xray_structure_simple(
crystal_symmetry=crystal_symmetry)
mmtbx.utils.setup_scattering_dictionaries(
scattering_table=params.maps.scattering_table,
xray_structure=xray_structure,
d_min=f_obs.d_min(),
log=log)
xray_structure.show_summary(f=log, prefix=" ")
print >> log, "-" * 79
print >> log, "Bulk solvent correction and anisotropic scaling:"
fmodel = mmtbx.utils.fmodel_simple(
#update_f_part1_for = "map",
xray_structures=[xray_structure],
scattering_table=params.maps.scattering_table,
f_obs=f_obs,
r_free_flags=r_free_flags,
outliers_rejection=params.maps.input.reflection_data.
outliers_rejection,
skip_twin_detection=params.maps.skip_twin_detection,
bulk_solvent_correction=params.maps.bulk_solvent_correction,
anisotropic_scaling=params.maps.anisotropic_scaling)
fmodel_info = fmodel.info()
fmodel_info.show_rfactors_targets_scales_overall(out=log)
print >> log, "-" * 79
print >> log, "Compute maps."
cmo = mmtbx.maps.compute_map_coefficients(
fmodel=fmodel,
params=params.maps.map_coefficients,
pdb_hierarchy=pdb_hierarchy,
log=log)
file_name_base = params.maps.input.pdb_file_name
if (file_name_base.count(".") > 0):
file_name_base = file_name_base[:file_name_base.index(".")]
self.map_coeff_file_name = file_name_base + "_map_coeffs.mtz"
r_free_flags_output = None
write_mtz_file_result = cmo.write_mtz_file(
file_name=self.map_coeff_file_name,
r_free_flags=r_free_flags_output)
print >> log, "All done."
if (write_mtz_file_result):
print >> log, "Map coefficients: %s" % os.path.basename(
self.map_coeff_file_name)
def run(args, command_name="iotbx.r_free_flags_accumulation"):
def raise_usage():
raise Usage("%s reflection_file [label=value]" % command_name)
if (len(args) == 0 or "--help" in args or "-h" in args):
raise_usage()
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="r_free_flags_accumulation")
reflection_files = []
for arg in args:
if (os.path.isfile(arg)):
refl_file = reflection_file_reader.any_reflection_file(
file_name=arg)
if (refl_file.file_type() is not None):
reflection_files.append(refl_file)
arg = None
if (arg is not None):
try:
command_line_params = argument_interpreter.process(arg=arg)
except KeyboardInterrupt:
raise
except Exception:
raise Sorry("Unknown file or keyword: %s" % arg)
else:
phil_objects.append(command_line_params)
params_scope = master_params.fetch(sources=phil_objects).extract()
params = params_scope.r_free_flags_accumulation
srv = reflection_file_utils.reflection_file_server(
reflection_files=reflection_files)
r_free_flags, test_flag_value = srv.get_r_free_flags(
file_name=params.file_name,
label=params.label,
test_flag_value=params.test_flag_value,
disable_suitability_test=params.disable_suitability_test,
parameter_scope="r_free_flags_accumulation")
params.file_name = r_free_flags.info().source
params.label = r_free_flags.info().label_string()
params.test_flag_value = test_flag_value
if (params.output is None):
params.output = os.path.basename(params.file_name) \
+ ".r_free_flags_accumulation"
working_params = master_params.format(python_object=params_scope)
working_params.show()
print
print "#phil __OFF__"
r_free_flags = r_free_flags.array(
data=r_free_flags.data() == params.test_flag_value)
r_free_flags.show_r_free_flags_info()
print
accu = r_free_flags \
.sort(by_value="resolution") \
.r_free_flags_accumulation()
print "Writing file: %s" % show_string(params.output)
print " 1. column: reflection counts, sorted by resolution"
print " 2. column: number of free reflections / total number of reflections"
sys.stdout.flush()
out = open(params.output, "w")
for line in params.plot_header:
print >> out, line
for c, f in zip(accu.reflection_counts, accu.free_fractions):
print >> out, c, f
out.close()
print
sys.stdout.flush()
def run(args, command_name="phenix.fobs_minus_fobs_map", log=None):
if (len(args) == 0): args = ["--help"]
examples = """Examples:
phenix.fobs_minus_fobs_map f_obs_1_file=data1.mtz f_obs_2_file=data2.sca \
f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 model.pdb
phenix.fobs_minus_fobs_map f_obs_1_file=data.mtz f_obs_2_file=data.mtz \
f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 phase_source=model.pdb \
high_res=2.0 sigma_cutoff=2 scattering_table=neutron"""
command_line = (iotbx_option_parser(
usage="%s [options]" % command_name,
description=examples).option("--silent",
action="store_true",
help="Suppress output to the screen.").
enable_symmetry_comprehensive()).process(args=args)
#
if (log is None):
log = sys.stdout
if (not command_line.options.silent):
utils.print_header("phenix.fobs_minus_fobs_map", out=log)
print("Command line arguments: ", file=log)
print(args, file=log)
print(file=log)
#
processed_args = utils.process_command_line_args(
args=command_line.args,
cmd_cs=command_line.symmetry,
master_params=fo_minus_fo_master_params(),
absolute_angle_tolerance=5,
absolute_length_tolerance=1,
log=log,
suppress_symmetry_related_errors=True)
working_phil = processed_args.params
if (not command_line.options.silent):
print("*** Parameters:", file=log)
working_phil.show(out=log)
print(file=log)
params = working_phil.extract()
consensus_symmetry = None
if (params.ignore_non_isomorphous_unit_cells):
if (None in [
params.f_obs_1_file_name, params.f_obs_2_file_name,
params.phase_source
]):
raise Sorry(
"The file parameters (f_obs_1_file_name, f_obs_2_file_name, " +
"phase_source) must be specified explicitly when " +
"ignore_non_isomorphous_unit_cells=True.")
symm_manager = iotbx.symmetry.manager()
pdb_in = iotbx.file_reader.any_file(params.phase_source,
force_type="pdb")
symm_manager.process_pdb_file(pdb_in)
hkl_in_1 = iotbx.file_reader.any_file(params.f_obs_1_file_name,
force_type="hkl")
sg_err_1, uc_err_1 = symm_manager.process_reflections_file(hkl_in_1)
hkl_in_2 = iotbx.file_reader.any_file(params.f_obs_2_file_name,
force_type="hkl")
sg_err_2, uc_err_2 = symm_manager.process_reflections_file(hkl_in_2)
out = StringIO()
symm_manager.show(out=out)
if (sg_err_1) or (sg_err_2):
raise Sorry((
"Incompatible space groups in input files:\n%s\nAll files " +
"must have the same point group (and ideally the same space group). "
+
"Please note that any symmetry information in the PDB file will be "
+ "used first.") % out.getvalue())
elif (uc_err_1) or (uc_err_2):
libtbx.call_back(
message="warn",
data=
("Crystal symmetry mismatch:\n%s\nCalculations will continue "
+
"using the symmetry in the PDB file (or if not available, the "
+
"first reflection file), but the maps should be treated with "
+ "extreme suspicion.") % out.getvalue())
crystal_symmetry = symm_manager.as_symmetry_object()
else:
processed_args = utils.process_command_line_args(
args=command_line.args,
cmd_cs=command_line.symmetry,
master_params=fo_minus_fo_master_params(),
suppress_symmetry_related_errors=False,
absolute_angle_tolerance=5,
absolute_length_tolerance=1,
log=StringIO())
crystal_symmetry = processed_args.crystal_symmetry
#
pdb_file_names = processed_args.pdb_file_names
if (len(processed_args.pdb_file_names) == 0):
if (params.phase_source is not None):
pdb_file_names = [params.phase_source]
else:
raise Sorry("No PDB file found.")
# Extaract Fobs1, Fobs2
f_obss = []
if (len(processed_args.reflection_files) == 2):
for reflection_file in processed_args.reflection_files:
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=[reflection_file],
err=null_out())
# XXX UGLY !!!
try:
parameters = utils.data_and_flags_master_params().extract()
if (params.f_obs_1_label is not None):
parameters.labels = [params.f_obs_1_label]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server=reflection_file_server,
keep_going=True,
parameters=parameters,
log=null_out())
except: # intentional
parameters = utils.data_and_flags_master_params().extract()
if (params.f_obs_2_label is not None):
parameters.labels = [params.f_obs_2_label]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server=reflection_file_server,
keep_going=True,
parameters=parameters,
log=null_out())
f_obss.append(determine_data_and_flags_result.f_obs)
else:
if ([params.f_obs_1_file_name,
params.f_obs_2_file_name].count(None) == 2):
raise Sorry("No reflection data file found.")
for file_name, label in zip(
[params.f_obs_1_file_name, params.f_obs_2_file_name],
[params.f_obs_1_label, params.f_obs_2_label]):
reflection_file = reflection_file_reader.any_reflection_file(
file_name=file_name, ensure_read_access=False)
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=[reflection_file],
err=null_out())
parameters = utils.data_and_flags_master_params().extract()
if (label is not None):
parameters.labels = [label]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server=reflection_file_server,
parameters=parameters,
keep_going=True,
log=null_out())
f_obss.append(determine_data_and_flags_result.f_obs)
if (len(f_obss) != 2):
raise Sorry(" ".join(errors))
if (not command_line.options.silent):
for ifobs, fobs in enumerate(f_obss):
print("*** Summary for data set %d:" % ifobs, file=log)
fobs.show_comprehensive_summary(f=log)
print(file=log)
pdb_combined = combine_unique_pdb_files(file_names=pdb_file_names)
pdb_combined.report_non_unique(out=log)
if (len(pdb_combined.unique_file_names) == 0):
raise Sorry("No coordinate file given.")
#
raw_recs = flex.std_string()
for rec in pdb_combined.raw_records:
if (rec.upper().count("CRYST1") == 0):
raw_recs.append(rec)
raw_recs.append(
iotbx.pdb.format_cryst1_record(crystal_symmetry=crystal_symmetry))
#
pdb_in = iotbx.pdb.input(source_info=None, lines=raw_recs)
model = mmtbx.model.manager(model_input=pdb_in)
d_min = min(f_obss[0].d_min(), f_obss[1].d_min())
model.setup_scattering_dictionaries(
scattering_table=params.scattering_table, d_min=d_min)
xray_structure = model.get_xray_structure()
hierarchy = model.get_hierarchy()
#
omit_sel = flex.bool(hierarchy.atoms_size(), False)
if (params.advanced.omit_selection is not None):
print("Will omit selection from phasing model:", file=log)
print(" " + params.advanced.omit_selection, file=log)
omit_sel = hierarchy.atom_selection_cache().selection(
params.advanced.omit_selection)
print("%d atoms selected for removal" % omit_sel.count(True), file=log)
del hierarchy
xray_structure = xray_structure.select(~omit_sel)
if (not command_line.options.silent):
print("*** Model summary:", file=log)
xray_structure.show_summary(f=log)
print(file=log)
info0 = f_obss[0].info()
info1 = f_obss[1].info()
f_obss[0] = f_obss[0].resolution_filter(
d_min=params.high_resolution,
d_max=params.low_resolution).set_info(info0)
f_obss[1] = f_obss[1].resolution_filter(
d_min=params.high_resolution,
d_max=params.low_resolution).set_info(info1)
if (params.sigma_cutoff is not None):
for i in [0, 1]:
if (f_obss[i].sigmas() is not None):
sel = f_obss[i].data(
) > f_obss[i].sigmas() * params.sigma_cutoff
f_obss[i] = f_obss[i].select(sel).set_info(info0)
for k, f_obs in enumerate(f_obss):
if (f_obs.indices().size() == 0):
raise Sorry(
"No data left in array %d (labels=%s) after filtering!" %
(k + 1, f_obs.info().label_string()))
output_file_name = params.output_file
if (output_file_name is None) and (params.file_name_prefix is not None):
output_file_name = "%s_%s.mtz" % (params.file_name_prefix,
params.job_id)
output_files = compute_fo_minus_fo_map(
data_arrays=f_obss,
xray_structure=xray_structure,
log=log,
silent=command_line.options.silent,
output_file=output_file_name,
peak_search=params.find_peaks_holes,
map_cutoff=params.map_cutoff,
peak_search_params=params.peak_search,
multiscale=params.advanced.multiscale,
anomalous=params.advanced.anomalous).file_names
return output_files
def run(args):
def vdw_radii_callback(option, opt_str, value, parser):
# create a dict from space separated string of element types and radii
radii = {}
items = value.split()
assert len(items) % 2 == 0
for i in range(int(len(items) / 2)):
radii.setdefault(items[i*2], float(items[i*2+1]))
setattr(parser.values, option.dest, radii)
command_line = (option_parser(
usage="smtbx.masks structure reflections [options]")
.enable_symmetry_comprehensive()
.option(None, "--solvent_radius",
action="store",
type="float",
default=1.3)
.option(None, "--shrink_truncation_radius",
action="store",
type="float",
default=1.3)
.option(None, "--debug",
action="store_true")
.option(None, "--verbose",
action="store_true")
.option(None, "--resolution_factor",
action="store",
type="float",
default=1/4)
.option(None, "--grid_step",
action="store",
type="float")
.option(None, "--d_min",
action="store",
type="float")
.option(None, "--two_theta_max",
action="store",
type="float")
.option(None, "--cb_op",
action="store",
type="string")
.option(None, "--vdw_radii",
action="callback",
callback=vdw_radii_callback,
type="string",
nargs=1)
.option(None, "--use_space_group_symmetry",
action="store_true")).process(args=args)
structure_file = command_line.args[0]
ext = os.path.splitext(structure_file)[-1].lower()
if ext in ('.res', '.ins'):
xs = xray.structure.from_shelx(filename=structure_file)
elif ext == '.cif':
xs = xray.structure.from_cif(filename=structure_file)
else:
print "%s: unsupported structure file format {shelx|cif}" %ext
return
reflections_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = xs.crystal_symmetry(),
reflection_files = [
reflection_file_reader.any_reflection_file(command_line.args[1])
]
)
fo_sq = reflections_server.get_miller_arrays(None)[0]
if command_line.options.cb_op is not None:
cb_op = sgtbx.change_of_basis_op(sgtbx.rt_mx(command_line.options.cb_op))
fo_sq = fo_sq.change_basis(cb_op).customized_copy(
crystal_symmetry=xs)
print "structure file: %s" %command_line.args[0]
print "reflection file: %s" %command_line.args[1]
if command_line.options.debug:
print "debug: %s" %command_line.options.debug
print
xs.show_summary()
print
d_min = command_line.options.d_min
two_theta_max = command_line.options.two_theta_max
assert [d_min, two_theta_max].count(None) > 0
if two_theta_max is not None:
d_min = uctbx.two_theta_as_d(two_theta_max, wavelength=0.71073, deg=True)
exercise_masks(
xs, fo_sq,
solvent_radius=command_line.options.solvent_radius,
shrink_truncation_radius=command_line.options.shrink_truncation_radius,
resolution_factor=command_line.options.resolution_factor,
grid_step=command_line.options.grid_step,
resolution_cutoff=d_min,
atom_radii_table=command_line.options.vdw_radii,
use_space_group_symmetry=command_line.options.use_space_group_symmetry,
debug=command_line.options.debug,
verbose=command_line.options.verbose)
print "OK"
def cmd_run(args, validated=False, out=sys.stdout):
if (len(args) == 0):
print >> out, "-" * 79
print >> out, " phenix.polder"
print >> out, "-" * 79
print >> out, legend
print >> out, "-" * 79
master_params.show(out=out)
return
log = multi_out()
log.register("stdout", out)
log_file_name = "polder.log"
logfile = open(log_file_name, "w")
log.register("logfile", logfile)
print >> log, "phenix.polder is running..."
print >> log, "input parameters:\n", args
parsed = master_params
inputs = mmtbx.utils.process_command_line_args(args=args,
master_params=parsed)
#inputs.params.show() #check
params = inputs.params.extract()
# check model file
if len(inputs.pdb_file_names) == 0:
if (params.model_file_name is None):
raise Sorry("No model file found.")
elif (len(inputs.pdb_file_names) == 1):
params.model_file_name = inputs.pdb_file_names[0]
else:
raise Sorry("Only one model file should be given")
# check reflection file
reflection_files = inputs.reflection_files
if (len(reflection_files) == 0):
if (params.reflection_file_name is None):
raise Sorry("No reflection file found.")
else:
hkl_in = file_reader.any_file(params.reflection_file_name,
force_type="hkl")
hkl_in.assert_file_type("hkl")
reflection_files = [hkl_in.file_object]
# crystal symmetry
crystal_symmetry = None
crystal_symmetry = inputs.crystal_symmetry
if (crystal_symmetry is None):
crystal_symmetries = []
for f in [
str(params.model_file_name),
str(params.reflection_file_name)
]:
cs = crystal_symmetry_from_any.extract_from(f)
if (cs is not None): crystal_symmetries.append(cs)
if (len(crystal_symmetries) == 1):
crystal_symmetry = crystal_symmetries[0]
elif (len(crystal_symmetries) == 0):
raise Sorry("No crystal symmetry found.")
else:
if (not crystal_symmetries[0].is_similar_symmetry(
crystal_symmetries[1])):
raise Sorry(
"Crystal symmetry mismatch between different files.")
crystal_symmetry = crystal_symmetries[0]
f_obs, r_free_flags = None, None
rfs = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=StringIO())
parameters = mmtbx.utils.data_and_flags_master_params().extract()
if (params.data_labels is not None):
parameters.labels = params.data_labels
if (params.r_free_flags_labels is not None):
parameters.r_free_flags.label = params.r_free_flags_labels
determined_data_and_flags = mmtbx.utils.determine_data_and_flags(
reflection_file_server=rfs,
parameters=parameters,
keep_going=True,
log=StringIO())
f_obs = determined_data_and_flags.f_obs
if (params.data_labels is None):
params.data_labels = f_obs.info().label_string()
if (params.reflection_file_name is None):
params.reflection_file_name = parameters.file_name
r_free_flags = determined_data_and_flags.r_free_flags
assert f_obs is not None
print >> log, "Input data:"
print >> log, " Iobs or Fobs:", f_obs.info().labels
if (r_free_flags is not None):
print >> log, " Free-R flags:", r_free_flags.info().labels
params.r_free_flags_labels = r_free_flags.info().label_string()
else:
print >> log, " Free-R flags: Not present"
model_basename = os.path.basename(params.model_file_name.split(".")[0])
if (len(model_basename) > 0 and params.output_file_name_prefix is None):
params.output_file_name_prefix = model_basename
print params.output_file_name_prefix
new_params = master_params.format(python_object=params)
new_params.show()
if (not validated):
validate_params(params)
pdb_input = iotbx.pdb.input(file_name=params.model_file_name)
pdb_hierarchy = pdb_input.construct_hierarchy()
xray_structure = pdb_hierarchy.extract_xray_structure(
crystal_symmetry=crystal_symmetry)
# DON'T USE:
# xray_structure = pdb_input.xray_structure_simple()
# atom order might be wrong
mmtbx.utils.setup_scattering_dictionaries(
scattering_table=params.scattering_table,
xray_structure=xray_structure,
d_min=f_obs.d_min())
#if f_obs is not None:
f_obs = f_obs.resolution_filter(d_min=params.high_resolution,
d_max=params.low_resolution)
if (r_free_flags is not None):
r_free_flags = r_free_flags.resolution_filter(
d_min=params.high_resolution, d_max=params.low_resolution)
# Grab case that data are anomalous
if (f_obs.anomalous_flag()):
f_obs, r_free_flags = prepare_f_obs_and_flags(
f_obs=f_obs, r_free_flags=r_free_flags)
cpm_obj = compute_polder_map(f_obs=f_obs,
r_free_flags=r_free_flags,
xray_structure=xray_structure,
pdb_hierarchy=pdb_hierarchy,
params=params,
log=log)
# Significance check
fmodel = mmtbx.f_model.manager(f_obs=f_obs,
r_free_flags=r_free_flags,
xray_structure=xray_structure)
fmodel.update_all_scales(remove_outliers=False, fast=True)
f_obs_1 = abs(fmodel.f_model())
fmodel.update_xray_structure(
xray_structure=cpm_obj.xray_structure_noligand,
update_f_calc=True,
update_f_mask=True,
force_update_f_mask=True)
# PVA: do we need it? fmodel.update_all_scales(remove_outliers=False)
f_obs_2 = abs(fmodel.f_model())
xrs_selected = cpm_obj.pdb_hierarchy_selected.extract_xray_structure(
crystal_symmetry=f_obs.crystal_symmetry())
f_calc = f_obs.structure_factors_from_scatterers(
xray_structure=cpm_obj.xray_structure_noligand).f_calc()
f_mask = f_obs.structure_factors_from_map(map=cpm_obj.mask_polder,
use_scale=True,
anomalous_flag=False,
use_sg=False)
def get_poler_diff_map(f_obs):
fmodel = mmtbx.f_model.manager(f_obs=f_obs,
r_free_flags=r_free_flags,
f_calc=f_calc,
f_mask=f_mask)
fmodel.update_all_scales(remove_outliers=False)
mc_diff = map_tools.electron_density_map(
fmodel=fmodel).map_coefficients(map_type="mFo-DFc",
isotropize=True,
fill_missing=False)
fft_map = miller.fft_map(crystal_gridding=cpm_obj.crystal_gridding,
fourier_coefficients=mc_diff)
fft_map.apply_sigma_scaling()
map_data = fft_map.real_map_unpadded()
return mmtbx.utils.extract_box_around_model_and_map(
xray_structure=xrs_selected, map_data=map_data, box_cushion=2.1)
box_1 = get_poler_diff_map(f_obs=f_obs_1)
box_2 = get_poler_diff_map(f_obs=f_obs_2)
box_3 = get_poler_diff_map(f_obs=f_obs)
sites_cart_box = box_1.xray_structure_box.sites_cart()
sel = maptbx.grid_indices_around_sites(
unit_cell=box_1.xray_structure_box.unit_cell(),
fft_n_real=box_1.map_box.focus(),
fft_m_real=box_1.map_box.all(),
sites_cart=sites_cart_box,
site_radii=flex.double(sites_cart_box.size(), 2.0))
b1 = box_1.map_box.select(sel).as_1d()
b2 = box_2.map_box.select(sel).as_1d()
b3 = box_3.map_box.select(sel).as_1d()
print >> log, "Map 1: calculated Fobs with ligand"
print >> log, "Map 2: calculated Fobs without ligand"
print >> log, "Map 3: real Fobs data"
print >> log, "CC(1,2): %6.4f" % flex.linear_correlation(
x=b1, y=b2).coefficient()
print >> log, "CC(1,3): %6.4f" % flex.linear_correlation(
x=b1, y=b3).coefficient()
print >> log, "CC(2,3): %6.4f" % flex.linear_correlation(
x=b2, y=b3).coefficient()
### D-function
b1 = maptbx.volume_scale_1d(map=b1, n_bins=10000).map_data()
b2 = maptbx.volume_scale_1d(map=b2, n_bins=10000).map_data()
b3 = maptbx.volume_scale_1d(map=b3, n_bins=10000).map_data()
print >> log, "Peak CC:"
print >> log, "CC(1,2): %6.4f" % flex.linear_correlation(
x=b1, y=b2).coefficient()
print >> log, "CC(1,3): %6.4f" % flex.linear_correlation(
x=b1, y=b3).coefficient()
print >> log, "CC(2,3): %6.4f" % flex.linear_correlation(
x=b2, y=b3).coefficient()
cutoffs = flex.double([i / 10. for i in range(1, 10)] +
[i / 100 for i in range(91, 100)])
d12 = maptbx.discrepancy_function(map_1=b1, map_2=b2, cutoffs=cutoffs)
d13 = maptbx.discrepancy_function(map_1=b1, map_2=b3, cutoffs=cutoffs)
d23 = maptbx.discrepancy_function(map_1=b2, map_2=b3, cutoffs=cutoffs)
print >> log, "q D(1,2) D(1,3) D(2,3)"
for c, d12_, d13_, d23_ in zip(cutoffs, d12, d13, d23):
print >> log, "%4.2f %6.4f %6.4f %6.4f" % (c, d12_, d13_, d23_)
###
if (params.debug):
box_1.write_ccp4_map(file_name="box_1_polder.ccp4")
box_2.write_ccp4_map(file_name="box_2_polder.ccp4")
box_3.write_ccp4_map(file_name="box_3_polder.ccp4")
cpm_obj.pdb_hierarchy_selected.adopt_xray_structure(
box_1.xray_structure_box)
cpm_obj.pdb_hierarchy_selected.write_pdb_file(
file_name="box_polder.pdb",
crystal_symmetry=box_1.box_crystal_symmetry)
#
polder_file_name = "polder_map_coeffs.mtz"
if (params.output_file_name_prefix is not None):
polder_file_name = params.output_file_name_prefix + "_" + polder_file_name
#
print >> log, '*' * 79
print >> log, 'File %s was written.' % polder_file_name
print >> log, "Finished."
return True
def run(args, command_name="phenix.twin_map_utils"):
log=sys.stdout
params=None
if (len(args) == 0 or "--help" in args or "--h" in args or "-h" in args):
print_help(command_name=command_name)
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="map_coefs")
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown file or keyword:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
"""
new_params = master_params.format(python_object=params)
new_params.show(out=log)
"""
# now get the unit cell from the pdb file
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.twin_utils.input.xray_data.file_name)
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.twin_utils.input.model.file_name)
phil_xs=None
if ([params.twin_utils.input.unit_cell,
params.twin_utils.input.space_group]).count(None)<2:
phil_xs = crystal.symmetry(
unit_cell=params.twin_utils.input.unit_cell,
space_group_info=params.twin_utils.input.space_group )
combined_xs = crystal.select_crystal_symmetry(
None,phil_xs, [pdb_xs],[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.twin_utils.input.unit_cell = combined_xs.unit_cell()
params.twin_utils.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
new_params = master_params.format(python_object=params)
new_params.show(out=log)
if params.twin_utils.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.twin_utils.input.space_group is None:
raise Sorry("space group not specified")
if params.twin_utils.input.xray_data.file_name is None:
raise Sorry("Xray data not specified")
if params.twin_utils.input.model.file_name is None:
raise Sorry("pdb file with model not specified")
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
phil_xs = crystal.symmetry(
unit_cell=params.twin_utils.input.unit_cell,
space_group_info=params.twin_utils.input.space_group )
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = phil_xs,
force_symmetry = True,
reflection_files=[])
miller_array = None
free_flags = None
tmp_params = utils.data_and_flags_master_params().extract()
# insert proper values please
tmp_params.file_name = params.twin_utils.input.xray_data.file_name
tmp_params.labels = params.twin_utils.input.xray_data.obs_labels
tmp_params.r_free_flags.file_name=params.twin_utils.input.xray_data.file_name
tmp_params.r_free_flags.label=params.twin_utils.input.xray_data.free_flag
tmp_object = utils.determine_data_and_flags( reflection_file_server = xray_data_server,
parameters = tmp_params, log=log )
miller_array = tmp_object.extract_data()
if miller_array.is_xray_intensity_array():
miller_array = miller_array.f_sq_as_f()
assert miller_array.is_xray_amplitude_array()
free_flags = tmp_object.extract_flags(data = miller_array)
print >> log
print >> log, "Attempting to extract Free R flags"
free_flags = free_flags.customized_copy( data = flex.bool( free_flags.data()==1 ) )
if free_flags is None:
free_flags = miller_array.generate_r_free_flags(use_lattice_symmetry=True)
assert miller_array.observation_type() is not None
print >> log
print >> log, "Summary info of observed data"
print >> log, "============================="
miller_array.show_summary(f=log)
print >> log
if miller_array.indices().size() == 0:
raise Sorry("No data available")
#----------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
model = pdb.input(file_name=params.twin_utils.input.model.file_name).xray_structure_simple(
crystal_symmetry=phil_xs)
print >> log, "Atomic model summary"
print >> log, "===================="
model.show_summary(f=log)
print >> log
#----------------------------------------------------------------
# step 3: get the twin laws for this xs
twin_laws = twin_analyses.twin_laws(
miller_array,
lattice_symmetry_max_delta=\
params.twin_utils.parameters.twinning.max_delta,
out=log)
print >> log
print >> log, "Preliminary data analyses"
print >> log, "=========================="
twin_laws.show(out=log)
#---------
# step 3:
# make twin model managers for all twin laws
print >> log
print >> log, "Overall and bulk solvent scale paranmeters and twin fraction estimation"
print >> log, "======================================================================="
twin_models = []
operator_count = 0
if params.twin_utils.parameters.twinning.twin_law is not None:
tmp_law = sgtbx.rt_mx( params.twin_utils.parameters.twinning.twin_law )
tmp_law = twin_analyses.twin_law(tmp_law,None,None,None,None,None)
twin_laws.operators = [ tmp_law ]
for twin_law in twin_laws.operators:
operator_count += 1
operator_hkl = sgtbx.change_of_basis_op( twin_law.operator ).as_hkl()
twin_model = twin_f_model.twin_model_manager(
f_obs=miller_array,
r_free_flags = free_flags,
xray_structure=model,
twin_law = twin_law.operator,
detwin_mode = params.twin_utils.parameters.twinning.detwin_mode,
out=log)
print >> log, "--- bulk solvent scaling ---"
twin_model.update_solvent_and_scale(update_f_part1=False)
twin_model.r_values()
twin_model.target()
twin_model.show_k_sol_b_sol_b_cart_target()
twin_model.show_essential()
wfofc = twin_model.map_coefficients(map_type="mFo-DFc" )
wtfofc = twin_model.map_coefficients(map_type="2mFo-DFc" )
grad = twin_model.map_coefficients(map_type="gradient" )
mtz_dataset = wtfofc.as_mtz_dataset(
column_root_label="FWT")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = wfofc,
column_root_label = "DFWT"
)
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = grad,
column_root_label = "GRAD"
)
name = params.twin_utils.output.map_coeffs_root+"_"+str(operator_count)+".mtz"
print >> log
print >> log, "Writing %s for twin law %s"%(name,operator_hkl)
print >> log
mtz_dataset.mtz_object().write(
file_name=name)
if params.twin_utils.output.obs_and_calc is not None:
# i want also a Fobs and Fmodel combined dataset please
mtz_dataset = miller_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = twin_model.f_model(),
column_root_label="FMODEL")
name = params.twin_utils.output.obs_and_calc
mtz_dataset.mtz_object().write(
file_name=name)
if len(twin_laws.operators)==0:
print >> log
print >> log, "No twin laws were found"
print >> log, "Performing maximum likelihood based bulk solvent scaling"
f_model_object = f_model.manager(
f_obs = miller_array,
r_free_flags = free_flags,
xray_structure = model )
f_model_object.update_solvent_and_scale(out=log)
tfofc = f_model_object.map_coefficients(map_type="2mFobs-DFmodel")
fofc = f_model_object.map_coefficients(map_type="mFobs-DFmodel")
mtz_dataset = tfofc.as_mtz_dataset(
column_root_label="FWT")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = fofc,
column_root_label = "DELFWT"
)
name = params.twin_utils.output.map_coeffs_root+"_ML.mtz"
mtz_dataset.mtz_object().write(
file_name=name)
if params.twin_utils.output.obs_and_calc is not None:
# i want also a Fobs and Fmodel combined dataset please
mtz_dataset = miller_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = f_model_object.f_model(),
column_root_label="FMODEL")
name = params.twin_utils.output.obs_and_calc
mtz_dataset.mtz_object().write(
file_name=name)
print >> log
print >> log
print >> log, "All done \n"
logfile = open(params.twin_utils.output.logfile,'w')
print >> logfile, string_buffer.getvalue()
print >> log
def __init__ (self,
args,
master_phil,
out=sys.stdout,
process_pdb_file=True,
require_data=True,
create_fmodel=True,
prefer_anomalous=None,
force_non_anomalous=False,
set_wavelength_from_model_header=False,
set_inelastic_form_factors=None,
usage_string=None,
create_log_buffer=False,
remove_unknown_scatterers=False,
generate_input_phil=False) :
import mmtbx.monomer_library.pdb_interpretation
import mmtbx.monomer_library.server
import mmtbx.utils
from iotbx import crystal_symmetry_from_any
from iotbx import file_reader
import iotbx.phil
if generate_input_phil :
assert isinstance(master_phil, basestring)
master_phil = generate_master_phil_with_inputs(phil_string=master_phil)
if isinstance(master_phil, str) :
master_phil = iotbx.phil.parse(master_phil)
if (usage_string is not None) :
if (len(args) == 0) or ("--help" in args) :
raise Usage("""%s\n\nFull parameters:\n%s""" % (usage_string,
master_phil.as_str(prefix=" ")))
if (force_non_anomalous) :
assert (not prefer_anomalous)
assert (set_inelastic_form_factors in [None, "sasaki", "henke"])
self.args = args
self.master_phil = master_phil
self.processed_pdb_file = self.pdb_inp = None
self.pdb_hierarchy = self.xray_structure = None
self.geometry = None
self.sequence = None
self.fmodel = None
self.f_obs = None
self.r_free_flags = None
self.intensity_flag = None
self.raw_data = None
self.raw_flags = None
self.test_flag_value = None
self.miller_arrays = None
self.hl_coeffs = None
self.cif_objects = []
self.log = out
if ("--quiet" in args) or ("quiet=True" in args) :
self.log = null_out()
elif create_log_buffer :
self.log = multi_out()
self.log.register(label="stdout", file_object=out)
self.log.register(label="log_buffer", file_object=StringIO())
make_header("Collecting inputs", out=self.log)
cmdline = iotbx.phil.process_command_line_with_files(
args=args,
master_phil=master_phil,
pdb_file_def="input.pdb.file_name",
reflection_file_def="input.xray_data.file_name",
cif_file_def="input.monomers.file_name",
seq_file_def="input.sequence")
self.working_phil = cmdline.work
params = self.working_phil.extract()
if len(params.input.pdb.file_name) == 0 :
raise Sorry("At least one PDB file is required as input.")
self.cif_file_names = params.input.monomers.file_name
self.pdb_file_names = params.input.pdb.file_name
# SYMMETRY HANDLING - PDB FILES
self.crystal_symmetry = pdb_symm = None
for pdb_file_name in params.input.pdb.file_name :
pdb_symm = crystal_symmetry_from_any.extract_from(pdb_file_name)
if (pdb_symm is not None) :
break
# DATA INPUT
data_and_flags = hkl_symm = hkl_in = None
if (params.input.xray_data.file_name is None) :
if (require_data) :
raise Sorry("At least one reflections file is required as input.")
else :
# FIXME this may still require that the data file has full crystal
# symmetry defined (although for MTZ input this will not be a problem)
make_sub_header("Processing X-ray data", out=self.log)
hkl_in = file_reader.any_file(params.input.xray_data.file_name)
hkl_in.check_file_type("hkl")
hkl_server = hkl_in.file_server
symm = hkl_server.miller_arrays[0].crystal_symmetry()
if ((symm is None) or
(symm.space_group() is None) or
(symm.unit_cell() is None)) :
if (pdb_symm is not None) :
from iotbx.reflection_file_utils import reflection_file_server
print >> self.log, \
"No symmetry in X-ray data file - using PDB symmetry:"
pdb_symm.show_summary(f=out, prefix=" ")
hkl_server = reflection_file_server(
crystal_symmetry=pdb_symm,
reflection_files=[hkl_in.file_object])
else :
raise Sorry("No crystal symmetry information found in input files.")
if (hkl_server is None) :
hkl_server = hkl_in.file_server
data_and_flags = mmtbx.utils.determine_data_and_flags(
reflection_file_server=hkl_server,
parameters=params.input.xray_data,
data_parameter_scope="input.xray_data",
flags_parameter_scope="input.xray_data.r_free_flags",
prefer_anomalous=prefer_anomalous,
force_non_anomalous=force_non_anomalous,
log=self.log)
self.intensity_flag = data_and_flags.intensity_flag
self.raw_data = data_and_flags.raw_data
self.raw_flags = data_and_flags.raw_flags
self.test_flag_value = data_and_flags.test_flag_value
self.f_obs = data_and_flags.f_obs
self.r_free_flags = data_and_flags.r_free_flags
self.miller_arrays = hkl_in.file_server.miller_arrays
hkl_symm = self.raw_data.crystal_symmetry()
if len(self.cif_file_names) > 0 :
for file_name in self.cif_file_names :
cif_obj = mmtbx.monomer_library.server.read_cif(file_name=file_name)
self.cif_objects.append((file_name, cif_obj))
# SYMMETRY HANDLING - COMBINED
if (hkl_symm is not None) :
use_symmetry = hkl_symm
from iotbx.symmetry import combine_model_and_data_symmetry
self.crystal_symmetry = combine_model_and_data_symmetry(
model_symmetry=pdb_symm,
data_symmetry=hkl_symm)
if (self.crystal_symmetry is not None) and (self.f_obs is not None) :
self.f_obs = self.f_obs.customized_copy(
crystal_symmetry=self.crystal_symmetry).eliminate_sys_absent().set_info(
self.f_obs.info())
self.r_free_flags = self.r_free_flags.customized_copy(
crystal_symmetry=self.crystal_symmetry).eliminate_sys_absent().set_info(
self.r_free_flags.info())
# EXPERIMENTAL PHASES
target_name = "ml"
if hasattr(params.input, "experimental_phases") :
flag = params.input.use_experimental_phases
if (flag in [True, Auto]) :
phases_file = params.input.experimental_phases.file_name
if (phases_file is None) :
phases_file = params.input.xray_data.file_name
phases_in = hkl_in
else :
phases_in = file_reader.any_file(phases_file)
phases_in.check_file_type("hkl")
phases_in.file_server.err = self.log # redirect error output
space_group = self.crystal_symmetry.space_group()
point_group = space_group.build_derived_point_group()
hl_coeffs = mmtbx.utils.determine_experimental_phases(
reflection_file_server = phases_in.file_server,
parameters = params.input.experimental_phases,
log = self.log,
parameter_scope = "input.experimental_phases",
working_point_group = point_group,
symmetry_safety_check = True)
if (hl_coeffs is not None) :
hl_coeffs = hl_coeffs.map_to_asu()
if hl_coeffs.anomalous_flag() :
if (not self.f_obs.anomalous_flag()) :
hl_coeffs = hl_coeffs.average_bijvoet_mates()
elif self.f_obs.anomalous_flag() :
hl_coeffs = hl_coeffs.generate_bijvoet_mates()
self.hl_coeffs = hl_coeffs.matching_set(other=self.f_obs,
data_substitute=(0,0,0,0))
target_name = "mlhl"
# PDB INPUT
self.unknown_residues_flag = False
self.unknown_residues_error_message = False
if process_pdb_file :
pdb_interp_params = getattr(params, "pdb_interpretation", None)
if (pdb_interp_params is None) :
pdb_interp_params = \
mmtbx.monomer_library.pdb_interpretation.master_params.extract()
make_sub_header("Processing PDB file(s)", out=self.log)
pdb_combined = mmtbx.utils.combine_unique_pdb_files(
file_names=params.input.pdb.file_name,)
pdb_combined.report_non_unique(out=self.log)
pdb_raw_records = pdb_combined.raw_records
processed_pdb_files_srv = mmtbx.utils.process_pdb_file_srv(
cif_objects=self.cif_objects,
pdb_interpretation_params=pdb_interp_params,
crystal_symmetry=self.crystal_symmetry,
use_neutron_distances=params.input.scattering_table=="neutron",
stop_for_unknowns=getattr(pdb_interp_params, "stop_for_unknowns",False),
log=self.log)
self.processed_pdb_file, self.pdb_inp = \
processed_pdb_files_srv.process_pdb_files(
raw_records = pdb_raw_records,
stop_if_duplicate_labels = False,
allow_missing_symmetry=\
(self.crystal_symmetry is None) and (not require_data))
error_msg = self.processed_pdb_file.all_chain_proxies.\
fatal_problems_message(
ignore_unknown_scattering_types=False,
ignore_unknown_nonbonded_energy_types=False)
if (error_msg is not None) :
self.unknown_residues_flag = True
self.unknown_residues_error_message = error_msg
self.geometry = self.processed_pdb_file.geometry_restraints_manager(
show_energies=False)
assert (self.geometry is not None)
self.xray_structure = self.processed_pdb_file.xray_structure()
chain_proxies = self.processed_pdb_file.all_chain_proxies
self.pdb_hierarchy = chain_proxies.pdb_hierarchy
else :
pdb_file_object = mmtbx.utils.pdb_file(
pdb_file_names=params.input.pdb.file_name,
cif_objects=self.cif_objects,
crystal_symmetry=self.crystal_symmetry,
log=self.log)
self.pdb_inp = pdb_file_object.pdb_inp
self.pdb_hierarchy = self.pdb_inp.construct_hierarchy()
if (remove_unknown_scatterers) :
known_sel = self.pdb_hierarchy.atom_selection_cache().selection(
"not element X")
if (known_sel.count(True) != len(known_sel)) :
self.pdb_hierarchy = self.pdb_hierarchy.select(known_sel)
self.xray_structure = self.pdb_hierarchy.extract_xray_structure(
crystal_symmetry=self.crystal_symmetry)
self.pdb_hierarchy.atoms().reset_i_seq()
if (self.xray_structure is None) :
self.xray_structure = self.pdb_inp.xray_structure_simple(
crystal_symmetry=self.crystal_symmetry)
# wavelength
if (params.input.energy is not None) :
if (params.input.wavelength is not None) :
raise Sorry("Both wavelength and energy have been specified!")
params.input.wavelength = 12398.424468024265 / params.input.energy
if (set_wavelength_from_model_header and params.input.wavelength is None) :
wavelength = self.pdb_inp.extract_wavelength()
if (wavelength is not None) :
print >> self.log, ""
print >> self.log, "Using wavelength = %g from PDB header" % wavelength
params.input.wavelength = wavelength
# set scattering table
if (data_and_flags is not None) :
self.xray_structure.scattering_type_registry(
d_min=self.f_obs.d_min(),
table=params.input.scattering_table)
if ((params.input.wavelength is not None) and
(set_inelastic_form_factors is not None)) :
self.xray_structure.set_inelastic_form_factors(
photon=params.input.wavelength,
table=set_inelastic_form_factors)
make_sub_header("xray_structure summary", out=self.log)
self.xray_structure.scattering_type_registry().show(out = self.log)
self.xray_structure.show_summary(f=self.log)
# FMODEL SETUP
if (create_fmodel) and (data_and_flags is not None) :
make_sub_header("F(model) initialization", out=self.log)
skip_twin_detection = getattr(params.input, "skip_twin_detection", None)
twin_law = getattr(params.input, "twin_law", None)
if (twin_law is Auto) :
if (self.hl_coeffs is not None) :
raise Sorry("Automatic twin law determination not supported when "+
"experimental phases are used.")
elif (skip_twin_detection is not None) :
twin_law = Auto
if (twin_law is Auto) :
print >> self.log, "Twinning will be detected automatically."
self.fmodel = mmtbx.utils.fmodel_simple(
xray_structures=[self.xray_structure],
scattering_table=params.input.scattering_table,
f_obs=self.f_obs,
r_free_flags=self.r_free_flags,
skip_twin_detection=skip_twin_detection,
target_name=target_name,
log=self.log)
else :
if ((twin_law is not None) and (self.hl_coeffs is not None)) :
raise Sorry("Automatic twin law determination not supported when "+
"experimental phases are used.")
self.fmodel = mmtbx.utils.fmodel_manager(
f_obs=self.f_obs,
xray_structure=self.xray_structure,
r_free_flags=self.r_free_flags,
twin_law=params.input.twin_law,
hl_coeff=self.hl_coeffs,
target_name=target_name)
self.fmodel.update_all_scales(
params=None,
log=self.log,
optimize_mask=True,
show=True)
self.fmodel.info().show_rfactors_targets_scales_overall(out=self.log)
# SEQUENCE
if (params.input.sequence is not None) :
seq_file = file_reader.any_file(params.input.sequence,
force_type="seq",
raise_sorry_if_errors=True)
self.sequence = seq_file.file_object
# UNMERGED DATA
self.unmerged_i_obs = None
if hasattr(params.input, "unmerged_data") :
if (params.input.unmerged_data.file_name is not None) :
self.unmerged_i_obs = load_and_validate_unmerged_data(
f_obs=self.f_obs,
file_name=params.input.unmerged_data.file_name,
data_labels=params.input.unmerged_data.labels,
log=self.log)
self.params = params
print >> self.log, ""
print >> self.log, "End of input processing"
def reflection_file_server(crystal_symmetry, reflection_files):
return reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=StringIO())
def cmd_run(args, validated=False, out=sys.stdout):
if (len(args) == 0):
print >> out, "-"*79
print >> out, " phenix.polder"
print >> out, "-"*79
print >> out, legend
print >> out, "-"*79
master_params.show(out=out)
return
log = multi_out()
log.register("stdout", out)
log_file_name = "polder.log"
logfile = open(log_file_name, "w")
log.register("logfile", logfile)
print >> log, "phenix.polder is running..."
print >> log, "input parameters:\n", args
parsed = master_params
inputs = mmtbx.utils.process_command_line_args(args = args,
master_params = parsed)
#inputs.params.show() #check
params = inputs.params.extract()
# check model file
if len(inputs.pdb_file_names) == 0:
if (params.model_file_name is None):
raise Sorry("No model file found.")
elif (len(inputs.pdb_file_names) == 1):
params.model_file_name = inputs.pdb_file_names[0]
else:
raise Sorry("Only one model file should be given")
# check reflection file
reflection_files = inputs.reflection_files
if (len(reflection_files) == 0):
if (params.reflection_file_name is None):
raise Sorry("No reflection file found.")
else:
hkl_in = file_reader.any_file(params.reflection_file_name,
force_type="hkl")
hkl_in.assert_file_type("hkl")
reflection_files = [ hkl_in.file_object ]
# crystal symmetry
crystal_symmetry = None
crystal_symmetry = inputs.crystal_symmetry
if (crystal_symmetry is None):
crystal_symmetries = []
for f in [str(params.model_file_name), str(params.reflection_file_name)]:
cs = crystal_symmetry_from_any.extract_from(f)
if(cs is not None): crystal_symmetries.append(cs)
if(len(crystal_symmetries) == 1): crystal_symmetry = crystal_symmetries[0]
elif(len(crystal_symmetries) == 0):
raise Sorry("No crystal symmetry found.")
else:
if(not crystal_symmetries[0].is_similar_symmetry(crystal_symmetries[1])):
raise Sorry("Crystal symmetry mismatch between different files.")
crystal_symmetry = crystal_symmetries[0]
f_obs, r_free_flags = None, None
rfs = reflection_file_utils.reflection_file_server(
crystal_symmetry = crystal_symmetry,
force_symmetry = True,
reflection_files = reflection_files,
err = StringIO())
parameters = mmtbx.utils.data_and_flags_master_params().extract()
if (params.data_labels is not None):
parameters.labels = params.data_labels
if (params.r_free_flags_labels is not None):
parameters.r_free_flags.label = params.r_free_flags_labels
determined_data_and_flags = mmtbx.utils.determine_data_and_flags(
reflection_file_server = rfs,
parameters = parameters,
keep_going = True,
log = StringIO())
f_obs = determined_data_and_flags.f_obs
if (params.data_labels is None):
params.data_labels = f_obs.info().label_string()
if (params.reflection_file_name is None):
params.reflection_file_name = parameters.file_name
r_free_flags = determined_data_and_flags.r_free_flags
assert f_obs is not None
print >> log, "Input data:"
print >> log, " Iobs or Fobs:", f_obs.info().labels
if (r_free_flags is not None):
print >> log, " Free-R flags:", r_free_flags.info().labels
params.r_free_flags_labels = r_free_flags.info().label_string()
else:
print >> log, " Free-R flags: Not present"
model_basename = os.path.basename(params.model_file_name.split(".")[0])
if (len(model_basename) > 0 and
params.output_file_name_prefix is None):
params.output_file_name_prefix = model_basename
print params.output_file_name_prefix
new_params = master_params.format(python_object=params)
new_params.show()
if (not validated):
validate_params(params)
pdb_input = iotbx.pdb.input(file_name = params.model_file_name)
pdb_hierarchy = pdb_input.construct_hierarchy()
xray_structure = pdb_hierarchy.extract_xray_structure(
crystal_symmetry = crystal_symmetry)
# DON'T USE:
# xray_structure = pdb_input.xray_structure_simple()
# atom order might be wrong
mmtbx.utils.setup_scattering_dictionaries(
scattering_table = params.scattering_table,
xray_structure = xray_structure,
d_min = f_obs.d_min())
#if f_obs is not None:
f_obs = f_obs.resolution_filter(
d_min = params.high_resolution,
d_max = params.low_resolution)
if (r_free_flags is not None):
r_free_flags = r_free_flags.resolution_filter(
d_min = params.high_resolution,
d_max = params.low_resolution)
# Grab case that data are anomalous
if (f_obs.anomalous_flag()):
f_obs, r_free_flags = prepare_f_obs_and_flags(
f_obs = f_obs,
r_free_flags = r_free_flags)
cpm_obj = compute_polder_map(
f_obs = f_obs,
r_free_flags = r_free_flags,
xray_structure = xray_structure,
pdb_hierarchy = pdb_hierarchy,
params = params,
log = log)
# Significance check
fmodel = mmtbx.f_model.manager(
f_obs = f_obs,
r_free_flags = r_free_flags,
xray_structure = xray_structure)
fmodel.update_all_scales(remove_outliers=False, fast=True)
f_obs_1 = abs(fmodel.f_model())
fmodel.update_xray_structure(xray_structure=cpm_obj.xray_structure_noligand,
update_f_calc=True, update_f_mask=True, force_update_f_mask=True)
# PVA: do we need it? fmodel.update_all_scales(remove_outliers=False)
f_obs_2 = abs(fmodel.f_model())
xrs_selected = cpm_obj.pdb_hierarchy_selected.extract_xray_structure(
crystal_symmetry = f_obs.crystal_symmetry())
f_calc = f_obs.structure_factors_from_scatterers(
xray_structure = cpm_obj.xray_structure_noligand).f_calc()
f_mask = f_obs.structure_factors_from_map(
map = cpm_obj.mask_polder,
use_scale = True,
anomalous_flag = False,
use_sg = False)
def get_poler_diff_map(f_obs):
fmodel = mmtbx.f_model.manager(
f_obs = f_obs,
r_free_flags = r_free_flags,
f_calc = f_calc,
f_mask = f_mask)
fmodel.update_all_scales(remove_outliers=False)
mc_diff = map_tools.electron_density_map(
fmodel = fmodel).map_coefficients(
map_type = "mFo-DFc",
isotropize = True,
fill_missing = False)
fft_map = miller.fft_map(
crystal_gridding = cpm_obj.crystal_gridding,
fourier_coefficients = mc_diff)
fft_map.apply_sigma_scaling()
map_data = fft_map.real_map_unpadded()
return mmtbx.utils.extract_box_around_model_and_map(
xray_structure = xrs_selected,
map_data = map_data,
box_cushion = 2.1)
box_1=get_poler_diff_map(f_obs = f_obs_1)
box_2=get_poler_diff_map(f_obs = f_obs_2)
box_3=get_poler_diff_map(f_obs = f_obs)
sites_cart_box = box_1.xray_structure_box.sites_cart()
sel = maptbx.grid_indices_around_sites(
unit_cell = box_1.xray_structure_box.unit_cell(),
fft_n_real = box_1.map_box.focus(),
fft_m_real = box_1.map_box.all(),
sites_cart = sites_cart_box,
site_radii = flex.double(sites_cart_box.size(), 2.0))
b1 = box_1.map_box.select(sel).as_1d()
b2 = box_2.map_box.select(sel).as_1d()
b3 = box_3.map_box.select(sel).as_1d()
print >> log, "Map 1: calculated Fobs with ligand"
print >> log, "Map 2: calculated Fobs without ligand"
print >> log, "Map 3: real Fobs data"
print >>log, "CC(1,2): %6.4f"%flex.linear_correlation(x=b1,y=b2).coefficient()
print >>log, "CC(1,3): %6.4f"%flex.linear_correlation(x=b1,y=b3).coefficient()
print >>log, "CC(2,3): %6.4f"%flex.linear_correlation(x=b2,y=b3).coefficient()
### D-function
b1 = maptbx.volume_scale_1d(map=b1, n_bins=10000).map_data()
b2 = maptbx.volume_scale_1d(map=b2, n_bins=10000).map_data()
b3 = maptbx.volume_scale_1d(map=b3, n_bins=10000).map_data()
print >> log, "Peak CC:"
print >>log, "CC(1,2): %6.4f"%flex.linear_correlation(x=b1,y=b2).coefficient()
print >>log, "CC(1,3): %6.4f"%flex.linear_correlation(x=b1,y=b3).coefficient()
print >>log, "CC(2,3): %6.4f"%flex.linear_correlation(x=b2,y=b3).coefficient()
cutoffs = flex.double(
[i/10. for i in range(1,10)]+[i/100 for i in range(91,100)])
d12 = maptbx.discrepancy_function(map_1=b1, map_2=b2, cutoffs=cutoffs)
d13 = maptbx.discrepancy_function(map_1=b1, map_2=b3, cutoffs=cutoffs)
d23 = maptbx.discrepancy_function(map_1=b2, map_2=b3, cutoffs=cutoffs)
print >> log, "q D(1,2) D(1,3) D(2,3)"
for c,d12_,d13_,d23_ in zip(cutoffs,d12,d13,d23):
print >> log, "%4.2f %6.4f %6.4f %6.4f"%(c,d12_,d13_,d23_)
###
if(params.debug):
box_1.write_ccp4_map(file_name="box_1_polder.ccp4")
box_2.write_ccp4_map(file_name="box_2_polder.ccp4")
box_3.write_ccp4_map(file_name="box_3_polder.ccp4")
cpm_obj.pdb_hierarchy_selected.adopt_xray_structure(
box_1.xray_structure_box)
cpm_obj.pdb_hierarchy_selected.write_pdb_file(file_name="box_polder.pdb",
crystal_symmetry=box_1.box_crystal_symmetry)
#
print >> log, "Finished."
return True
def simul_utils(args):
if len(args) == 0:
print_help()
elif ("--help" in args):
print_help()
elif ("--h" in args):
print_help()
elif ("-h" in args):
print_help()
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="map_coefs")
print("#phil __OFF__", file=log)
print("======================", file=log)
print(" SIMUL ", file=log)
print("A data simulation tool", file=log)
print("======================", file=log)
print(file=log)
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
if not arg_is_processed:
print("##----------------------------------------------##",
file=log)
print("## Unknown file or keyword:", arg, file=log)
print("##----------------------------------------------##",
file=log)
print(file=log)
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
"""
new_params = master_params.format(python_object=params)
new_params.show(out=log)
"""
# now get the unit cell from the pdb file
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.simul_utils.input.xray_data.file_name)
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.simul_utils.input.model.file_name)
phil_xs = crystal.symmetry(
unit_cell=params.simul_utils.input.unit_cell,
space_group_info=params.simul_utils.input.space_group)
combined_xs = select_crystal_symmetry(None, phil_xs, [pdb_xs],
[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.simul_utils.input.unit_cell = combined_xs.unit_cell()
params.simul_utils.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
print("#phil __ON__", file=log)
new_params = master_params.format(python_object=params)
new_params.show(out=log)
print("#phil __END__", file=log)
if params.simul_utils.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.simul_utils.input.space_group is None:
raise Sorry("space group not specified")
if params.simul_utils.input.xray_data.file_name is None:
raise Sorry("Xray data not specified")
if params.simul_utils.input.model.file_name is None:
raise Sorry("pdb file with model not specified")
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
phil_xs = crystal.symmetry(
unit_cell=params.simul_utils.input.unit_cell,
space_group_info=params.simul_utils.input.space_group)
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=phil_xs, force_symmetry=True, reflection_files=[])
miller_array = None
miller_array = xray_data_server.get_xray_data(
file_name=params.simul_utils.input.xray_data.file_name,
labels=params.simul_utils.input.xray_data.labels,
ignore_all_zeros=True,
parameter_scope='simul_utils.input.xray_data',
parameter_name='labels')
info = miller_array.info()
miller_array = miller_array.map_to_asu()
miller_array = miller_array.select(miller_array.indices() != (0, 0, 0))
miller_array = miller_array.select(miller_array.data() > 0)
if miller_array.sigmas() is not None:
miller_array = miller_array.select(miller_array.sigmas() > 0)
if (miller_array.is_xray_intensity_array()):
miller_array = miller_array.f_sq_as_f()
elif (miller_array.is_complex_array()):
miller_array = abs(miller_array)
miller_array.set_info(info)
print(file=log)
print("Summary info of observed data", file=log)
print("=============================", file=log)
miller_array.show_summary(f=log)
print(file=log)
free_flags = miller_array.generate_r_free_flags()
#--------------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
model = pdb.input(file_name=params.simul_utils.input.model.file_name
).xray_structure_simple(crystal_symmetry=phil_xs, )
print("Atomic model summary", file=log)
print("====================", file=log)
model.show_summary()
print(file=log)
#-------------------------------------------------------------------
# Step 3: make an F_model object to get model phases and amplitudes
#
print("Performing bulk solvent scaling", file=log)
print("===============================", file=log)
print(file=log)
print(file=log)
f_model_object = f_model.manager(f_obs=miller_array,
r_free_flags=free_flags,
xray_structure=model)
f_model_object.update_all_scales(log=log)
fmodel = abs(
f_model_object.f_model()).set_observation_type(miller_array)
mockfmodel = None
if params.simul_utils.input.mock_model.file_name is not None:
print("Reading in mock model", file=log)
print("=====================", file=log)
print(file=log)
print(file=log)
mock_model = pdb.input(file_name=params.simul_utils.input.
mock_model.file_name).xray_structure_simple(
crystal_symmetry=phil_xs)
mock_f_model = f_model.manager(f_obs=miller_array,
r_free_flags=free_flags,
xray_structure=mock_model)
mock_f_model.update(k_sol=f_model_object.k_sol(),
b_sol=f_model_object.b_sol(),
b_cart=f_model_object.b_cart())
mockfmodel = abs(
mock_f_model.f_model()).set_observation_type(miller_array)
else:
mockfmodel = fmodel.deep_copy()
print("Making new data", file=log)
print("===============", file=log)
print(file=log)
print(file=log)
new_data_builder = error_swap(miller_array, fmodel, mockfmodel)
new_data = new_data_builder.new_obs
# we now have to write the data actually
print("Writing new data set", file=log)
print("====================", file=log)
mtz_dataset = new_data.as_mtz_dataset(column_root_label="FOBS")
mtz_dataset.mtz_object().write(
file_name=params.simul_utils.output.hklout)
def run(args):
if len(args)==0:
master_params.show(expert_level=100)
elif ( "--help" in args ):
print "no help available"
elif ( "--h" in args ):
print "no help available"
elif ( "--show_defaults" in args ):
master_params.show(expert_level=0)
elif ( "--show_defaults_all" in args ):
master_params.show(expert_level=10)
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
log_plots = StringIO()
print >> log,"#phil __OFF__"
print >> log
print >> log, date_and_time()
print >> log
print >> log
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="scaling")
reflection_file = None
for arg in args:
command_line_params = None
arg_is_processed = False
if arg == '--quiet':
arg_is_processed = True
## The associated action with this keyword is implemented above
if (os.path.isfile(arg)): ## is this a file name?
## Check if this is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
except KeyboardInterrupt: raise
except Exception : pass
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
## Check if this file is a reflection file
if command_line_params is None:
reflection_file = reflection_file_reader.any_reflection_file(
file_name=arg, ensure_read_access=False)
if (reflection_file is not None):
reflection_file = arg
arg_is_processed = True
## If it is not a file, it must be a phil command
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown phil-file or phil-command:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file format or phil command: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
## Now please read in the reflections files
## get symmetry and cell data first please
## By default, the native cell and symmetry are used
## as reference
crystal_symmetry_nat = None
print params.scaling.input.xray_data.wavelength1.file_name
crystal_symmetry_nat = crystal_symmetry_from_any.extract_from(
file_name=params.scaling.input.xray_data.wavelength1.file_name)
if params.scaling.input.xray_data.space_group is None:
params.scaling.input.xray_data.space_group =\
crystal_symmetry_nat.space_group_info()
print >> log, "Using symmetry of native data"
if params.scaling.input.xray_data.unit_cell is None:
params.scaling.input.xray_data.unit_cell =\
crystal_symmetry_nat.unit_cell()
print >> log, "Using cell of native data"
## Check if a unit cell is defined
if params.scaling.input.xray_data.space_group is None:
raise Sorry("No space group defined")
if params.scaling.input.xray_data.unit_cell is None:
raise Sorry("No unit cell defined")
crystal_symmetry = crystal_symmetry = crystal.symmetry(
unit_cell = params.scaling.input.xray_data.unit_cell,
space_group_symbol = str(
params.scaling.input.xray_data.space_group) )
effective_params = master_params.fetch(sources=phil_objects)
new_params = master_params.format(python_object=params)
print >> log, "Effective parameters"
print >> log, "#phil __ON__"
new_params.show(out=log,expert_level=params.scaling.input.expert_level)
print >> log, "#phil __END__"
print >> log
## define a xray data server
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = crystal_symmetry,
force_symmetry = True,
reflection_files=[])
## Read in native data and make appropriate selections
miller_array_w1 = None
miller_array_w1 = xray_data_server.get_xray_data(
file_name = params.scaling.input.xray_data.wavelength1.file_name,
labels = params.scaling.input.xray_data.wavelength1.labels,
ignore_all_zeros = True,
parameter_scope = 'scaling.input.SIR_scale.xray_data.native'
)
info_native = miller_array_w1.info()
miller_array_w1=miller_array_w1.map_to_asu().select(
miller_array_w1.indices()!=(0,0,0) )
miller_array_w1 = miller_array_w1.select(
miller_array_w1.data() > 0 )
## Convert to amplitudes
if (miller_array_w1.is_xray_intensity_array()):
miller_array_w1 = miller_array_w1.f_sq_as_f()
elif (miller_array_w1.is_complex_array()):
miller_array_w1 = abs(miller_array_w1)
if not miller_array_w1.is_real_array():
raise Sorry("miller_array_native is not a real array")
miller_array_w1.set_info(info = info_native)
## Read in derivative data and make appropriate selections
miller_array_w2 = None
miller_array_w2 = xray_data_server.get_xray_data(
file_name = params.scaling.input.xray_data.wavelength2.file_name,
labels = params.scaling.input.xray_data.wavelength2.labels,
ignore_all_zeros = True,
parameter_scope = 'scaling.input.SIR_scale.xray_data.derivative'
)
info_w2 = miller_array_w2.info()
miller_array_w2=miller_array_w2.map_to_asu().select(
miller_array_w2.indices()!=(0,0,0) )
miller_array_w2 = miller_array_w2.select(
miller_array_w2.data() > 0 )
## Convert to amplitudes
if (miller_array_w2.is_xray_intensity_array()):
miller_array_w2 = miller_array_w2.f_sq_as_f()
elif (miller_array_w2.is_complex_array()):
miller_array_w2 = abs(miller_array_w2)
if not miller_array_w2.is_real_array():
raise Sorry("miller_array_derivative is not a real array")
miller_array_w2.set_info(info = info_w2)
## Make sure we have anomalous diffs in both files
assert miller_array_w1.anomalous_flag()
assert miller_array_w2.anomalous_flag()
## Print info
print >> log
print >> log, "Wavelength 1"
print >> log, "============"
miller_array_w1.show_comprehensive_summary(f=log)
print >> log
w1_pre_scale = pre_scale.pre_scaler(
miller_array_w1,
params.scaling.input.scaling_strategy.pre_scaler_protocol,
params.scaling.input.basic)
miller_array_w1 = w1_pre_scale.x1.deep_copy()
del w1_pre_scale
print >> log
print >> log, "Wavelength 2"
print >> log, "============"
miller_array_w2.show_comprehensive_summary(f=log)
print >> log
w2_pre_scale = pre_scale.pre_scaler(
miller_array_w2,
params.scaling.input.scaling_strategy.pre_scaler_protocol,
params.scaling.input.basic)
miller_array_w2 = w2_pre_scale.x1.deep_copy()
del w2_pre_scale
print >> log
print >> log, "Checking for possible reindexing schemes"
print >> log, "----------------------------------------"
print >> log
print >> log, "Reindexing operator derived as described in:"
print >> log, "Grosse-Kunstleve, Afonine, Sauter & Adams. (2005)."
print >> log, " IUCr Computing Commission Newsletter 5."
print >> log
reindex_object = pair_analyses.reindexing(
set_a=miller_array_w1,
set_b=miller_array_w2,
out=log)
miller_array_w2 = reindex_object.select_and_transform()
miller_array_w2.map_to_asu()
print >> log
print >> log, "Relative scaling of 2-wavelength mad data"
print >> log, "-----------------------------------------"
print >> log
scaler = fa_estimation.combined_scaling(
miller_array_w1,
miller_array_w2,
params.scaling.input.scaling_strategy.iso_protocol)
miller_array_w1 = scaler.x1.deep_copy()
miller_array_w2 = scaler.x2.deep_copy()
del scaler
print >> log
print >> log, "Estimating f\" and f' ratios"
print >> log, "----------------------------"
print >> log
# now things are scaled see if we can guestimate the ratio
fdpratio = pair_analyses.f_double_prime_ratio(
miller_array_w1,
miller_array_w2)
fpfdpratio = pair_analyses.delta_f_prime_f_double_prime_ratio(
miller_array_w1,
miller_array_w2)
k1 = fdpratio.ratio
k2 = fpfdpratio.ratio
if k1 is not None:
print >> log
print >> log, " The estimate of f\"(w1)/f\"(w2) is %3.2f"\
%(fdpratio.ratio)
if k2 is not None:
print >> log, " The estimate of (f'(w1)-f'(w2))/f\"(w2) is %3.2f"\
%(fpfdpratio.ratio)
print >> log
print >> log, " The quality of these estimates depends to a large extend"
print >> log, " on the quality of the data. If user supplied values"
print >> log, " of f\" and f' are given, they will be used instead "
print >> log, " of the estimates."
print >> log
if params.scaling.input.xray_data.wavelength1.f_double_prime is not None:
if params.scaling.input.xray_data.wavelength2.f_double_prime is not None:
k1 = (params.scaling.input.xray_data.wavelength1.f_double_prime/
params.scaling.input.xray_data.wavelength2.f_double_prime)
print >> log, " Using user specified f\" values"
print >> log, " user specified f\"(w1)/f\"(w2) is %3.2f"\
%(k1)
print >> log
if params.scaling.input.xray_data.wavelength1.f_prime is not None:
if params.scaling.input.xray_data.wavelength2.f_prime is not None:
if params.scaling.input.xray_data.wavelength2.f_double_prime is not None:
k2 = (params.scaling.input.xray_data.wavelength1.f_prime-
params.scaling.input.xray_data.wavelength2.f_prime)\
/params.scaling.input.xray_data.wavelength2.f_double_prime
print >> log, " Using user specified f\" and f' values"
print >> log, " user specified f\"(w1)/f\"(w2) is %3.2f"\
%(k2)
print >> log
fa_gen = fa_estimation.twmad_fa_driver(miller_array_w1,
miller_array_w2,
k1,
k2,
params.scaling.input.fa_estimation)
print >> log
print >> log, "writing mtz file"
print >> log, "----------------"
print >> log
## Please write out the abs_delta_f array
fa = fa_gen.fa_values
mtz_dataset = fa.as_mtz_dataset(
column_root_label='F'+params.scaling.input.output.outlabel)
mtz_dataset.mtz_object().write(
file_name=params.scaling.input.output.hklout)
def run(args):
if len(args) == 0:
master_params.show(expert_level=0)
elif ("--help" in args):
print("no help available")
elif ("--h" in args):
print("no help available")
elif ("--show_defaults" in args):
master_params.show(expert_level=0)
elif ("--show_defaults_all" in args):
master_params.show(expert_level=10)
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
log_plots = StringIO()
print("#phil __OFF__", file=log)
print(file=log)
print(date_and_time(), file=log)
print(file=log)
print(file=log)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="scaling")
reflection_file = None
for arg in args:
command_line_params = None
arg_is_processed = False
if arg == '--quiet':
arg_is_processed = True
## The associated action with this keyword is implemented above
if (os.path.isfile(arg)): ## is this a file name?
## Check if this is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
except KeyboardInterrupt:
raise
except Exception:
pass
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
## Check if this file is a reflection file
if command_line_params is None:
reflection_file = reflection_file_reader.any_reflection_file(
file_name=arg, ensure_read_access=False)
if (reflection_file is not None):
reflection_file = arg
arg_is_processed = True
## If it is not a file, it must be a phil command
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
if not arg_is_processed:
print("##----------------------------------------------##",
file=log)
print("## Unknown phil-file or phil-command:", arg, file=log)
print("##----------------------------------------------##",
file=log)
print(file=log)
raise Sorry("Unknown file format or phil command: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
## Now please read in the reflections files
## get symmetry and cell data first please
## By default, the native cell and symmetry are used
## as reference
crystal_symmetry_nat = None
crystal_symmetry_nat = crystal_symmetry_from_any.extract_from(
file_name=params.scaling.input.xray_data.native.file_name)
if params.scaling.input.xray_data.space_group is None:
params.scaling.input.xray_data.space_group =\
crystal_symmetry_nat.space_group_info()
print("Using symmetry of native data", file=log)
if params.scaling.input.xray_data.unit_cell is None:
params.scaling.input.xray_data.unit_cell =\
crystal_symmetry_nat.unit_cell()
print("Using cell of native data", file=log)
## Check if a unit cell is defined
if params.scaling.input.xray_data.space_group is None:
raise Sorry("No space group defined")
if params.scaling.input.xray_data.unit_cell is None:
raise Sorry("No unit cell defined")
crystal_symmetry = crystal_symmetry = crystal.symmetry(
unit_cell=params.scaling.input.xray_data.unit_cell,
space_group_symbol=str(params.scaling.input.xray_data.space_group))
effective_params = master_params.fetch(sources=phil_objects)
new_params = master_params.format(python_object=params)
print("Effective parameters", file=log)
print("#phil __ON__", file=log)
new_params.show(out=log,
expert_level=params.scaling.input.expert_level)
print("#phil __END__", file=log)
print(file=log)
## define a xray data server
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=[])
## Read in native data and make appropriatre selections
miller_array_native = None
miller_array_native = xray_data_server.get_xray_data(
file_name=params.scaling.input.xray_data.native.file_name,
labels=params.scaling.input.xray_data.native.labels,
ignore_all_zeros=True,
parameter_scope='scaling.input.SIR_scale.xray_data.native')
info_native = miller_array_native.info()
miller_array_native = miller_array_native.map_to_asu().select(
miller_array_native.indices() != (0, 0, 0))
miller_array_native = miller_array_native.select(
miller_array_native.data() > 0)
## Convert to amplitudes
if (miller_array_native.is_xray_intensity_array()):
miller_array_native = miller_array_native.f_sq_as_f()
elif (miller_array_native.is_complex_array()):
miller_array_native = abs(miller_array_native)
if not miller_array_native.is_real_array():
raise Sorry("miller_array_native is not a real array")
miller_array_native.set_info(info=info_native)
## Read in derivative data and make appropriate selections
miller_array_derivative = None
miller_array_derivative = xray_data_server.get_xray_data(
file_name=params.scaling.input.xray_data.derivative.file_name,
labels=params.scaling.input.xray_data.derivative.labels,
ignore_all_zeros=True,
parameter_scope='scaling.input.SIR_scale.xray_data.derivative')
info_derivative = miller_array_derivative.info()
miller_array_derivative = miller_array_derivative.map_to_asu().select(
miller_array_derivative.indices() != (0, 0, 0))
miller_array_derivative = miller_array_derivative.select(
miller_array_derivative.data() > 0)
## Convert to amplitudes
if (miller_array_derivative.is_xray_intensity_array()):
miller_array_derivative = miller_array_derivative.f_sq_as_f()
elif (miller_array_derivative.is_complex_array()):
miller_array_derivative = abs(miller_array_derivative)
if not miller_array_derivative.is_real_array():
raise Sorry("miller_array_derivative is not a real array")
miller_array_derivative.set_info(info=info_derivative)
## As this is a SIR case, we will remove any anomalous pairs
if miller_array_derivative.anomalous_flag():
miller_array_derivative = miller_array_derivative.average_bijvoet_mates()\
.set_observation_type( miller_array_derivative )
if miller_array_native.anomalous_flag():
miller_array_native = miller_array_native.average_bijvoet_mates()\
.set_observation_type( miller_array_native )
## Print info
print(file=log)
print("Native data", file=log)
print("===========", file=log)
miller_array_native.show_comprehensive_summary(f=log)
print(file=log)
native_pre_scale = pre_scale.pre_scaler(
miller_array_native,
params.scaling.input.scaling_strategy.pre_scaler_protocol,
params.scaling.input.basic)
miller_array_native = native_pre_scale.x1.deep_copy()
del native_pre_scale
print(file=log)
print("Derivative data", file=log)
print("===============", file=log)
miller_array_derivative.show_comprehensive_summary(f=log)
print(file=log)
derivative_pre_scale = pre_scale.pre_scaler(
miller_array_derivative,
params.scaling.input.scaling_strategy.pre_scaler_protocol,
params.scaling.input.basic)
miller_array_derivative = derivative_pre_scale.x1.deep_copy()
del derivative_pre_scale
scaler = fa_estimation.combined_scaling(
miller_array_native, miller_array_derivative,
params.scaling.input.scaling_strategy.iso_protocol)
miller_array_native = scaler.x1.deep_copy()
miller_array_derivative = scaler.x2.deep_copy()
del scaler
print(file=log)
print("Making delta f's", file=log)
print("----------------", file=log)
print(file=log)
delta_gen = pair_analyses.delta_generator(miller_array_native,
miller_array_derivative)
print(file=log)
print("writing mtz file", file=log)
print("----------------", file=log)
print(file=log)
## some assertions to make sure nothing went weerd
assert miller_array_native.observation_type() is not None
assert miller_array_derivative.observation_type() is not None
assert delta_gen.abs_delta_f.observation_type() is not None
## Please write out the abs_delta_f array
mtz_dataset = delta_gen.abs_delta_f.as_mtz_dataset(
column_root_label='F' + params.scaling.input.output.outlabel)
mtz_dataset.mtz_object().write(
file_name=params.scaling.input.output.hklout)
def run(args, command_name = "phenix.fobs_minus_fobs_map", log=None):
if(len(args) == 0): args = ["--help"]
examples = """Examples:
phenix.fobs_minus_fobs_map f_obs_1_file=data1.mtz f_obs_2_file=data2.sca \
f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 model.pdb
phenix.fobs_minus_fobs_map f_obs_1_file=data.mtz f_obs_2_file=data.mtz \
f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 phase_source=model.pdb \
high_res=2.0 sigma_cutoff=2 scattering_table=neutron"""
command_line = (iotbx_option_parser(
usage="%s [options]" % command_name,
description=examples)
.option("--silent",
action="store_true",
help="Suppress output to the screen.")
.enable_symmetry_comprehensive()
).process(args=args)
#
if (log is None) :
log = sys.stdout
if(not command_line.options.silent):
utils.print_header("phenix.fobs_minus_fobs_map", out = log)
print >> log, "Command line arguments: "
print >> log, args
print >> log
#
processed_args = utils.process_command_line_args(
args=command_line.args,
cmd_cs=command_line.symmetry,
master_params=fo_minus_fo_master_params(),
absolute_angle_tolerance=5,
absolute_length_tolerance=1,
log=log,
suppress_symmetry_related_errors=True)
working_phil = processed_args.params
if(not command_line.options.silent):
print >> log, "*** Parameters:"
working_phil.show(out = log)
print >> log
params = working_phil.extract()
consensus_symmetry = None
if (params.ignore_non_isomorphous_unit_cells) :
if (None in [params.f_obs_1_file_name, params.f_obs_2_file_name,
params.phase_source]):
raise Sorry("The file parameters (f_obs_1_file_name, f_obs_2_file_name, "+
"phase_source) must be specified explicitly when "+
"ignore_non_isomorphous_unit_cells=True.")
symm_manager = iotbx.symmetry.manager()
pdb_in = iotbx.file_reader.any_file(params.phase_source, force_type="pdb")
symm_manager.process_pdb_file(pdb_in)
hkl_in_1 = iotbx.file_reader.any_file(params.f_obs_1_file_name,
force_type="hkl")
sg_err_1, uc_err_1 = symm_manager.process_reflections_file(hkl_in_1)
hkl_in_2 = iotbx.file_reader.any_file(params.f_obs_2_file_name,
force_type="hkl")
sg_err_2, uc_err_2 = symm_manager.process_reflections_file(hkl_in_2)
out = StringIO()
symm_manager.show(out=out)
if (sg_err_1) or (sg_err_2) :
raise Sorry(("Incompatible space groups in input files:\n%s\nAll files "+
"must have the same point group (and ideally the same space group). "+
"Please note that any symmetry information in the PDB file will be "+
"used first.") % out.getvalue())
elif (uc_err_1) or (uc_err_2) :
libtbx.call_back(message="warn",
data=("Crystal symmetry mismatch:\n%s\nCalculations will continue "+
"using the symmetry in the PDB file (or if not available, the "+
"first reflection file), but the maps should be treated with "+
"extreme suspicion.") % out.getvalue())
crystal_symmetry = symm_manager.as_symmetry_object()
else :
processed_args = utils.process_command_line_args(
args=command_line.args,
cmd_cs=command_line.symmetry,
master_params=fo_minus_fo_master_params(),
absolute_angle_tolerance=5,
absolute_length_tolerance=1,
log=StringIO())
crystal_symmetry = processed_args.crystal_symmetry
#
pdb_file_names = processed_args.pdb_file_names
if(len(processed_args.pdb_file_names) == 0):
if(params.phase_source is not None):
pdb_file_names = [params.phase_source]
else:
raise Sorry("No PDB file found.")
# Extaract Fobs1, Fobs2
f_obss = []
if(len(processed_args.reflection_files)==2):
for reflection_file in processed_args.reflection_files:
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = crystal_symmetry,
force_symmetry = True,
reflection_files = [reflection_file],
err = null_out())
# XXX UGLY !!!
try:
parameters = utils.data_and_flags_master_params().extract()
if(params.f_obs_1_label is not None):
parameters.labels = [params.f_obs_1_label]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server = reflection_file_server,
keep_going = True,
parameters = parameters,
log = null_out())
except: # intentional
parameters = utils.data_and_flags_master_params().extract()
if(params.f_obs_2_label is not None):
parameters.labels = [params.f_obs_2_label]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server = reflection_file_server,
keep_going = True,
parameters = parameters,
log = null_out())
f_obss.append(determine_data_and_flags_result.f_obs)
else:
if([params.f_obs_1_file_name,params.f_obs_2_file_name].count(None)==2):
raise Sorry("No reflection data file found.")
for file_name, label in zip([params.f_obs_1_file_name,params.f_obs_2_file_name],
[params.f_obs_1_label,params.f_obs_2_label]):
reflection_file = reflection_file_reader.any_reflection_file(
file_name = file_name, ensure_read_access = False)
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = crystal_symmetry,
force_symmetry = True,
reflection_files = [reflection_file],
err = null_out())
parameters = utils.data_and_flags_master_params().extract()
if(label is not None):
parameters.labels = [label]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server = reflection_file_server,
parameters = parameters,
keep_going = True,
log = null_out())
f_obss.append(determine_data_and_flags_result.f_obs)
if(len(f_obss)!=2):
raise Sorry(" ".join(errors))
if(not command_line.options.silent):
for ifobs, fobs in enumerate(f_obss):
print >> log, "*** Summary for data set %d:"%ifobs
fobs.show_comprehensive_summary(f = log)
print >> log
pdb_combined = combine_unique_pdb_files(file_names = pdb_file_names)
pdb_combined.report_non_unique(out = log)
if(len(pdb_combined.unique_file_names) == 0):
raise Sorry("No coordinate file given.")
#
raw_recs = flex.std_string()
for rec in pdb_combined.raw_records:
if(rec.upper().count("CRYST1")==0):
raw_recs.append(rec)
raw_recs.append(iotbx.pdb.format_cryst1_record(
crystal_symmetry = crystal_symmetry))
#
pdb_in = iotbx.pdb.input(source_info = None, lines = raw_recs)
hierarchy = pdb_in.construct_hierarchy()
omit_sel = flex.bool(hierarchy.atoms_size(), False)
if (params.advanced.omit_selection is not None) :
print >> log, "Will omit selection from phasing model:"
print >> log, " " + params.advanced.omit_selection
omit_sel = hierarchy.atom_selection_cache().selection(
params.advanced.omit_selection)
print >> log, "%d atoms selected for removal" % omit_sel.count(True)
xray_structure = pdb_in.xray_structure_simple()
xray_structure = xray_structure.select(~omit_sel)
if(not command_line.options.silent):
print >> log, "*** Model summary:"
xray_structure.show_summary(f = log)
print >> log
info0 = f_obss[0].info()
info1 = f_obss[1].info()
f_obss[0] = f_obss[0].resolution_filter(d_min = params.high_resolution,
d_max = params.low_resolution).set_info(info0)
f_obss[1] = f_obss[1].resolution_filter(d_min = params.high_resolution,
d_max = params.low_resolution).set_info(info1)
if(params.sigma_cutoff is not None):
for i in [0,1]:
if(f_obss[i].sigmas() is not None):
sel = f_obss[i].data() > f_obss[i].sigmas()*params.sigma_cutoff
f_obss[i] = f_obss[i].select(sel).set_info(info0)
for k, f_obs in enumerate(f_obss) :
if (f_obs.indices().size() == 0) :
raise Sorry("No data left in array %d (labels=%s) after filtering!" % (k+1,
f_obs.info().label_string()))
output_file_name = params.output_file
if (output_file_name is None) and (params.file_name_prefix is not None) :
output_file_name = "%s_%s.mtz" % (params.file_name_prefix, params.job_id)
output_files = compute_fo_minus_fo_map(
data_arrays = f_obss,
xray_structure = xray_structure,
log = log,
silent = command_line.options.silent,
output_file = output_file_name,
peak_search=params.find_peaks_holes,
map_cutoff=params.map_cutoff,
peak_search_params=params.peak_search,
multiscale=params.advanced.multiscale,
anomalous=params.advanced.anomalous).file_names
return output_files
