def sfcalc(names, miller_arrays, xray_structure, parameters, out):
from mmtbx import f_model
f_obs = None
if parameters.fobs is None:
if parameters.output not in ["complex_fcalc", "abs_fcalc", "intensities" ]:
raise Sorry("Experimental data is needed for %s coefficients.\n Please supply Fobs")
else:
f_obs = abs(xray_structure.structure_factors(
d_min = parameters.bulk_and_scale_parameters.d_min,
anomalous_flag = False).f_calc())
else:
f_obs = miller_arrays[ names[parameters.fobs] ].deep_copy()
if f_obs.is_xray_intensity_array():
f_obs = f_obs.f_sq_as_f()
flags = f_obs.generate_r_free_flags(fraction = 0.1,
max_free = 99999999)
b_cart = [parameters.bulk_and_scale_parameters.overall.b_cart.b_11,
parameters.bulk_and_scale_parameters.overall.b_cart.b_22,
parameters.bulk_and_scale_parameters.overall.b_cart.b_33,
parameters.bulk_and_scale_parameters.overall.b_cart.b_12,
parameters.bulk_and_scale_parameters.overall.b_cart.b_13,
parameters.bulk_and_scale_parameters.overall.b_cart.b_23 ]
fmodel = f_model.manager( xray_structure = xray_structure,
r_free_flags = flags,
target_name = "ls_wunit_k1",
f_obs = f_obs,
b_cart = b_cart,
k_sol = parameters.bulk_and_scale_parameters.solvent.k_sol,
b_sol = parameters.bulk_and_scale_parameters.solvent.b_sol)
if parameters.use_bulk_and_scale == "as_estimated":
if parameters.fobs is not None:
fmodel.update_solvent_and_scale(out=out)
result = None
if parameters.output in ["complex_fcalc", "abs_fcalc", "intensities" ]:
result = fmodel.f_model()
if parameters.output == "complex_fcalc":
result = result
if parameters.output == "abs_fcalc":
result = abs( result )
if parameters.output == "intensities":
result = abs(result).f_as_f_sq()
else:
if parameters.output == "2mFo-DFc":
result = fmodel.electron_density_map().map_coefficients(map_type = "2m*Fobs-D*Fmodel")
if parameters.output == "mFo-DFc":
# XXX BUG ?
result = fmodel.electron_density_map().map_coefficients(map_type = "2m*Fobs-D*Fmodel")
assert result is not None
return result
def get_r_factor(self,f_obs,f_calc):
'''
When using f_model to get r_factor,
Note that f_model requires that
f_obs is real and that f_calc is complex
'''
fmodel = f_model.manager(
f_obs = f_obs,
f_calc = f_calc,
f_mask = f_calc.customized_copy(data = f_calc.data()*0))
r_factor = fmodel.r_work()
return r_factor
def generate_f_model_manager(self):
f_model_manager = fm.manager(f_calc = self.f_calc,
f_obs = self.f_obs,
r_free_flags = self.flags_plus_minus,
xray_structure=self.xray_structure,
target_name='ls')
f_model = f_model_manager.f_model()
f_model_manager.update_solvent_and_scale()
f_model_manager.show()
k = f_model_manager.k_masks()
print(dir(k))
return
def simul_utils(args):
if len(args) == 0:
print_help()
elif ("--help" in args):
print_help()
elif ("--h" in args):
print_help()
elif ("-h" in args):
print_help()
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="map_coefs")
print("#phil __OFF__", file=log)
print("======================", file=log)
print(" SIMUL ", file=log)
print("A data simulation tool", file=log)
print("======================", file=log)
print(file=log)
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
if not arg_is_processed:
print("##----------------------------------------------##",
file=log)
print("## Unknown file or keyword:", arg, file=log)
print("##----------------------------------------------##",
file=log)
print(file=log)
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
"""
new_params = master_params.format(python_object=params)
new_params.show(out=log)
"""
# now get the unit cell from the pdb file
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.simul_utils.input.xray_data.file_name)
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.simul_utils.input.model.file_name)
phil_xs = crystal.symmetry(
unit_cell=params.simul_utils.input.unit_cell,
space_group_info=params.simul_utils.input.space_group)
combined_xs = select_crystal_symmetry(None, phil_xs, [pdb_xs],
[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.simul_utils.input.unit_cell = combined_xs.unit_cell()
params.simul_utils.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
print("#phil __ON__", file=log)
new_params = master_params.format(python_object=params)
new_params.show(out=log)
print("#phil __END__", file=log)
if params.simul_utils.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.simul_utils.input.space_group is None:
raise Sorry("space group not specified")
if params.simul_utils.input.xray_data.file_name is None:
raise Sorry("Xray data not specified")
if params.simul_utils.input.model.file_name is None:
raise Sorry("pdb file with model not specified")
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
phil_xs = crystal.symmetry(
unit_cell=params.simul_utils.input.unit_cell,
space_group_info=params.simul_utils.input.space_group)
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=phil_xs, force_symmetry=True, reflection_files=[])
miller_array = None
miller_array = xray_data_server.get_xray_data(
file_name=params.simul_utils.input.xray_data.file_name,
labels=params.simul_utils.input.xray_data.labels,
ignore_all_zeros=True,
parameter_scope='simul_utils.input.xray_data',
parameter_name='labels')
info = miller_array.info()
miller_array = miller_array.map_to_asu()
miller_array = miller_array.select(miller_array.indices() != (0, 0, 0))
miller_array = miller_array.select(miller_array.data() > 0)
if miller_array.sigmas() is not None:
miller_array = miller_array.select(miller_array.sigmas() > 0)
if (miller_array.is_xray_intensity_array()):
miller_array = miller_array.f_sq_as_f()
elif (miller_array.is_complex_array()):
miller_array = abs(miller_array)
miller_array.set_info(info)
print(file=log)
print("Summary info of observed data", file=log)
print("=============================", file=log)
miller_array.show_summary(f=log)
print(file=log)
free_flags = miller_array.generate_r_free_flags()
#--------------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
model = pdb.input(file_name=params.simul_utils.input.model.file_name
).xray_structure_simple(crystal_symmetry=phil_xs, )
print("Atomic model summary", file=log)
print("====================", file=log)
model.show_summary()
print(file=log)
#-------------------------------------------------------------------
# Step 3: make an F_model object to get model phases and amplitudes
#
print("Performing bulk solvent scaling", file=log)
print("===============================", file=log)
print(file=log)
print(file=log)
f_model_object = f_model.manager(f_obs=miller_array,
r_free_flags=free_flags,
xray_structure=model)
f_model_object.update_all_scales(log=log)
fmodel = abs(
f_model_object.f_model()).set_observation_type(miller_array)
mockfmodel = None
if params.simul_utils.input.mock_model.file_name is not None:
print("Reading in mock model", file=log)
print("=====================", file=log)
print(file=log)
print(file=log)
mock_model = pdb.input(file_name=params.simul_utils.input.
mock_model.file_name).xray_structure_simple(
crystal_symmetry=phil_xs)
mock_f_model = f_model.manager(f_obs=miller_array,
r_free_flags=free_flags,
xray_structure=mock_model)
mock_f_model.update(k_sol=f_model_object.k_sol(),
b_sol=f_model_object.b_sol(),
b_cart=f_model_object.b_cart())
mockfmodel = abs(
mock_f_model.f_model()).set_observation_type(miller_array)
else:
mockfmodel = fmodel.deep_copy()
print("Making new data", file=log)
print("===============", file=log)
print(file=log)
print(file=log)
new_data_builder = error_swap(miller_array, fmodel, mockfmodel)
new_data = new_data_builder.new_obs
# we now have to write the data actually
print("Writing new data set", file=log)
print("====================", file=log)
mtz_dataset = new_data.as_mtz_dataset(column_root_label="FOBS")
mtz_dataset.mtz_object().write(
file_name=params.simul_utils.output.hklout)
def sfcalc(args):
if len(args) == 0:
print_help()
elif ("--help" in args):
print_help()
elif ("--h" in args):
print_help()
elif ("-h" in args):
print_help()
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="sfcalc")
print("#phil __OFF__", file=log)
print("=================", file=log)
print(" SFCALC ", file=log)
print("=================", file=log)
print(file=log)
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
if not arg_is_processed:
print("##----------------------------------------------##",
file=log)
print("## Unknown file or keyword:", arg, file=log)
print("##----------------------------------------------##",
file=log)
print(file=log)
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
# now get the unit cell from the files
hkl_xs = None
pdb_xs = None
if params.sfcalc.input.model.file_name is not None:
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.sfcalc.input.model.file_name)
phil_xs = crystal.symmetry(
unit_cell=params.sfcalc.input.unit_cell,
space_group_info=params.sfcalc.input.space_group)
combined_xs = crystal.select_crystal_symmetry(None, phil_xs, [pdb_xs],
[None])
combined_xs.show_summary()
if combined_xs.unit_cell() is None:
raise Sorry("Unit cell not defined")
if combined_xs.space_group() is None:
raise Sorry("Space group not defined")
# inject the unit cell and symmetry in the phil scope please
params.sfcalc.input.unit_cell = combined_xs.unit_cell()
params.sfcalc.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
print("#phil __ON__", file=log)
new_params = master_params.format(python_object=params)
new_params.show(out=log)
print("#phil __END__", file=log)
pdb_model = None
if params.sfcalc.input.model.file_name is not None:
pdb_model = pdb.input(
file_name=params.sfcalc.input.model.file_name)
model = pdb_model.xray_structure_simple(crystal_symmetry=phil_xs)
print("Atomic model summary", file=log)
print("====================", file=log)
model.show_summary()
print(file=log)
#make an f_model object please
b_cart = params.sfcalc.parameters.overall.b_cart
b_cart = [
b_cart.b_11, b_cart.b_22, b_cart.b_33, b_cart.b_12,
b_cart.b_13, b_cart.b_23
]
dummy = abs(
model.structure_factors(d_min=params.sfcalc.parameters.d_min,
anomalous_flag=False).f_calc())
flags = dummy.generate_r_free_flags(fraction=0.1,
max_free=99999999)
fmodel = f_model.manager(
xray_structure=model,
r_free_flags=flags,
target_name="ls_wunit_k1",
f_obs=dummy,
b_cart=b_cart,
k_sol=params.sfcalc.parameters.solvent.k_sol,
b_sol=params.sfcalc.parameters.solvent.b_sol)
calc_data_with_solvent_contrib = fmodel.f_model()
calc_data_with_solvent_contrib = calc_data_with_solvent_contrib.array(
data=calc_data_with_solvent_contrib.data() *
params.sfcalc.parameters.overall.k_overall)
result = None
label = None
if params.sfcalc.parameters.output_type == "complex":
result = calc_data_with_solvent_contrib
label = "FMODEL"
if params.sfcalc.parameters.output_type == "amplitudes":
result = abs(calc_data_with_solvent_contrib)
label = "FMODEL"
if params.sfcalc.parameters.output_type == "intensities":
result = abs(calc_data_with_solvent_contrib)
result = result.f_as_f_sq()
label = "IMODEL"
#write an mtz file with the data
mtz_dataset = result.as_mtz_dataset(column_root_label=label)
mtz_dataset.mtz_object().write(
file_name=params.sfcalc.output.hklout)
#write the logfile
logger = open(params.sfcalc.output.logfile, 'w')
print("writing log file with name %s" % (params.sfcalc.output.logfile),
file=log)
print(file=log)
print(string_buffer.getvalue(), file=logger)
def run(args, command_name="phenix.remove_outliers"):
if (len(args) == 0 or "--help" in args or "--h" in args or "-h" in args):
print_help(command_name=command_name)
else:
log = multi_out()
plot_out = None
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="outlier_detection")
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if arg == "--quiet":
arg_is_processed = True
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown file or keyword:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
if not os.path.exists(params.outlier_utils.input.xray_data.file_name):
raise Sorry("File %s can not be found" %
(params.outlier_utils.input.xray_data.file_name))
if params.outlier_utils.input.model.file_name is not None:
if not os.path.exists(params.outlier_utils.input.model.file_name):
raise Sorry("File %s can not be found" %
(params.outlier_utils.input.model.file_name))
# now get the unit cell from the pdb file
hkl_xs = None
if params.outlier_utils.input.xray_data.file_name is not None:
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.outlier_utils.input.xray_data.file_name)
pdb_xs = None
if params.outlier_utils.input.model.file_name is not None:
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.outlier_utils.input.model.file_name)
phil_xs = crystal.symmetry(
unit_cell=params.outlier_utils.input.unit_cell,
space_group_info=params.outlier_utils.input.space_group)
phil_xs.show_summary()
hkl_xs.show_summary()
combined_xs = crystal.select_crystal_symmetry(None, phil_xs, [pdb_xs],
[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.outlier_utils.input.unit_cell = combined_xs.unit_cell()
params.outlier_utils.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
new_params = master_params.format(python_object=params)
new_params.show(out=log)
if params.outlier_utils.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.outlier_utils.input.space_group is None:
raise Sorry("space group not specified")
if params.outlier_utils.input.xray_data.file_name is None:
raise Sorry("Xray data not specified")
if params.outlier_utils.input.model.file_name is None:
print "PDB file not specified. Basic wilson outlier rejections only."
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
phil_xs = crystal.symmetry(
unit_cell=params.outlier_utils.input.unit_cell,
space_group_info=params.outlier_utils.input.space_group)
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=phil_xs, force_symmetry=True, reflection_files=[])
miller_array = None
miller_array = xray_data_server.get_xray_data(
file_name=params.outlier_utils.input.xray_data.file_name,
labels=params.outlier_utils.input.xray_data.obs_labels,
ignore_all_zeros=True,
parameter_scope='outlier_utils.input.xray_data',
parameter_name='obs_labels')
info = miller_array.info()
miller_array = miller_array.map_to_asu()
miller_array = miller_array.select(miller_array.indices() != (0, 0, 0))
#we have to check if the sigma's make any sense at all
if not miller_array.sigmas_are_sensible():
miller_array = miller_array.customized_copy(
data=miller_array.data(),
sigmas=None).set_observation_type(miller_array)
miller_array = miller_array.select(miller_array.data() > 0)
if miller_array.sigmas() is not None:
miller_array = miller_array.select(miller_array.sigmas() > 0)
if (miller_array.is_xray_intensity_array()):
miller_array = miller_array.f_sq_as_f()
elif (miller_array.is_complex_array()):
miller_array = abs(miller_array)
miller_array.set_info(info)
merged_anomalous = False
if miller_array.anomalous_flag():
miller_array = miller_array.average_bijvoet_mates(
).set_observation_type(miller_array)
merged_anomalous = True
miller_array = miller_array.map_to_asu()
# get the free reflections please
free_flags = None
if params.outlier_utils.input.xray_data.free_flags is None:
free_flags = miller_array.generate_r_free_flags(
fraction=params.outlier_utils.\
additional_parameters.free_flag_generation.fraction,
max_free=params.outlier_utils.\
additional_parameters.free_flag_generation.max_number,
lattice_symmetry_max_delta=params.outlier_utils.\
additional_parameters.free_flag_generation.lattice_symmetry_max_delta,
use_lattice_symmetry=params.outlier_utils.\
additional_parameters.free_flag_generation.use_lattice_symmetry
)
else:
free_flags = xray_data_server.get_xray_data(
file_name=params.outlier_utils.input.xray_data.file_name,
labels=params.outlier_utils.input.xray_data.free_flags,
ignore_all_zeros=True,
parameter_scope='outlier_utils.input.xray_data',
parameter_name='free_flags')
if free_flags.anomalous_flag():
free_flags = free_flags.average_bijvoet_mates()
merged_anomalous = True
free_flags = free_flags.customized_copy(data=flex.bool(
free_flags.data() == 1))
free_flags = free_flags.map_to_asu()
free_flags = free_flags.common_set(miller_array)
print >> log
print >> log, "Summary info of observed data"
print >> log, "============================="
miller_array.show_summary(f=log)
if merged_anomalous:
print >> log, "For outlier detection purposes, the Bijvoet pairs have been merged."
print >> log
print >> log, "Constructing an outlier manager"
print >> log, "==============================="
print >> log
outlier_manager = outlier_rejection.outlier_manager(miller_array,
free_flags,
out=log)
basic_array = None
extreme_array = None
model_based_array = None
basic_array = outlier_manager.basic_wilson_outliers(
p_basic_wilson = params.outlier_utils.outlier_detection.\
parameters.basic_wilson.level,
return_data = True)
extreme_array = outlier_manager.extreme_wilson_outliers(
p_extreme_wilson = params.outlier_utils.outlier_detection.parameters.\
extreme_wilson.level,
return_data = True)
beamstop_array = outlier_manager.beamstop_shadow_outliers(
level = params.outlier_utils.outlier_detection.parameters.\
beamstop.level,
d_min = params.outlier_utils.outlier_detection.parameters.\
beamstop.d_min,
return_data=True)
#----------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
if params.outlier_utils.input.model.file_name is not None:
model = pdb.input(file_name=params.outlier_utils.input.model.
file_name).xray_structure_simple(
crystal_symmetry=phil_xs)
print >> log, "Atomic model summary"
print >> log, "===================="
model.show_summary(f=log)
print >> log
# please make an f_model object for bulk solvent scaling etc etc
f_model_object = f_model.manager(f_obs=miller_array,
r_free_flags=free_flags,
xray_structure=model)
print >> log, "Bulk solvent scaling of the data"
print >> log, "================================"
print >> log, "Maximum likelihood bulk solvent scaling."
print >> log
f_model_object.update_all_scales(log=log, remove_outliers=False)
plot_out = StringIO()
model_based_array = outlier_manager.model_based_outliers(
f_model_object.f_model(),
level=params.outlier_utils.outlier_detection.parameters.
model_based.level,
return_data=True,
plot_out=plot_out)
#check what needs to be put out please
if params.outlier_utils.output.hklout is not None:
if params.outlier_utils.outlier_detection.protocol == "model":
if params.outlier_utils.input.model.file_name == None:
print >> log, "Model based rejections requested. No model was supplied."
print >> log, "Switching to writing out rejections based on extreme value Wilson statistics."
params.outlier_utils.outlier_detection.protocol = "extreme"
output_array = None
print >> log
if params.outlier_utils.outlier_detection.protocol == "basic":
print >> log, "Non-outliers found by the basic wilson statistics"
print >> log, "protocol will be written out."
output_array = basic_array
new_set_of_free_flags = free_flags.common_set(basic_array)
if params.outlier_utils.outlier_detection.protocol == "extreme":
print >> log, "Non-outliers found by the extreme value wilson statistics"
print >> log, "protocol will be written out."
output_array = extreme_array
new_set_of_free_flags = free_flags.common_set(extreme_array)
if params.outlier_utils.outlier_detection.protocol == "model":
print >> log, "Non-outliers found by the model based"
print >> log, "protocol will be written out to the file:"
print >> log, params.outlier_utils.output.hklout
print >> log
output_array = model_based_array
new_set_of_free_flags = free_flags.common_set(
model_based_array)
if params.outlier_utils.outlier_detection.protocol == "beamstop":
print >> log, "Outliers found for the beamstop shadow"
print >> log, "problems detection protocol will be written to the file:"
print >> log, params.outlier_utils.output.hklout
print >> log
output_array = model_based_array
new_set_of_free_flags = free_flags.common_set(
model_based_array)
mtz_dataset = output_array.as_mtz_dataset(column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array=new_set_of_free_flags,
column_root_label="Free_R_Flag")
mtz_dataset.mtz_object().write(
file_name=params.outlier_utils.output.hklout)
if (params.outlier_utils.output.logfile is not None):
final_log = StringIO()
print >> final_log, string_buffer.getvalue()
print >> final_log
if plot_out is not None:
print >> final_log, plot_out.getvalue()
outfile = open(params.outlier_utils.output.logfile, 'w')
outfile.write(final_log.getvalue())
print >> log
print >> log, "A logfile named %s was created." % (
params.outlier_utils.output.logfile)
print >> log, "This logfile contains the screen output and"
print >> log, "(possibly) some ccp4 style loggraph plots"
def run(args, command_name="phenix.twin_map_utils"):
log = sys.stdout
params = None
if (len(args) == 0 or "--help" in args or "--h" in args or "-h" in args):
print_help(command_name=command_name)
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="map_coefs")
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
if not arg_is_processed:
print("##----------------------------------------------##",
file=log)
print("## Unknown file or keyword:", arg, file=log)
print("##----------------------------------------------##",
file=log)
print(file=log)
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
"""
new_params = master_params.format(python_object=params)
new_params.show(out=log)
"""
# now get the unit cell from the pdb file
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.twin_utils.input.xray_data.file_name)
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.twin_utils.input.model.file_name)
phil_xs = None
if ([
params.twin_utils.input.unit_cell,
params.twin_utils.input.space_group
]).count(None) < 2:
phil_xs = crystal.symmetry(
unit_cell=params.twin_utils.input.unit_cell,
space_group_info=params.twin_utils.input.space_group)
combined_xs = crystal.select_crystal_symmetry(None, phil_xs, [pdb_xs],
[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.twin_utils.input.unit_cell = combined_xs.unit_cell()
params.twin_utils.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
new_params = master_params.format(python_object=params)
new_params.show(out=log)
if params.twin_utils.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.twin_utils.input.space_group is None:
raise Sorry("space group not specified")
if params.twin_utils.input.xray_data.file_name is None:
raise Sorry("Xray data not specified")
if params.twin_utils.input.model.file_name is None:
raise Sorry("pdb file with model not specified")
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
phil_xs = crystal.symmetry(
unit_cell=params.twin_utils.input.unit_cell,
space_group_info=params.twin_utils.input.space_group)
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=phil_xs, force_symmetry=True, reflection_files=[])
miller_array = None
free_flags = None
tmp_params = utils.data_and_flags_master_params().extract()
# insert proper values please
tmp_params.file_name = params.twin_utils.input.xray_data.file_name
tmp_params.labels = params.twin_utils.input.xray_data.obs_labels
tmp_params.r_free_flags.file_name = params.twin_utils.input.xray_data.file_name
tmp_params.r_free_flags.label = params.twin_utils.input.xray_data.free_flag
tmp_object = utils.determine_data_and_flags(
reflection_file_server=xray_data_server,
parameters=tmp_params,
log=log)
miller_array = tmp_object.extract_data()
if miller_array.is_xray_intensity_array():
miller_array = miller_array.f_sq_as_f()
assert miller_array.is_xray_amplitude_array()
free_flags = tmp_object.extract_flags(data=miller_array)
print(file=log)
print("Attempting to extract Free R flags", file=log)
free_flags = free_flags.customized_copy(data=flex.bool(
free_flags.data() == 1))
if free_flags is None:
free_flags = miller_array.generate_r_free_flags(
use_lattice_symmetry=True)
assert miller_array.observation_type() is not None
print(file=log)
print("Summary info of observed data", file=log)
print("=============================", file=log)
miller_array.show_summary(f=log)
print(file=log)
if miller_array.indices().size() == 0:
raise Sorry("No data available")
#----------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
model = pdb.input(file_name=params.twin_utils.input.model.file_name
).xray_structure_simple(crystal_symmetry=phil_xs)
print("Atomic model summary", file=log)
print("====================", file=log)
model.show_summary(f=log)
print(file=log)
#----------------------------------------------------------------
# step 3: get the twin laws for this xs
twin_laws = twin_analyses.twin_laws(
miller_array,
lattice_symmetry_max_delta=\
params.twin_utils.parameters.twinning.max_delta,
out=log)
print(file=log)
print("Preliminary data analyses", file=log)
print("==========================", file=log)
twin_laws.show(out=log)
#---------
# step 3:
# make twin model managers for all twin laws
print(file=log)
print(
"Overall and bulk solvent scale paranmeters and twin fraction estimation",
file=log)
print(
"=======================================================================",
file=log)
twin_models = []
operator_count = 0
if params.twin_utils.parameters.twinning.twin_law is not None:
tmp_law = sgtbx.rt_mx(
params.twin_utils.parameters.twinning.twin_law)
tmp_law = twin_analyses.twin_law(tmp_law, None, None, None, None,
None)
twin_laws.operators = [tmp_law]
for twin_law in twin_laws.operators:
operator_count += 1
operator_hkl = sgtbx.change_of_basis_op(twin_law.operator).as_hkl()
twin_model = twin_f_model.twin_model_manager(
f_obs=miller_array,
r_free_flags=free_flags,
xray_structure=model,
twin_law=twin_law.operator,
detwin_mode=params.twin_utils.parameters.twinning.detwin_mode,
out=log)
print("--- bulk solvent scaling ---", file=log)
twin_model.update_all_scales()
twin_model.r_values()
twin_model.target()
twin_model.show_k_sol_b_sol_b_cart_target()
twin_model.show_essential()
wfofc = twin_model.map_coefficients(map_type="mFo-DFc")
wtfofc = twin_model.map_coefficients(map_type="2mFo-DFc")
grad = twin_model.map_coefficients(map_type="gradient")
mtz_dataset = wtfofc.as_mtz_dataset(column_root_label="FWT")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array=wfofc, column_root_label="DFWT")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array=grad, column_root_label="GRAD")
name = params.twin_utils.output.map_coeffs_root + "_" + str(
operator_count) + ".mtz"
print(file=log)
print("Writing %s for twin law %s" % (name, operator_hkl),
file=log)
print(file=log)
mtz_dataset.mtz_object().write(file_name=name)
if params.twin_utils.output.obs_and_calc is not None:
# i want also a Fobs and Fmodel combined dataset please
mtz_dataset = miller_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array=twin_model.f_model(),
column_root_label="FMODEL")
name = params.twin_utils.output.obs_and_calc
mtz_dataset.mtz_object().write(file_name=name)
if len(twin_laws.operators) == 0:
print(file=log)
print("No twin laws were found", file=log)
print("Performing maximum likelihood based bulk solvent scaling",
file=log)
f_model_object = f_model.manager(f_obs=miller_array,
r_free_flags=free_flags,
xray_structure=model)
f_model_object.update_all_scales(log=log)
tfofc = f_model_object.map_coefficients(map_type="2mFobs-DFmodel")
fofc = f_model_object.map_coefficients(map_type="mFobs-DFmodel")
mtz_dataset = tfofc.as_mtz_dataset(column_root_label="FWT")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array=fofc, column_root_label="DELFWT")
name = params.twin_utils.output.map_coeffs_root + "_ML.mtz"
mtz_dataset.mtz_object().write(file_name=name)
if params.twin_utils.output.obs_and_calc is not None:
# i want also a Fobs and Fmodel combined dataset please
mtz_dataset = miller_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array=f_model_object.f_model(),
column_root_label="FMODEL")
name = params.twin_utils.output.obs_and_calc
mtz_dataset.mtz_object().write(file_name=name)
print(file=log)
print(file=log)
print("All done \n", file=log)
logfile = open(params.twin_utils.output.logfile, 'w')
print(string_buffer.getvalue(), file=logfile)
print(file=log)
def run(args, command_name="phenix.remove_outliers"):
if (len(args)==0 or "--help" in args or "--h" in args or "-h" in args):
print_help(command_name=command_name)
else:
log = multi_out()
plot_out = None
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="outlier_detection")
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if arg=="--quiet":
arg_is_processed = True
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown file or keyword:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
if not os.path.exists( params.outlier_utils.input.xray_data.file_name ) :
raise Sorry("File %s can not be found"%(params.outlier_utils.input.xray_data.file_name) )
if params.outlier_utils.input.model.file_name is not None:
if not os.path.exists( params.outlier_utils.input.model.file_name ):
raise Sorry("File %s can not be found"%(params.outlier_utils.input.model.file_name) )
# now get the unit cell from the pdb file
hkl_xs = None
if params.outlier_utils.input.xray_data.file_name is not None:
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.outlier_utils.input.xray_data.file_name)
pdb_xs = None
if params.outlier_utils.input.model.file_name is not None:
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.outlier_utils.input.model.file_name)
phil_xs = crystal.symmetry(
unit_cell=params.outlier_utils.input.unit_cell,
space_group_info=params.outlier_utils.input.space_group )
phil_xs.show_summary()
hkl_xs.show_summary()
combined_xs = crystal.select_crystal_symmetry(
None,phil_xs, [pdb_xs],[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.outlier_utils.input.unit_cell = combined_xs.unit_cell()
params.outlier_utils.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
new_params = master_params.format(python_object=params)
new_params.show(out=log)
if params.outlier_utils.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.outlier_utils.input.space_group is None:
raise Sorry("space group not specified")
if params.outlier_utils.input.xray_data.file_name is None:
raise Sorry("Xray data not specified")
if params.outlier_utils.input.model.file_name is None:
print "PDB file not specified. Basic wilson outlier rejections only."
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
phil_xs = crystal.symmetry(
unit_cell=params.outlier_utils.input.unit_cell,
space_group_info=params.outlier_utils.input.space_group )
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = phil_xs,
force_symmetry = True,
reflection_files=[])
miller_array = None
miller_array = xray_data_server.get_xray_data(
file_name = params.outlier_utils.input.xray_data.file_name,
labels = params.outlier_utils.input.xray_data.obs_labels,
ignore_all_zeros = True,
parameter_scope = 'outlier_utils.input.xray_data',
parameter_name = 'obs_labels'
)
info = miller_array.info()
miller_array = miller_array.map_to_asu()
miller_array = miller_array.select(
miller_array.indices() != (0,0,0))
#we have to check if the sigma's make any sense at all
if not miller_array.sigmas_are_sensible():
miller_array = miller_array.customized_copy(
data = miller_array.data(),
sigmas=None).set_observation_type(miller_array)
miller_array = miller_array.select(
miller_array.data() > 0 )
if miller_array.sigmas() is not None:
miller_array = miller_array.select(
miller_array.sigmas() > 0 )
if (miller_array.is_xray_intensity_array()):
miller_array = miller_array.f_sq_as_f()
elif (miller_array.is_complex_array()):
miller_array = abs(miller_array)
miller_array.set_info(info)
merged_anomalous=False
if miller_array.anomalous_flag():
miller_array = miller_array.average_bijvoet_mates().set_observation_type(
miller_array )
merged_anomalous=True
miller_array = miller_array.map_to_asu()
# get the free reflections please
free_flags = None
if params.outlier_utils.input.xray_data.free_flags is None:
free_flags = miller_array.generate_r_free_flags(
fraction=params.outlier_utils.\
additional_parameters.free_flag_generation.fraction,
max_free=params.outlier_utils.\
additional_parameters.free_flag_generation.max_number,
lattice_symmetry_max_delta=params.outlier_utils.\
additional_parameters.free_flag_generation.lattice_symmetry_max_delta,
use_lattice_symmetry=params.outlier_utils.\
additional_parameters.free_flag_generation.use_lattice_symmetry
)
else:
free_flags = xray_data_server.get_xray_data(
file_name = params.outlier_utils.input.xray_data.file_name,
labels = params.outlier_utils.input.xray_data.free_flags,
ignore_all_zeros = True,
parameter_scope = 'outlier_utils.input.xray_data',
parameter_name = 'free_flags'
)
if free_flags.anomalous_flag():
free_flags = free_flags.average_bijvoet_mates()
merged_anomalous=True
free_flags = free_flags.customized_copy(
data = flex.bool( free_flags.data() == 1 ))
free_flags = free_flags.map_to_asu()
free_flags = free_flags.common_set( miller_array )
print >> log
print >> log, "Summary info of observed data"
print >> log, "============================="
miller_array.show_summary(f=log)
if merged_anomalous:
print >> log, "For outlier detection purposes, the Bijvoet pairs have been merged."
print >> log
print >> log, "Constructing an outlier manager"
print >> log, "==============================="
print >> log
outlier_manager = outlier_rejection.outlier_manager(
miller_array,
free_flags,
out=log)
basic_array = None
extreme_array = None
model_based_array = None
basic_array = outlier_manager.basic_wilson_outliers(
p_basic_wilson = params.outlier_utils.outlier_detection.\
parameters.basic_wilson.level,
return_data = True)
extreme_array = outlier_manager.extreme_wilson_outliers(
p_extreme_wilson = params.outlier_utils.outlier_detection.parameters.\
extreme_wilson.level,
return_data = True)
beamstop_array = outlier_manager.beamstop_shadow_outliers(
level = params.outlier_utils.outlier_detection.parameters.\
beamstop.level,
d_min = params.outlier_utils.outlier_detection.parameters.\
beamstop.d_min,
return_data=True)
#----------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
if params.outlier_utils.input.model.file_name is not None:
model = pdb.input(file_name=params.outlier_utils.input.model.file_name).xray_structure_simple(
crystal_symmetry=phil_xs)
print >> log, "Atomic model summary"
print >> log, "===================="
model.show_summary(f=log)
print >> log
# please make an f_model object for bulk solvent scaling etc etc
f_model_object = f_model.manager(
f_obs = miller_array,
r_free_flags = free_flags,
xray_structure = model )
print >> log, "Bulk solvent scaling of the data"
print >> log, "================================"
print >> log, "Maximum likelihood bulk solvent scaling."
print >> log
f_model_object.update_solvent_and_scale(out=log)
plot_out = StringIO()
model_based_array = outlier_manager.model_based_outliers(
f_model_object.f_model(),
level=params.outlier_utils.outlier_detection.parameters.model_based.level,
return_data=True,
plot_out=plot_out)
#check what needs to be put out please
if params.outlier_utils.output.hklout is not None:
if params.outlier_utils.outlier_detection.protocol == "model":
if params.outlier_utils.input.model.file_name == None:
print >> log, "Model based rejections requested. No model was supplied."
print >> log, "Switching to writing out rejections based on extreme value Wilson statistics."
params.outlier_utils.outlier_detection.protocol="extreme"
output_array = None
print >> log
if params.outlier_utils.outlier_detection.protocol == "basic":
print >> log, "Non-outliers found by the basic wilson statistics"
print >> log, "protocol will be written out."
output_array = basic_array
new_set_of_free_flags = free_flags.common_set( basic_array )
if params.outlier_utils.outlier_detection.protocol == "extreme":
print >> log, "Non-outliers found by the extreme value wilson statistics"
print >> log, "protocol will be written out."
output_array = extreme_array
new_set_of_free_flags = free_flags.common_set( extreme_array )
if params.outlier_utils.outlier_detection.protocol == "model":
print >> log, "Non-outliers found by the model based"
print >> log, "protocol will be written out to the file:"
print >> log, params.outlier_utils.output.hklout
print >> log
output_array = model_based_array
new_set_of_free_flags = free_flags.common_set( model_based_array )
if params.outlier_utils.outlier_detection.protocol == "beamstop":
print >> log, "Outliers found for the beamstop shadow"
print >> log, "problems detection protocol will be written to the file:"
print >> log, params.outlier_utils.output.hklout
print >> log
output_array = model_based_array
new_set_of_free_flags = free_flags.common_set( model_based_array )
mtz_dataset = output_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = new_set_of_free_flags,
column_root_label = "Free_R_Flag"
)
mtz_dataset.mtz_object().write(
file_name=params.outlier_utils.output.hklout)
if (params.outlier_utils.output.logfile is not None):
final_log = StringIO()
print >> final_log, string_buffer.getvalue()
print >> final_log
if plot_out is not None:
print >> final_log, plot_out.getvalue()
outfile = open( params.outlier_utils.output.logfile, 'w' )
outfile.write( final_log.getvalue() )
print >> log
print >> log, "A logfile named %s was created."%(
params.outlier_utils.output.logfile)
print >> log, "This logfile contains the screen output and"
print >> log, "(possibly) some ccp4 style loggraph plots"
def sfcalc(names, miller_arrays, xray_structure, parameters, out):
from mmtbx import f_model
f_obs = None
if parameters.fobs is None:
if parameters.output not in [
"complex_fcalc", "abs_fcalc", "intensities"
]:
raise Sorry(
"Experimental data is needed for %s coefficients.\n Please supply Fobs"
)
else:
f_obs = abs(
xray_structure.structure_factors(
d_min=parameters.bulk_and_scale_parameters.d_min,
anomalous_flag=False).f_calc())
else:
f_obs = miller_arrays[names[parameters.fobs]].deep_copy()
if f_obs.is_xray_intensity_array():
f_obs = f_obs.f_sq_as_f()
flags = f_obs.generate_r_free_flags(fraction=0.1, max_free=99999999)
b_cart = [
parameters.bulk_and_scale_parameters.overall.b_cart.b_11,
parameters.bulk_and_scale_parameters.overall.b_cart.b_22,
parameters.bulk_and_scale_parameters.overall.b_cart.b_33,
parameters.bulk_and_scale_parameters.overall.b_cart.b_12,
parameters.bulk_and_scale_parameters.overall.b_cart.b_13,
parameters.bulk_and_scale_parameters.overall.b_cart.b_23
]
fmodel = f_model.manager(
xray_structure=xray_structure,
r_free_flags=flags,
target_name="ls_wunit_k1",
f_obs=f_obs,
b_cart=b_cart,
k_sol=parameters.bulk_and_scale_parameters.solvent.k_sol,
b_sol=parameters.bulk_and_scale_parameters.solvent.b_sol)
if parameters.use_bulk_and_scale == "as_estimated":
if parameters.fobs is not None:
fmodel.update_all_scales(log=out)
result = None
if parameters.output in ["complex_fcalc", "abs_fcalc", "intensities"]:
result = fmodel.f_model()
if parameters.output == "complex_fcalc":
result = result
if parameters.output == "abs_fcalc":
result = abs(result)
if parameters.output == "intensities":
result = abs(result).f_as_f_sq()
else:
if parameters.output == "2mFo-DFc":
result = fmodel.electron_density_map().map_coefficients(
map_type="2m*Fobs-D*Fmodel")
if parameters.output == "mFo-DFc":
# XXX BUG ?
result = fmodel.electron_density_map().map_coefficients(
map_type="2m*Fobs-D*Fmodel")
assert result is not None
return result
def simul_utils(args):
if len(args)==0:
print_help()
elif ( "--help" in args ):
print_help()
elif ( "--h" in args ):
print_help()
elif ("-h" in args ):
print_help()
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="map_coefs")
print >> log, "#phil __OFF__"
print >> log, "======================"
print >> log, " SIMUL "
print >> log, "A data simulation tool"
print >> log, "======================"
print >> log
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown file or keyword:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
"""
new_params = master_params.format(python_object=params)
new_params.show(out=log)
"""
# now get the unit cell from the pdb file
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.simul_utils.input.xray_data.file_name)
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.simul_utils.input.model.file_name)
phil_xs = crystal.symmetry(
unit_cell=params.simul_utils.input.unit_cell,
space_group_info=params.simul_utils.input.space_group )
combined_xs = select_crystal_symmetry(
None,phil_xs, [pdb_xs],[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.simul_utils.input.unit_cell = combined_xs.unit_cell()
params.simul_utils.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
print >> log, "#phil __ON__"
new_params = master_params.format(python_object=params)
new_params.show(out=log)
print >> log, "#phil __END__"
if params.simul_utils.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.simul_utils.input.space_group is None:
raise Sorry("space group not specified")
if params.simul_utils.input.xray_data.file_name is None:
raise Sorry("Xray data not specified")
if params.simul_utils.input.model.file_name is None:
raise Sorry("pdb file with model not specified")
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
phil_xs = crystal.symmetry(
unit_cell=params.simul_utils.input.unit_cell,
space_group_info=params.simul_utils.input.space_group )
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = phil_xs,
force_symmetry = True,
reflection_files=[])
miller_array = None
miller_array = xray_data_server.get_xray_data(
file_name = params.simul_utils.input.xray_data.file_name,
labels = params.simul_utils.input.xray_data.labels,
ignore_all_zeros = True,
parameter_scope = 'simul_utils.input.xray_data',
parameter_name = 'labels'
)
info = miller_array.info()
miller_array = miller_array.map_to_asu()
miller_array = miller_array.select(
miller_array.indices() != (0,0,0))
miller_array = miller_array.select(
miller_array.data() > 0 )
if miller_array.sigmas() is not None:
miller_array = miller_array.select(
miller_array.sigmas() > 0 )
if (miller_array.is_xray_intensity_array()):
miller_array = miller_array.f_sq_as_f()
elif (miller_array.is_complex_array()):
miller_array = abs(miller_array)
miller_array.set_info(info)
print >> log
print >> log, "Summary info of observed data"
print >> log, "============================="
miller_array.show_summary(f=log)
print >> log
free_flags = miller_array.generate_r_free_flags()
#--------------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
model = pdb.input(file_name=params.simul_utils.input.model.file_name).xray_structure_simple(
crystal_symmetry=phil_xs,
)
print >> log, "Atomic model summary"
print >> log, "===================="
model.show_summary()
print >> log
#-------------------------------------------------------------------
# Step 3: make an F_model object to get model phases and amplitudes
#
print >> log, "Performing bulk solvent scaling"
print >> log, "==============================="
print >> log
print >> log
f_model_object = f_model.manager(
f_obs = miller_array,
r_free_flags = free_flags,
xray_structure = model )
f_model_object.update_solvent_and_scale(out=log)
fmodel = abs( f_model_object.f_model() ).set_observation_type( miller_array )
mockfmodel = None
if params.simul_utils.input.mock_model.file_name is not None:
print >> log, "Reading in mock model"
print >> log, "====================="
print >> log
print >> log
mock_model = pdb.input(file_name=params.simul_utils.input.mock_model.file_name).xray_structure_simple(
crystal_symmetry=phil_xs)
mock_f_model = f_model.manager(
f_obs = miller_array,
r_free_flags = free_flags,
xray_structure = mock_model )
mock_f_model.update(
k_sol = f_model_object.k_sol() ,
b_sol = f_model_object.b_sol() ,
b_cart = f_model_object.b_cart()
)
mockfmodel = abs( mock_f_model.f_model() ).set_observation_type( miller_array )
else:
mockfmodel = fmodel.deep_copy()
print >> log, "Making new data"
print >> log, "==============="
print >> log
print >> log
new_data_builder = error_swap( miller_array,
fmodel,
mockfmodel )
new_data = new_data_builder.new_obs
# we now have to write the data actually
print >> log, "Writing new data set"
print >> log, "===================="
mtz_dataset = new_data.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset.mtz_object().write(
file_name=params.simul_utils.output.hklout)
def sfcalc(args):
if len(args)==0:
print_help()
elif ( "--help" in args ):
print_help()
elif ( "--h" in args ):
print_help()
elif ("-h" in args ):
print_help()
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="sfcalc")
print >> log, "#phil __OFF__"
print >> log, "================="
print >> log, " SFCALC "
print >> log, "================="
print >> log
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown file or keyword:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
# now get the unit cell from the files
hkl_xs = None
pdb_xs = None
if params.sfcalc.input.model.file_name is not None:
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.sfcalc.input.model.file_name)
phil_xs = crystal.symmetry(
unit_cell=params.sfcalc.input.unit_cell,
space_group_info=params.sfcalc.input.space_group )
combined_xs = crystal.select_crystal_symmetry(
None,phil_xs, [pdb_xs],[None])
combined_xs.show_summary()
if combined_xs.unit_cell() is None:
raise Sorry("Unit cell not defined")
if combined_xs.space_group() is None:
raise Sorry("Space group not defined")
# inject the unit cell and symmetry in the phil scope please
params.sfcalc.input.unit_cell = combined_xs.unit_cell()
params.sfcalc.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
print >> log, "#phil __ON__"
new_params = master_params.format(python_object=params)
new_params.show(out=log)
print >> log, "#phil __END__"
pdb_model = None
if params.sfcalc.input.model.file_name is not None:
pdb_model = pdb.input(file_name=params.sfcalc.input.model.file_name)
model = pdb_model.xray_structure_simple(crystal_symmetry=phil_xs)
print >> log, "Atomic model summary"
print >> log, "===================="
model.show_summary()
print >> log
#make an f_model object please
b_cart = params.sfcalc.parameters.overall.b_cart
b_cart = [b_cart.b_11,
b_cart.b_22,
b_cart.b_33,
b_cart.b_12,
b_cart.b_13,
b_cart.b_23]
dummy = abs(model.structure_factors(
d_min = params.sfcalc.parameters.d_min,
anomalous_flag = False).f_calc())
flags = dummy.generate_r_free_flags(fraction = 0.1,
max_free = 99999999)
fmodel = f_model.manager( xray_structure = model,
r_free_flags = flags,
target_name = "ls_wunit_k1",
f_obs = dummy,
b_cart = b_cart,
k_sol = params.sfcalc.parameters.solvent.k_sol,
b_sol = params.sfcalc.parameters.solvent.b_sol )
calc_data_with_solvent_contrib = fmodel.f_model()
calc_data_with_solvent_contrib = calc_data_with_solvent_contrib.array(
data=calc_data_with_solvent_contrib.data()*params.sfcalc.parameters.overall.k_overall)
result = None
label = None
if params.sfcalc.parameters.output_type == "complex":
result = calc_data_with_solvent_contrib
label="FMODEL"
if params.sfcalc.parameters.output_type == "amplitudes":
result = abs(calc_data_with_solvent_contrib)
label="FMODEL"
if params.sfcalc.parameters.output_type == "intensities":
result = abs(calc_data_with_solvent_contrib)
result = result.f_as_f_sq()
label="IMODEL"
#write an mtz file with the data
mtz_dataset = result.as_mtz_dataset(
column_root_label=label)
mtz_dataset.mtz_object().write(
file_name=params.sfcalc.output.hklout)
#write the logfile
logger = open( params.sfcalc.output.logfile, 'w')
print >> log, "writing log file with name %s"%(params.sfcalc.output.logfile)
print >> log
print >> logger, string_buffer.getvalue()
def run(args, command_name="phenix.twin_map_utils"):
log=sys.stdout
params=None
if (len(args) == 0 or "--help" in args or "--h" in args or "-h" in args):
print_help(command_name=command_name)
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="map_coefs")
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown file or keyword:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
"""
new_params = master_params.format(python_object=params)
new_params.show(out=log)
"""
# now get the unit cell from the pdb file
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.twin_utils.input.xray_data.file_name)
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.twin_utils.input.model.file_name)
phil_xs=None
if ([params.twin_utils.input.unit_cell,
params.twin_utils.input.space_group]).count(None)<2:
phil_xs = crystal.symmetry(
unit_cell=params.twin_utils.input.unit_cell,
space_group_info=params.twin_utils.input.space_group )
combined_xs = crystal.select_crystal_symmetry(
None,phil_xs, [pdb_xs],[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.twin_utils.input.unit_cell = combined_xs.unit_cell()
params.twin_utils.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
new_params = master_params.format(python_object=params)
new_params.show(out=log)
if params.twin_utils.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.twin_utils.input.space_group is None:
raise Sorry("space group not specified")
if params.twin_utils.input.xray_data.file_name is None:
raise Sorry("Xray data not specified")
if params.twin_utils.input.model.file_name is None:
raise Sorry("pdb file with model not specified")
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
phil_xs = crystal.symmetry(
unit_cell=params.twin_utils.input.unit_cell,
space_group_info=params.twin_utils.input.space_group )
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = phil_xs,
force_symmetry = True,
reflection_files=[])
miller_array = None
free_flags = None
tmp_params = utils.data_and_flags_master_params().extract()
# insert proper values please
tmp_params.file_name = params.twin_utils.input.xray_data.file_name
tmp_params.labels = params.twin_utils.input.xray_data.obs_labels
tmp_params.r_free_flags.file_name=params.twin_utils.input.xray_data.file_name
tmp_params.r_free_flags.label=params.twin_utils.input.xray_data.free_flag
tmp_object = utils.determine_data_and_flags( reflection_file_server = xray_data_server,
parameters = tmp_params, log=log )
miller_array = tmp_object.extract_data()
if miller_array.is_xray_intensity_array():
miller_array = miller_array.f_sq_as_f()
assert miller_array.is_xray_amplitude_array()
free_flags = tmp_object.extract_flags(data = miller_array)
print >> log
print >> log, "Attempting to extract Free R flags"
free_flags = free_flags.customized_copy( data = flex.bool( free_flags.data()==1 ) )
if free_flags is None:
free_flags = miller_array.generate_r_free_flags(use_lattice_symmetry=True)
assert miller_array.observation_type() is not None
print >> log
print >> log, "Summary info of observed data"
print >> log, "============================="
miller_array.show_summary(f=log)
print >> log
if miller_array.indices().size() == 0:
raise Sorry("No data available")
#----------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
model = pdb.input(file_name=params.twin_utils.input.model.file_name).xray_structure_simple(
crystal_symmetry=phil_xs)
print >> log, "Atomic model summary"
print >> log, "===================="
model.show_summary(f=log)
print >> log
#----------------------------------------------------------------
# step 3: get the twin laws for this xs
twin_laws = twin_analyses.twin_laws(
miller_array,
lattice_symmetry_max_delta=\
params.twin_utils.parameters.twinning.max_delta,
out=log)
print >> log
print >> log, "Preliminary data analyses"
print >> log, "=========================="
twin_laws.show(out=log)
#---------
# step 3:
# make twin model managers for all twin laws
print >> log
print >> log, "Overall and bulk solvent scale paranmeters and twin fraction estimation"
print >> log, "======================================================================="
twin_models = []
operator_count = 0
if params.twin_utils.parameters.twinning.twin_law is not None:
tmp_law = sgtbx.rt_mx( params.twin_utils.parameters.twinning.twin_law )
tmp_law = twin_analyses.twin_law(tmp_law,None,None,None,None,None)
twin_laws.operators = [ tmp_law ]
for twin_law in twin_laws.operators:
operator_count += 1
operator_hkl = sgtbx.change_of_basis_op( twin_law.operator ).as_hkl()
twin_model = twin_f_model.twin_model_manager(
f_obs=miller_array,
r_free_flags = free_flags,
xray_structure=model,
twin_law = twin_law.operator,
detwin_mode = params.twin_utils.parameters.twinning.detwin_mode,
out=log)
print >> log, "--- bulk solvent scaling ---"
twin_model.update_solvent_and_scale(update_f_part1=False)
twin_model.r_values()
twin_model.target()
twin_model.show_k_sol_b_sol_b_cart_target()
twin_model.show_essential()
wfofc = twin_model.map_coefficients(map_type="mFo-DFc" )
wtfofc = twin_model.map_coefficients(map_type="2mFo-DFc" )
grad = twin_model.map_coefficients(map_type="gradient" )
mtz_dataset = wtfofc.as_mtz_dataset(
column_root_label="FWT")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = wfofc,
column_root_label = "DFWT"
)
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = grad,
column_root_label = "GRAD"
)
name = params.twin_utils.output.map_coeffs_root+"_"+str(operator_count)+".mtz"
print >> log
print >> log, "Writing %s for twin law %s"%(name,operator_hkl)
print >> log
mtz_dataset.mtz_object().write(
file_name=name)
if params.twin_utils.output.obs_and_calc is not None:
# i want also a Fobs and Fmodel combined dataset please
mtz_dataset = miller_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = twin_model.f_model(),
column_root_label="FMODEL")
name = params.twin_utils.output.obs_and_calc
mtz_dataset.mtz_object().write(
file_name=name)
if len(twin_laws.operators)==0:
print >> log
print >> log, "No twin laws were found"
print >> log, "Performing maximum likelihood based bulk solvent scaling"
f_model_object = f_model.manager(
f_obs = miller_array,
r_free_flags = free_flags,
xray_structure = model )
f_model_object.update_solvent_and_scale(out=log)
tfofc = f_model_object.map_coefficients(map_type="2mFobs-DFmodel")
fofc = f_model_object.map_coefficients(map_type="mFobs-DFmodel")
mtz_dataset = tfofc.as_mtz_dataset(
column_root_label="FWT")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = fofc,
column_root_label = "DELFWT"
)
name = params.twin_utils.output.map_coeffs_root+"_ML.mtz"
mtz_dataset.mtz_object().write(
file_name=name)
if params.twin_utils.output.obs_and_calc is not None:
# i want also a Fobs and Fmodel combined dataset please
mtz_dataset = miller_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = f_model_object.f_model(),
column_root_label="FMODEL")
name = params.twin_utils.output.obs_and_calc
mtz_dataset.mtz_object().write(
file_name=name)
print >> log
print >> log
print >> log, "All done \n"
logfile = open(params.twin_utils.output.logfile,'w')
print >> logfile, string_buffer.getvalue()
print >> log
